This is how I did it:
function OnClientValidationFailed(sender, args) {
var allowedExtensions = $(".uploadFileExtensions").html();
alert('Sorry but that file extension is not allowed. Only the following file extensions are allowed ' + allowedExtensions);
var upload = document.getElementById('<%=radUpload.ClientID %>');
var index = $(args.get_row()).index();
sender.deleteFileInputAt(index);
}
Make sure you set the OnClientValidationFailed event up on the control:
<telerik:RadAsyncUpload ID="radUpload" runat="server" AllowedFileExtensions="pdf,jpeg,jpg,png,ppt" OnClientValidationFailed="OnClientValidationFailed" />