Seems like you are going to have limited options since you want the check to occur before the upload. I think the best you are going to get is to use javascript to validate the extension of the file. You could build a hash of valid extensions and then look to see if the extension of the file being uploaded existed in the hash.
HTML:
<input type="file" name="FILENAME" size="20" onchange="check_extension(this.value,"upload");"/>
<input type="submit" id="upload" name="upload" value="Attach" disabled="disabled" />
Javascript:
var hash = {
'xls' : 1,
'xlsx' : 1,
};
function check_extension(filename,submitId) {
var re = /\..+$/;
var ext = filename.match(re);
var submitEl = document.getElementById(submitId);
if (hash[ext]) {
submitEl.disabled = false;
return true;
} else {
alert("Invalid filename, please select another file");
submitEl.disabled = true;
return false;
}
}