As the ajax call is asynchronous, you have to prevent the form from submitting, and then when a result is returned, you check if it matches the condition and submit the form with the native submit handler, avoiding the preventDefault()
in the jQuery event handler :
$("form").submit(function(e) {
e.preventDefault();
var self = this,
tray = $('select[name=tray_id]').val();
$.ajax({
type: "POST",
url: "modules/reserve-check.php",
data: {tray_id: tray},
cache: false
}).done(function(result) {
if (result == "") self.submit();
}).fail(function() {
alert('error');
});
});