Try this, looks for WOW64 (32-bit on 64-bit) or Win64 (native 64-bit) in the user-agent string.
if (navigator.userAgent.indexOf("WOW64") != -1 ||
navigator.userAgent.indexOf("Win64") != -1 ){
alert("This is a 64 bit OS");
} else {
alert("Not a 64 bit OS");
}