I finally figured out the problem:

//         function submitbutton_click() {
//             if (!ValidatorOnSubmit())
//                 return false;
//             var submitbutton = document.getElementById('<%=btSubmit.ClientID %>');
//             var uploadobj = document.getElementById('<%=AjaxUpload.ClientID %>');
//             if (!window.filesuploaded) {
//                 if (uploadobj.getqueuecount() > 0) {
//                     uploadobj.startupload();
//                 }
//                 return false;
//             }
//             window.filesuploaded = false;
//             return true;
//         }

         function submitbutton_click() {
             if (!ValidatorOnSubmit())
                 return false;

             var uploadobj = document.getElementById('<%=AjaxUpload.ClientID %>');
             if (uploadobj.getqueuecount() == 0)
                 return false;

             try {
                 var submit = document.getElementById('<%=btSubmit.ClientID %>');
                 var cancel = document.getElementById('<%=btCancelUpload.ClientID %>');
                 var span = document.getElementById('<%=span_upload.ClientID %>');
                 submit.style.visibility = 'hidden';
                 cancel.style.visibility = 'visible';
                 span.removeChild(submit);
                 span.removeChild(cancel);
                 span.appendChild(cancel);
                 span.appendChild(submit);
             } catch (e) {
                 alert(e);
             }

             if (!window.filesuploaded) {
                 if (uploadobj.getqueuecount() > 0) {
                     uploadobj.startupload();
                 }
                 return false;
             }
             window.filesuploaded = false;
             return true;
         }

the first version breaks the upload, the seccond one doesnt. I was trying to replace the upload button with the cancel button. is there any way to do this without breaking the upload?