You can use a multipart form post instead. You can create a file using the string contents:

var form = window.FormData();
form.set('fileName', 'foo.txt');
form.set('fileContent', '...');

LABKEY.Ajax.request({
    url: LABKEY.ActionURL.buildURL("assay", "assayFileUpload", null, { protocolId: assay.id }),
    form: form
});

Or you can let the user upload a file using a file input:

<form id='myform'>
  <input name='myfile' type='file'>
</form>
<button onclick='uploadMe();return false;'>Upload</button>

<script>
function uploadMe()
{
  var form = document.getElementById('myform');
  LABKEY.Ajax.request({
    url: LABKEY.ActionURL.buildURL("assay", "assayFileUpload", null, {protocolId: assay.id }),
    form: form
  });
}
</scrit>