Angular/Node.js File Upload only works on files < 64k

Question

I'm having an issue getting file uploads to work on a project of mine (PS I'm relatively new to Web programming). I'm using NodeJS/Express/Mongoose/Mongo with busboy for the API/backend, and Angular with ng-file-upload for the front end.

I am trying to implement a basic file upload/download through which the file is stored in a Mongoose Schema:

var mongoose = require('mongoose');
var Schema = mongoose.Schema;

var FileSchema = new Schema({
  filename: { type: String },
  owner: { type: String },
  contentType: { type: String },
  length: { type: Number },
  uploaddate: { type: Date, default: Date.now },
  metadata: {},
  file: { data: Buffer, contentType: String }
});

module.exports = mongoose.model('Files', FileSchema);

The Node API:

api.route('/files')
  .post(function (req, res) {
   if (req.busboy) {
     var newFile = new File();
     var part = 0, buf = [], len = 0;
     var meta = {};
     var md5 = crypto.createHash('md5');

     req.busboy.on('field', function (key, value, keyTruncated, valueTruncated) {
       console.log('[FIELD] KEY:' + key + ' VALUE:' + value);
       if (key == 'owner') newFile.owner = value;
       else meta[key] = value;
   });

   req.busboy.on('file', function (fieldname, file, filename, encoding, mimetype) {
     console.log('File [' + fieldname + ']: filename: ' + filename + ', encoding: ' + encoding + ', mimetype: ' + mimetype);
     newFile.filename = filename;
     //newFile.contentType = mimetype;

     file.on('data', function (data) {
       console.log('File [' + fieldname + '] got ' + data.length + ' bytes');
       buf.push(data);
       len = len + data.length;
       part++;
     });
     file.on('end', function () {

       newFile.file.data = Buffer.concat(buf, part);
       md5.update(newFile.file.data);
       newFile.file.contentType = mimetype;
       newFile.length = len;
       console.log('File [' + fieldname + '] Finished');
       meta['md5'] = md5.digest('hex');
       newFile.metadata = meta;
       newFile.save(function (err, data) {
         if (err) console.log(err);
         res.json({ _id: newFile._id });
       })
     });

   });
   req.pipe(req.busboy);
 }
});

This works perfectly fine for any files <64KB, or within one upload chuck. I've verified the file MD5 against the stored values and retrieved from the DB and images load fine. The problem is when the upload is broken into chunks. Something happens causing it to be transformed somehow and break the file. The original file's MD5 and the stored MD5 of files >64kb or more than one chunk are different...

What am I messing up?

Or if anyone has any suggestions of how to better execute this functionality.

Thanks!

mscdex · Accepted Answer · 2015-08-28 17:08:32Z

2

The second (optional) argument to Buffer.concat() is the total byte length of all Buffers in the array, not simply buf.length. This is an optimization since otherwise Buffer.concat() will have to loop through the Buffers to calculate the total size for the new Buffer.

With that in mind you'd need to change:

newFile.file.data = Buffer.concat(buf, part);

to:

newFile.file.data = Buffer.concat(buf, len);

answered Aug 28, 2015 at 17:08

mscdex

107k15 gold badges201 silver badges159 bronze badges

Sign up to request clarification or add additional context in comments.

1 Comment

Eric Over a year ago

Worked like a charm! Thanks sooo much!

Collectives™ on Stack Overflow

Angular/Node.js File Upload only works on files < 64k

1 Answer 1

1 Comment

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

1 Comment

Your Answer

Sign up or log in

Post as a guest

Related