2

Hi I am trying to upload multiple files, file writing is proper. but I want to move the files into mongo grid fs, the 1st file is only moving properly rest of the file are not moving into grid fs and not unlinking also.

Here is my code:

for (var i = 0; i < files.length; i++) {
    (function(i) {
        var singleFile = files[i];
        var fileData = JSON.parse(files[i].name);
        var fileName = fileData.fileName;
        var fileType = files[i].type;
        var uniqId = fileData.attachmentId;
        var targetPath = "./attachments/" + uniqId;
        var tmp_path = files[i].path;

    console.log("uniqId : " + uniqId);
    //uploadFile(tmp_path,targetPath,fileName,uniqId,fileType);

    fs.readFile(tmp_path, function(err, data) {
        fs.writeFile(targetPath, data, function(err) {
            if (!err) {
                mongoose.connect(configSettings.mongodb.ip, configSettings.mongodb.databasename, configSettings.mongodb.port, function(err, succ) {
                    var id = new ObjectID(uniqId);
                    new GridStore(mongoose.connection.db, id, fileName, 'w', {
                        'content_type': 'application/octet-stream'
                    }).open(function(err, gs) {
                        gs.writeFile(targetPath, function(err) {
                            if (err) {
                                console.log("err");
                            }
                            gs.close(function(err) {
                                console.log("success");
                                fs.unlink("./attachments/" + uniqId, function(err) {
                                    console.log("err : " + err);
                                    console.log(uniqId + ' ::successfully deleted ');
                                });
                                mongoose.connection.close();
                            });
                        });
                    });
                });
            }
        });
    });
   })(i);
}

    res.send("success");

This one i have tried with closure function, even i have tried with creating separate function also but the same result comes(only one file inserted and unlinked).

what i have to do in this case to store all files in gridfs and unlink from the temporary folder?

1
  • does mongodb support all these concurrent opening and closing? shouldn't it be better to first establish the connexion to mongobd and open the grid, then, in this opening succes function, place the loop to launch all the files processing? Commented Aug 23, 2017 at 14:07

3 Answers 3

1

Here is my guess. Since your calls are deferred to later, your function ends and the file list is probably erased by Node or whatever, you can see a simplified example here :

let myList = [1, 2, 3, 4, 5];

function doWork(list) {
  console.log('Start of function');
  
  for (let i = 0; i < list.length; i++) {
    (function(i) {
      setTimeout(function() {
        console.log('Working on index', i);
        console.log('list[i] :', list[i]);
      }, 100);
    })(i);
  }

  console.log('End of function');
}

doWork(myList);

myList[4] = null;

The solution is to enclose the file (or the list/index couple) itself to keep it alive in your context :

let myList = [1, 2, 3, 4, 5];

function doWork(list) {
  console.log('Start of function');
  
  for (let i = 0; i < list.length; i++) {
    (function(num) {
      setTimeout(function() {
        console.log('Num :', num);
      }, 100);
    })(list[i]);
  }

  console.log('End of function');
}

doWork(myList);

myList[4] = null;

Sign up to request clarification or add additional context in comments.

Comments

1

Actually the problem is with the gridFs connection, files are uploading inside loop, the grid connection is up and writing the first file and the loop continues parallel. once connection is on from the gridfs it should close before inserting another file and have to reconnect for the next time properly.

Here is the working code check it :

var files = req.files.fileData[0];
var count = "";

if(files instanceof Array){
    count = files.length;
}
else{
    count = 1;
}

if(!fs.existsSync('./attachments')){
    console.log("Creating the attachments folder.");
    fs.mkdir('./attachments');
}   

function moveToGrid(gridArr,callback){      
    var uniqId = gridArr.uniqId, 
        filePath = gridArr.targetPath, 
        fileName = gridArr.fileName,
        fileType = gridArr.fileType;

        console.log(filePath+' '+fileName+' '+fileType + ' '+uniqId);

    mongoose.connect(configSettings.mongodb.ip, configSettings.mongodb.databasename, configSettings.mongodb.port, function (err, succ){

        var uploadResponse   = function(uniqId){
            // Delete the tmp file
            fs.unlink("./attachments/" + uniqId, function (err) {
                console.log(uniqId + ' ::temp file successfully deleted ');
            });
             mongoose.connection.close();
             console.log("=========request END time==========::::" + (new Date()).toString() + "::" +(new Date()).getMilliseconds() );
             callback();
        };
        var id = new ObjectID(uniqId);
        new GridStore(mongoose.connection.db,id,fileName, 'w',{'content_type':'application/octet-stream'}).open(function(err, gs) {
            gs.writeFile(filePath, function(err) {
                if(err){
                console.log("err : "+err);
                }
                gs.close(function(err) {
                    uploadResponse(uniqId);
                });
            });
        });
    });
}

var recursive = function(gridArr){  
    function uploader(i){
      if( i < gridArr.length ){
        moveToGrid(gridArr[i],
             function(){
              uploader(i+1)
             });
       }
       else{
        res.send("success");
       }
    }
    uploader(0); 
}

if(count > 1){
    var gridArr = [];
    for(var i=0; i<files.length; i++){
        (function(i){
            var singleFile = files[i];
                var fileData = JSON.parse(singleFile.name);
                var fileName = fileData.fileName;
                var fileType = singleFile.type;
                var uniqId   =  fileData.attachmentId;
                var targetPath =  "./attachments/" + uniqId;
                var tmp_path = singleFile.path;

                fs.readFile(tmp_path,function(err,data){
                    fs.writeFile(targetPath,data,function(err){
                        if(!err){
                            gridArr.push({"uniqId":uniqId, "targetPath":targetPath, "fileName":fileName, "fileType":fileType});
                            count --;
                            if(count == 0){
                                recursive(gridArr);
                            }
                        }
                    });
                });
        }(i));                  
    }
}
else{
    var gridArr = [];
    var singleFile = files;
    var fileData = JSON.parse(singleFile.name);
    var fileName = fileData.fileName;
    var fileType = singleFile.type;
    var uniqId   =  fileData.attachmentId;
    var targetPath =  "./attachments/" + uniqId;
    var tmp_path = singleFile.path;

    fs.readFile(tmp_path,function(err,data){
        fs.writeFile(targetPath,data,function(err){
            if(!err){
                gridArr.push({"uniqId":uniqId, "targetPath":targetPath, "fileName":fileName, "fileType":fileType});
                count --;
                if(count == 0){
                    recursive(gridArr);
                }
            }
        });
    });     
}

Using this code, we can insert multiple files to mongo gridFs.. Hope this helps!

Comments

0

short answer learn here how to async function inside loop

(function loop(i) {
if(i==4) return
setTimeout(function() {
  console.log('Working on index', i);
  loop(++i)
 }, 1000);
})(0);

TL;DR https://mongodb.github.io/node-mongodb-native/api-generated/gridstore.html

Comments

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.