Correct way to insert many records into Mongodb with Node.js

Question

I was wondering what is the correct way to do bulk inserts into Mongodb (although could be any other database) with Node.js

I have written the following code as an example, although I believe it is floored as db.close() may be run before all the asynchronous collection.insert calls have completed.

MongoClient.connect('mongodb://127.0.0.1:27017/test', function (err, db) {
    var i, collection;
    if (err) {
        throw err;
    }
    collection = db.collection('entries');
    for (i = 0; i < entries.length; i++) {
        collection.insert(entries[i].entry);
    }
    db.close();
});

chridam · Accepted Answer · 2018-03-12 15:43:50Z

26

If your MongoDB server is 2.6 or newer, it would be better to take advantage of using a write commands Bulk API that allow for the execution of bulk insert operations which are simply abstractions on top of the server to make it easy to build bulk operations and thus get perfomance gains with your update over large collections.

Sending the bulk insert operations in batches results in less traffic to the server and thus performs efficient wire transactions by not sending everything all in individual statements, but rather breaking up into manageable chunks for server commitment. There is also less time waiting for the response in the callback with this approach.

These bulk operations come mainly in two flavours:

Ordered bulk operations. These operations execute all the operation in order and error out on the first write error.
Unordered bulk operations. These operations execute all the operations in parallel and aggregates up all the errors. Unordered bulk operations do not guarantee order of execution.

Note, for older servers than 2.6 the API will downconvert the operations. However it's not possible to downconvert 100% so there might be some edge cases where it cannot correctly report the right numbers.

In your case, you could implement the Bulk API insert operation in batches of 1000 like this:

For MongoDB 3.2+ using bulkWrite

var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost:27017/test';
var entries = [ ... ] // a huge array containing the entry objects

var createNewEntries = function(db, entries, callback) {

    // Get the collection and bulk api artefacts
    var collection = db.collection('entries'),          
        bulkUpdateOps = [];    

    entries.forEach(function(doc) {
        bulkUpdateOps.push({ "insertOne": { "document": doc } });

        if (bulkUpdateOps.length === 1000) {
            collection.bulkWrite(bulkUpdateOps).then(function(r) {
                // do something with result
            });
            bulkUpdateOps = [];
        }
    })

    if (bulkUpdateOps.length > 0) {
        collection.bulkWrite(bulkUpdateOps).then(function(r) {
            // do something with result
        });
    }
};

For MongoDB <3.2

var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost:27017/test';
var entries = [ ... ] // a huge array containing the entry objects

var createNewEntries = function(db, entries, callback) {

    // Get the collection and bulk api artefacts
    var collection = db.collection('entries'),          
        bulk = collection.initializeOrderedBulkOp(), // Initialize the Ordered Batch
        counter = 0;    

    // Execute the forEach method, triggers for each entry in the array
    entries.forEach(function(obj) {         

        bulk.insert(obj);           
        counter++;

        if (counter % 1000 == 0 ) {
            // Execute the operation
            bulk.execute(function(err, result) {  
                // re-initialise batch operation           
                bulk = collection.initializeOrderedBulkOp();
                callback();
            });
        }
    });             

    if (counter % 1000 != 0 ){
        bulk.execute(function(err, result) {
            // do something with result 
            callback();             
        }); 
    } 
};

Call the createNewEntries() function.

MongoClient.connect(url, function(err, db) {
    createNewEntries(db, entries, function() {
        db.close();
    });
});

edited Mar 12, 2018 at 15:43

answered Dec 30, 2015 at 13:50

chridam

104k26 gold badges246 silver badges243 bronze badges

Sign up to request clarification or add additional context in comments.

6 Comments

Richard Hensman Over a year ago

How would you close the db if counter % 1000 == 0

chridam Over a year ago

You may have to add the db.close(); statement after the if (counter % 1000 != 0 ){ ... } statement block to close the db after all.

Richard Hensman Over a year ago

Do you not then risk calling db.close() whilst the bulk.execute calls from the forEach are still running?

chridam Over a year ago

You won't because here bulk.execute() is a mongodb write operation and its an asynchronous IO call. This allows node.js to proceed with the event loop before bulk.execute() is done with its db writes and calls back. I've updated my answer with this callback approach.

Ong Kong Tat Over a year ago

This is so much more convenient and it works beautifully! Thanks for this answer.

|

Arjan Frans · Accepted Answer · 2015-12-30 13:24:12Z

11

You can use insertMany. It accepts an array of objects. Check the API.

answered Dec 30, 2015 at 13:24

Arjan Frans

4994 silver badges10 bronze badges

3 Comments

Richard Hensman Over a year ago

That is ok if you have a small number of records to insert, but what if you had many thousands?

HDB Over a year ago

In bulk operation mongoDB(3.x) takes 1000 document batch in single group and for more documents it create groups and executes it. Please refer docs.mongodb.com/v3.2/reference/method/db.collection.insertMany/…

Karthic Raghupathi Over a year ago

Another place in the documentation where the earlier comment is re-iterated regarding insertMany: PyMongo will automatically split the batch into smaller sub-batches based on the maximum message size accepted by MongoDB, supporting very large bulk insert operations.

Dimuthu · Accepted Answer · 2017-10-22 03:16:29Z

2

New in version 3.2.

The db.collection.bulkWrite() method provides the ability to perform bulk insert, update, and remove operations. MongoDB also supports bulk insert through the db.collection.insertMany().

In bulkWrite it is supporting only insertOne, updateOne, updateMany, replaceOne, deleteOne, deleteMany

In your case to insert data using single line of code, it can use insertMany option.

 MongoClient.connect('mongodb://127.0.0.1:27017/test', function (err, db) {
            var i, collection;
            if (err) {
                throw err;
            }
            collection = db.collection('entries');
            collection.insertMany(entries)
            db.close();
        });

answered Oct 22, 2017 at 3:16

Dimuthu

1,6811 gold badge14 silver badges16 bronze badges

Comments

Mrityunjay pandey · Accepted Answer · 2017-09-21 17:54:21Z

1

var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost:27017/test';
var data1={
    name:'Data1',
    work:'student',
    No:4355453,
    Date_of_birth:new Date(1996,10,17)
};

var data2={
    name:'Data2',
    work:'student',
    No:4355453,
    Date_of_birth:new Date(1996,10,17)
};

MongoClient.connect(url, function(err, db) {
    if(err!=null){
        return console.log(err.message)
    }

    //insertOne
    db.collection("App").insertOne(data1,function (err,data) {

        if(err!=null){
            return console.log(err);
        }
        console.log(data.ops[0]);
    });

    //insertMany

var Data=[data1,data2];

db.collection("App").insertMany(Data,forceServerObjectId=true,function (err,data) {

        if(err!=null){
            return console.log(err);
        }
        console.log(data.ops);
    });
    db.close();
});

edited Sep 21, 2017 at 17:54

answered Sep 21, 2017 at 15:51

Mrityunjay pandey

1291 silver badge5 bronze badges

3 Comments

Donald Duck is with Ukraine Over a year ago

While this code may answer the question, providing additional context regarding why and/or how this code answers the question improves its long-term value.

Mrityunjay pandey Over a year ago

This code only for insert many record or a single record example

Aquafreax Over a year ago

"forceServerObjectId=true" worked for Duplicate Key Error id, Thanks

Collectives™ on Stack Overflow

Correct way to insert many records into Mongodb with Node.js

4 Answers 4

6 Comments

3 Comments

Comments

3 Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

4 Answers 4

6 Comments

3 Comments

Comments

3 Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related