How to insert millions of document using batch process in MongoDB with NodeJS

Question

I've written nodejs script to read the JSON file and insert multiple records into mongo collection, I've millions of records and I don't want to insert all documents in one short. I would like to insert 300 documents per second and sleep for 30 sec and insert another 300 and so on? I'm new with NodeJS - can you please how can I achieve this with my below code? Appreciated your help and support.

app.js

const mongoClient = require("mongodb").MongoClient;
const util = require('util');
const fs = require('fs');

let database = null;
new mongoClient('mongodb://localhost:3000/', {
    auth: {
        user: 'admin',
        password: 'password',
    }
}).connect(
    (err, db) => {
        if (err) return console.error(err);
        database = db.db('myDB');
        fs.readFile('data.json', 'utf8', function(err, data) {
            if (err) throw err;
            var json = JSON.parse(data);
            database.collection("test").insertMany(json, function(err, doc) {
                console.log("Documents inserting");
                if (err) throw err;
            });
            //db.close();
        });
    });

Sample Data: - I've millions of records like this in a single file.

 [{
    "firstName": "Ariel",
    "lastName": "Bailey"
 }, {
    "firstName": "Lura",
    "lastName": "Buckridge"
 }, {
    "firstName": "Milton",
    "lastName": "Macejkovic"
 }, {
    "firstName": "Carolyn",
    "lastName": "Hegmann"
 }, {
    "firstName": "Sid",
    "lastName": "Beer"
 }]

@yeya JSON file will be 40-45mb and sleep point is after each 300 record insert. — learn groovy
– learn groovy, Commented Dec 18, 2018 at 2:24
@PyaePhyoeShein I got stuck how to read millions of records and insert in a batch of 300 and pause for some time and insert again after 30 sec. This process continues until all records insert. My current code is inserting all millions record in one shot - which I don't want. — learn groovy
– learn groovy, Commented Dec 18, 2018 at 2:34

ack_inc · Accepted Answer · 2018-12-19 03:24:34Z

2

You can use setTimeout to 'batch' your inserts like this:

fs.readFile('data.json', 'utf8', function (err, data) {
  if (err) throw err;
  var json = JSON.parse(data);
  processRecords(0, 300, 30 * 1000);

  function processRecords(startIdx, n, delay) {
    if (startIdx >= json.length) return db.close();

    database.collection("test").insertMany(json.slice(startIdx, startIdx + n), function (err, doc) {
      if (err) throw err;

      setTimeout(() => processRecords(startIdx + n, n, delay), delay);
    });
  }
});

edited Dec 19, 2018 at 3:24

answered Dec 18, 2018 at 3:24

ack_inc

1,1111 gold badge7 silver badges17 bronze badges

Sign up to request clarification or add additional context in comments.

2 Comments

learn groovy Over a year ago

Thanks, it works but sleep is happening only for the first 300 records- after 30 seconds other records are inserting in one go.. :(

ack_inc Over a year ago

Forgot to give processRecords its third argument inside the setTimeout call. Try now.

Pyae Phyoe Shein · Accepted Answer · 2018-12-18 03:14:34Z

Here is basic concept of what you want to get. Honestly, it's not 100% perfect, the rest is your effort.

var ids = 0, offset = 10000;

function readJson() {
    var json = /* read JSON file */;
    return json;
}

function splitWithBatch(ids, offset) {
    var jsonObj = {};
    for(var i = ids; i < offset; i++){
        jsonObj.push(json[i]);
    }
    return Q.resolve(jsonObj);
}

function callSending(ids) {
    return splitWithBatch(ids, 0).then(ProcessToSave);
}

function ProcessToSave(json) {
    var quantityLimit = 1000;
    return SendToMongo(json).then(doNextBatch);
    function doNextBatch() {
        if (json.length === quantityLimit) {
            return splitWithBatch(ids, offset + quantityLimit);
        } else {
            return Q.resolve(null);
        }
    }
}

function SendToMongo(json) {
    database.collection('test').insertMany(json, function(err, doc) {
        if (err) throw err;
    });
}

readJson().then(callSending).then(
    function(){
        console.log('done');
    },
    function (err){
        console.log('err', err.stack, err);
    }
);

Collectives™ on Stack Overflow

How to insert millions of document using batch process in MongoDB with NodeJS

2 Answers 2

2 Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

2 Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related