Migrate large json file into Mongodb using nodejs

Question

I'm using nodejs to migrate data into mongodb, the json file which i'm trying to migrate is too large, when I try to migrate just some data (about 8000 docs), it works, otherwise I'm getting this issue :

MongooseError: Operation `products.insertOne()` buffering timed out after 10000ms

at Timeout.<anonymous> (\migration- 
module\node_modules\mongoose\lib\drivers\node-mongodb-native\collection.js:148:23)

at listOnTimeout (internal/timers.js:557:17)

at processTimers (internal/timers.js:500:7)

MongooseError: Operation `products.insertOne()` buffering timed out after 10000ms

 at Timeout.<anonymous> (\migration- 
module\node_modules\mongoose\lib\drivers\node-mongodb-native\collection.js:148:23)

at listOnTimeout (internal/timers.js:557:17)

at processTimers (internal/timers.js:500:7)

the script :

/** @format */
fs = require('fs')
var mongoose = require('mongoose')
 mongoose.connect('mongodb://host')
 mongoose.Promise = global.Promise
 var schema = new mongoose.Schema({
 //some fields
  })
  var d = fs.readFileSync('./data/file.json', 'utf8', (err, data) => {
   if (err) throw err
   return d
   })
  var e = JSON.parse(d)
  var Product = mongoose.model('product', schema)
  //console.log(e)
  for (var i = 0; i < e.length; i++) {
   // data process
   // another data process
  var product = new Product(e[i])
  product.save(function (err) {
  if (err) return console.log(err)
  })
 }
 e.length === i ? console.log('Migration Done successfully') : ''

any solution to get ride of this problem please ?

Check this : stackoverflow.com/questions/40585705/… - You need to configure timeouts or turn them off — Goran.it
– Goran.it, Commented Feb 25, 2022 at 9:06
It's not connection, it's buffering. Turn it off on schema level, or globally mongoose.set('bufferCommands', false); — Alex Blex
– Alex Blex, Commented Feb 25, 2022 at 9:30
I put this command under mongoose connect but I got this error :MongooseError: Cannot call products.insertOne() before initial connection is complete if bufferCommands = false. Make sure you await mongoose.connect() if you have bufferCommands = false. — Java user
– Java user, Commented Feb 25, 2022 at 9:38

Alex Blex · Accepted Answer · 2022-02-25 10:41:49Z

0

Disabling buffering and few other fixes:

fs = require('fs')
var mongoose = require('mongoose')

mongoose.set('bufferCommands', false);
var schema = new mongoose.Schema({
 //some fields
})
var Product = mongoose.model('product', schema)

var d = fs.readFileSync('./data/file.json', 'utf8', (err, data) => {
   if (err) throw err
     return data   // <= corrected here
})

var e = JSON.parse(d);
  //console.log(e)

async function main() {
  await mongoose.connect('mongodb://host')   // <= corrected here
  for (var i = 0; i < e.length; i++) {
     // data process
     // another data process
     var product = new Product(e[i]);
     await product.save();     // <= corrected here
  })
}

main().then(()=>{console.log('Migration Done successfully')})

answered Feb 25, 2022 at 10:41

Alex Blex

37.3k7 gold badges53 silver badges86 bronze badges

Sign up to request clarification or add additional context in comments.

3 Comments

Java user Over a year ago

it worked for a while (35 min) then it gave this error :(node:396) UnhandledPromiseRejectionWarning: MongoNetworkError: connection 1 to @ip:27017 closed. , is it normal ?

Alex Blex Over a year ago

No, unhandled rejections are never normal. Please don't just copy-paste the code. It's here to illustrate the answer to the original question. In real-life application you wrap all async calls in try-catch and gracefully shut down on exception. This particular one says the database has gone. You will need to check database logs to understand why it happened.

Java user Over a year ago

you are right :)

Collectives™ on Stack Overflow

Migrate large json file into Mongodb using nodejs

1 Answer 1

3 Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

3 Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related