How to use dynamoDB batchWriteItem with nodejs sdk?

Question

I have a use case where I need to perform a batch_write operation on dynamodb. I referred this article which has a good solution for similar use case. I implemented it with few cleanup in my code and it works as expected.

const _ = require('lodash');

// helper methods

async function asyncForEach(array, cb) {
  await Promise.all(array.map(async (item) => {
    await cb(item, array);
  }));
}

function to(promise) {
  return promise.then((data) => [null, data])
    .catch((err) => [err]);
}
const call = function (params) {
  return dynamoDb.batchWriteItem(params).promise();
};
async function batchWrite25(arrayOf25, tableName) {
  // 25 is as many as you can write in one time
  const itemsArray = [];
  _.forEach(arrayOf25, (item) => {
    itemsArray.push({
      PutRequest: {
        Item: item,
      },
    });
  });
    const params = {
        RequestItems: {
        [tableName]: itemsArray,
        },
    };

  await to(call(params));
}
async function batchWrite(itemArray, tableName) {
  let mainIndex = 0;
  let subIndex = 0;
  let arrayOf25 = [];
  const arrayLength = itemArray.length;
  await asyncForEach(itemArray, async (item) => {
    arrayOf25.push(item);
    subIndex += 1;
    mainIndex += 1;

    // 25 is as many as you can write in one time
    if (subIndex % 25 === 0 || mainIndex === arrayLength) {
      await to(batchWrite25(arrayOf25, tableName));
      subIndex = 0; // reset
      arrayOf25 = [];
    }
  });
}

module.exports = {
  batchWrite,
};

However, the code looks a bit complicated here with so many callbacks involved. Is there a cleaner way of writing the same thing without using -- call or asyncForEach or to methods ?

jarmod · Accepted Answer · 2022-07-08 14:32:11Z

3

Here's one simple way to batch the items:

const BATCH_MAX = 25;

const batchWrite = async (items, table_name) => {
  const BATCHES = Math.floor((items.length + BATCH_MAX - 1) / BATCH_MAX);

  for (let batch = 0; batch < BATCHES; batch++) {
    const itemsArray = [];

    for (let ii = 0; ii < BATCH_MAX; ii++) {
      const index = batch * BATCH_MAX + ii;

      if (index >= items.length) break;

      itemsArray.push({
        PutRequest: {
          Item: items[index],
        },
      });
    }

    const params = {
      RequestItems: {
        [table_name]: itemsArray,
      },
    };

    console.log("Batch", batch, "write", itemsArray.length, "items");
    await dynamodb.batchWriteItem(params).promise();
  }
};

To make the entire process asynchronous, you can convert this function to return an array of promises and later call Promise.all(promises) on that array. For example:

const batchWrite = (items, table_name) => {
  const promises = [];
  const BATCHES = Math.floor((items.length + BATCH_MAX - 1) / BATCH_MAX);

  for (let batch = 0; batch < BATCHES; batch++) {
    // same code as above here ...
    promises.push(dynamodb.batchWriteItem(params).promise());
  }

  return promises;
};

edited Jul 8, 2022 at 14:32

answered Jul 7, 2022 at 17:23

jarmod

79.8k18 gold badges132 silver badges137 bronze badges

Sign up to request clarification or add additional context in comments.

2 Comments

Naxi Over a year ago

await inside a for loop makes it synchronous. how to make this asynchronous ?

jarmod Over a year ago

Updated answer to accumulate individual promises.

Naxi · Accepted Answer · 2022-07-11 14:06:55Z

0

A much cleaner way using lodash that worked for me is listed below. Hope this helps somone.

batchWrite=async ()=> {
    const batchSplitArr=_.chunk(this.dynamoPayload,25); //dynamoPayload has the entire payload in the desired format for dynamodb insertion.
    await Promise.all(
        batchSplitArr.map(async (item) => {
            const params = {
            RequestItems: {
              [this.tableName]: item,
            },
          };
            await this.dynamoDb.batchWriteItem(params).promise();
        })
      );
  };

answered Jul 11, 2022 at 14:06

Naxi

2,2647 gold badges50 silver badges102 bronze badges

2 Comments

jarmod Over a year ago

Note that this results in a slightly different outcome. Instead of a single RequestItems array of items targeting the table, this lodash solution results in multiple RequestItems, one per item, each targeting the same table. This might impact performance under the hood, unless DynamoDB merges them.

Naxi Over a year ago

works as expected.

Collectives™ on Stack Overflow

How to use dynamoDB batchWriteItem with nodejs sdk?

2 Answers 2

2 Comments

2 Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

2 Comments

2 Comments

Your Answer

Sign up or log in

Post as a guest

Related