load S3 file in Lambda with Node.js version 8.10

Question

I have been trying (and failing) to load a file from an S3 bucket using s3.getObject in Node version 8.10.

I found a great post with a reply that almost worked here but the syntax doesn't quite work in 8.10 and no matter how I re-arrange the code I can't get it to work.

var AWS = require('aws-sdk')
var s3 = new AWS.S3();

var fileData = null;

exports.handler = (event, context, callback) => {
    console.log('I am in the main procedure');
    var params = {
        Bucket: "change_for_your_bucket",
        Key:  "change_to_your_json_file"
    };

    fetchDataFromS3(params);
    console.log('I am in the main procedure, the function above should be waiting but it is not');
    waitForFileLoadBeforeDoingSomething(event, context, callback);

    const s = JSON.stringify(fileData.Body.toString('utf-8'));
    console.log(`this is the file: ${s}`);
    console.log('I have the file!(dg.2)');    

};

function fetchDataFromS3(params)
{
    console.log('-------------- fetchDataFromS3:Start -------------------------');
    // gets the object from s3 => promise
    const uploadPromise = s3.getObject(params).promise();

    // returns the body of the s3 object
    uploadPromise
            .then(function(data) {
                console.log("successfully downloaded data");
                fileData = data.Body.toString();

            })
            .catch(function download(err) {console.log(err,err.stack); throw err;});


    console.log('-------------- fetchDataFromS3:Done -------------------------');
}



function waitForFileLoadBeforeDoingSomething(event, context, callback){
    if(!fileData){
        console.log('No file available to me as yet, lets sleep for a bit');
        setTimeout(function(){
            waitForFileLoadBeforeDoingSomething(event, context, callback);
        }, 300);
    } 
}

the output is as follows.

Function Logs:
START RequestId: cb16f155-c0d7-11e8-ad01-f5991c5adaaf Version: $LATEST
2018-09-25T15:29:29.759Z    cb16f155-c0d7-11e8-ad01-f5991c5adaaf    I am in the main procedure
2018-09-25T15:29:29.759Z    cb16f155-c0d7-11e8-ad01-f5991c5adaaf    -------------- fetchDataFromS3:Start -------------------------
2018-09-25T15:29:29.811Z    cb16f155-c0d7-11e8-ad01-f5991c5adaaf    -------------- fetchDataFromS3:Done -------------------------
2018-09-25T15:29:29.811Z    cb16f155-c0d7-11e8-ad01-f5991c5adaaf    I am in the main procedure, the function above should be waiting but it is not
2018-09-25T15:29:29.811Z    cb16f155-c0d7-11e8-ad01-f5991c5adaaf    No file available to me as yet, lets sleep for a bit
2018-09-25T15:29:29.812Z    cb16f155-c0d7-11e8-ad01-f5991c5adaaf    TypeError: Cannot read property 'Body' of null
    at exports.handler (/var/task/dg3.js:17:39)

You can see that I'm not hitting the "successfully downloaded data" line and I can't work out if I've made a mistake and the function is still running asynchronously or if I have got the syntax of the promise wrong.

Nikolay Vetrov · Accepted Answer · 2018-09-25 16:31:03Z

3

First of all, you have to change your entry point method. Like you mention before you trying to use an 8.10 node runtime, then the following part of code:

exports.handler = (event, context, callback) => {}

You have to change to:

export async function <function_name>(event) {}

ref: https://aws.amazon.com/blogs/compute/node-js-8-10-runtime-now-available-in-aws-lambda/

Then your path to that function should be:

<module_name>.<function_name>

Also, you don't need the following part of code:

function waitForFileLoadBeforeDoingSomething(event, context, callback){
    if(!fileData){
        console.log('No file available to me as yet, lets sleep for a bit');
        setTimeout(function(){
            waitForFileLoadBeforeDoingSomething(event, context, callback);
        }, 300);
    } 
}

Then get rid off var declaration. Don't mess with scope. Simply use:

const AWS = require('aws-sdk');

The next step is to create an S3 instance:

const S3 = new AWS.S3({region: process.env.AWS_REGION, apiVersion: '2006-03-01'});
// with region of your AWS account and current API verstion;

Declare params for your fetch method:

const params = 
{
  Bucket: 'STRING_VALUE', // a path to your Bucket
  Key: 'STRING_VALUE' // a key (literally a path to your file)
}

ref: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getObject-property

And you don't have to stringify your event, because it's already stringified:

const s = JSON.stringify(fileData.Body.toString('utf-8'));

And finally:

try
{
    const result = await S3.getObject(params).promise();
    // if successful then:
    console.log(`Check the result: ${result}`);
}
catch (ex) // if an error occured
{
     console.error(ex);
}

Also, make sure that a runtime 5 minutes (it's only for debugging purpose after you can adjust) and increase lambda's memory (also for testing purpose).

answered Sep 25, 2018 at 16:31

Nikolay Vetrov

6346 silver badges17 bronze badges

Sign up to request clarification or add additional context in comments.

2 Comments

Chris B Over a year ago

Thanks for a great response, if really helped with my understanding I briefly got stuck again separating the S3 getobject call into a separate function before I figured it out, and I had to change the event handler declaration slightly to get it work (I don't know why). I will post an updated version of the code I've created.

Chris B Over a year ago

As I can't post long code snippets in a comment here is a link to a working version of the code. link

Collectives™ on Stack Overflow

load S3 file in Lambda with Node.js version 8.10

1 Answer 1

2 Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

2 Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related