0

I'm using Back4app.

My Profile class schema has 4 File columns containing pictures. So when I retrieve an object , I have to make an HTTP request for each file URL and get the byte data like this.

const data = await Parse.Cloud.httpRequest({url:profilePhoto.url()});
return data.buffer.toString('base64');

But for all four files I have to do 4 HTTP requests to the server. Is there anyway to do a batch HTTP request so that with just 1 request I can get data for all 4 files ? My main aim is to do the least amount of requests to the server as possible.

4
  • you could concatenate the stream of the files and then the client must know where to split them or, since you are doing a base64 encoding return an array of file in base64 format. HTTP 1.1 out of the box doesn't have this feature. HTTP2 has it instead Commented Jun 29, 2020 at 9:08
  • HI @ManuelSpigolon thankyou for the comment , but could you put up a bit of code as to how go about it ? I'm fairly new to all this so a code example would help a lot Commented Jun 29, 2020 at 9:40
  • I don't know parse, I know raw node.js and backend arch Commented Jun 29, 2020 at 10:15
  • you could get a write stream , writing multiple files to the single writestream , while using a pipe to connect the single stream to the http response . github.com/aheckmann/gridfs-stream/issues/18 is P1 stackoverflow.com/questions/11906198/… is P2 Commented Jul 5, 2020 at 20:44

2 Answers 2

0

There is no out-of-the-box way to retrieve multiple files with one request in Parse Server.

You could implement your own Parse Cloud Code function to retrieve multiple files, but you would have to manually combine them server side and separate them client side.

As a starting point you could look at packages like multistream that allow you to combine multiple file streams into one to get some inspiration.

Sign up to request clarification or add additional context in comments.

1 Comment

Yes I too found out that there is no way to retrieve multiple files with one request because we are talking about files and to each file must happen one request.
0

You might be able to do something similar to what I've done in cloud code.

I had to load up a bunch of information at the start of my application, requiring many round trips to the server.

So I wrote a function called getUserData().

This does many unrelated queries, and jams all of the results into one big object. I then return the object from the function.

Here is the entire function:


console.log("startig getUserData");
var callCount = 0;
var lastLoadTime=0;

// Given a user, load all friends.  Save the objects to ret.objects,
// and save the objectIds to ret.friends
// 
// Note:  we always load the exhaustive friend list, because
//        otherwise, we would have no way of recognizing
//        removed friendships.
//
async function loadFriends(user, ret) {
  const friendQuery = user.relation("friends").query();
  const friends = await findFully(friendQuery);
  for(var i=0;i<friends.length;i++){
    ret.friends[friends[i].id]=1;
    ret.objects[friends[i].id]=friends[i];
  };
}
// Given a user, load all owned cells.  Save the objects to ret.owned,
// and save their objectIds to ret.ownedCells.
//
// Also, save the ids of members, which we will use to flesh out ret.objects with
// the objects who are not friends, but share a cell with the current user.

async function loadPublicCells(user, ret, memberIds) {
  const ownedCellQ = new Parse.Query('PublicCell');
  ownedCellQ.equalTo('owner',user);

  const joinedCellQ = new Parse.Query('PublicCell');
  joinedCellQ.equalTo('members',user);

  const publicCellQ = Parse.Query.or(ownedCellQ,joinedCellQ);
  publicCellQ.greaterThan("updatedAt",new Date(lastLoadTime));

  const publicCells=await findFully(publicCellQ);

  for(var i=0;i<publicCells.length;i++) {
    const cell = publicCells[i];
    ret.ownedCells[cell.id]=cell;
    const owner = cell.get("owner");
    if(owner==null)
      continue;
    
    ret.objects[cell.id]=cell;
    if(owner.id === user.id) {
      ret.ownedCells[cell.id]=1;
    } else {
      ret.joinedCells[cell.id]=1;
    };
    const memberQ = cell.relation("members").query();
    const members = await findFully(memberQ);
    if(ret.memberMap[cell.id]==null)
      ret.memberMap[cell.id]={};
    const map = ret.memberMap[cell.id];
    for(var j=0;j<members.length;j++){
      const member=members[j];
      map[member.id]=1;
      ret.objects[member.id]=member;
    };
  };
};
// given a list of all members of all cells, load those objects and store
// them in ret.objects.  We do not have to record which cells they belong
// to, because that information is in ret.memberMap
async function loadMembers(memberIds, ret) {
  const memberQ = new Parse.Query(Parse.User);
  var partIds;
  while(memberIds.length){
    partIds = memberIds.splice(0,100);
    memberQ.containedIn('objectId',partIds);
    const part = await findFully(memberQ);
    for(var i=0;i<part.length;i++) {
      ret.objects[part[i].id]=part[i];
    }
  };
};
// given a user, save all of the objectIds of people who have annoyed him with
// spam.  We save only the ids, they don't go on ret.objects, because we only
// need to filter them out of things.  The objectIds are sufficient.
//
// We always send all spam objects, otherwise we would not recognize deletions
async function loadUserSpams(user, ret) {
  const userSpamsQ = new Parse.Query("_User");
  userSpamsQ.equalTo("spamUsers",user);
  userSpamsQ.greaterThan("updatedAt", new Date(lastLoadTime));
  const userSpams = await findFully(userSpamsQ);
  for(var i=0;i<userSpams.length;i++){
    ret.userSpams[userSpams[i].id]=1;
  };
};
// given a user, save all of the objectIds of people who have been annoyed *BY*
// him with spam.  We save only the ids, they don't go on ret.objects, because we
// only need to filter them out of things.  The objectIds are sufficient.
//
// We always send all spam objects, otherwise we would not recognize deletions
async function loadSpamUsers(user, ret) {
  const spamUserR = user.relation('spamUsers');
  const spamUserQ = spamUserR.query();
  spamUserQ.greaterThan("updatedAt", new Date(lastLoadTime));
  const spamUsers = await findFully(spamUserQ);
  for(var i=0;i<spamUsers.length;i++){
    ret.spamUsers[spamUsers[i].id]=1;
  };
};
// given a user, save all of the objectIds of people to whom he has sent a
// friend request which is still pending.  We save only the ids, they don't go
// on ret.objects, because we only need to filter them out of things.  The
// objectIds are sufficient.
async function loadPendingFriends(user, ret) {
  const request1Q = new Parse.Query('Request');
  request1Q.equalTo("owner",user);
  const request2Q = new Parse.Query('Request');
  request2Q.equalTo("sentTo",user);
  const requestQ = Parse.Query.or(request1Q,request2Q);
  requestQ.equalTo("status",'PENDING');
  const requests = await findFully(requestQ);
  for(var i=0;i<requests.length;i++){
    const request = requests[i];
    const sentBy = request.get("owner");
    if(sentBy==null){
      console.warn("sentBy==null");
      continue;
    };
    const sentTo = request.get("sentTo");
    if(sentTo==null){
      console.warn("sentTo==null");
      continue;
    };
    console.dump({sentTo,sentBy});
    if(sentBy.id==user.id){
      ret["pendingFriends"][sentTo.id]=sentTo;
    } else if ( sentTo.id==user.id ) {
      ret["friendingPends"][sentBy.id]=sentBy;
    };
  };
};
// given a user, load all of his private cells.  We do not store
// the user objects, because only friends will be in your private cells.
async function loadPrivateCells(user, ret) {
  const privateCellQ = new Parse.Query('PrivateCell');
  privateCellQ.equalTo("owner", user);
  privateCellQ.greaterThan("updatedAt", new Date(lastLoadTime));
  const privateCells = await findFully(privateCellQ);
  for(var i=0;i<privateCells.length;i++) {
    const cell = privateCells[i]; 
    ret.objects[cell.id]=cell;
    ret.privateCells[cell.id]=cell;
    if(ret.memberMap[cell.id]==null)
      ret.memberMap[cell.id]={};
    const map = ret.memberMap[cell.id];
    const memberQ = cell.relation("members").query();
    const members = await findFully(memberQ);
    for(var j=0;j<members.length;j++){
      const member=members[j];
      map[member.id]=1;
      ret.objects[member.id]=member;
    };
  };
  //});
}
// we use objects as maps to weed out duplicate objects and cells.
// when we are done, we use this function to replace the object
// with an array of objects.  we don't need to send the keys, since
// they already exist within the objects.
function objToValueList(k,ret){
  const objs = [];
  for( var id in ret[k] )
    objs.push(ret[k][id]);
  ret[k]=objs;
  ret.counts[k]=objs.length;
};
// convert the objects which have been used to accumulate key lists
// to arrays of objectIds.  k is the name of the list we are working
// on.  ret[k] is the list itself.
function objToKeyList(k,ret) {
  const objs = [];
  for( var id in ret[k] ) {
    objs.push(id);
  };
  ret[k]=objs;
  ret.counts[k]=objs.length;
};
async function checkUserConsent(user){
  const query = new Parse.Query("PrivacyPolicy");
  query.descending("createdAt");
  query.limit(1);
  const res = await query.find();
  if(res.length==0) {
    return true;
  };
  const policy=res[0];
  console.dump(policy);
  console.log(policy);
  const userConsent=user.get("lastConsent");
  return userConsent!=null && userConsent.id == policy.id;
};
async function loadAlerts(user,ret) {
  const q1 = new Parse.Query("Alert");
  q1.equalTo("owner", user);
  const q2 = new Parse.Query("Response");
  q2.equalTo("owner", user);
  const q3 = new Parse.Query("Alert");
  q3.matchesKeyInQuery("objectId", "alert", q2);
  const q = Parse.Query.or(q1,q3);
  const list = await q.find();
  var time = new Date().getTime();
  time -= 1000*86400;
  time=Math.max(lastLoadTime, time);
  q.greaterThan("updatedAt",time);
  for(var i=0;i<list.length;i++) {
    const item=list[i];
    ret.alerts[item.id]=1;
    ret.objects[item.id]=item;
  };
}
async function doGetUserData(user) {
  if(!user)
    return {fatal:  'not logged in!' };
  const ret = {
    owner: {},
    privateCells: {},
    friends: {},
    alerts: {},
    objects: {},
    ownedCells: {},
    joinedCells: {},
    spamUsers: {},
    userSpams: {},
    pendingFriends: {},
    friendingPends: {},
    memberMap: {},
    loadTime: lastLoadTime,
    counts: {callCount: callCount++},
  };

  {
    user.fetch();
    ret.owner=user.id;
    const memberIds={};
    ret.objects[user.id]=user;
    console.log("loadFriends");
    await loadFriends(user,ret);

    console.log("loadPrivateCells");
    await loadPrivateCells(user,ret,memberIds);
    console.log("loadPublicCells");
    await loadPublicCells(user,ret,memberIds);
    
    console.log("loadPendingFriends");
    await loadPendingFriends(user,ret);
  
    console.log("loadUserSpams");
    await loadUserSpams(user,ret);
    console.log("loadSpamUsers");
    await loadSpamUsers(user,ret);

    console.log("loadAlerts");
    await loadAlerts(user,ret);

    const memberList=[];
    for( var id in memberIds ) {
      console.log(ret.objects[id]);
      memberList.push(id);
    };
    console.log("loadMembers");
    await loadMembers(memberList,ret);

  }

  for(var cell in ret.memberMap) {
    var map = ret.memberMap[cell];
    var list = [];
    ret.memberMap[cell]=list;
    for(var member in map) {
      list.push(member);
    };
  }
  delete ret.objects[user.id];
  [ 
    'friends',       "friendingPends",  'pendingFriends',
    'privateCells',  'ownedCells',      'joinedCells',
    'userSpams',     'spamUsers',       "alerts"
  ].forEach((k)=>{
    objToKeyList(k,ret);
  });
  objToValueList('objects',ret);
  delete ret.counts;
  return ret;  
}

async function getUserData(req) {
  try {
    var nextLoadTime=new Date().getTime();
    const user = req.user;
    console.log(user);
    lastLoadTime = req.params.lastLoadTime;
    if(lastLoadTime==null)
      lastLoadTime=0;
    lastLoadTime = new Date(lastLoadTime);
    const ret = await doGetUserData(user);
    ret.loadTime=nextLoadTime;
    return ret;
  } catch ( err ) {
    console.log(err);
    try {
    console.log(err.stack());
    } catch ( xxx ) {
      console.log(err);
    };
    throw (`error getting data: ${err}`);
  };
};
Parse.Cloud.define("getUserData", getUserData);

Something like this could easily be done to get your data for you. Like this solution, it is unlikely to be entirely pretty, but it would probably work.

1 Comment

If you find the friendingPends and the userSpams confusing, they are my slightly humorous way of indicating that I'm getting the reverse side of the relationship. So if you send somebody a friend request, he's a pendingFriend ... but if he sends it to you, he's a friendingPend.

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.