Remove duplicates values in array Google Apps Script

Question

I would like to know how to remove duplicates values from 2 arrays, combined into one main array.

This values must NOT be removed from the sheets or document, just in the array, thats why I didnt use clear() or clearContents() built in functions;

Ive also tried to modify the removeDuplicates() function from the GAS tutorials, but it throws me rows inside columns from A to Z, instead filtered rows...a total mess.

Notes:

Parameters from getClients() are from others functions, and works ok.

newClients list clients from the sheet 'Users' and newUsers list users from another sheet called 'Data'.

Boths sheets belongs to the same spreadsheet.

newClients and newUsers: both arrays only contains strings (usernames), and in both there are duplicated values.

So the goal is identified and remove those values, the original and the duplicate.

Should be easier I think, but Im new in JS, so everything Ive been tried, didnt worked.

Thanks

The Code

function getAllClientsFromData(body,project){

  var ss = SpreadsheetApp.getActiveSpreadsheet();

  // Active the Sheets
  var sheet= ss.getSheets()[1,3];

  // Access data in Sheet "Data"
  var rangeC = ss.getRangeByName("clients"); //A:2:C273  
  var rangeU = ss.getRangeByName("names"); //A5:A

  var clients = rangeC.getValues();
  var users   = rangeU.getValues();

  var lastRow = sheet.getLastRow()-2;

  body += "<h2>Total of " + lastRow + " clients in " + project + "</h2>" 
  body += "<table style=" + STYLE.TABLE + ">";
  body += getClients(ss,clients,users,project);
  body += "</table>";

  return body;  

}

function getClients(ss,clients,users,project){

  var body = "";

  var newClients = [];

  for( var g = 0; g < clients.length; g++ ){

    var currentProject = clients[g][2];

    if( clients[g]!= "" ){

       newClients.push(clients[g][0]);

    } 

  } // end for

  var newUsers = []; 

  for( var u = 0; u < users.length; u++ ){

    if( users[u] != "" ){    

      newUsers.push(users[u][0]);

    }

  } // end for

  var allData = newUsers.concat(newClients);

  var uniqueData = allData.sort();

  body += "<tr><td style=" + STYLE.TD + ">" + uniqueData.join("</td><tr><td style=" + STYLE.TD + ">") + "</td></tr></tr>";

  return body;

}

UPDATES!

The answers works great filtering, but Im getting the same result as on my previous tries: displaying the filtered results. I need to remove them from the array. example:

var array = ['aa','bb','aa','ff','pp', 'pp'];
filtering code...
var array = ['bb','ff'];

I try to add splice() js method but the params I pass, does not working ok.

when you say "it didn't work", what was the actual output or error? — Tim
– Tim, Commented Sep 9, 2014 at 4:38

Serge insas · Accepted Answer · 2014-09-10 06:12:31Z

7

The array you are working on is not a 2D array anymore since you extracted the fields before sorting... so you can use a very simple duplicate removal function as shown below with an example and some added Logger.log to see how it works.

function test(){
  var array = ['aa','bb','cc','aa','dd','cc']
  Logger.log(removeDups(array));
}

function removeDups(array) {
  var outArray = [];
  array.sort():
  outArray.push(array[0]);
  for(var n in array){
    Logger.log(outArray[outArray.length-1]+'  =  '+array[n]+' ?');
    if(outArray[outArray.length-1]!=array[n]){
      outArray.push(array[n]);
    }
  }
  return outArray;
}

in your code this would replace the line

var uniqueData = allData.sort();

that would become :

var uniqueData = removeDups(allData);

EDIT :

If letter case is an issue, you can modify this code to ignore it. You should change the condition and the sort function so that they both ignore the case in your names but preferably keep the original letter case.

This could be achieved with the code below :

function test(){
  var array = ['aa','bb','Cc','AA','dd','CC'];// an example with Upper and Lower case
  Logger.log(removeDups(array));
}

function removeDups(array) {
  var outArray = [];
  array.sort(lowerCase);
  function lowerCase(a,b){
    return a.toLowerCase()>b.toLowerCase() ? 1 : -1;// sort function that does not "see" letter case
  }
  outArray.push(array[0]);
  for(var n in array){
    Logger.log(outArray[outArray.length-1]+'  =  '+array[n]+' ?');
    if(outArray[outArray.length-1].toLowerCase()!=array[n].toLowerCase()){
      outArray.push(array[n]);
    }
  }
  return outArray;
}

Logger result :

enter image description here

EDIT 2 :

Here is another version that keeps only unique values (I didn't understand correctly your request in the first version as it kept one element from the duplicates...)

I simply added an else if condition to remove the elements that were part of a group of duplicates.(I kept the case insensitive version but you can remove it easily)

function test(){
  var array = ['aa','dd','hh','aa','bb','Cc','cc','cc','bb','nn','bb','AA','dd','CC'];// an example with Upper and Lower case
  Logger.log('original array = '+array);
  Logger.log('unique result = '+removeDups2(array));
}


function removeDups2(array) {
  var uniqueArray = []
  array.sort(lowerCase);
  function lowerCase(a,b){
    return a.toLowerCase()>b.toLowerCase() ? 1 : -1;// sort function that does not "see" letter case
  }
  var temp = array[0];
  for(var n=1 ;n<array.length ; n++){
    Logger.log(temp+'  =  '+array[n]+' ?');
    if(temp.toLowerCase()!=array[n].toLowerCase()){
      uniqueArray.push(array[n]);
      temp = array[n];
    }else if(uniqueArray[uniqueArray.length-1]==temp){
      uniqueArray.pop();// remove it from result if one of the duplicate values
    }
  }
  return uniqueArray;
}

Logger result new code :

enter image description here

edited Sep 10, 2014 at 6:12

answered Sep 9, 2014 at 8:43

Serge insas

46.9k7 gold badges113 silver badges135 bronze badges

Sign up to request clarification or add additional context in comments.

2 Comments

Guilherme Henrique Mendes Over a year ago

Only worked for me after applied .toString() at the end of outArray[outArray.length-1] and array[n] in the last if statement.

TC76 Over a year ago

Thank you very much for this! I had to make a change/addition (I know next to nothing about JS). I kept getting

function () {         return this.filter(function (value, index, self) { return self.indexOf(value) === index; });     } ?

so I changed if (outArray[outArray.length - 1] != array[n]) to if (outArray[outArray.length - 1] != array[n] && n != "unique"). I don't know enough to know what caused this issue or if my "fix" is suitable, so I wanted to run it by you. Thanks again!

Ido Green · Accepted Answer · 2014-09-09 06:41:40Z

2

In your code you are not doing anything to filter the duplicate values. This line will just sort the data and won't give you unique data.

var uniqueData = allData.sort();

You can do something like this on your merged array, after you 'installing' 2DArray lib: https://sites.google.com/site/scriptsexamples/custom-methods/2d-arrays-library

var uniqueData = unique(allData);

Another option is to create a loop and check for duplicate values, but you should remember to transform all the values of the string to lowercase before you do these matches.

answered Sep 9, 2014 at 6:41

Ido Green

2,8131 gold badge19 silver badges26 bronze badges

Comments

Hung Nguyen Viet · Accepted Answer · 2021-07-12 04:26:13Z

2

I created this function and it worked.

function removeDups(data) {
  var newData = [];

  data.forEach(function(value) {
      if (newData.indexOf(value) == -1) {
        newData.push(value);
    }
  });
  return newData;
}

answered Jul 12, 2021 at 4:26

Hung Nguyen Viet

313 bronze badges

1 Comment

greybeard Over a year ago

Welcome to stackoverflow. Considerate of you to leave a comment with the code confirming that it does work. Please make it a habit to document in your code a) what it accomplishes and b) how, if not blatantly obvious. Please revisit the examples in the last paragraph of the question for an idea why anyone would vote your answer not useful.

William Belini · Accepted Answer · 2023-09-20 16:13:28Z

Serge insas you are the best man!

A thing I improve in your code, removeDups2, an error caused because the first element, even if its not duplicate, it does´t appears in the uniquearray. To solved this problem, just declarate for the first time =>

uniqueArray = [array[0]]

So de code stay look like this, and work very well

function test(){
  var array = ['aa','dd','hh','aa','bb','Cc','cc','cc','bb','nn','bb','AA','dd','CC'];// an example with Upper and Lower case
  Logger.log('original array = '+array);
  Logger.log('unique result = '+removeDups2(array));
}


    function removeDups2(array) {
      var uniqueArray = **[array [0]]**
      array. Sort(lowercase);
      function lowerCase(a,b){
        return a.toLowerCase()>b.toLowerCase() ? 1 : -1;// sort function that does not "see" letter case
      }
      var temp = array[0];
      for(var n=1 ;n<array.length ; n++){
        Logger.log(temp+'  =  '+array[n]+' ?');
        if(temp.toLowerCase()!=array[n].toLowerCase()){
          uniqueArray.push(array[n]);
          temp = array[n];
        }else if(uniqueArray[uniqueArray.length-1]==temp){
          uniqueArray.pop();// remove it from result if one of the duplicate values
        }
      }
      return uniqueArray;
    }

Laltan · Accepted Answer · 2022-01-30 08:08:45Z

0

Yet another solution:

function removeDups(array) {
  array.sort()
  var lastValue = !array[0]
  var outArray = array.filter(function(value) {
    if (value == lastValue)
      return false
    lastValue = value
    return true
  })
  return outArray
}

This also works correctly for empty arrays whereas some earlier solutions yield [null] in this special case.

edited Jan 30, 2022 at 8:08

answered Jan 26, 2022 at 10:33

Laltan

12 bronze badges

Collectives™ on Stack Overflow

Remove duplicates values in array Google Apps Script

5 Answers 5

EDIT 2 :

2 Comments

Comments

1 Comment

Comments

Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

5 Answers 5

EDIT 2 :

2 Comments

Comments

1 Comment

Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related