Get all unique values in a JavaScript array (remove duplicates) [duplicate]

Using object keys to make unique array, I have tried following

function uniqueArray( ar ) {
  var j = {};

  ar.forEach( function(v) {
    j[v+ '::' + typeof v] = v;
  });


  return Object.keys(j).map(function(v){
    return j[v];
  });
}   

uniqueArray(["1",1,2,3,4,1,"foo", false, false, null,1]);

Which returns ["1", 1, 2, 3, 4, "foo", false, null]

Finding unique in Array of objects using One Liner

const uniqueBy = (x,f)=>Object.values(x.reduce((a,b)=>((a[f(b)]=b),a),{}));
// f -> should must return string because it will be use as key

const data = [
  { comment: "abc", forItem: 1, inModule: 1 },
  { comment: "abc", forItem: 1, inModule: 1 },
  { comment: "xyz", forItem: 1, inModule: 2 },
  { comment: "xyz", forItem: 1, inModule: 2 },
];

uniqueBy(data, (x) => x.forItem +'-'+ x.inModule); // find unique by item with module
// output
// [
//   { comment: "abc", forItem: 1, inModule: 1 },
//   { comment: "xyz", forItem: 1, inModule: 2 },
// ];

// can also use for strings and number or other primitive values

uniqueBy([1, 2, 2, 1], (v) => v); // [1, 2]
uniqueBy(["a", "b", "a"], (v) => v); // ['a', 'b']

uniqueBy(
  [
    { id: 1, name: "abc" },
    { id: 2, name: "xyz" },
    { id: 1, name: "abc" },
  ],
  (v) => v.id
);
// output
// [
//   { id: 1, name: "abc" },
//   { id: 2, name: "xyz" },
// ];

The task is to get a unique array from an array consisted of arbitrary types (primitive and non primitive).

The approach based on using new Set(...) is not new. Here it is leveraged by JSON.stringify(...), JSON.parse(...) and [].map method. The advantages are universality (applicability for an array of any types), short ES6 notation and probably performance for this case:

const dedupExample = [
    { a: 1 },
    { a: 1 },
    [ 1, 2 ],
    [ 1, 2 ],
    1,
    1,
    '1',
    '1'
]

const getUniqArrDeep = arr => {
    const arrStr = arr.map(item => JSON.stringify(item))
    return [...new Set(arrStr)]
        .map(item => JSON.parse(item))
}

console.info(getUniqArrDeep(dedupExample))
   /* [ {a: 1}, [1, 2], 1, '1' ] */

As explained already, [...new Set(values)] is the best option, if that's available to you.

Otherwise, here's a one-liner that doesn't iterate the array for every index:

values.sort().filter((val, index, arr) => index === 0 ? true : val !== arr[index - 1]);

That simply compares each value to the one before it. The result will be sorted.

Example:

let values = [ 1, 2, 3, 3, 4, 5, 5, 5, 4, 4, 4, 5, 1, 1, 1, 3, 3 ];
let unique = values.sort().filter((val, index, arr) => index === 0 ? true : val !== arr[index - 1]);
console.log(unique);

Building on other answers, here's another variant that takes an optional flag to choose a strategy (keep first occurrence or keep last):

Without extending Array.prototype

function unique(arr, keepLast) {
  return arr.filter(function (value, index, array) {
    return keepLast ? array.indexOf(value, index + 1) < 0 : array.indexOf(value) === index;
  });
};

// Usage
unique(['a', 1, 2, '1', 1, 3, 2, 6]); // -> ['a', 1, 2, '1', 3, 6]
unique(['a', 1, 2, '1', 1, 3, 2, 6], true); // -> ['a', '1', 1, 3, 2, 6]

Extending Array.prototype

Array.prototype.unique = function (keepLast) {
  return this.filter(function (value, index, array) {
    return keepLast ? array.indexOf(value, index + 1) < 0 : array.indexOf(value) === index;
  });
};

// Usage
['a', 1, 2, '1', 1, 3, 2, 6].unique(); // -> ['a', 1, 2, '1', 3, 6]
['a', 1, 2, '1', 1, 3, 2, 6].unique(true); // -> ['a', '1', 1, 3, 2, 6]

Using ES6 (one-liner)

Array of Primitive values

let originalArr= ['a', 1, 'a', 2, '1'];

let uniqueArr = [...new Set(originalArr)];

Array of Objects

let uniqueObjArr = [...new Map(originalObjArr.map((item) => [item["propertyName"], item])).values()];

const ObjArray = [
    {
        name: "Eva Devore",
        character: "Evandra",
        episodes: 15,
    },
    {
        name: "Alessia Medina",
        character: "Nixie",
        episodes: 15,
    },
    {
        name: "Kendall Drury",
        character: "DM",
        episodes: 15,
    },
    {
        name: "Thomas Taufan",
        character: "Antrius",
        episodes: 14,
    },
    {
        name: "Alessia Medina",
        character: "Nixie",
        episodes: 15,
    },
];

let uniqueObjArray = [...new Map(ObjArray.map((item) => [item["id"], item])).values()];

You can use a Set to eliminate the duplicates.

const originalNumbers = [1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 1, 2, 9];
const uniqueNumbersSet = new Set(originalNumbers);

/** get the array back from the set */
const uniqueNumbersArray = Array.from(uniqueNumbersSet);

/** uniqueNumbersArray outputs to: [1, 2, 3, 4, 5, 9] */

Learn more about set: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set

A simple code as this:

let arr = [1,'k',12,1,1,'k','12'];
let distictArr=arr.filter((item, index, arr) => arr.indexOf(item) === index);

console.log(distictArr); // [1, 'k', 12, '12']

You can also use jQuery

var a = [1,5,1,6,4,5,2,5,4,3,1,2,6,6,3,3,2,4];

// note: jQuery's filter params are opposite of javascript's native implementation :(
var unique = $.makeArray($(a).filter(function(i,itm){ 
    // note: 'index', not 'indexOf'
    return i == $(a).index(itm);
}));

// unique: [1, 5, 6, 4, 2, 3]

Originally answered at: jQuery function to get all unique elements from an array?

If anyone using knockoutjs

ko.utils.arrayGetDistinctValues()

BTW have look at all ko.utils.array* utilities.

Look at this. Jquery provides uniq method: https://api.jquery.com/jQuery.unique/

var ids_array = []

$.each($(my_elements), function(index, el) {
    var id = $(this).attr("id")
    ids_array.push(id)
});

var clean_ids_array = jQuery.unique(ids_array)

$.each(clean_ids_array, function(index, id) {
   elment = $("#" + id)   // my uniq element
   // TODO WITH MY ELEMENT
});

Do it with lodash and identity lambda function, just define it before use your object

const _ = require('lodash');
...    
_.uniqBy([{a:1,b:2},{a:1,b:2},{a:1,b:3}], v=>v.a.toString()+v.b.toString())
_.uniq([1,2,3,3,'a','a','x'])

and will have:

[{a:1,b:2},{a:1,b:3}]
[1,2,3,'a','x']

(this is the simplest way )

Deduplication usually requires an equality operator for the given type. However, using an eq function stops us from utilizing a Set to determine duplicates in an efficient manner, because Set falls back to ===. As you know for sure, === doesn't work for reference types. So we're kind if stuck, right?

The way out is simply using a transformer function that allows us to transform a (reference) type into something we can actually lookup using a Set. We could use a hash function, for instance, or JSON.stringify the data structure, if it doesn't contain any functions.

Often we only need to access a property, which we can then compare instead of the Object's reference.

Here are two combinators that meet these requirements:

const dedupeOn = k => xs => {
  const s = new Set();

  return xs.filter(o =>
    s.has(o[k])
      ? null
      : (s.add(o[k]), o[k]));
};

const dedupeBy = f => xs => {
  const s = new Set();

  return xs.filter(x => {
    const r = f(x);
    
    return s.has(r)
      ? null
      : (s.add(r), x);
  });
};

const xs = [{foo: "a"}, {foo: "b"}, {foo: "A"}, {foo: "b"}, {foo: "c"}];

console.log(
  dedupeOn("foo") (xs)); // [{foo: "a"}, {foo: "b"}, {foo: "A"}, {foo: "c"}]

console.log(
  dedupeBy(o => o.foo.toLowerCase()) (xs)); // [{foo: "a"}, {foo: "b"}, {foo: "c"}]

With these combinators we're extremely flexible in handling all kinds of deduplication issues. It's not the fastes approach, but the most expressive and most generic one.

Here is an almost one-liner that is O(n), keeps the first element, and where you can keep the field you are uniq'ing on separate.

This is a pretty common technique in functional programming - you use reduce to build up an array that you return. Since we build the array like this, we guarantee that we get a stable ordering, unlike the [...new Set(array)] approach. We still use a Set to ensure we don't have duplicates, so our accumulator contains both a Set and the array we are building.

const removeDuplicates = (arr) =>
  arr.reduce(
    ([set, acc], item) => set.has(item) ? [set, acc] : [set.add(item), (acc.push(item), acc)],
    [new Set(), []]
  )[1]

The above will work for simple values, but not for objects, similarly to how [...new Set(array)] breaks down. If the items are objects that contain an id property, you'd do:

const removeDuplicates = (arr) =>
  arr.reduce(
    ([set, acc], item) => set.has(item.id) ? [set, acc] : [set.add(item.id), (acc.push(item), acc)],
    [new Set(), []]
  )[1]

For removing the duplicates there could be 2 situations. first, all the data are not objects, secondly all the data are objects.

If all the data are any kind of primitive data type like int, float, string etc then you can follow this one

const uniqueArray = [...new Set(oldArray)]

But suppose your array consist JS objects like bellow

{
    id: 1,
    name: 'rony',
    email: '[email protected]'
}

then to get all the unique objects you can follow this

let uniqueIds = [];
const uniqueUsers = oldArray.filter(item => {
    if(uniqueIds.includes(item.id)){
        return false;
    }else{
        uniqueIds.push(item.id);
        return true;
    }
})

You can also use this method to make any kind of array to make unique. Just keep the tracking key on the uniqueIds array.

Not really a direct literal answer to the original question, because I preferred to have the duplicate values never in the array in the first place. So here's my UniqueArray:

class UniqueArray extends Array {
    constructor(...args) {
        super(...new Set(args));
    }
    push(...args) {
        for (const a of args) if (!this.includes(a)) super.push(a);
        return this.length;
    }
    unshift(...args) {
        for (const a of args.reverse()) if (!this.includes(a)) super.unshift(a);
        return this.length;
    }
    concat(...args) {
        var r = new UniqueArray(...this);
        for (const a of args) r.push(...a);
        return r;
    }
}

> a = new UniqueArray(1,2,3,1,2,4,5,1)
UniqueArray(5) [ 1, 2, 3, 4, 5 ]
> a.push(1,4,6)
6
> a
UniqueArray(6) [ 1, 2, 3, 4, 5, 6 ]
> a.unshift(1)
6
> a
UniqueArray(6) [ 1, 2, 3, 4, 5, 6 ]
> a.unshift(0)
7
> a
UniqueArray(7) [
  0, 1, 2, 3,
  4, 5, 6
]
> a.concat(2,3,7)
UniqueArray(8) [
  0, 1, 2, 3,
  4, 5, 6, 7
]

Always remember, The build-in methods are easy to use. But keep in mind that they have a complexity.

The basic logic is best. There is no hidden complexity.

let list = [1, 1, 2, 100, 2] // your array
let check = {}
list = list.filter(item => {
    if(!check[item]) {
        check[item] = true
        return true;
    }
})

or use, let check = [] if you need future traverse to checked items (waste of memory though)

In ES6/ES2015 and later, you can achieve a succinct one-liner for filtering out duplicate items in an array using the Set and the Spread Operator:

let uniqueItems = Array.from(new Set(items));

This solution maintains clarity by explicitly converting the Set back into an array using Array.from(). The end result is an array containing only unique items:

[4, 5, 6, 3, 2, 23, 1]

You can also use sugar.js:

[1,2,2,3,1].unique() // => [1,2,3]

[{id:5, name:"Jay"}, {id:6, name:"Jay"}, {id: 5, name:"Jay"}].unique('id') 
  // => [{id:5, name:"Jay"}, {id:6, name:"Jay"}]

Yet another solution for the pile.

I recently needed to make a sorted list unique and I did it using filter that keeps track of the previous item in an object like this:

uniqueArray = sortedArray.filter(function(e) { 
    if(e==this.last) 
      return false; 
    this.last=e; return true;  
  },{last:null});

The version that accepts selector, should be pretty fast and concise:

function unique(xs, f) {
  var seen = {};
  return xs.filter(function(x) {
    var fx = (f && f(x)) || x;
    return !seen[fx] && (seen[fx] = 1);
  });
}

var a = [1,4,2,7,1,5,9,2,4,7,2]
var b = {}, c = {};
var len = a.length;
for(var i=0;i<len;i++){
  a[i] in c ? delete b[a[i]] : b[a[i]] = true;
  c[a[i]] = true;
} 

// b contains all unique elements

This one is not pure, it will modify the array, but this is the fastest one. If yours is faster, then please write in the comments ;)

http://jsperf.com/unique-array-webdeb

Array.prototype.uniq = function(){
    for(var i = 0, l = this.length; i < l; ++i){
        var item = this[i];
        var duplicateIdx = this.indexOf(item, i + 1);
        while(duplicateIdx != -1) {
            this.splice(duplicateIdx, 1);
            duplicateIdx = this.indexOf(item, duplicateIdx);
            l--;
        }
    }

    return this;
}

[
 "",2,4,"A","abc",
 "",2,4,"A","abc",
 "",2,4,"A","abc",
 "",2,4,"A","abc",
 "",2,4,"A","abc",
 "",2,4,"A","abc",
 "",2,4,"A","abc",
 "",2,4,"A","abc"
].uniq() //  ["",2,4,"A","abc"]

For a array of strings:

function removeDuplicatesFromArray(arr) {
  const unique = {};
  arr.forEach((word) => {
    unique[word] = 1; // it doesn't really matter what goes here
  });
  return Object.keys(unique);
}

You can use Ramda.js, a functional javascript library to do this:

var unique = R.uniq([1, 2, 1, 3, 1, 4])
console.log(unique)

<script src="https://cdnjs.cloudflare.com/ajax/libs/ramda/0.25.0/ramda.js"></script>

A lot of people have already mentioned using...

[...new Set(arr)];

And this is a great solution, but my preference is a solution that works with .filter. In my opinion filter is a more natural way to get unique values. You're effectively removing duplicates, and removing elements from an array is exactly what filter is meant for. It also lets you chain off of .map, .reduce and other .filter calls. I devised this solution...

const unique = () => {
  let cache;  
  return (elem, index, array) => {
    if (!cache) cache = new Set(array);
    return cache.delete(elem);
  };
};

myArray.filter(unique());

The caveat is that you need a closure, but I think this is a worthy tradeoff. In terms of performance, it is more performant than the other solutions I have seen posted that use .filter, but worse performing than [...new Set(arr)].

See also my github package youneek

in my solution, I sort data before filtering :

const uniqSortedArray = dataArray.sort().filter((v, idx, t) => idx==0 || v != t[idx-1]); 

If you want to only get the unique elements and remove the elements which repeats even once, you can do this:

let array = [2, 3, 4, 1, 2, 8, 1, 1, 2, 9, 3, 5, 3, 4, 8, 4];

function removeDuplicates(inputArray) {
  let output = [];
  let countObject = {};

  for (value of array) {
    countObject[value] = (countObject[value] || 0) + 1;
  }

  for (key in countObject) {
    if (countObject[key] === 1) {
      output.push(key);
    }
  }

  return output;
}

console.log(removeDuplicates(array));