Logstash: Flatten nested JSON, combine fields inside array

Question

I have a JSON looking like this:

{
  "foo": {
    "bar": {
      "type": "someType",
      "id": "ga241ghs"
    },
    "tags": [
      {
        "@tagId": "123",
        "tagAttributes": {
          "attr1": "AAA",
          "attr2": "111"
        }
      },
      {
        "@tagId": "456",
        "tagAttributes": {
          "attr1": "BBB",
          "attr2": "222"
        }
      }
    ]
  },
  "text": "My text"
}

Actually it's not split to multiple lines (just did it to give a better overview), so it's looking like this:

{"foo":{"bar":{"type":"someType","id":"ga241ghs"},"tags":[{"@tagId":"123","tagAttributes":{"attr1":404,"attr2":416}},{"@tagId":"456","tagAttributes":{"attr1":1096,"attr2":1103}}]},"text":"My text"}

I want to insert this JSON with Logstash to an Elasticsearch index. However, I want to insert a flattened JSON with the fields in the array combined like this:

"foo.bar.tags.tagId": ["123", "456"]
"foo.tags.tagAttributs.attr1": ["AAA", "BBB"]
"foo.tags.tagAttributs.attr2": ["111", "222"]

In total, the data inserted to Elasticsearch should look like this:

"foo.bar.type": "someType"
"foo.bar.id": "ga241ghs"
"foo.tags.tagId": ["123", "456"]
"foo.tags.tagAttributs.attr1": ["AAA", "BBB"]
"foo.tags.tagAttributs.attr2": ["111", "222"]
"foo.text": "My text"

This is my current Logstash .conf; I am able to split the "tags" array, but now I am getting 2 entries as a result.

How can I now join all tagIds to one field, attr1 values of the array to one field, and all attr2 values to another?

input {
  file {
    codec => json
    path => ["/path/to/my/data/*.json"]
    mode => "read"
    file_completed_action => "log"
    file_completed_log_path => ["/path/to/my/logfile"]
    sincedb_path => "/dev/null"
  }
}

filter {
  split {
    field => "[foo][tags]"
  }
}

output {
  stdout { codec => rubydebug }
}

Thanks a lot!

Shouldn't "foo.bar.tags.tagId": "[123, 456]" be "foo.bar.tags.tagId": [123, 456]? — 3limin4t0r
– 3limin4t0r, Commented Aug 29, 2019 at 9:55
I think you would have to write that in ruby. There is a ruby script that will do part of the job here. You would have to extend that to handle arrays. — Badger
– Badger, Commented Aug 29, 2019 at 13:49

Jan · Accepted Answer · 2019-08-30 09:36:43Z

Nice example for my JSON iterator IIFE - no need for complex algos, just pick DepthFirst, sligthly modified path (new "raw" version) and that is it. In case you like this JS answer, mind ticking accept flag under voting buttons.

In case you want different language, have also C# parser with similar iterators on same GitHub.

var src = {"foo":{"bar":{"type":"someType","id":"ga241ghs"},"tags":[{"@tagId":"123","tagAttributes":{"attr1":"AAA","attr2":"111"}},{"@tagId":"456","tagAttributes":{"attr1":"BBB","attr2":"222"}}],"text":"My text"}};
//console.log(JSON.stringify(src, null, 2));
function traverse(it) {
    var dest = {};
    var i=0;
    do {
        if (it.Current().HasStringValue()) {
            var pathKey = it.Path(true).join('.');
            var check = dest[pathKey];
            if (check) {
                if (!(check instanceof Array)) dest[pathKey] = [check];
                dest[pathKey].push(it.Value());
            } else {
                dest[pathKey] = it.Value();
            }
        }
        //console.log(it.Level + '\t' + it.Path(1).join('.') + '\t' + it.KeyDots(), (it.Value() instanceof Object) ? "-" : it.Value());
    } while (it.DepthFirst());

    console.log(JSON.stringify(dest, null, 2));
    return dest;
}

/*
 * https://github.com/eltomjan/ETEhomeTools/blob/master/HTM_HTA/JSON_Iterator_IIFE.js
 * +new raw Path feature
 */
'use strict';
var JNode = (function (jsNode) {

    function JNode(_parent, _pred, _key, _value) {
        this.parent = _parent;
        this.pred = _pred;
        this.node = null;
        this.next = null;
        this.key = _key;
        this.value = _value;
    }
    JNode.prototype.HasOwnKey = function () { return this.key && (typeof this.key != "number"); }
    JNode.prototype.HasStringValue = function () { return !(this.value instanceof Object); }

    return JNode;
})();

var JIterator = (function (json) {
    var root, current, maxLevel = -1;

    function JIterator(json, parent) {
        if (parent === undefined) parent = null;
        var pred = null, localCurrent;
        for (var child in json) {
            var obj = json[child] instanceof Object;
            if (json instanceof Array) child = parseInt(child); // non-associative array
            if (!root) root = localCurrent = new JNode(parent, null, child, json[child]);
            else {
                localCurrent = new JNode(parent, pred, child, obj ? ((json[child] instanceof Array) ? [] : {}) : json[child]);
            }
            if (pred) pred.next = localCurrent;
            if (parent && parent.node == null) parent.node = localCurrent;
            pred = localCurrent;
            if (obj) {
                var memPred = pred;
                JIterator(json[child], pred);
                pred = memPred;
            }
        }
        if (this) {
            current = root;
            this.Level = 0;
        }
    }

    JIterator.prototype.Current = function () { return current; }
    JIterator.prototype.SetCurrent = function (newCurrent) {
        current = newCurrent;
        this.Level = 0;
        while(newCurrent = newCurrent.parent) this.Level++;
    }
    JIterator.prototype.Parent = function () {
        var retVal = current.parent;
        if (retVal == null) return false;
        this.Level--;
        return current = retVal;
    }
    JIterator.prototype.Pred = function () {
        var retVal = current.pred;
        if (retVal == null) return false;
        return current = retVal;
    }
    JIterator.prototype.Node = function () {
        var retVal = current.node;
        if (retVal == null) return false;
        this.Level++;
        return current = retVal;
    }
    JIterator.prototype.Next = function () {
        var retVal = current.next;
        if (retVal == null) return false;
        return current = retVal;
    }
    JIterator.prototype.Key = function () { return current.key; }
    JIterator.prototype.KeyDots = function () { return (typeof (current.key) == "number") ? "" : (current.key + ':'); }
    JIterator.prototype.Value = function () { return current.value; }
    JIterator.prototype.Reset = function () {
        current = root;
        this.Level = 0;
    }
    JIterator.prototype.RawPath = function () {
        var steps = [], level = current;
        do {
            if (level != null && level.value instanceof Object) {
                steps.push(level.key + (level.value instanceof Array ? "[]" : "{}"));
            } else {
                if (level != null) steps.push(level.key);
                else break;
            }
            level = level.parent;
        } while (level != null);
        var retVal = "";
        retVal = steps.reverse();
        return retVal;
    }
    JIterator.prototype.Path = function (raw) {
        var steps = [], level = current;
        do {
            if (level != null && level.value instanceof Object) {
                var size = 0;
                var items = level.node;
                if (typeof (level.key) == "number" && !raw) steps.push('[' + level.key + ']');
                else {
                    if(raw) {
                        if (typeof (level.key) != "number") steps.push(level.key);
                    } else {
                        while (items) {
                            size++;
                            items = items.next;
                        }
                        var type = (level.value instanceof Array ? "[]" : "{}");
                        var prev = steps[steps.length - 1];
                        if (prev && prev[0] == '[') {
                            var last = prev.length - 1;
                            if (prev[last] == ']') {
                                last--;
                                if (!isNaN(prev.substr(1, last))) {
                                    steps.pop();
                                    size += '.' + prev.substr(1, last);
                                }
                            }
                        }
                        steps.push(level.key + type[0] + size + type[1]);
                    }
                }
            } else {
                if (level != null) {
                    if (typeof (level.key) == "number") steps.push('[' + level.key + ']');
                    else steps.push(level.key);
                }
                else break;
            }
            level = level.parent;
        } while (level != null);
        var retVal = "";
        retVal = steps.reverse();
        return retVal;
    }
    JIterator.prototype.DepthFirst = function () {
        if (current == null) return 0; // exit sign
        if (current.node != null) {
            current = current.node;
            this.Level++;
            if (maxLevel < this.Level) maxLevel = this.Level;
            return 1; // moved down
        } else if (current.next != null) {
            current = current.next;
            return 2; // moved right
        } else {
            while (current != null) {
                if (current.next != null) {
                    current = current.next;
                    return 3; // returned up & moved next
                }
                this.Level--;
                current = current.parent;
            }
        }
        return 0; // exit sign
    }
    JIterator.prototype.BreadthFirst = function () {
        if (current == null) return 0; // exit sign
        if (current.next) {
            current = current.next;
            return 1; // moved right
        } else if (current.parent) {
            var level = this.Level, point = current;
            while (this.DepthFirst() && level != this.Level);
            if (current) return 2; // returned up & moved next
            do {
                this.Reset();
                level++;
                while (this.DepthFirst() && level != this.Level);
                if (current) return 3; // returned up & moved next
            } while (maxLevel >= level);
            return current != null ? 3 : 0;
        } else if (current.node) {
            current = current.node;
            return 3;
        } else if (current.pred) {
            while (current.pred) current = current.pred;
            while (current && !current.node) current = current.next;
            if (!current) return null;
            else return this.DepthFirst();
        }
    }
    JIterator.prototype.ReadArray = function () {
        var retVal = {};
        var item = current;
        do {
            if (item.value instanceof Object) {
                if (item.value.length == 0) retVal[item.key] = item.node;
                else retVal[item.key] = item;
            } else retVal[item.key] = item.value;
            item = item.next;
        } while (item != null);
        return retVal;
    }
    JIterator.prototype.FindKey = function (key) {
        var pos = current;
        while (current && current.key != key) this.DepthFirst();
        if (current.key == key) {
            var retVal = current;
            current = pos;
            return retVal;
        } else {
            current = pos;
            return null;
        }
    }

    return JIterator;
})();

traverse(new JIterator(src));

Your short JSON version was different, now using this one, which looks like your required results (attrs changed and text moved from root under foo):

{
  "foo": {
    "bar": {
      "type": "someType",
      "id": "ga241ghs"
    },
    "tags": [
      {
        "@tagId": "123",
        "tagAttributes": {
          "attr1": "AAA",
          "attr2": "111"
        }
      },
      {
        "@tagId": "456",
        "tagAttributes": {
          "attr1": "BBB",
          "attr2": "222"
        }
      }
    ],
    "text": "My text"
  }
}

There is recursion used behind curtains - hidden in JIterator construction. So you do not need to care. Call with instantiation of iterator is @end when it is known to keep logic on top, but it is simple call: traverse(new JIterator(src));
Bad - no JavaScript around, but node.js can help here too (created in).

Alec · Accepted Answer · 2019-08-30 09:16:57Z

0

Figured it out how to do it with a Ruby filter directly in Logstash - for all searching for this in future, here is one example on how to do it for @tagId:

filter {
        ruby { code => '
            i = 0
            tagId_array = Array.new
            while i < event.get( "[foo][tags]" ).length do
                tagId_array = tagId_array.push(event.get( "[foo][tags][" + i.to_s + "][@tagId]" ))
                i += 1
                end
            event.set( "foo.tags.tagId", tagId_array )
        '
        }
}

answered Aug 30, 2019 at 9:16

Alec

1001 silver badge11 bronze badges

Collectives™ on Stack Overflow

Logstash: Flatten nested JSON, combine fields inside array

2 Answers 2

2 Comments

Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

2 Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related