Filter data using nodejs and elasticsearch

Question

I'm currently facing an issue with my datatable implemented in ReactJS. I'm retrieving data from elasticsearch and populating the datatable with it. The data retrieval process works fine without the filter applied, however, when I apply filters to the data, the datatable remains empty, even though the data _source has matching records.

The structure of the parameters I am sending is as follows:

{
  pageIndex: 1,
  pageSize: 10,
  sort: { order: '', key: '' },
  query: '',
  filterData: {
    analysis: [ '0', '1', '2', '3' ],
    threat_level_id: [ '1', '2', '3', '4' ],
  }
}

EndPoint:

POST /api/v1/events/public/list

Controller:

exports.getPublicEvents = async (req, res) => {
  try {
    client.ping()
    const { pageIndex, pageSize, sort, query, filterData } = req.body
    let esQuery = {
      index: 'ns_*',
      body: {
        query: {
          bool: {
            must: [
              {
                match_all: {},
              },
            ],
            filter: [],
          },
        },
        from: (pageIndex - 1) * pageSize,
        size: pageSize,
      },
    }
    if (query) {
      esQuery.body.query.bool.must = [
        {
          match: {
            'Event.info': {
              query: query,
              fuzziness: 'AUTO',
            },
          },
        },
      ]
    }
    if (filterData.analysis.length > 0) {
      esQuery.body.query.bool.filter.push({
        terms: {
          'Event.analysis': filterData.analysis,
        },
      })
    }
    if (filterData.threat_level_id.length > 0) {
      esQuery.body.query.bool.filter.push({
        terms: {
          'Event.threat_level_id': filterData.threat_level_id,
        },
      })
    }
    let esResponse = await client.search(esQuery)
    let data = esResponse.hits.hits.map((hit) => hit._source)
    let total = esResponse.hits.total.value

    res.status(200).json({
      status: 'success',
      data: data,
      total: total,
    })
  } catch (error) {
    res.status(500).json({
      error: 'Error connecting to Elasticsearch',
      errorMessage: error.message,
    })
  }
}

The controller below is without filters and it works just fine.

exports.getPublicEvents = async (req, res) => {
  try {
    client.ping()
    const { pageIndex, pageSize, sort, query } = req.body
    let esQuery = {
      index: 'ns_*',
      body: {
        query: {
          match_all: {},
        },
        from: (pageIndex - 1) * pageSize,
        size: pageSize,
      },
    }
    if (query) {
      esQuery.body.query = {
        match: {
          'Event.info': {
            query: query,
            fuzziness: 'AUTO',
          },
        },
      }
    }
    let esResponse = await client.search(esQuery)
    let data = esResponse.hits.hits.map((hit) => hit._source)
    let total = esResponse.hits.total.value

    res.status(200).json({
      status: 'success',
      data: data,
      total: total,
    })
  } catch (error) {
    res.status(500).json({
      error: 'Error connecting to Elasticsearch',
      errorMessage: error.message,
    })
  }
}

ElasticSearech version: 7.17.8

Result of: console.log(JSON.stringify(esQuery))

{
  "index": "INDEX_NAME",
  "body": {
    "query": {
      "bool": {
        "must": [{ "match_all": {} }],
        "filter": [
          { "terms": { "Event.analysis": ["0", "1", "2"] } },
          { "terms": { "Event.threat_level_id": ["1", "2", "3", "4"] } }
        ]
      }
    },
    "from": 0,
    "size": 10
  }
}

Data in elascticsearch schema

{
    "@version": "1",
    "@timestamp": "2023-02-01T14:43:09.997Z",
    "Event": {
        "info": ".......................",
        
        "description": ".......................",
        "analysis": 0,
        "threat_level_id": "4",
        "created_at": 1516566351,
        "uuid": "5a64f74f0e543738c12bc973322",
        "updated_at": 1675262417
    }
}

Index Mapping

{
    "index_patterns": ["INDEX_NAME"],
    "template": "TEMPLATE_NAME",
    "settings": {
      "number_of_replicas": 0,
      "index.mapping.nested_objects.limit": 10000000
      },
    "mappings": {
      "dynamic": false,
      "properties": {
          "@timestamp": {
          "type": "date"
        },
        "Event": {
          "type": "nested",
          "properties": {
            "date_occured": {
              "type": "date"
            },
            "threat_level_id": {
              "type": "integer"
            },
            "description": {
              "type": "text"
            },
            "is_shared": {
              "type": "boolean"
            },
            "analysis": {
              "type": "integer"
            },
            "uuid": {
              "type": "text"
            },
            "created_at": {
              "type": "date"
            },
            "info": {
              "type": "text"
            },
            "shared_with": {
                "type": "nested",
                 "properties": {
                  "_id": {
                    "type": "text"
                }
              }
            },
            "updated_at": {
              "type": "date"
            },
            "author": {
              "type": "text"
            },
            "Attributes": {
              "type": "nested",
              "properties": {
                "data": {
                  "type": "text"
                },
                "type": {
                  "type": "text"
                },
                "uuid": {
                  "type": "text"
                },
                "comment": {
                  "type": "text"
                },
                "category": {
                  "type": "text"
                },
                "value": {
                  "type": "text"
                },
                "timestamp": {
                  "type": "date"
                }
              }
            }, 
            "organisation": {
              "type": "nested",
              "properties": {
                "name": {
                  "type": "text"
                },
                "uuid": {
                  "type": "text"
                }
              }
            },
            "Tags": {
              "type": "nested",
              "properties": {
                "color": {
                  "type": "text"
                },
                "name": {
                  "type": "text"
                }
              }
            },
            "TLP": {
              "type": "nested",
              "properties": {
                "color": {
                  "type": "text"
                },
                "name": {
                  "type": "text"
                }
              }
            }
          }
        }  
      }
    }
  }

What are some values you are sending in query in your controller ? is it the first snippet ? Do you not have errors from es client ? Logs we could analyze ? — Paulo
– Paulo, Commented Jan 31, 2023 at 15:18
The params Im sending are mensionned in my question and the data returned is empty and get no errors in es client — Richard Branson
– Richard Branson, Commented Feb 1, 2023 at 10:48
Can you print out the query that is generated when you apply filters? like JSON.stringify(esQuery). Also can you specify which version of the ES client you're using because in recent versions, the body parameter disappeared and the query should be specified at the top-level? — Val
– Val, Commented Feb 1, 2023 at 12:05
I updated my question and added the ES version and JSON.stringify(esQuery) result — Richard Branson
– Richard Branson, Commented Feb 1, 2023 at 16:38

Val · Accepted Answer · 2023-02-07 13:17:41Z

1

+50

Event is a nested field, so you need to use nested queries, like this:

{
  "index": "INDEX_NAME",
  "body": {
    "query": {
      "bool": {
        "must": [{ "match_all": {} }],
        "filter": [
          {
            "nested": {
              "path": "Event",
              "query": {"terms": { "Event.analysis": ["0", "1", "2"] }}
            } 
          },
          {
            "nested": {
              "path": "Event",
              "query": {"terms": { "Event.threat_level_id": ["1", "2", "3", "4"] }}
            } 
          }
        ]
      }
    },
    "from": 0,
    "size": 10
  }
}

answered Feb 7, 2023 at 13:17

Val

218k14 gold badges377 silver badges384 bronze badges

Sign up to request clarification or add additional context in comments.

1 Comment

Richard Branson Over a year ago

Thank you Mr @Val for your help. It's working now :)

Collectives™ on Stack Overflow

Filter data using nodejs and elasticsearch

1 Answer 1

1 Comment

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

1 Comment

Your Answer

Sign up or log in

Post as a guest

Related