Skip to content
Fix Code Error

Search match multiple values in single field in Elasticsearch

June 24, 2021 by Code Error
Posted By: Anonymous

I’m using Elasticsearch for search for all document has string REQUEST and partnerId=2960 and customerId= in message field
I’m using this query but nothing return

{
  "query": {
    "bool": {
      "must": [
        {
          "match_phrase": {
            "message": "REQUEST"
          }
        },
        {
          "match_phrase": {
            "message": "partnerId=2960"
          }
        },
        {
          "match_phrase": {
            "message": "customerId="
          }
        }
      ]
    }
  }
}

response:

{
  "took" : 213,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 0,
      "relation" : "eq"
    },
    "max_score" : null,
    "hits" : [ ]
  }
}

My data:

{
  "_index": "demoapp-log_2021.05.31",
  "_type": "_doc",
  "_id": "epU5wXkBQRhN_CfKNayH",
  "_version": 1,
  "_score": null,
  "_source": {
    "tags": [
      "beats_input_codec_plain_applied",
      "demoapp-log"
    ],
    "message": [
      "2021-05-31 14:00:21,433 INFO  c.m.v.w.r.CollectionService_v_2_0_0 Line 434 - Inquire to business [b57add0e-4819-4130-9051-0628979b468f] CollectionInquiryRequestMsg_V_1_0_6{action=INQUIRE, transactionId=b57add0e-4819-4130-9051-0628979b468f, partnerId=2960, merchantId=VNP, customerId=091**59, billMonth=, partnerRequestId=6762e6fc-544b-4821-95f0-f4b81b547f3f, additionalInfo={"ACTION":"INQUIRE","VERSION":"2.0.0","PARTNER_ID":2960,"PROVINCE_ID":"VNP","CUSTOMER_ID":"091**59","BILL_MONTH":"","TRANS_REQUEST_ID":"6762e6fc-544b-4821-95f0-f4b81b547f3f","CHANNEL_ID":"1","TRANS_DATE_TIME":"20210531140021","SECURE_CODE":"3354327f5af3ec9e0285c81bf5a4f72e3d2dd3afce703fffd9e2fbec1666e93d"}, owner=, provinceCode=, paymentCode=, channelId=1, clientIp=, createDate=Mon May 31 14:00:21 ICT 2021, lastUpdate=Mon May 31 14:00:21 ICT 2021, collectionPartnerDate=Mon May 31 14:00:21 ICT 2021, merchantDate=null}",
      "Inquire to business [b57add0e-4819-4130-9051-0628979b468f] CollectionInquiryRequestMsg_V_1_0_6{action=INQUIRE, transactionId=b57add0e-4819-4130-9051-0628979b468f, partnerId=2960, merchantId=VNP, customerId=091**59, billMonth=, partnerRequestId=6762e6fc-544b-4821-95f0-f4b81b547f3f, additionalInfo={"ACTION":"INQUIRE","VERSION":"2.0.0","PARTNER_ID":2960,"PROVINCE_ID":"VNP","CUSTOMER_ID":"091**59","BILL_MONTH":"","TRANS_REQUEST_ID":"6762e6fc-544b-4821-95f0-f4b81b547f3f","CHANNEL_ID":"1","TRANS_DATE_TIME":"20210531140021","SECURE_CODE":"3354327f5af3ec9e0285c81bf5a4f72e3d2dd3afce703fffd9e2fbec1666e93d"}, owner=, provinceCode=, paymentCode=, channelId=1, clientIp=, createDate=Mon May 31 14:00:21 ICT 2021, lastUpdate=Mon May 31 14:00:21 ICT 2021, collectionPartnerDate=Mon May 31 14:00:21 ICT 2021, merchantDate=null}"
    ],
    "type": "beats",
    "agent": {
      "hostname": "web01"
    },
    "host": {},
    "loglevel": "INFO",
    "logger": "c.m.v.w.r.CollectionService_v_2_0_0",
    "line": "434",
    "ecs": {},
    "input": {
      "type": "log"
    },
    "@version": "1",
    "fields": {
      "service": "payapp",
      "app_id": "demoapp-log"
    },
    "log": {
      "file": {
        "path": "/home/collection/live/logs/c_api.log"
      }
    },
    "@timestamp": "2021-05-31T07:00:21.433Z",
    "index_day": "2021.05.31"
  },
  "fields": {
    "@timestamp": [
      "2021-05-31T07:00:21.433Z"
    ]
  },
  "sort": [
    1622444421433
  ]
}

(https://pastebin.com/UZWmXLpq )

My mapping for index

{
  "demoapp-log_2021.05.31" : {
    "mappings" : {
      "properties" : {
        "@timestamp" : {
          "type" : "date"
        },
        "@version" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "agent" : {
          "properties" : {
            "ephemeral_id" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "hostname" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "id" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "name" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "type" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "version" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "ecs" : {
          "properties" : {
            "version" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "fields" : {
          "properties" : {
            "app_id" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "max_bytes" : {
              "type" : "long"
            },
            "service" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "host" : {
          "properties" : {
            "name" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "index_day" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "input" : {
          "properties" : {
            "type" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "line" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "log" : {
          "properties" : {
            "file" : {
              "properties" : {
                "path" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                }
              }
            },
            "flags" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "offset" : {
              "type" : "long"
            }
          }
        },
        "logger" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "loglevel" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "message" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "tags" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "type" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        }
      }
    }
  }
}

(https://pastebin.com/8CMm94MY )

Solution

(I updated my answer below based on your comment)

Your search doesn’t work because "REQUEST" doesn’t appear in your message as a standalone word but attached to other word like: "partnerRequestId" or "_REQUEST_".

So if you want "REQUEST" to match with "_REQUEST_" in a case sensitive manner, you must change the analyzer of the message field.
By default the analyzer doesn’t split on underscore.

First, you need to create a new index with a custom mapping (you can later reindex your existing index into this new one).

Example of an index with only the "message" field and an analyser that split on underscore and non word character (see the doc for more on tokenizer and analyzer):

PUT my-index
{
  "settings": {
    "analysis": {
      "analyzer": {
        "my_analyzer": {
          "tokenizer": "underscore"
        }
      },
      "tokenizer": {
        "underscore": {
          "type": "pattern",
          "pattern": "[_\W]+"
        }
      }
    }
  },
  "mappings" : {
    "properties" : {
      "message" : {
        "type" : "text",
         "analyzer":"my_analyzer"
        }
      }
    }
}

Then you can add your document to this new index:

POST my-index/_doc
{
  "message": [
      "2021-05-31 14:00:21,433 INFO  c.m.v.w.r.CollectionService_v_2_0_0 Line 434 - Inquire to business [b57add0e-4819-4130-9051-0628979b468f] CollectionInquiryRequestMsg_V_1_0_6{action=INQUIRE, transactionId=b57add0e-4819-4130-9051-0628979b468f, partnerId=2960, merchantId=VNP, customerId=091**59, billMonth=, partnerRequestId=6762e6fc-544b-4821-95f0-f4b81b547f3f, additionalInfo={"ACTION":"INQUIRE","VERSION":"2.0.0","PARTNER_ID":2960,"PROVINCE_ID":"VNP","CUSTOMER_ID":"091**59","BILL_MONTH":"","TRANS_REQUEST_ID":"6762e6fc-544b-4821-95f0-f4b81b547f3f","CHANNEL_ID":"1","TRANS_DATE_TIME":"20210531140021","SECURE_CODE":"3354327f5af3ec9e0285c81bf5a4f72e3d2dd3afce703fffd9e2fbec1666e93d"}, owner=, provinceCode=, paymentCode=, channelId=1, clientIp=, createDate=Mon May 31 14:00:21 ICT 2021, lastUpdate=Mon May 31 14:00:21 ICT 2021, collectionPartnerDate=Mon May 31 14:00:21 ICT 2021, merchantDate=null}",
      "Inquire to business [b57add0e-4819-4130-9051-0628979b468f] CollectionInquiryRequestMsg_V_1_0_6{action=INQUIRE, transactionId=b57add0e-4819-4130-9051-0628979b468f, partnerId=2960, merchantId=VNP, customerId=091**59, billMonth=, partnerRequestId=6762e6fc-544b-4821-95f0-f4b81b547f3f, additionalInfo={"ACTION":"INQUIRE","VERSION":"2.0.0","PARTNER_ID":2960,"PROVINCE_ID":"VNP","CUSTOMER_ID":"091**59","BILL_MONTH":"","TRANS_REQUEST_ID":"6762e6fc-544b-4821-95f0-f4b81b547f3f","CHANNEL_ID":"1","TRANS_DATE_TIME":"20210531140021","SECURE_CODE":"3354327f5af3ec9e0285c81bf5a4f72e3d2dd3afce703fffd9e2fbec1666e93d"}, owner=, provinceCode=, paymentCode=, channelId=1, clientIp=, createDate=Mon May 31 14:00:21 ICT 2021, lastUpdate=Mon May 31 14:00:21 ICT 2021, collectionPartnerDate=Mon May 31 14:00:21 ICT 2021, merchantDate=null}"
    ]
}

And finally run this search that does the job:

GET my-index/_search
{
  "query": {
    "bool": {
      "must": [
        {
          "match": {
            "message": "partnerId=2960"
          }
        },
        {
          "match": {
            "message": "customerId"
          }
        },
        {
          "match":{
              "message": "REQUEST"
          }
        }
      ]
    }
  }
}

Also I don’t know if it is intentional or not but the "message" field is indexed as both a text and a keyword. Looks to me like you don’t need the keyword on this field.

Answered By: Anonymous

Related Articles

  • How to properly do JSON API GET requests and assign output…
  • How to parse JSON with XE2 dbxJSON
  • Azure Availability Zone ARM Config
  • The 'compilation' argument must be an instance of…
  • Event Snippet for Google only shows one event while testing…
  • Avoid creating new session on each axios request laravel
  • Why does this Azure Resource Manager Template fail…
  • mongodb group values by multiple fields
  • loop and eliminate unwanted lines with beautiful soup
  • NullpointerException error while working with choiceBox and…

Disclaimer: This content is shared under creative common license cc-by-sa 3.0. It is generated from StackExchange Website Network.

Post navigation

Previous Post:

Creating a 2D list from a 2D list using list comprehension

Next Post:

Zoho Sign Webhook – Receiving Response Using PHP

Leave a Reply Cancel reply

Your email address will not be published. Required fields are marked *

  • Get code errors & solutions at akashmittal.com
© 2022 Fix Code Error