从未满足针对@version 的正则表达式条件

Question

我的应用程序堆栈有一个 docker-compose 项目。我正在使用 Logspout 将所有服务的 docker 日志转发到 Logstash 实例。基于几个正则表达式条件，我想使用不同的 groks 和其他过滤器从日志消息中获取相关数据。

问题是 Logstash 对于正则表达式比较总是 returns false。我的条件永远不会触发。

我的管道配置是这样的：

input {
    tcp {
        port => 5000
    }
}

filter {
    if [@version] =~ /1/ {
        mutate {
            add_field => { "regex" => "Version 1" }
        }
    }

    if [@version] == "1" {
        mutate {
            add_field => { "equality" => "Version 1" }
        }
    }
}

output {
    stdout {
    }
}

从未添加 "regex" => "Version 1" 字段。

示例输出：

logstash_1       | {
logstash_1       |       "@version" => "1",
logstash_1       |     "@timestamp" => 2019-12-04T12:00:07.441Z,
logstash_1       |        "message" => "{\"hostname\": \"localhost\", \"container\": \"/deploy_scheduler_1\", \"timestamp\": \"2019-12-04T12:00:07Z\", \"source\" : \"stdout\", \"message\": \"Scheduler tick 2019-12-04T12:00:07.440Z\" }",
logstash_1       |           "host" => "deploy_logspout_1.deploy_default",
logstash_1       |           "port" => 51786,
logstash_1       |       "equality" => "Version 1"
logstash_1       | }
logstash_1       | {
logstash_1       |       "@version" => "1",
logstash_1       |     "@timestamp" => 2019-12-04T12:01:07.505Z,
logstash_1       |        "message" => "{\"hostname\": \"localhost\", \"container\": \"/deploy_scheduler_1\", \"timestamp\": \"2019-12-04T12:01:07Z\", \"source\" : \"stdout\", \"message\": \"Scheduler tick 2019-12-04T12:01:07.504Z\" }",
logstash_1       |           "host" => "deploy_logspout_1.deploy_default",
logstash_1       |           "port" => 51786,
logstash_1       |       "equality" => "Version 1"
logstash_1       | }
logstash_1       | {
logstash_1       |       "@version" => "1",
logstash_1       |     "@timestamp" => 2019-12-04T12:01:36.457Z,
logstash_1       |        "message" => "{\"hostname\": \"localhost\", \"container\": \"/deploy_app_1\", \"timestamp\": \"2019-12-04T12:01:36Z\", \"source\" : \"stderr\", \"message\": \"ERROR GET /article-asset/co-je-to-oop-a-proc-se-to-mam-ucit/oop-header.jpg/360x200 AssetError: No such file or directory: /home/teri/projects/docs/blog/co-je-to-oop-a-proc-se-to-mam-ucit/oop-header.jpg\\n    at getDocFilePath (/portal/packages/services/app/packages/server/routes/assetRoute.ts:64:15)\\n    at processTicksAndRejections (internal/process/task_queues.js:93:5)\\n    at getAssetPath (/portal/packages/services/app/packages/server/routes/assetRoute.ts:76:23)\\n    at articleAssetsRoute (/portal/packages/services/app/packages/server/routes/assetRoute.ts:122:23)\" }",
logstash_1       |           "host" => "deploy_logspout_1.deploy_default",
logstash_1       |           "port" => 51786,
logstash_1       |       "equality" => "Version 1"
logstash_1       | }
logstash_1       | {
logstash_1       |       "@version" => "1",
logstash_1       |     "@timestamp" => 2019-12-04T12:01:36.489Z,
logstash_1       |        "message" => "{\"hostname\": \"localhost\", \"container\": \"/deploy_app_1\", \"timestamp\": \"2019-12-04T12:01:36Z\", \"source\" : \"stderr\", \"message\": \"ERROR GET /article-asset/proc-je-dobre-umet-programovaci-jazyk-java/java-header.jpg/360x200 AssetError: No such file or directory: /home/teri/projects/docs/blog/proc-je-dobre-umet-programovaci-jazyk-java/java-header.jpg\\n    at getDocFilePath (/portal/packages/services/app/packages/server/routes/assetRoute.ts:64:15)\\n    at processTicksAndRejections (internal/process/task_queues.js:93:5)\\n    at getAssetPath (/portal/packages/services/app/packages/server/routes/assetRoute.ts:76:23)\\n    at articleAssetsRoute (/portal/packages/services/app/packages/server/routes/assetRoute.ts:122:23)\" }",
logstash_1       |           "host" => "deploy_logspout_1.deploy_default",
logstash_1       |           "port" => 51786,
logstash_1       |       "equality" => "Version 1"
logstash_1       | }

Answer 1

您的正则表达式条件是正确的，问题是您正在针对 @version 对其进行测试，这是一个内部 logstash 字段。

如果您创建一个值为 @version 的字段并在您的正则表达式中测试该字段，您会发现它有效。

以下管道表明您的正则表达式正在运行。

input {
    generator {
        lines => [
          "test message line"
        ]
        count => 1
    }
}

filter {
    mutate {
        add_field => {"test1" => "%{[@version]}" }
        add_field => {"test2" => 2 }
    }
    if [test1] =~ /1/ {
        mutate {
            add_field => { "regex1" => "regex test 1" }
        }
    }
    if [test2] =~ /2/ {
        mutate {
            add_field => { "regex2" => "regex test 2" }
        }
    }
    if [@version] =~ /1/ {
        mutate {
            add_field => { "regex" => "regex Version 1" }
        }
    }
    if [@version] == "1" {
        mutate {
            add_field => { "equality" => "Version 1" }
        }
    }
}

output {
    stdout { }
}

你的输出将是这样的

{
         "test2" => "2",
          "host" => "elk",
        "regex2" => "regex teste 2",
         "test1" => "1",
      "equality" => "Version 1",
      "sequence" => 0,
       "message" => "test message line",
      "@version" => "1",
        "regex1" => "regex test 1",
    "@timestamp" => 2019-12-10T02:04:23.021Z
}

字段 regex1 和 regex2 的存在表明正则表达式有效，只是由于某些未知原因它不适用于 @version 字段，该字段用于 logstash仅供内部使用，可以在您的管道中安全地移除。

如果您想在管道中使用正则表达式，您应该将其与自定义字段或文档中存在的字段一起使用。

从未满足针对@version 的正则表达式条件

regex condition against @version is never fulfilled

logstash

elastic-stack