使用 Nest 搜索未产生预期结果

Search with Nest not yielding expected result

我正在使用以下代码创建索引:

        var ElasticSettings = new ConnectionSettings(new Uri(ConnectionString))
            .DefaultIndex(_IndexName)
        .DefaultMappingFor<PictureObject>(M => M
            .Ignore(_ => _._id)
            .Ignore(_ => _.Log))
            .DefaultFieldNameInferrer(_ => _);

    _ElasticClient = new ElasticClient(ElasticSettings);

    if (!_ElasticClient.IndexExists(_IndexName).Exists)
    {
        var I = _ElasticClient.CreateIndex(_IndexName, Ci => Ci
            .Settings(S => S
                .Analysis(A => A
                    .CharFilters(Cf => Cf.Mapping("expressions",
                        E => E.Mappings(ExpressionsList))
                    )
                    .TokenFilters(Tf => Tf.Synonym("synonyms",
                        Descriptor => new SynonymTokenFilter
                        {
                            Synonyms = SynonymsList,
                            Tokenizer = "whitespace"
                        })
                    )
                    .Analyzers(Analyzer => Analyzer
                        .Custom("index", C => C
                            .CharFilters("expressions")
                            .Tokenizer("standard")
                            .Filters("synonyms", "standard", "lowercase", "stop")
                        )
                        .Custom("search", C => C
                            .CharFilters("expressions")
                            .Tokenizer("standard")
                            .Filters("synonyms", "standard", "lowercase", "stop")
                        )
                    )
                )
            )
            .Mappings(Mapping => Mapping
                .Map<PictureObject>(Map => Map
                    .AutoMap()
                    .Properties(P => P
                        .Text(T => T
                            .Name(N => N.Title)
                            .Analyzer("index")
                            .SearchAnalyzer("search")
                        )
                        .Text(T => T
                            .Name(N => N.Tags)
                            .Analyzer("index")
                            .SearchAnalyzer("search")
                        )
                    )
                )
            )
        );

我要搜索的字段是'title'和'tags'

我的同义词是这样的格式:

[ "big => large, huge", "small => tiny, minuscule", ]

我的表情是这样的:

[ "stormy weather => storm", "happy day => joy", ]

我正在用这两种方法进行测试:

var Test1 = _ElasticClient.Search<PictureObject>(S => S
        .From(From)
        .Size(Take)
        .Query(_ => _.Fuzzy(Fuzz => Fuzz.Field(F => F.Tags).Field(T => T.Title).Value(Terms).MaxExpansions(2)))).Documents;

var resTest2 = _ElasticClient.Search<PictureObject>(S => S
        .Query(_ => _.QueryString(F => F.Query(Terms)))
        .From(From)
        .Size(Take));

当尝试完全匹配标签字段中的字词时,这两个函数 return 会产生不同的结果。 尝试使用同义词时,结果又有所不同。

(最终,我也想处理拼写错误,但现在我只是用逐字字符串进行测试)

我错过了什么? (我对API的理解还很模糊,所以错误可能很明显)

编辑: 这是一个可以编译的完整工作示例。

namespace Test
{
    using System;
    using System.Collections.Generic;
    using Nest;

    public class MyData
    {
        public string Id;
        public string Title;
        public string Tags;
    }

    public static class Program
    {
        public static void Main()
        {
            const string INDEX_NAME = "testindex";

            var ExpressionsList = new[]
            {
                "bad weather => storm",
                "happy day => sun"
            };

            var SynonymsList = new[]
            {
                "big => large, huge",
                "small => tiny, minuscule",
                "sun => sunshine, shiny, sunny"
            };

            // connect
            var ElasticSettings = new ConnectionSettings(new Uri("http://elasticsearch:9200"))
                .DefaultIndex(INDEX_NAME)
                .DefaultFieldNameInferrer(_ => _) // stop the camel case
                .DefaultMappingFor<MyData>(M => M.IdProperty("Id"));

            var Client = new ElasticClient(ElasticSettings);

            // erase the old index, if any
            if (Client.IndexExists(INDEX_NAME).Exists) Client.DeleteIndex(INDEX_NAME);

            // create the index
            var I = Client.CreateIndex(INDEX_NAME, Ci => Ci
                .Settings(S => S
                    .Analysis(A => A
                        .CharFilters(Cf => Cf.Mapping("expressions",
                            E => E.Mappings(ExpressionsList))
                        )
                        .TokenFilters(Tf => Tf.Synonym("synonyms",
                            Descriptor => new SynonymTokenFilter
                            {
                                Synonyms = SynonymsList,
                                Tokenizer = "whitespace"
                            })
                        )
                        .Analyzers(Analyzer => Analyzer
                            .Custom("index", C => C
                                .CharFilters("expressions")
                                .Tokenizer("standard")
                                .Filters("synonyms", "standard", "lowercase", "stop")
                            )
                            .Custom("search", C => C
                                .CharFilters("expressions")
                                .Tokenizer("standard")
                                .Filters("synonyms", "standard", "lowercase", "stop")
                            )
                        )
                    )
                )
                .Mappings(Mapping => Mapping
                    .Map<MyData>(Map => Map
                        .AutoMap()
                        .Properties(P => P
                            .Text(T => T
                                .Name(N => N.Title)
                                .Analyzer("index")
                                .SearchAnalyzer("search")
                            )
                            .Text(T => T
                                .Name(N => N.Tags)
                                .Analyzer("index")
                                .SearchAnalyzer("search")
                            )
                        )
                    )
                )
            );

            // add some data
            var Data = new List<MyData>
            {
                new MyData { Id = "1", Title = "nice stormy weather", Tags = "storm nice" },
                new MyData { Id = "2", Title = "a large storm with sunshine", Tags = "storm large sunshine" },
                new MyData { Id = "3", Title = "a storm during a sunny day", Tags = "sun storm" }
            };

            Client.IndexMany(Data);
            Client.Refresh(INDEX_NAME);


            // do some queries
            var TestA1 = Client.Search<MyData>(S => S.Query(_ => _.Fuzzy(Fuzz => Fuzz.Field(F => F.Tags).Field(T => T.Title).Value("stormy sunny").MaxExpansions(2)))).Documents;
            var TestA2 = Client.Search<MyData>(S => S.Query(_ => _.Fuzzy(Fuzz => Fuzz.Field(F => F.Tags).Field(T => T.Title).Value("stromy sunny").MaxExpansions(2)))).Documents;

            var TestB1 = Client.Search<MyData>(S => S.Query(_ => _.QueryString(F => F.Query("stormy sunny")))).Documents;
            // expected to return documents 1, 2, 3 because of synonyms: sun => sunny, shiny, sunshine

            var TestB2 = Client.Search<MyData>(S => S.Query(_ => _.QueryString(F => F.Query("bad weather")))).Documents;
            var TestB3 = Client.Search<MyData>(S => S.Query(_ => _.QueryString(F => F.Query("a large happy day")))).Documents;

            /*
             * I'm expecting the fuzzy queries to handle misspellings
             * Also, I'm expecting the expressions and synonyms to do the substitutions as they're written
             *
             * Ideally I'd like to handle:
             *  - expressions
             *  - synonyms
             *  - misspellings
             *
             * all together
             *
             * I have tried a lot of string examples while debugging and it's really hit or miss.
             * Unfortunately, I haven't kept the strings, but it was enough to see that there is something
             * wrong with my approach in this code.
             */
        }
    }
}

这里有一些提示可以帮助您走上正轨

字符过滤器

var ExpressionsList = new[]
{
    "bad weather => storm",
    "happy day => sun"
};

考虑这些是否应该是字符过滤器;它们可能是,但通常,字符过滤器用于分词器可能分词不正确的地方,例如

  • 分词前剥离 HTML 个标签
  • 当我们理想地希望在字符过滤器中保留并替换为 and 时,标准分词器会删除 &
  • 标准分词器将 c# 分词为 c,理想情况下我们希望在字符过滤器
  • 中保留并替换为 csharp

可能是你想进行字符过滤,但是在多词的情况下,用同义词或同义词图可能会更好。

自定义分析器

indexsearch 自定义分析器是相同的,您可以删除一个。同样,如果未明确设置,text 数据类型字段的 search_analyzer 将是配置的 analyzer,因此这会稍微简化一些事情。

同义词

var SynonymsList = new[]
{
    "big => large, huge",
    "small => tiny, minuscule",
    "sun => sunshine, shiny, sunny"
};

这是一个 directional synonym map 即左侧的匹配项将 替换为 右侧的所有替代项。如果所有人都应该被视为彼此相同的同义词,那么您可能不需要方向图,即

var SynonymsList = new[]
{
    "big, large, huge",
    "small, tiny, minuscule",
    "sun, sunshine, shiny, sunny"
};

这将 return 全部 3 个文档

var TestB1 = Client.Search<MyData>(S => S.Query(_ => _.QueryString(F => F.Query("stormy sunny")))).Documents;
// expected to return documents 1, 2, 3 because of synonyms: sun => sunny, shiny, sunshine

令牌过滤器

.Custom("index", C => C
    .CharFilters("expressions")
    .Tokenizer("standard")
    .Filters("synonyms", "standard", "lowercase", "stop")
)
.Custom("search", C => C
    .CharFilters("expressions")
    .Tokenizer("standard")
    .Filters("synonyms", "standard", "lowercase", "stop")
)

标记过滤器的顺序很重要,因此您想 运行 同义词过滤器 小写过滤器

之后

模糊查询

Fuzzy queries 是术语级查询,因此查询输入不经过分析,这意味着如果您 运行 它针对在索引时分析的字段,模糊查询输入将需要匹配索引时分析文档的术语输出。如果查询输入是在索引时被标记为多个术语的查询输入,即模糊查询输入将被视为一个完整的术语,但目标文档字段的索引时间值可能有被拆分成多个术语。

查看权威指南中的 Fuzziness section - 它适用于 Elasticsearch 2.x,但在很大程度上仍与更高版本相关。您可能希望使用支持模糊性并在查询时执行分析的全文查询,例如 query_stringmatchmulti_match 查询。

一个例子

将这些放在一起,这是一个在开发时可以使用的示例

public class MyData
{
    public string Id;
    public string Title;
    public string Tags;
}

public static void Main()
{
    const string INDEX_NAME = "testindex";

    var expressions = new[]
    {
            "bad weather => storm",
            "happy day => sun"
    };

    var synonyms = new[]
    {
            "big, large, huge",
            "small, tiny, minuscule",
            "sun, sunshine, shiny, sunny"
    };

    // connect
    var settings = new ConnectionSettings(new Uri("http://localhost:9200"))
        .DefaultIndex(INDEX_NAME)
        .DefaultFieldNameInferrer(s => s) // stop the camel case
        .DefaultMappingFor<MyData>(m => m.IdProperty("Id"))
        .DisableDirectStreaming()
        .PrettyJson()
        .OnRequestCompleted(callDetails =>
        {
            if (callDetails.RequestBodyInBytes != null)
            {
                Console.WriteLine(
                    $"{callDetails.HttpMethod} {callDetails.Uri} \n" +
                    $"{Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}");
            }
            else
            {
                Console.WriteLine($"{callDetails.HttpMethod} {callDetails.Uri}");
            }

            Console.WriteLine();

            if (callDetails.ResponseBodyInBytes != null)
            {
                Console.WriteLine($"Status: {callDetails.HttpStatusCode}\n" +
                         $"{Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}\n" +
                         $"{new string('-', 30)}\n");
            }
            else
            {
                Console.WriteLine($"Status: {callDetails.HttpStatusCode}\n" +
                         $"{new string('-', 30)}\n");
            }
        });

    var Client = new ElasticClient(settings);

    // erase the old index, if any
    if (Client.IndexExists(INDEX_NAME).Exists) Client.DeleteIndex(INDEX_NAME);

    // create the index
    var createIndexResponse = Client.CreateIndex(INDEX_NAME, c => c
        .Settings(s => s
            .Analysis(a => a
                .CharFilters(cf => cf
                    .Mapping("expressions", E => E
                        .Mappings(expressions)
                    )
                )
                .TokenFilters(tf => tf
                    .Synonym("synonyms", sy => sy
                        .Synonyms(synonyms)
                        .Tokenizer("whitespace")
                    )
                )
                .Analyzers(an => an
                    .Custom("index", ca => ca
                        .CharFilters("expressions")
                        .Tokenizer("standard")
                        .Filters("standard", "lowercase", "synonyms",  "stop")
                    )
                )
            )
        )
        .Mappings(m => m
            .Map<MyData>(mm => mm
                .AutoMap()
                .Properties(p => p
                    .Text(t => t
                        .Name(n => n.Title)
                        .Analyzer("index")
                    )
                    .Text(t => t
                        .Name(n => n.Tags)
                        .Analyzer("index")
                    )
                )
            )
        )
    );

    // add some data
    var data = new List<MyData>
        {
            new MyData { Id = "1", Title = "nice stormy weather", Tags = "storm nice" },
            new MyData { Id = "2", Title = "a large storm with sunshine", Tags = "storm large sunshine" },
            new MyData { Id = "3", Title = "a storm during a sunny day", Tags = "sun storm" }
        };

    Client.IndexMany(data);
    Client.Refresh(INDEX_NAME);

    //var query = "stormy sunny";
    var query = "stromy sunny";
    // var query = "bad weather";
    // var query = "a large happy day";

    var testA1 = Client.Search<MyData>(s => s
        .Query(q => q
            .MultiMatch(fu => fu
                .Fields(f => f
                    .Field(ff => ff.Tags)
                    .Field(ff => ff.Title)
                )           
                .Query(query)
                .Fuzziness(Fuzziness.EditDistance(2))
            )
        )
    ).Documents;
}

我在连接设置中添加了 .DisableDirectStreaming().PrettyJson() 和一个 .OnRequestCompleted(...) 处理程序,以便您可以看到写入控制台的请求和响应。这些在开发时很有用,但您可能希望在生产中删除它们,因为它们会增加开销。在这里开发时,像 Linqpad 这样的小应用程序会有所帮助:)

该示例使用了一个 multi_match 查询,启用了编辑距离 2 的模糊性(可能只想在这里使用自动模糊性,它做了一个明智的工作),运行ning 在 TagsTitle 字段。所有三个文档都针对(拼写错误的)查询 returned "stromy sunny"