|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +// score() over MATCH on a keyword (untokenized) inverted index. Like an Elasticsearch match or term |
| 19 | +// query on a keyword field, it scores BM25 with unit frequency and length, so every matching row |
| 20 | +// scores the idf of the whole value, the same as SEARCH on that field; it used to score 0. An |
| 21 | +// equality predicate stays a pure filter. |
| 22 | +suite("test_match_keyword_score", "p0") { |
| 23 | + // MATCH and SEARCH need index execution even when CI fuzzes these session defaults. |
| 24 | + sql "set enable_common_expr_pushdown = true" |
| 25 | + sql "set enable_inverted_index_query = true" |
| 26 | + |
| 27 | + for (def storageFormat : ["V2", "V3"]) { |
| 28 | + sql "DROP TABLE IF EXISTS test_match_keyword_score" |
| 29 | + sql """ |
| 30 | + CREATE TABLE test_match_keyword_score ( |
| 31 | + id INT NOT NULL, |
| 32 | + body STRING NULL, |
| 33 | + tag STRING NULL, |
| 34 | + tag_none STRING NULL, |
| 35 | + INDEX idx_body(body) USING INVERTED |
| 36 | + PROPERTIES("parser" = "english", "support_phrase" = "true"), |
| 37 | + INDEX idx_tag(tag) USING INVERTED PROPERTIES("ignore_above" = "1024"), |
| 38 | + INDEX idx_tag_none(tag_none) USING INVERTED |
| 39 | + PROPERTIES("parser" = "none", "support_phrase" = "true") |
| 40 | + ) ENGINE=OLAP |
| 41 | + DUPLICATE KEY(id) |
| 42 | + DISTRIBUTED BY HASH(id) BUCKETS 1 |
| 43 | + PROPERTIES ( |
| 44 | + "replication_allocation" = "tag.location.default:1", |
| 45 | + "inverted_index_storage_format" = "${storageFormat}" |
| 46 | + ) |
| 47 | + """ |
| 48 | + // The first row alone is the reported case: one document scores idf = ln(4/3) = 0.287682. |
| 49 | + sql """INSERT INTO test_match_keyword_score VALUES (1, 'apple apple', 'BP', 'Hello World')""" |
| 50 | + sql "sync" |
| 51 | + quickTest("${storageFormat}_single_row_match", """ |
| 52 | + SELECT id, ROUND(s, 6) FROM ( |
| 53 | + SELECT id, score() AS s FROM test_match_keyword_score |
| 54 | + WHERE tag MATCH 'BP' ORDER BY s DESC LIMIT 10 |
| 55 | + ) hits ORDER BY id |
| 56 | + """, true) |
| 57 | + |
| 58 | + sql """INSERT INTO test_match_keyword_score VALUES |
| 59 | + (2, 'apple pear', 'BP', 'hello world'), (3, 'pear', 'other', 'Hello World'), |
| 60 | + (4, 'apple', 'x', 'Hello'), (5, 'apple', 'BP', NULL), (6, 'apple', NULL, 'Hello World') |
| 61 | + """ |
| 62 | + sql "sync" |
| 63 | + |
| 64 | + // The whole value is one term: MATCH, MATCH_ALL and MATCH_PHRASE score as SEARCH does. |
| 65 | + for (def query : [ |
| 66 | + ["match", "tag MATCH 'BP'"], |
| 67 | + ["match_all", "tag MATCH_ALL 'BP'"], |
| 68 | + ["match_phrase", "tag MATCH_PHRASE 'BP'"], |
| 69 | + ["search", "SEARCH('tag:BP')"], |
| 70 | + // Case and spaces are kept, and phrase support inherited by parser=none is ignored. |
| 71 | + ["none_match", "tag_none MATCH 'Hello World'"], |
| 72 | + ["none_match_phrase", "tag_none MATCH_PHRASE 'Hello World'"], |
| 73 | + ["none_search", "SEARCH('tag_none:\"Hello World\"')"], |
| 74 | + // Both leaves score: idf(apple) from body plus idf(BP) from tag. |
| 75 | + ["analyzed_and_keyword", "body MATCH 'apple' AND tag MATCH 'BP'"], |
| 76 | + ["search_and", "SEARCH('body:apple AND tag:BP')"], |
| 77 | + // An equality is a filter: only the body leaf scores. |
| 78 | + ["analyzed_and_equality", "body MATCH 'apple' AND tag = 'BP'"]]) { |
| 79 | + quickTest("${storageFormat}_${query[0]}", """ |
| 80 | + SELECT id, ROUND(s, 6) FROM ( |
| 81 | + SELECT id, score() AS s FROM test_match_keyword_score |
| 82 | + WHERE ${query[1]} ORDER BY s DESC LIMIT 10 |
| 83 | + ) hits ORDER BY id |
| 84 | + """, true) |
| 85 | + } |
| 86 | + } |
| 87 | +} |
0 commit comments