Skip to content

Commit

Permalink
[fix](inverted index) Fix the issue with incorrect seek results in DI…
Browse files Browse the repository at this point in the history
…CT_COMPRESS (#45738)

Related PR: #44414

Problem Summary:
In inverted index version 3 mode, using dictionary compression may
lead to incorrect results after a seek operation.
  • Loading branch information
zzzxl1993 authored Dec 23, 2024
1 parent 49d397b commit 0fe8d7b
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 1 deletion.
12 changes: 12 additions & 0 deletions regression-test/data/inverted_index_p0/test_inverted_index_v3.out
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,15 @@
-- !sql --
105

-- !sql --
238

-- !sql --
104

-- !sql --
104

-- !sql --
105

Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@
suite("test_inverted_index_v3", "p0"){
def indexTbName1 = "test_inverted_index_v3_1"
def indexTbName2 = "test_inverted_index_v3_2"
def indexTbName3 = "test_inverted_index_v3_3"

sql "DROP TABLE IF EXISTS ${indexTbName1}"
sql "DROP TABLE IF EXISTS ${indexTbName2}"
sql "DROP TABLE IF EXISTS ${indexTbName3}"

sql """
CREATE TABLE ${indexTbName1} (
Expand Down Expand Up @@ -59,6 +61,24 @@ suite("test_inverted_index_v3", "p0"){
);
"""

sql """
CREATE TABLE ${indexTbName3} (
`@timestamp` int(11) NULL COMMENT "",
`clientip` varchar(20) NULL COMMENT "",
`request` text NULL COMMENT "",
`status` int(11) NULL COMMENT "",
`size` int(11) NULL COMMENT "",
INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true", "dict_compression" = "true") COMMENT ''
) ENGINE=OLAP
DUPLICATE KEY(`@timestamp`)
COMMENT "OLAP"
DISTRIBUTED BY RANDOM BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"inverted_index_storage_format" = "V3"
);
"""

def load_httplogs_data = {table_name, label, read_flag, format_flag, file_name, ignore_failure=false,
expected_succ_rows = -1, load_to_single_tablet = 'true' ->

Expand Down Expand Up @@ -99,6 +119,7 @@ suite("test_inverted_index_v3", "p0"){
try {
load_httplogs_data.call(indexTbName1, indexTbName1, 'true', 'json', 'documents-1000.json')
load_httplogs_data.call(indexTbName2, indexTbName2, 'true', 'json', 'documents-1000.json')
load_httplogs_data.call(indexTbName3, indexTbName3, 'true', 'json', 'documents-1000.json')

sql "sync"

Expand All @@ -112,6 +133,11 @@ suite("test_inverted_index_v3", "p0"){
qt_sql """ select count() from ${indexTbName2} where request match_phrase 'hm bg'; """
qt_sql """ select count() from ${indexTbName2} where request match_phrase_prefix 'hm bg'; """

qt_sql """ select count() from ${indexTbName3} where request match_any 'hm bg'; """
qt_sql """ select count() from ${indexTbName3} where request match_all 'hm bg'; """
qt_sql """ select count() from ${indexTbName3} where request match_phrase 'hm bg'; """
qt_sql """ select count() from ${indexTbName3} where request match_phrase_prefix 'hm bg'; """

} finally {
}
}

0 comments on commit 0fe8d7b

Please sign in to comment.