diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 34d4638cc..07fe9de2c 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -5,7 +5,7 @@ luwak-parent com.github.flaxsearch - 1.6.0-SNAPSHOT + 1.7.0-SNAPSHOT 4.0.0 diff --git a/luwak/pom.xml b/luwak/pom.xml index 772cf3264..a9066be8e 100644 --- a/luwak/pom.xml +++ b/luwak/pom.xml @@ -5,7 +5,7 @@ luwak-parent com.github.flaxsearch - 1.6.0-SNAPSHOT + 1.7.0-SNAPSHOT luwak diff --git a/luwak/src/main/java/uk/co/flax/luwak/Monitor.java b/luwak/src/main/java/uk/co/flax/luwak/Monitor.java index e0af334f4..c613bf0c8 100644 --- a/luwak/src/main/java/uk/co/flax/luwak/Monitor.java +++ b/luwak/src/main/java/uk/co/flax/luwak/Monitor.java @@ -231,7 +231,8 @@ private void prepareQueryCache(boolean storeQueries) throws IOException { } seenIds.add(id); - BytesRef serializedMQ = dataValues.mq.get(dataValues.doc); + dataValues.mq.advanceExact(dataValues.doc); + BytesRef serializedMQ = dataValues.mq.binaryValue(); MonitorQuery mq = MonitorQuery.deserialize(serializedMQ); BytesRef hash = mq.hash(); @@ -494,7 +495,8 @@ public MonitorQuery getQuery(final String queryId) throws IOException { throw new IllegalStateException("Cannot call getQuery() as queries are not stored"); final MonitorQuery[] queryHolder = new MonitorQuery[]{ null }; queryIndex.search(new TermQuery(new Term(FIELDS.id, queryId)), (id, query, dataValues) -> { - BytesRef serializedMQ = dataValues.mq.get(dataValues.doc); + dataValues.mq.advanceExact(dataValues.doc); + BytesRef serializedMQ = dataValues.mq.binaryValue(); queryHolder[0] = MonitorQuery.deserialize(serializedMQ); }); return queryHolder[0]; diff --git a/luwak/src/main/java/uk/co/flax/luwak/QueryIndex.java b/luwak/src/main/java/uk/co/flax/luwak/QueryIndex.java index b0416b8ea..485b96ff7 100644 --- a/luwak/src/main/java/uk/co/flax/luwak/QueryIndex.java +++ b/luwak/src/main/java/uk/co/flax/luwak/QueryIndex.java @@ -30,7 +30,7 @@ class QueryIndex { // NB this is not final because it can be replaced by purgeCache() // package-private for testing - final Map termFilters = new HashMap<>(); + final Map termFilters = new HashMap<>(); QueryIndex(IndexWriter indexWriter) throws IOException { this.writer = indexWriter; @@ -46,8 +46,8 @@ private class TermsHashBuilder extends SearcherFactory { public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) throws IOException { IndexSearcher searcher = super.newSearcher(reader, previousReader); searcher.setQueryCache(null); - termFilters.put(reader, new QueryTermFilter(reader)); - reader.addReaderClosedListener(termFilters::remove); + termFilters.put(reader.getReaderCacheHelper().getKey(), new QueryTermFilter(reader)); + reader.getReaderCacheHelper().addClosedListener(termFilters::remove); return searcher; } } @@ -108,7 +108,9 @@ long search(QueryBuilder queryBuilder, QueryCollector matcher) throws IOExceptio MonitorQueryCollector collector = new MonitorQueryCollector(queries, matcher); long buildTime = System.nanoTime(); - Query query = queryBuilder.buildQuery(termFilters.get(searcher.getIndexReader())); + Query query = queryBuilder.buildQuery(termFilters.get(searcher.getIndexReader() + .getReaderCacheHelper() + .getKey())); buildTime = System.nanoTime() - buildTime; searcher.search(query, collector); return buildTime; @@ -241,8 +243,10 @@ public void setScorer(Scorer scorer) throws IOException { @Override public void collect(int doc) throws IOException { - BytesRef hash = dataValues.hash.get(doc); - BytesRef id = dataValues.id.get(doc); + dataValues.hash.advanceExact(doc); + dataValues.id.advanceExact(doc); + BytesRef hash = dataValues.hash.binaryValue(); + BytesRef id = dataValues.id.binaryValue(); QueryCacheEntry query = queries.get(hash); dataValues.doc = doc; matcher.matchQuery(id.utf8ToString(), query, dataValues); diff --git a/luwak/src/main/java/uk/co/flax/luwak/presearcher/FieldFilterPresearcherComponent.java b/luwak/src/main/java/uk/co/flax/luwak/presearcher/FieldFilterPresearcherComponent.java index 206e3cc90..2a512c0d3 100644 --- a/luwak/src/main/java/uk/co/flax/luwak/presearcher/FieldFilterPresearcherComponent.java +++ b/luwak/src/main/java/uk/co/flax/luwak/presearcher/FieldFilterPresearcherComponent.java @@ -75,7 +75,7 @@ public Query adjustPresearcherQuery(LeafReader reader, Query presearcherQuery) t private Query buildFilterClause(LeafReader reader) throws IOException { - Terms terms = reader.fields().terms(field); + Terms terms = reader.terms(field); if (terms == null) return null; diff --git a/luwak/src/main/java/uk/co/flax/luwak/presearcher/MultipassTermFilteredPresearcher.java b/luwak/src/main/java/uk/co/flax/luwak/presearcher/MultipassTermFilteredPresearcher.java index 3cea30138..78fbcb8a7 100644 --- a/luwak/src/main/java/uk/co/flax/luwak/presearcher/MultipassTermFilteredPresearcher.java +++ b/luwak/src/main/java/uk/co/flax/luwak/presearcher/MultipassTermFilteredPresearcher.java @@ -9,12 +9,13 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.TermsQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefHash; + import uk.co.flax.luwak.analysis.TermsEnumTokenStream; import uk.co.flax.luwak.termextractor.querytree.QueryTree; import uk.co.flax.luwak.termextractor.querytree.QueryTreeViewer; @@ -138,7 +139,15 @@ public void addTerm(String field, BytesRef term) throws IOException { public Query build() { BooleanQuery.Builder parent = new BooleanQuery.Builder(); for (int i = 0; i < passes; i++) { - parent.add(new TermsQuery(terms.get(i)), BooleanClause.Occur.MUST); + if (terms.get(i).size() == 1) { + parent.add(new TermQuery(terms.get(i).iterator().next()), BooleanClause.Occur.MUST); + } else { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + for (Term term : terms.get(i)) { + bq.add(new TermQuery(term), BooleanClause.Occur.SHOULD); + } + parent.add(bq.build(), BooleanClause.Occur.MUST); + } } return parent.build(); } diff --git a/luwak/src/main/java/uk/co/flax/luwak/presearcher/TermFilteredPresearcher.java b/luwak/src/main/java/uk/co/flax/luwak/presearcher/TermFilteredPresearcher.java index dc95ec03c..60da3b5a2 100644 --- a/luwak/src/main/java/uk/co/flax/luwak/presearcher/TermFilteredPresearcher.java +++ b/luwak/src/main/java/uk/co/flax/luwak/presearcher/TermFilteredPresearcher.java @@ -27,7 +27,6 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.*; -import org.apache.lucene.queries.TermsQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; @@ -83,18 +82,18 @@ public TermFilteredPresearcher() { public final Query buildQuery(LeafReader reader, QueryTermFilter queryTermFilter) { try { DocumentQueryBuilder queryBuilder = getQueryBuilder(); - for (String field : reader.fields()) { + for (FieldInfo fieldInfo : reader.getFieldInfos()) { - TokenStream ts = new TermsEnumTokenStream(reader.terms(field).iterator()); + TokenStream ts = new TermsEnumTokenStream(reader.terms(fieldInfo.name).iterator()); for (PresearcherComponent component : components) { - ts = component.filterDocumentTokens(field, ts); + ts = component.filterDocumentTokens(fieldInfo.name, ts); } - ts = new BytesRefFilteredTokenFilter(ts, queryTermFilter.getTerms(field)); + ts = new BytesRefFilteredTokenFilter(ts, queryTermFilter.getTerms(fieldInfo.name)); TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); while (ts.incrementToken()) { - queryBuilder.addTerm(field, BytesRef.deepCopyOf(termAtt.getBytesRef())); + queryBuilder.addTerm(fieldInfo.name, BytesRef.deepCopyOf(termAtt.getBytesRef())); } ts.close(); @@ -130,7 +129,15 @@ public void addTerm(String field, BytesRef term) throws IOException { @Override public Query build() { - return new TermsQuery(terms); + if (terms.size() == 1) { + return new TermQuery(terms.iterator().next()); + } else { + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + for (Term term : terms) { + builder.add(new TermQuery(term), BooleanClause.Occur.SHOULD); + } + return builder.build(); + } } }; } diff --git a/luwak/src/main/java/uk/co/flax/luwak/termextractor/treebuilder/TermsQueryTreeBuilder.java b/luwak/src/main/java/uk/co/flax/luwak/termextractor/treebuilder/TermsQueryTreeBuilder.java deleted file mode 100644 index ceb5b553f..000000000 --- a/luwak/src/main/java/uk/co/flax/luwak/termextractor/treebuilder/TermsQueryTreeBuilder.java +++ /dev/null @@ -1,42 +0,0 @@ -package uk.co.flax.luwak.termextractor.treebuilder; -/* - * Copyright (c) 2016 Lemur Consulting Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.queries.TermsQuery; -import uk.co.flax.luwak.termextractor.QueryAnalyzer; -import uk.co.flax.luwak.termextractor.QueryTreeBuilder; -import uk.co.flax.luwak.termextractor.querytree.QueryTree; - -public class TermsQueryTreeBuilder extends QueryTreeBuilder { - - public static final TermsQueryTreeBuilder INSTANCE = new TermsQueryTreeBuilder(); - - private TermsQueryTreeBuilder() { - super(TermsQuery.class); - } - - @Override - public QueryTree buildTree(QueryAnalyzer builder, TermsQuery query) { - try { - return builder.buildTree(query.rewrite(null)); - } catch (IOException e) { - throw new RuntimeException(e); // should never happen - } - } - -} diff --git a/luwak/src/main/java/uk/co/flax/luwak/termextractor/treebuilder/TreeBuilders.java b/luwak/src/main/java/uk/co/flax/luwak/termextractor/treebuilder/TreeBuilders.java index 51645768b..c5489d637 100644 --- a/luwak/src/main/java/uk/co/flax/luwak/termextractor/treebuilder/TreeBuilders.java +++ b/luwak/src/main/java/uk/co/flax/luwak/termextractor/treebuilder/TreeBuilders.java @@ -69,7 +69,6 @@ public QueryTree buildTree(QueryAnalyzer builder, Query query) { newFilteringQueryBuilder(BoostedQuery.class, BoostedQuery::getQuery), newDisjunctionBuilder(DisjunctionMaxQuery.class, (b, q) -> q.getDisjuncts().stream().map(b::buildTree).collect(Collectors.toList())), - TermsQueryTreeBuilder.INSTANCE, TermInSetQueryTreeBuilder.INSTANCE, new QueryTreeBuilder(SpanWithinQuery.class) { @Override diff --git a/luwak/src/main/java/uk/co/flax/luwak/util/ForceNoBulkScoringQuery.java b/luwak/src/main/java/uk/co/flax/luwak/util/ForceNoBulkScoringQuery.java index 1692ce3a6..8a5da4903 100644 --- a/luwak/src/main/java/uk/co/flax/luwak/util/ForceNoBulkScoringQuery.java +++ b/luwak/src/main/java/uk/co/flax/luwak/util/ForceNoBulkScoringQuery.java @@ -62,9 +62,9 @@ public Query getWrappedQuery() { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { - final Weight innerWeight = inner.createWeight(searcher, needsScores); + final Weight innerWeight = inner.createWeight(searcher, needsScores, boost); return new Weight(ForceNoBulkScoringQuery.this) { @Override @@ -78,18 +78,14 @@ public Explanation explain(LeafReaderContext leafReaderContext, int i) throws IO } @Override - public float getValueForNormalization() throws IOException { - return innerWeight.getValueForNormalization(); - } - - @Override - public void normalize(float v, float v1) { - innerWeight.normalize(v, v1); + public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException { + return innerWeight.scorer(leafReaderContext); } @Override - public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException { - return innerWeight.scorer(leafReaderContext); + public boolean isCacheable(LeafReaderContext ctx) { + // TODO Auto-generated method stub + return false; } }; } diff --git a/luwak/src/main/java/uk/co/flax/luwak/util/SpanOffsetReportingQuery.java b/luwak/src/main/java/uk/co/flax/luwak/util/SpanOffsetReportingQuery.java index 7ff7c5b57..68da81270 100644 --- a/luwak/src/main/java/uk/co/flax/luwak/util/SpanOffsetReportingQuery.java +++ b/luwak/src/main/java/uk/co/flax/luwak/util/SpanOffsetReportingQuery.java @@ -87,8 +87,8 @@ public int hashCode() { } @Override - public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return new SpanOffsetWeight(searcher, in.createWeight(searcher, needsScores)); + public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return new SpanOffsetWeight(searcher, in.createWeight(searcher, needsScores, boost), boost); } /** @@ -107,8 +107,8 @@ private class SpanOffsetWeight extends SpanWeight { private final SpanWeight in; - private SpanOffsetWeight(IndexSearcher searcher, SpanWeight in) throws IOException { - super(SpanOffsetReportingQuery.this, searcher, termContexts(in)); + private SpanOffsetWeight(IndexSearcher searcher, SpanWeight in, float boost) throws IOException { + super(SpanOffsetReportingQuery.this, searcher, termContexts(in), boost); this.in = in; } @@ -126,5 +126,11 @@ public Spans getSpans(LeafReaderContext ctx, Postings requiredPostings) throws I public void extractTerms(Set terms) { in.extractTerms(terms); } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + // TODO Auto-generated method stub + return false; + } } } diff --git a/luwak/src/main/java/uk/co/flax/luwak/util/SpanRewriter.java b/luwak/src/main/java/uk/co/flax/luwak/util/SpanRewriter.java index a58a67f56..8642ad837 100644 --- a/luwak/src/main/java/uk/co/flax/luwak/util/SpanRewriter.java +++ b/luwak/src/main/java/uk/co/flax/luwak/util/SpanRewriter.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.PrefixCodedTerms; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.TermsQuery; import org.apache.lucene.search.*; import org.apache.lucene.search.spans.*; import org.apache.lucene.util.BytesRef; @@ -50,8 +49,6 @@ public Query rewrite(Query in, IndexSearcher searcher) throws RewriteException, return rewriteDisjunctionMaxQuery((DisjunctionMaxQuery) in, searcher); if (in instanceof TermInSetQuery) return rewriteTermInSetQuery((TermInSetQuery) in); - if (in instanceof TermsQuery) - return rewrite(in.rewrite(null), null); if (in instanceof BoostQuery) return rewrite(((BoostQuery) in).getQuery(), searcher); // we don't care about boosts for rewriting purposes if (in instanceof PhraseQuery) diff --git a/luwak/src/test/java/uk/co/flax/luwak/TestSlowLog.java b/luwak/src/test/java/uk/co/flax/luwak/TestSlowLog.java index 58b9c9547..d7a0328a8 100644 --- a/luwak/src/test/java/uk/co/flax/luwak/TestSlowLog.java +++ b/luwak/src/test/java/uk/co/flax/luwak/TestSlowLog.java @@ -7,9 +7,9 @@ import org.apache.lucene.search.*; import org.apache.lucene.util.Bits; import org.junit.Test; + import uk.co.flax.luwak.matchers.SimpleMatcher; import uk.co.flax.luwak.presearcher.MatchAllPresearcher; - import static org.assertj.core.api.Assertions.assertThat; /** @@ -48,13 +48,38 @@ public String toString(String s) { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) { try { Thread.sleep(delay); } catch (InterruptedException e) { throw new RuntimeException(e); } - return new RandomAccessWeight(this) { + return new ConstantScoreWeight(this, boost) { + @Override + public final Scorer scorer(LeafReaderContext context) throws IOException { + final Bits matchingDocs = getMatchingDocs(context); + if (matchingDocs == null || matchingDocs instanceof Bits.MatchNoBits) { + return null; + } + final DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); + final TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { + + @Override + public boolean matches() throws IOException { + final int doc = approximation.docID(); + + return matchingDocs.get(doc); + } + + @Override + public float matchCost() { + return 10; // TODO: use some cost of matchingDocs + } + }; + + return new ConstantScoreScorer(this, score(), twoPhase); + } + protected Bits getMatchingDocs(LeafReaderContext context) throws IOException { return new Bits.MatchAllBits(context.reader().maxDoc()); } @@ -62,6 +87,12 @@ protected Bits getMatchingDocs(LeafReaderContext context) throws IOException { public String toString() { return "weight(MatchAllDocs)"; } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + // TODO Auto-generated method stub + return false; + } }; } diff --git a/luwak/src/test/java/uk/co/flax/luwak/analysis/TestSuffixingNGramTokenizer.java b/luwak/src/test/java/uk/co/flax/luwak/analysis/TestSuffixingNGramTokenizer.java index f40e4a032..0014d6e6f 100644 --- a/luwak/src/test/java/uk/co/flax/luwak/analysis/TestSuffixingNGramTokenizer.java +++ b/luwak/src/test/java/uk/co/flax/luwak/analysis/TestSuffixingNGramTokenizer.java @@ -139,7 +139,7 @@ public static void main(String... args) throws IOException { // Cannot use try-with-resources here as we assign to ts in the block. LeafReader reader = batch.getIndexReader(); - TokenStream ts = new TermsEnumTokenStream(reader.fields().terms("f").iterator()); + TokenStream ts = new TermsEnumTokenStream(reader.terms("f").iterator()); try { ts = new SuffixingNGramTokenFilter(ts, "XX", "__WILDCARD__", 20); //ts = new DuplicateRemovalTokenFilter(ts); diff --git a/luwak/src/test/java/uk/co/flax/luwak/matchers/ConcurrentMatcherTestBase.java b/luwak/src/test/java/uk/co/flax/luwak/matchers/ConcurrentMatcherTestBase.java index 96f3e9f42..897fc9308 100644 --- a/luwak/src/test/java/uk/co/flax/luwak/matchers/ConcurrentMatcherTestBase.java +++ b/luwak/src/test/java/uk/co/flax/luwak/matchers/ConcurrentMatcherTestBase.java @@ -87,7 +87,7 @@ public void testMatchesAreDisambiguated() throws IOException, UpdateException { // 'test' and 'doc' parts will match. 'test' will have a higher score, // because of it's lower termfreq. We need to check that each query ends // up with the score for the 'test' subquery, not the 'doc' subquery - assertThat(match.getScore()).isEqualTo(2.5316024f); + assertThat(match.getScore()).isEqualTo(2.8768208f); } } } @@ -98,6 +98,7 @@ public void testParallelSlowLog() throws IOException, UpdateException { ExecutorService executor = Executors.newCachedThreadPool(); try (Monitor monitor = new Monitor(new TestSlowLog.SlowQueryParser(250), new MatchAllPresearcher())) { + monitor.setSlowLogLimit(20000000); monitor.update(new MonitorQuery("1", "slow"), new MonitorQuery("2", "fast"), new MonitorQuery("3", "slow")); DocumentBatch batch = DocumentBatch.of(InputDocument.builder("doc1").build()); diff --git a/luwak/src/test/java/uk/co/flax/luwak/matchers/TestHighlightingMatcher.java b/luwak/src/test/java/uk/co/flax/luwak/matchers/TestHighlightingMatcher.java index 558edc0b8..c49af85f1 100644 --- a/luwak/src/test/java/uk/co/flax/luwak/matchers/TestHighlightingMatcher.java +++ b/luwak/src/test/java/uk/co/flax/luwak/matchers/TestHighlightingMatcher.java @@ -374,8 +374,8 @@ public int hashCode() { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return inner.createWeight(searcher, needsScores); + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return inner.createWeight(searcher, needsScores, boost); } }, new MatchAllPresearcher()); diff --git a/luwak/src/test/java/uk/co/flax/luwak/presearcher/PresearcherTestBase.java b/luwak/src/test/java/uk/co/flax/luwak/presearcher/PresearcherTestBase.java index f6c0e6585..1b782e3ea 100644 --- a/luwak/src/test/java/uk/co/flax/luwak/presearcher/PresearcherTestBase.java +++ b/luwak/src/test/java/uk/co/flax/luwak/presearcher/PresearcherTestBase.java @@ -4,7 +4,6 @@ import java.util.Map; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.LegacyNumericTokenStream; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; @@ -215,41 +214,6 @@ public void testNonStringTermHandling() throws IOException, UpdateException { } - @Test - @SuppressWarnings("deprecation") - public void filtersOnNumericTermQueries() throws IOException, UpdateException { - - // Rudimentary query parser which returns numeric encoded BytesRefs - try (Monitor numeric_monitor = new Monitor(new MonitorQueryParser() { - @Override - public Query parse(String queryString, Map metadata) throws Exception - { - BytesRefBuilder brb = new BytesRefBuilder(); - LegacyNumericUtils.intToPrefixCoded(Integer.parseInt(queryString), 0, brb); - - Term t = new Term(TEXTFIELD, brb.get()); - return new TermQuery(t); - } - }, presearcher)) { - - for (int i = 8; i <= 15; i++) { - numeric_monitor.update(new MonitorQuery("query" + i, "" + i)); - } - - for (int i = 8; i <= 15; i++) { - LegacyNumericTokenStream nts = new LegacyNumericTokenStream(1); - nts.setIntValue(i); - InputDocument doc = InputDocument.builder("doc" + i) - .addField(new TextField(TEXTFIELD, nts)).build(); - assertThat(numeric_monitor.match(doc, SimpleMatcher.FACTORY)) - .matchesDoc("doc" + i) - .hasMatchCount("doc" + i, 1) - .matchesQuery("query" + i, "doc" + i); - } - - } - } - public static BooleanClause must(Query q) { return new BooleanClause(q, BooleanClause.Occur.MUST); } diff --git a/luwak/src/test/java/uk/co/flax/luwak/presearcher/TestMultipassPresearcher.java b/luwak/src/test/java/uk/co/flax/luwak/presearcher/TestMultipassPresearcher.java index f847a65f1..45e502150 100644 --- a/luwak/src/test/java/uk/co/flax/luwak/presearcher/TestMultipassPresearcher.java +++ b/luwak/src/test/java/uk/co/flax/luwak/presearcher/TestMultipassPresearcher.java @@ -4,17 +4,16 @@ import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.index.*; -import org.apache.lucene.queries.TermsQuery; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.assertj.core.api.Assertions; import org.junit.Test; + import uk.co.flax.luwak.*; import uk.co.flax.luwak.matchers.SimpleMatcher; import uk.co.flax.luwak.queryparsers.LuceneQueryParser; - import static uk.co.flax.luwak.assertions.MatchesAssert.assertThat; /** @@ -95,10 +94,10 @@ public void testQueryBuilder() throws IOException, UpdateException { BooleanQuery q = (BooleanQuery) presearcher.buildQuery(docs.getIndexReader(), new QueryTermFilter(reader)); BooleanQuery expected = new BooleanQuery.Builder() .add(should(new BooleanQuery.Builder() - .add(must(new TermsQuery(new Term("f_0", "test")))) - .add(must(new TermsQuery(new Term("f_1", "test")))) - .add(must(new TermsQuery(new Term("f_2", "test")))) - .add(must(new TermsQuery(new Term("f_3", "test")))) + .add(must(new TermQuery(new Term("f_0", "test")))) + .add(must(new TermQuery(new Term("f_1", "test")))) + .add(must(new TermQuery(new Term("f_2", "test")))) + .add(must(new TermQuery(new Term("f_3", "test")))) .build())) .add(should(new TermQuery(new Term("__anytokenfield", "__ANYTOKEN__")))) .build(); diff --git a/luwak/src/test/java/uk/co/flax/luwak/presearcher/TestTermPresearcher.java b/luwak/src/test/java/uk/co/flax/luwak/presearcher/TestTermPresearcher.java index 5759d1902..b67a66324 100644 --- a/luwak/src/test/java/uk/co/flax/luwak/presearcher/TestTermPresearcher.java +++ b/luwak/src/test/java/uk/co/flax/luwak/presearcher/TestTermPresearcher.java @@ -6,7 +6,6 @@ import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.*; -import org.apache.lucene.queries.TermsQuery; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TermQuery; @@ -15,11 +14,11 @@ import org.apache.lucene.util.BytesRefHash; import org.assertj.core.api.Assertions; import org.junit.Test; + import uk.co.flax.luwak.*; import uk.co.flax.luwak.matchers.SimpleMatcher; import uk.co.flax.luwak.queryparsers.LuceneQueryParser; import uk.co.flax.luwak.termextractor.querytree.QueryTree; - import static uk.co.flax.luwak.assertions.MatchesAssert.assertThat; /* @@ -121,7 +120,7 @@ public void testQueryBuilder() throws IOException, UpdateException { BooleanQuery q = (BooleanQuery) presearcher.buildQuery(batch.getIndexReader(), new QueryTermFilter(reader)); BooleanQuery expected = new BooleanQuery.Builder() - .add(should(new TermsQuery(new Term("f", "test")))) + .add(should(new TermQuery( new Term("f", "test")))) .add(should(new TermQuery(new Term("__anytokenfield", "__ANYTOKEN__")))) .build(); diff --git a/luwak/src/test/java/uk/co/flax/luwak/termextractor/TestCoreLuceneQueryExtractors.java b/luwak/src/test/java/uk/co/flax/luwak/termextractor/TestCoreLuceneQueryExtractors.java index 26a2cd123..b303b1b28 100644 --- a/luwak/src/test/java/uk/co/flax/luwak/termextractor/TestCoreLuceneQueryExtractors.java +++ b/luwak/src/test/java/uk/co/flax/luwak/termextractor/TestCoreLuceneQueryExtractors.java @@ -44,7 +44,10 @@ public class TestCoreLuceneQueryExtractors { "org.apache.lucene.search.FieldValueQuery", "org.apache.lucene.search.BlendedTermQuery", "org.apache.lucene.search.AutomatonQuery", - "org.apache.lucene.queries.BoostingQuery" + "org.apache.lucene.queries.BoostingQuery", + "org.apache.lucene.search.NormsFieldExistsQuery", + "org.apache.lucene.search.CoveringQuery", + "org.apache.lucene.search.DocValuesFieldExistsQuery" ); public static Set unhandledQueries = ImmutableSet.of( diff --git a/luwak/src/test/java/uk/co/flax/luwak/termextractor/TestExtractors.java b/luwak/src/test/java/uk/co/flax/luwak/termextractor/TestExtractors.java index a29ba63bb..87d638a69 100644 --- a/luwak/src/test/java/uk/co/flax/luwak/termextractor/TestExtractors.java +++ b/luwak/src/test/java/uk/co/flax/luwak/termextractor/TestExtractors.java @@ -2,7 +2,6 @@ import com.google.common.collect.ImmutableList; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.TermsQuery; import org.apache.lucene.search.*; import org.junit.Test; import uk.co.flax.luwak.termextractor.querytree.TreeWeightor; @@ -49,26 +48,6 @@ public void testRegexpExtractor() { .containsExactly(new QueryTerm("field", "heXX", QueryTerm.Type.CUSTOM, "WILDCARD")); } - - @Test - @SuppressWarnings("deprecation") - public void testRangeQueriesReturnAnyToken() { - - LegacyNumericRangeQuery nrq = LegacyNumericRangeQuery.newLongRange("field", 0l, 10l, true, true); - - assertThat(treeBuilder.collectTerms(nrq)) - .hasSize(1) - .extracting("type") - .containsExactly(QueryTerm.Type.ANY); - - BooleanQuery.Builder bq = new BooleanQuery.Builder(); - bq.add(nrq, BooleanClause.Occur.MUST); - bq.add(new TermQuery(new Term("field", "term")), BooleanClause.Occur.MUST); - - assertThat(treeBuilder.collectTerms(bq.build())) - .containsExactly(new QueryTerm("field", "term", QueryTerm.Type.EXACT)); - } - @Test public void testConstantScoreQueryExtractor() { @@ -117,13 +96,6 @@ public void testDisjunctionMaxExtractor() { .containsExactly(new QueryTerm("f", "t1", QueryTerm.Type.EXACT), new QueryTerm("f", "t2", QueryTerm.Type.EXACT)); } - @Test - public void testTermsQueryExtractor() { - Query q = new TermsQuery(new Term("f1", "t1"), new Term("f2", "t2")); - assertThat(treeBuilder.collectTerms(q)) - .containsExactly(new QueryTerm("f1", "t1", QueryTerm.Type.EXACT), new QueryTerm("f2", "t2", QueryTerm.Type.EXACT)); - } - @Test public void testBooleanExtractsFilter() { Query q = new BooleanQuery.Builder() diff --git a/luwak/src/test/java/uk/co/flax/luwak/util/TestSpanRewriter.java b/luwak/src/test/java/uk/co/flax/luwak/util/TestSpanRewriter.java index 61ab20c9c..f3ef90723 100644 --- a/luwak/src/test/java/uk/co/flax/luwak/util/TestSpanRewriter.java +++ b/luwak/src/test/java/uk/co/flax/luwak/util/TestSpanRewriter.java @@ -16,7 +16,6 @@ */ import org.apache.lucene.index.Term; -import org.apache.lucene.queries.TermsQuery; import org.apache.lucene.search.*; import org.apache.lucene.search.spans.SpanNearQuery; import org.apache.lucene.search.spans.SpanTermQuery; @@ -26,15 +25,6 @@ public class TestSpanRewriter { - @Test - public void testTermsQueryWithMultipleFields() throws Exception { - - TermsQuery tq = new TermsQuery(new Term("field1", "term1"), new Term("field2", "term1"), new Term("field2", "term2")); - - Query q = new SpanRewriter().rewrite(tq, null); - assertThat(q).isInstanceOf(BooleanQuery.class); - } - @Test public void testBoostQuery() throws Exception { diff --git a/pom.xml b/pom.xml index c4f49ad48..fcddac8ec 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.github.flaxsearch luwak-parent pom - 1.6.0-SNAPSHOT + 1.7.0-SNAPSHOT benchmark luwak @@ -20,7 +20,7 @@ org.apache.lucene - 6.5.0 + 7.3.1 UTF-8 UTF-8 1.8