Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Make compatible with Lucene 7.3.1 #170

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Make compatible with Lucene 7.3.1 #170

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion benchmark/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>luwak-parent</artifactId>
<groupId>com.github.flaxsearch</groupId>
<version>1.6.0-SNAPSHOT</version>
<version>1.7.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion luwak/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>luwak-parent</artifactId>
<groupId>com.github.flaxsearch</groupId>
<version>1.6.0-SNAPSHOT</version>
<version>1.7.0-SNAPSHOT</version>
</parent>

<artifactId>luwak</artifactId>
Expand Down
6 changes: 4 additions & 2 deletions luwak/src/main/java/uk/co/flax/luwak/Monitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,8 @@ private void prepareQueryCache(boolean storeQueries) throws IOException {
}
seenIds.add(id);

BytesRef serializedMQ = dataValues.mq.get(dataValues.doc);
dataValues.mq.advanceExact(dataValues.doc);
BytesRef serializedMQ = dataValues.mq.binaryValue();
MonitorQuery mq = MonitorQuery.deserialize(serializedMQ);

BytesRef hash = mq.hash();
Expand Down Expand Up @@ -494,7 +495,8 @@ public MonitorQuery getQuery(final String queryId) throws IOException {
throw new IllegalStateException("Cannot call getQuery() as queries are not stored");
final MonitorQuery[] queryHolder = new MonitorQuery[]{ null };
queryIndex.search(new TermQuery(new Term(FIELDS.id, queryId)), (id, query, dataValues) -> {
BytesRef serializedMQ = dataValues.mq.get(dataValues.doc);
dataValues.mq.advanceExact(dataValues.doc);
BytesRef serializedMQ = dataValues.mq.binaryValue();
queryHolder[0] = MonitorQuery.deserialize(serializedMQ);
});
return queryHolder[0];
Expand Down
16 changes: 10 additions & 6 deletions luwak/src/main/java/uk/co/flax/luwak/QueryIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ class QueryIndex {
// NB this is not final because it can be replaced by purgeCache()

// package-private for testing
final Map<IndexReader, QueryTermFilter> termFilters = new HashMap<>();
final Map<IndexReader.CacheKey, QueryTermFilter> termFilters = new HashMap<>();

QueryIndex(IndexWriter indexWriter) throws IOException {
this.writer = indexWriter;
Expand All @@ -46,8 +46,8 @@ private class TermsHashBuilder extends SearcherFactory {
public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) throws IOException {
IndexSearcher searcher = super.newSearcher(reader, previousReader);
searcher.setQueryCache(null);
termFilters.put(reader, new QueryTermFilter(reader));
reader.addReaderClosedListener(termFilters::remove);
termFilters.put(reader.getReaderCacheHelper().getKey(), new QueryTermFilter(reader));
reader.getReaderCacheHelper().addClosedListener(termFilters::remove);
return searcher;
}
}
Expand Down Expand Up @@ -108,7 +108,9 @@ long search(QueryBuilder queryBuilder, QueryCollector matcher) throws IOExceptio

MonitorQueryCollector collector = new MonitorQueryCollector(queries, matcher);
long buildTime = System.nanoTime();
Query query = queryBuilder.buildQuery(termFilters.get(searcher.getIndexReader()));
Query query = queryBuilder.buildQuery(termFilters.get(searcher.getIndexReader()
.getReaderCacheHelper()
.getKey()));
buildTime = System.nanoTime() - buildTime;
searcher.search(query, collector);
return buildTime;
Expand Down Expand Up @@ -241,8 +243,10 @@ public void setScorer(Scorer scorer) throws IOException {

@Override
public void collect(int doc) throws IOException {
BytesRef hash = dataValues.hash.get(doc);
BytesRef id = dataValues.id.get(doc);
dataValues.hash.advanceExact(doc);
dataValues.id.advanceExact(doc);
BytesRef hash = dataValues.hash.binaryValue();
BytesRef id = dataValues.id.binaryValue();
QueryCacheEntry query = queries.get(hash);
dataValues.doc = doc;
matcher.matchQuery(id.utf8ToString(), query, dataValues);
Expand Down
2 changes: 1 addition & 1 deletion luwak/src/main/java/uk/co/flax/luwak/presearcher/FieldFilterPresearcherComponent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public Query adjustPresearcherQuery(LeafReader reader, Query presearcherQuery) t

private Query buildFilterClause(LeafReader reader) throws IOException {

Terms terms = reader.fields().terms(field);
Terms terms = reader.terms(field);
if (terms == null)
return null;

Expand Down
13 changes: 11 additions & 2 deletions luwak/src/main/java/uk/co/flax/luwak/presearcher/MultipassTermFilteredPresearcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,13 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.TermsQuery;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;

import uk.co.flax.luwak.analysis.TermsEnumTokenStream;
import uk.co.flax.luwak.termextractor.querytree.QueryTree;
import uk.co.flax.luwak.termextractor.querytree.QueryTreeViewer;
Expand Down Expand Up @@ -138,7 +139,15 @@ public void addTerm(String field, BytesRef term) throws IOException {
public Query build() {
BooleanQuery.Builder parent = new BooleanQuery.Builder();
for (int i = 0; i < passes; i++) {
parent.add(new TermsQuery(terms.get(i)), BooleanClause.Occur.MUST);
if (terms.get(i).size() == 1) {
parent.add(new TermQuery(terms.get(i).iterator().next()), BooleanClause.Occur.MUST);
} else {
BooleanQuery.Builder bq = new BooleanQuery.Builder();
for (Term term : terms.get(i)) {
bq.add(new TermQuery(term), BooleanClause.Occur.SHOULD);
}
parent.add(bq.build(), BooleanClause.Occur.MUST);
}
}
return parent.build();
}
Expand Down
21 changes: 14 additions & 7 deletions luwak/src/main/java/uk/co/flax/luwak/presearcher/TermFilteredPresearcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.*;
import org.apache.lucene.queries.TermsQuery;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
Expand Down Expand Up @@ -83,18 +82,18 @@ public TermFilteredPresearcher() {
public final Query buildQuery(LeafReader reader, QueryTermFilter queryTermFilter) {
try {
DocumentQueryBuilder queryBuilder = getQueryBuilder();
for (String field : reader.fields()) {
for (FieldInfo fieldInfo : reader.getFieldInfos()) {

TokenStream ts = new TermsEnumTokenStream(reader.terms(field).iterator());
TokenStream ts = new TermsEnumTokenStream(reader.terms(fieldInfo.name).iterator());
for (PresearcherComponent component : components) {
ts = component.filterDocumentTokens(field, ts);
ts = component.filterDocumentTokens(fieldInfo.name, ts);
}

ts = new BytesRefFilteredTokenFilter(ts, queryTermFilter.getTerms(field));
ts = new BytesRefFilteredTokenFilter(ts, queryTermFilter.getTerms(fieldInfo.name));

TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
while (ts.incrementToken()) {
queryBuilder.addTerm(field, BytesRef.deepCopyOf(termAtt.getBytesRef()));
queryBuilder.addTerm(fieldInfo.name, BytesRef.deepCopyOf(termAtt.getBytesRef()));
}
ts.close();

Expand Down Expand Up @@ -130,7 +129,15 @@ public void addTerm(String field, BytesRef term) throws IOException {

@Override
public Query build() {
return new TermsQuery(terms);
if (terms.size() == 1) {
return new TermQuery(terms.iterator().next());
} else {
BooleanQuery.Builder builder = new BooleanQuery.Builder();
for (Term term : terms) {
builder.add(new TermQuery(term), BooleanClause.Occur.SHOULD);
}
return builder.build();
}
}
};
}
Expand Down
42 changes: 0 additions & 42 deletions luwak/src/main/java/uk/co/flax/luwak/termextractor/treebuilder/TermsQueryTreeBuilder.java
View file
Open in desktop

This file was deleted.

1 change: 0 additions & 1 deletion luwak/src/main/java/uk/co/flax/luwak/termextractor/treebuilder/TreeBuilders.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ public QueryTree buildTree(QueryAnalyzer builder, Query query) {
newFilteringQueryBuilder(BoostedQuery.class, BoostedQuery::getQuery),
newDisjunctionBuilder(DisjunctionMaxQuery.class,
(b, q) -> q.getDisjuncts().stream().map(b::buildTree).collect(Collectors.toList())),
TermsQueryTreeBuilder.INSTANCE,
TermInSetQueryTreeBuilder.INSTANCE,
new QueryTreeBuilder<SpanWithinQuery>(SpanWithinQuery.class) {
@Override
Expand Down
18 changes: 7 additions & 11 deletions luwak/src/main/java/uk/co/flax/luwak/util/ForceNoBulkScoringQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,9 @@ public Query getWrappedQuery() {
}

@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {

final Weight innerWeight = inner.createWeight(searcher, needsScores);
final Weight innerWeight = inner.createWeight(searcher, needsScores, boost);

return new Weight(ForceNoBulkScoringQuery.this) {
@Override
Expand All @@ -78,18 +78,14 @@ public Explanation explain(LeafReaderContext leafReaderContext, int i) throws IO
}

@Override
public float getValueForNormalization() throws IOException {
return innerWeight.getValueForNormalization();
}

@Override
public void normalize(float v, float v1) {
innerWeight.normalize(v, v1);
public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException {
return innerWeight.scorer(leafReaderContext);
}

@Override
public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException {
return innerWeight.scorer(leafReaderContext);
public boolean isCacheable(LeafReaderContext ctx) {
// TODO Auto-generated method stub
return false;
}
};
}
Expand Down
14 changes: 10 additions & 4 deletions luwak/src/main/java/uk/co/flax/luwak/util/SpanOffsetReportingQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@ public int hashCode() {
}

@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
return new SpanOffsetWeight(searcher, in.createWeight(searcher, needsScores));
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
return new SpanOffsetWeight(searcher, in.createWeight(searcher, needsScores, boost), boost);
}

/**
Expand All @@ -107,8 +107,8 @@ private class SpanOffsetWeight extends SpanWeight {

private final SpanWeight in;

private SpanOffsetWeight(IndexSearcher searcher, SpanWeight in) throws IOException {
super(SpanOffsetReportingQuery.this, searcher, termContexts(in));
private SpanOffsetWeight(IndexSearcher searcher, SpanWeight in, float boost) throws IOException {
super(SpanOffsetReportingQuery.this, searcher, termContexts(in), boost);
this.in = in;
}

Expand All @@ -126,5 +126,11 @@ public Spans getSpans(LeafReaderContext ctx, Postings requiredPostings) throws I
public void extractTerms(Set<Term> terms) {
in.extractTerms(terms);
}

@Override
public boolean isCacheable(LeafReaderContext ctx) {
// TODO Auto-generated method stub
return false;
}
}
}
3 changes: 0 additions & 3 deletions luwak/src/main/java/uk/co/flax/luwak/util/SpanRewriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@

import org.apache.lucene.index.PrefixCodedTerms;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.TermsQuery;
import org.apache.lucene.search.*;
import org.apache.lucene.search.spans.*;
import org.apache.lucene.util.BytesRef;
Expand All @@ -50,8 +49,6 @@ public Query rewrite(Query in, IndexSearcher searcher) throws RewriteException,
return rewriteDisjunctionMaxQuery((DisjunctionMaxQuery) in, searcher);
if (in instanceof TermInSetQuery)
return rewriteTermInSetQuery((TermInSetQuery) in);
if (in instanceof TermsQuery)
return rewrite(in.rewrite(null), null);
if (in instanceof BoostQuery)
return rewrite(((BoostQuery) in).getQuery(), searcher); // we don't care about boosts for rewriting purposes
if (in instanceof PhraseQuery)
Expand Down
37 changes: 34 additions & 3 deletions luwak/src/test/java/uk/co/flax/luwak/TestSlowLog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
import org.apache.lucene.search.*;
import org.apache.lucene.util.Bits;
import org.junit.Test;

import uk.co.flax.luwak.matchers.SimpleMatcher;
import uk.co.flax.luwak.presearcher.MatchAllPresearcher;

import static org.assertj.core.api.Assertions.assertThat;

/**
Expand Down Expand Up @@ -48,20 +48,51 @@ public String toString(String s) {
}

@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores) {
public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) {
try {
Thread.sleep(delay);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
return new RandomAccessWeight(this) {
return new ConstantScoreWeight(this, boost) {
@Override
public final Scorer scorer(LeafReaderContext context) throws IOException {
final Bits matchingDocs = getMatchingDocs(context);
if (matchingDocs == null || matchingDocs instanceof Bits.MatchNoBits) {
return null;
}
final DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc());
final TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) {

@Override
public boolean matches() throws IOException {
final int doc = approximation.docID();

return matchingDocs.get(doc);
}

@Override
public float matchCost() {
return 10; // TODO: use some cost of matchingDocs
}
};

return new ConstantScoreScorer(this, score(), twoPhase);
}

protected Bits getMatchingDocs(LeafReaderContext context) throws IOException {
return new Bits.MatchAllBits(context.reader().maxDoc());
}

public String toString() {
return "weight(MatchAllDocs)";
}

@Override
public boolean isCacheable(LeafReaderContext ctx) {
// TODO Auto-generated method stub
return false;
}
};
}

Expand Down
2 changes: 1 addition & 1 deletion luwak/src/test/java/uk/co/flax/luwak/analysis/TestSuffixingNGramTokenizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ public static void main(String... args) throws IOException {

// Cannot use try-with-resources here as we assign to ts in the block.
LeafReader reader = batch.getIndexReader();
TokenStream ts = new TermsEnumTokenStream(reader.fields().terms("f").iterator());
TokenStream ts = new TermsEnumTokenStream(reader.terms("f").iterator());
try {
ts = new SuffixingNGramTokenFilter(ts, "XX", "__WILDCARD__", 20);
//ts = new DuplicateRemovalTokenFilter(ts);
Expand Down
Loading