From 5d5f180d455c9f3210cbc2031d96f8dd7212544c Mon Sep 17 00:00:00 2001 From: karthikalle <5334600+karthikalle@users.noreply.github.com> Date: Fri, 16 Oct 2020 07:14:29 -0700 Subject: [PATCH] Add geo_bounding_box query for LatLonFields (#178) * Add geo_bounding_box query for LatLonFields * Add GeoQueryable interface * Throw exception when field is not searchable --- .../main/proto/yelp/nrtsearch/search.proto | 9 ++ .../server/luceneserver/QueryNodeMapper.java | 15 ++ .../luceneserver/field/LatLonFieldDef.java | 19 ++- .../field/properties/GeoQueryable.java | 33 ++++ .../field/LatLonFieldDefTest.java | 149 ++++++++++++++++++ .../resources/field/registerFieldsLatLon.json | 26 +++ 6 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/field/properties/GeoQueryable.java create mode 100644 src/test/java/com/yelp/nrtsearch/server/luceneserver/field/LatLonFieldDefTest.java create mode 100644 src/test/resources/field/registerFieldsLatLon.json diff --git a/clientlib/src/main/proto/yelp/nrtsearch/search.proto b/clientlib/src/main/proto/yelp/nrtsearch/search.proto index 531b1346e..28000a787 100644 --- a/clientlib/src/main/proto/yelp/nrtsearch/search.proto +++ b/clientlib/src/main/proto/yelp/nrtsearch/search.proto @@ -159,6 +159,13 @@ message RangeQuery { string upper = 3; // Upper bound, inclusive } +// A query that matches documents with values within the specified range. The lower and upper values though provided as strings will be converted to the type of the field. This works with INT, LONG, FLOAT, DOUBLE and DATE_TIME field types. +message GeoBoundingBoxQuery { + string field = 1; // Field in the document to query + google.type.LatLng topLeft = 2; // top left corner of the geo box + google.type.LatLng bottomRight = 3; // bottom right corner of the geo box +} + // Defines different types of QueryNodes. enum QueryType { NONE = 0; @@ -172,6 +179,7 @@ enum QueryType { MATCH_PHRASE = 8; MULTI_MATCH = 9; RANGE = 10; + GEO_BOUNDING_BOX = 11; } // Defines a full query consisting of a QueryNode which may be one of several types. @@ -190,6 +198,7 @@ message Query { MatchPhraseQuery matchPhraseQuery = 10; MultiMatchQuery multiMatchQuery = 11; RangeQuery rangeQuery = 12; + GeoBoundingBoxQuery geoBoundingBoxQuery = 13; } } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/QueryNodeMapper.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/QueryNodeMapper.java index 0090becc3..b70407ebf 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/QueryNodeMapper.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/QueryNodeMapper.java @@ -20,6 +20,7 @@ import com.yelp.nrtsearch.server.grpc.*; import com.yelp.nrtsearch.server.luceneserver.analysis.AnalyzerCreator; import com.yelp.nrtsearch.server.luceneserver.field.FieldDef; +import com.yelp.nrtsearch.server.luceneserver.field.properties.GeoQueryable; import com.yelp.nrtsearch.server.luceneserver.field.properties.RangeQueryable; import com.yelp.nrtsearch.server.luceneserver.field.properties.TermQueryable; import com.yelp.nrtsearch.server.luceneserver.script.ScoreScript; @@ -85,6 +86,8 @@ private Query getQueryNode(com.yelp.nrtsearch.server.grpc.Query query, IndexStat return getMultiMatchQuery(query.getMultiMatchQuery(), state); case RANGEQUERY: return getRangeQuery(query.getRangeQuery(), state); + case GEOBOUNDINGBOXQUERY: + return getGeoBoundingBoxQuery(query.getGeoBoundingBoxQuery(), state); default: throw new UnsupportedOperationException( "Unsupported query type received: " + query.getQueryNodeCase()); @@ -275,6 +278,18 @@ private Query getRangeQuery(RangeQuery rangeQuery, IndexState state) { return ((RangeQueryable) field).getRangeQuery(rangeQuery); } + private Query getGeoBoundingBoxQuery(GeoBoundingBoxQuery geoBoundingBoxQuery, IndexState state) { + String fieldName = geoBoundingBoxQuery.getField(); + FieldDef field = state.getField(fieldName); + + if (!(field instanceof GeoQueryable)) { + throw new IllegalArgumentException( + "Field: " + fieldName + " does not support GeoBoundingBoxQuery"); + } + + return ((GeoQueryable) field).getGeoBoundingBoxQuery(geoBoundingBoxQuery); + } + private Map initializeOccurMapping() { return Arrays.stream(com.yelp.nrtsearch.server.grpc.BooleanClause.Occur.values()) diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/LatLonFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/LatLonFieldDef.java index ba499853c..524798b15 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/LatLonFieldDef.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/LatLonFieldDef.java @@ -18,9 +18,11 @@ import static com.yelp.nrtsearch.server.luceneserver.analysis.AnalyzerCreator.hasAnalyzer; import com.yelp.nrtsearch.server.grpc.Field; +import com.yelp.nrtsearch.server.grpc.GeoBoundingBoxQuery; import com.yelp.nrtsearch.server.grpc.Point; import com.yelp.nrtsearch.server.grpc.SortType; import com.yelp.nrtsearch.server.luceneserver.doc.LoadedDocValues; +import com.yelp.nrtsearch.server.luceneserver.field.properties.GeoQueryable; import com.yelp.nrtsearch.server.luceneserver.field.properties.Sortable; import java.io.IOException; import java.util.List; @@ -32,10 +34,11 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; /** Field class for 'LAT_LON' field type. */ -public class LatLonFieldDef extends IndexableFieldDef implements Sortable { +public class LatLonFieldDef extends IndexableFieldDef implements Sortable, GeoQueryable { public LatLonFieldDef(String name, Field requestField) { super(name, requestField); } @@ -122,4 +125,18 @@ public SortField getSortField(SortType type) { return LatLonDocValuesField.newDistanceSort( getName(), origin.getLatitude(), origin.getLongitude()); } + + @Override + public Query getGeoBoundingBoxQuery(GeoBoundingBoxQuery geoBoundingBoxQuery) { + if (!this.isSearchable()) { + throw new IllegalArgumentException( + String.format("field %s is not searchable", this.getName())); + } + return LatLonPoint.newBoxQuery( + geoBoundingBoxQuery.getField(), + geoBoundingBoxQuery.getBottomRight().getLatitude(), + geoBoundingBoxQuery.getTopLeft().getLatitude(), + geoBoundingBoxQuery.getTopLeft().getLongitude(), + geoBoundingBoxQuery.getBottomRight().getLongitude()); + } } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/properties/GeoQueryable.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/properties/GeoQueryable.java new file mode 100644 index 000000000..74745c4ca --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/properties/GeoQueryable.java @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.field.properties; + +import com.yelp.nrtsearch.server.grpc.GeoBoundingBoxQuery; +import org.apache.lucene.search.Query; + +/** + * Trait interface for {@link com.yelp.nrtsearch.server.luceneserver.field.FieldDef} types that can + * be queried by geo-related queries. + */ +public interface GeoQueryable { + /** + * Build a geo bounding box query for this field type with the given configuration. + * + * @param geoBoundingBoxQuery geo bounding box query configuration + * @return lucene box query + */ + Query getGeoBoundingBoxQuery(GeoBoundingBoxQuery geoBoundingBoxQuery); +} diff --git a/src/test/java/com/yelp/nrtsearch/server/luceneserver/field/LatLonFieldDefTest.java b/src/test/java/com/yelp/nrtsearch/server/luceneserver/field/LatLonFieldDefTest.java new file mode 100644 index 000000000..f7443df34 --- /dev/null +++ b/src/test/java/com/yelp/nrtsearch/server/luceneserver/field/LatLonFieldDefTest.java @@ -0,0 +1,149 @@ +/* + * Copyright 2020 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.field; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.google.type.LatLng; +import com.yelp.nrtsearch.server.grpc.AddDocumentRequest; +import com.yelp.nrtsearch.server.grpc.AddDocumentRequest.MultiValuedField; +import com.yelp.nrtsearch.server.grpc.FieldDefRequest; +import com.yelp.nrtsearch.server.grpc.GeoBoundingBoxQuery; +import com.yelp.nrtsearch.server.grpc.Query; +import com.yelp.nrtsearch.server.grpc.SearchRequest; +import com.yelp.nrtsearch.server.grpc.SearchResponse; +import com.yelp.nrtsearch.server.grpc.SearchResponse.Hit; +import com.yelp.nrtsearch.server.luceneserver.ServerTestCase; +import io.grpc.StatusRuntimeException; +import io.grpc.testing.GrpcCleanupRule; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Test; + +public class LatLonFieldDefTest extends ServerTestCase { + + @ClassRule public static final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule(); + + protected List getIndices() { + return Collections.singletonList(DEFAULT_TEST_INDEX); + } + + protected FieldDefRequest getIndexDef(String name) throws IOException { + return getFieldsFromResourceFile("/field/registerFieldsLatLon.json"); + } + + protected void initIndex(String name) throws Exception { + List docs = new ArrayList<>(); + // Business in Fremont + AddDocumentRequest fremontDocRequest = + AddDocumentRequest.newBuilder() + .setIndexName(name) + .putFields("doc_id", MultiValuedField.newBuilder().addValue("1").build()) + .putFields( + "lat_lon_multi", + MultiValuedField.newBuilder() + .addValue("37.476220") + .addValue("-121.904404") + .addValue("37.506900") + .addValue("-121.933121") + .build()) + .build(); + // Business in SF + AddDocumentRequest sfDocRequest = + AddDocumentRequest.newBuilder() + .setIndexName(name) + .putFields("doc_id", MultiValuedField.newBuilder().addValue("2").build()) + .putFields( + "lat_lon_multi", + MultiValuedField.newBuilder().addValue("37.781158").addValue("-122.414011").build()) + .build(); + docs.add(fremontDocRequest); + docs.add(sfDocRequest); + addDocuments(docs.stream()); + } + + @Test + public void testGeoBoxQuery() { + GeoBoundingBoxQuery fremontGeoBoundingBoxQuery = + GeoBoundingBoxQuery.newBuilder() + .setField("lat_lon_multi") + .setTopLeft( + LatLng.newBuilder().setLatitude(37.589207).setLongitude(-122.019474).build()) + .setBottomRight( + LatLng.newBuilder().setLatitude(37.419254).setLongitude(-121.836704).build()) + .build(); + queryAndVerifyIds(fremontGeoBoundingBoxQuery, "1"); + + GeoBoundingBoxQuery sfGeoBoundingBoxQuery = + GeoBoundingBoxQuery.newBuilder() + .setField("lat_lon_multi") + .setTopLeft( + LatLng.newBuilder().setLatitude(37.793321).setLongitude(-122.430808).build()) + .setBottomRight( + LatLng.newBuilder().setLatitude(37.759250).setLongitude(-122.383569).build()) + .build(); + queryAndVerifyIds(sfGeoBoundingBoxQuery, "2"); + + // No results in Mountain View + GeoBoundingBoxQuery mountainViewGeoBoxQuery = + GeoBoundingBoxQuery.newBuilder() + .setField("lat_lon_multi") + .setTopLeft( + LatLng.newBuilder().setLatitude(37.409778).setLongitude(-122.116884).build()) + .setBottomRight( + LatLng.newBuilder().setLatitude(37.346484).setLongitude(-121.984961).build()) + .build(); + queryAndVerifyIds(mountainViewGeoBoxQuery); + } + + @Test(expected = StatusRuntimeException.class) + public void testGeoBoxQueryNotSearchable() { + GeoBoundingBoxQuery fremontGeoBoundingBoxQuery = + GeoBoundingBoxQuery.newBuilder() + .setField("lat_lon_not_searchable") + .setTopLeft( + LatLng.newBuilder().setLatitude(37.589207).setLongitude(-122.019474).build()) + .setBottomRight( + LatLng.newBuilder().setLatitude(37.419254).setLongitude(-121.836704).build()) + .build(); + queryAndVerifyIds(fremontGeoBoundingBoxQuery); + } + + private void queryAndVerifyIds(GeoBoundingBoxQuery geoBoundingBoxQuery, String... expectedIds) { + Query query = Query.newBuilder().setGeoBoundingBoxQuery(geoBoundingBoxQuery).build(); + SearchResponse response = + getGrpcServer() + .getBlockingStub() + .search( + SearchRequest.newBuilder() + .setIndexName(DEFAULT_TEST_INDEX) + .setStartHit(0) + .setTopHits(10) + .setQuery(query) + .addRetrieveFields("doc_id") + .build()); + List idList = Arrays.asList(expectedIds); + assertEquals(idList.size(), response.getHitsCount()); + for (Hit hit : response.getHitsList()) { + assertTrue(idList.contains(hit.getFieldsOrThrow("doc_id").getFieldValue(0).getTextValue())); + } + } +} diff --git a/src/test/resources/field/registerFieldsLatLon.json b/src/test/resources/field/registerFieldsLatLon.json new file mode 100644 index 000000000..d0bdd5247 --- /dev/null +++ b/src/test/resources/field/registerFieldsLatLon.json @@ -0,0 +1,26 @@ +{ + "indexName": "test_index", + "field": [ + { + "name": "doc_id", + "type": "ATOM", + "storeDocValues": true + }, + { + "name": "lat_lon_multi", + "type": "LAT_LON", + "multiValued": true, + "storeDocValues": true, + "sort": true, + "search": true + }, + { + "name": "lat_lon_not_searchable", + "type": "LAT_LON", + "multiValued": true, + "storeDocValues": true, + "sort": true, + "search": false + } + ] +}