Merge remote-tracking branch 'origin/main' into sarthakn_refactor_rep…

…lication_server_handlers # Conflicts: # src/main/java/com/yelp/nrtsearch/server/grpc/LuceneServer.java # src/main/java/com/yelp/nrtsearch/server/luceneserver/AddReplicaHandler.java # src/main/java/com/yelp/nrtsearch/server/luceneserver/CopyFilesHandler.java # src/main/java/com/yelp/nrtsearch/server/luceneserver/GetNodesInfoHandler.java # src/main/java/com/yelp/nrtsearch/server/luceneserver/Handler.java # src/main/java/com/yelp/nrtsearch/server/luceneserver/NewNRTPointHandler.java # src/main/java/com/yelp/nrtsearch/server/luceneserver/RecvCopyStateHandler.java # src/main/java/com/yelp/nrtsearch/server/luceneserver/ReplicaCurrentSearchingVersionHandler.java # src/main/java/com/yelp/nrtsearch/server/luceneserver/WriteNRTPointHandler.java # src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/Handler.java
Yelp · Oct 4, 2024 · f3c2b8d · f3c2b8d
2 parents c8005df + cdbdfb0
commit f3c2b8d
Show file tree

Hide file tree

Showing 193 changed files with 486 additions and 560 deletions.
diff --git a/src/main/java/com/yelp/nrtsearch/server/config/ThreadPoolConfiguration.java b/src/main/java/com/yelp/nrtsearch/server/config/ThreadPoolConfiguration.java
@@ -16,8 +16,8 @@
 package com.yelp.nrtsearch.server.config;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.yelp.nrtsearch.server.luceneserver.concurrent.ThreadPoolExecutorFactory;
 import com.yelp.nrtsearch.server.utils.JsonUtils;
-import com.yelp.nrtsearch.server.utils.ThreadPoolExecutorFactory;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -56,7 +56,7 @@ public class ThreadPoolConfiguration {
       Math.max(100, 2 * DEFAULT_VECTOR_MERGE_THREADS);
 
   /**
-   * Settings for a {@link com.yelp.nrtsearch.server.utils.ThreadPoolExecutorFactory.ExecutorType}.
+   * Settings for a {@link ThreadPoolExecutorFactory.ExecutorType}.
    *
    * @param maxThreads max number of threads
    * @param maxBufferedItems max number of buffered items

diff --git a/src/main/java/com/yelp/nrtsearch/server/grpc/GrpcServerExecutorSupplier.java b/src/main/java/com/yelp/nrtsearch/server/grpc/GrpcServerExecutorSupplier.java
@@ -15,7 +15,7 @@
  */
 package com.yelp.nrtsearch.server.grpc;
 
-import com.yelp.nrtsearch.server.utils.ThreadPoolExecutorFactory;
+import com.yelp.nrtsearch.server.luceneserver.concurrent.ThreadPoolExecutorFactory;
 import io.grpc.Metadata;
 import io.grpc.ServerCall;
 import io.grpc.ServerCallExecutorSupplier;

diff --git a/src/main/java/com/yelp/nrtsearch/server/grpc/LuceneServer.java b/src/main/java/com/yelp/nrtsearch/server/grpc/LuceneServer.java
@@ -25,18 +25,17 @@
 import com.google.protobuf.Any;
 import com.google.protobuf.Empty;
 import com.google.protobuf.util.JsonFormat;
-import com.yelp.nrtsearch.LuceneServerModule;
 import com.yelp.nrtsearch.server.config.LuceneServerConfiguration;
 import com.yelp.nrtsearch.server.config.QueryCacheConfig;
 import com.yelp.nrtsearch.server.luceneserver.AddReplicaHandler;
 import com.yelp.nrtsearch.server.luceneserver.CopyFilesHandler;
 import com.yelp.nrtsearch.server.luceneserver.GetNodesInfoHandler;
-import com.yelp.nrtsearch.server.luceneserver.GlobalState;
 import com.yelp.nrtsearch.server.luceneserver.NewNRTPointHandler;
 import com.yelp.nrtsearch.server.luceneserver.RecvCopyStateHandler;
 import com.yelp.nrtsearch.server.luceneserver.ReplicaCurrentSearchingVersionHandler;
 import com.yelp.nrtsearch.server.luceneserver.WriteNRTPointHandler;
 import com.yelp.nrtsearch.server.luceneserver.analysis.AnalyzerCreator;
+import com.yelp.nrtsearch.server.luceneserver.concurrent.ThreadPoolExecutorFactory;
 import com.yelp.nrtsearch.server.luceneserver.custom.request.CustomRequestProcessor;
 import com.yelp.nrtsearch.server.luceneserver.field.FieldDefCreator;
 import com.yelp.nrtsearch.server.luceneserver.handler.AddDocumentHandler;
@@ -75,6 +74,9 @@
 import com.yelp.nrtsearch.server.luceneserver.handler.StopIndexHandler;
 import com.yelp.nrtsearch.server.luceneserver.handler.UpdateFieldsHandler;
 import com.yelp.nrtsearch.server.luceneserver.highlights.HighlighterService;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
 import com.yelp.nrtsearch.server.luceneserver.index.handlers.LiveSettingsV2Handler;
 import com.yelp.nrtsearch.server.luceneserver.index.handlers.SettingsV2Handler;
 import com.yelp.nrtsearch.server.luceneserver.logging.HitsLoggerCreator;
@@ -84,6 +86,8 @@
 import com.yelp.nrtsearch.server.luceneserver.search.cache.NrtQueryCache;
 import com.yelp.nrtsearch.server.luceneserver.search.collectors.CollectorCreator;
 import com.yelp.nrtsearch.server.luceneserver.similarity.SimilarityCreator;
+import com.yelp.nrtsearch.server.luceneserver.state.GlobalState;
+import com.yelp.nrtsearch.server.modules.LuceneServerModule;
 import com.yelp.nrtsearch.server.monitoring.Configuration;
 import com.yelp.nrtsearch.server.monitoring.DeadlineMetrics;
 import com.yelp.nrtsearch.server.monitoring.DirSizeCollector;
@@ -99,7 +103,6 @@
 import com.yelp.nrtsearch.server.plugins.Plugin;
 import com.yelp.nrtsearch.server.plugins.PluginsService;
 import com.yelp.nrtsearch.server.remote.RemoteBackend;
-import com.yelp.nrtsearch.server.utils.ThreadPoolExecutorFactory;
 import com.yelp.nrtsearch.tools.cli.VersionProvider;
 import io.grpc.Server;
 import io.grpc.ServerBuilder;

diff --git a/src/main/java/com/yelp/nrtsearch/server/grpc/ReplicationServerClient.java b/src/main/java/com/yelp/nrtsearch/server/grpc/ReplicationServerClient.java
@@ -20,7 +20,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.yelp.nrtsearch.server.grpc.ReplicationServerGrpc.ReplicationServerBlockingStub;
 import com.yelp.nrtsearch.server.grpc.discovery.PrimaryFileNameResolverProvider;
-import com.yelp.nrtsearch.server.luceneserver.SimpleCopyJob.FileChunkStreamingIterator;
+import com.yelp.nrtsearch.server.luceneserver.nrt.SimpleCopyJob.FileChunkStreamingIterator;
 import io.grpc.Deadline;
 import io.grpc.ManagedChannel;
 import io.grpc.ManagedChannelBuilder;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/AddReplicaHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/AddReplicaHandler.java
@@ -18,6 +18,8 @@
 import com.yelp.nrtsearch.server.grpc.AddReplicaRequest;
 import com.yelp.nrtsearch.server.grpc.AddReplicaResponse;
 import com.yelp.nrtsearch.server.grpc.ReplicationServerClient;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
 import com.yelp.nrtsearch.server.luceneserver.handler.Handler;
 import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager;
 import io.grpc.Status;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/CopyFilesHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/CopyFilesHandler.java
@@ -18,6 +18,9 @@
 import com.yelp.nrtsearch.server.grpc.CopyFiles;
 import com.yelp.nrtsearch.server.grpc.TransferStatus;
 import com.yelp.nrtsearch.server.grpc.TransferStatusCode;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
+import com.yelp.nrtsearch.server.luceneserver.nrt.NRTReplicaNode;
 import com.yelp.nrtsearch.server.luceneserver.handler.Handler;
 import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager;
 import com.yelp.nrtsearch.server.monitoring.NrtMetrics;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/GetNodesInfoHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/GetNodesInfoHandler.java
@@ -19,6 +19,9 @@
 import com.yelp.nrtsearch.server.grpc.GetNodesResponse;
 import com.yelp.nrtsearch.server.grpc.NodeInfo;
 import com.yelp.nrtsearch.server.luceneserver.handler.Handler;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
+import com.yelp.nrtsearch.server.luceneserver.nrt.NRTPrimaryNode;
 import com.yelp.nrtsearch.server.utils.HostPort;
 import io.grpc.Status;
 import io.grpc.stub.StreamObserver;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/IndexBackupUtils.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/IndexBackupUtils.java
diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/NewNRTPointHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/NewNRTPointHandler.java
@@ -23,6 +23,8 @@
 import io.grpc.Status;
 import io.grpc.StatusRuntimeException;
 import io.grpc.stub.StreamObserver;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
 import java.io.IOException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/RecvCopyStateHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/RecvCopyStateHandler.java
@@ -25,6 +25,9 @@
 import io.grpc.Status;
 import io.grpc.StatusRuntimeException;
 import io.grpc.stub.StreamObserver;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
+import com.yelp.nrtsearch.server.luceneserver.nrt.NRTPrimaryNode;
 import java.io.IOException;
 import java.util.Map;
 import org.apache.lucene.replicator.nrt.FileMetaData;

diff --git a/...in/java/com/yelp/nrtsearch/server/luceneserver/ReplicaCurrentSearchingVersionHandler.java b/...in/java/com/yelp/nrtsearch/server/luceneserver/ReplicaCurrentSearchingVersionHandler.java
@@ -20,6 +20,8 @@
 import com.yelp.nrtsearch.server.luceneserver.handler.Handler;
 import io.grpc.Status;
 import io.grpc.stub.StreamObserver;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
 import java.io.IOException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/StartIndexHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/StartIndexHandler.java
@@ -22,7 +22,9 @@
 import com.yelp.nrtsearch.server.grpc.StartIndexRequest;
 import com.yelp.nrtsearch.server.grpc.StartIndexResponse;
 import com.yelp.nrtsearch.server.luceneserver.handler.Handler.HandlerException;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
 import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
 import com.yelp.nrtsearch.server.luceneserver.nrt.NrtDataManager;
 import com.yelp.nrtsearch.server.luceneserver.state.BackendGlobalState;
 import com.yelp.nrtsearch.server.remote.RemoteBackend;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/WriteNRTPointHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/WriteNRTPointHandler.java
@@ -20,12 +20,15 @@
 import com.yelp.nrtsearch.server.grpc.SearcherVersion;
 import com.yelp.nrtsearch.server.luceneserver.handler.Handler;
 import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
+import com.yelp.nrtsearch.server.luceneserver.nrt.NRTPrimaryNode;
 import io.grpc.Status;
 import io.grpc.StatusRuntimeException;
 import io.grpc.stub.StreamObserver;
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Iterator;
-import java.util.Queue;
 import org.apache.lucene.facet.taxonomy.SearcherTaxonomyManager;
 import org.apache.lucene.index.DirectoryReader;
 import org.slf4j.Logger;
@@ -81,8 +84,8 @@ private SearcherVersion handle(IndexState indexState, String indexId) {
         // before notifying all replicas, at which point we have a newer version index than client
         // knew about?
         long version = shardState.nrtPrimaryNode.getCopyStateVersion();
-        Queue<NRTPrimaryNode.ReplicaDetails> replicasInfos =
-            shardState.nrtPrimaryNode.replicasInfos;
+        Collection<NRTPrimaryNode.ReplicaDetails> replicasInfos =
+            shardState.nrtPrimaryNode.getNodesInfo();
         shardState.nrtPrimaryNode.message(
             "send flushed version=" + version + " replica count " + replicasInfos.size());
 

diff --git a/...arch/server/luceneserver/ServerCodec.java → ...erver/luceneserver/codec/ServerCodec.java b/...arch/server/luceneserver/ServerCodec.java → ...erver/luceneserver/codec/ServerCodec.java
@@ -13,11 +13,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.yelp.nrtsearch.server.luceneserver;
+package com.yelp.nrtsearch.server.luceneserver.codec;
 
 import com.yelp.nrtsearch.server.luceneserver.field.FieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.VectorFieldDef;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
 import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;

diff --git a/...rver/utils/ThreadPoolExecutorFactory.java → ...concurrent/ThreadPoolExecutorFactory.java b/...rver/utils/ThreadPoolExecutorFactory.java → ...concurrent/ThreadPoolExecutorFactory.java
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.yelp.nrtsearch.server.utils;
+package com.yelp.nrtsearch.server.luceneserver.concurrent;
 
 import com.yelp.nrtsearch.server.config.ThreadPoolConfiguration;
 import com.yelp.nrtsearch.server.monitoring.ThreadPoolCollector;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/doc/LoadedDocValues.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/doc/LoadedDocValues.java
@@ -26,6 +26,8 @@
 import com.yelp.nrtsearch.server.grpc.SearchResponse.Hit.FieldValue.Vector;
 import com.yelp.nrtsearch.server.grpc.SearchResponse.Hit.FieldValue.Vector.Builder;
 import com.yelp.nrtsearch.server.luceneserver.geo.GeoPoint;
+import com.yelp.nrtsearch.server.luceneserver.vector.ByteVectorType;
+import com.yelp.nrtsearch.server.luceneserver.vector.FloatVectorType;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.FloatBuffer;
@@ -951,9 +953,9 @@ public int size() {
     }
   }
 
-  public static final class SingleVector extends LoadedDocValues<VectorType> {
+  public static final class SingleVector extends LoadedDocValues<FloatVectorType> {
     private final BinaryDocValues docValues;
-    private VectorType value;
+    private FloatVectorType value;
 
     public SingleVector(BinaryDocValues docValues) {
       this.docValues = docValues;
@@ -972,18 +974,18 @@ public void setDocId(int docID) throws IOException {
     }
 
     /** Decodes binary doc value to float array and wraps it into a VectorType */
-    private static VectorType decodeBytesRefToVectorType(BytesRef bytesRef) {
+    private static FloatVectorType decodeBytesRefToVectorType(BytesRef bytesRef) {
       float[] floats = new float[bytesRef.length / Float.BYTES];
       FloatBuffer fb =
           ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length).asFloatBuffer();
       fb.get(floats);
-      return new VectorType(floats);
+      return new FloatVectorType(floats);
     }
 
     /** Provide field value containing the doc value data for a given index */
     @Override
     public FieldValue toFieldValue(int index) {
-      VectorType vector = get(index);
+      FloatVectorType vector = get(index);
       Builder vectorBuilder = Vector.newBuilder();
       for (float value : vector.getVectorData()) {
         vectorBuilder.addValue(value);
@@ -994,7 +996,7 @@ public FieldValue toFieldValue(int index) {
     }
 
     @Override
-    public VectorType get(int index) {
+    public FloatVectorType get(int index) {
       if (value == null) {
         throw new IllegalStateException("No doc values for document");
       } else if (index != 0) {
@@ -1008,7 +1010,7 @@ public int size() {
       return value == null ? 0 : 1;
     }
 
-    public VectorType getValue() {
+    public FloatVectorType getValue() {
       return get(0);
     }
   }
@@ -1071,9 +1073,9 @@ public ByteVectorType getValue() {
    * Doc value interface for vector data loaded from index vector values indexed for vector search.
    * Calls to {@link #setDocId(int)} must provide ids in increasing order.
    */
-  public static final class SingleSearchVector extends LoadedDocValues<VectorType> {
+  public static final class SingleSearchVector extends LoadedDocValues<FloatVectorType> {
     private final FloatVectorValues vectorValues;
-    private VectorType value = null;
+    private FloatVectorType value = null;
 
     public SingleSearchVector(FloatVectorValues vectorValues) {
       this.vectorValues = vectorValues;
@@ -1086,7 +1088,7 @@ public void setDocId(int docID) throws IOException {
           vectorValues.advance(docID);
         }
         if (vectorValues.docID() == docID) {
-          value = new VectorType(vectorValues.vectorValue());
+          value = new FloatVectorType(vectorValues.vectorValue());
         } else {
           value = null;
         }
@@ -1095,7 +1097,7 @@ public void setDocId(int docID) throws IOException {
 
     @Override
     public FieldValue toFieldValue(int index) {
-      VectorType vector = get(index);
+      FloatVectorType vector = get(index);
       Builder vectorBuilder = Vector.newBuilder();
       for (float value : vector.getVectorData()) {
         vectorBuilder.addValue(value);
@@ -1106,7 +1108,7 @@ public FieldValue toFieldValue(int index) {
     }
 
     @Override
-    public VectorType get(int index) {
+    public FloatVectorType get(int index) {
       if (value == null) {
         throw new IllegalStateException("No doc values for document");
       } else if (index != 0) {
@@ -1120,7 +1122,7 @@ public int size() {
       return value == null ? 0 : 1;
     }
 
-    public VectorType getValue() {
+    public FloatVectorType getValue() {
       return get(0);
     }
   }

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/facet/DrillSidewaysImpl.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/facet/DrillSidewaysImpl.java
@@ -19,8 +19,6 @@
 import com.yelp.nrtsearch.server.grpc.Facet;
 import com.yelp.nrtsearch.server.grpc.NumericRangeType;
 import com.yelp.nrtsearch.server.grpc.SearchResponse.Diagnostics;
-import com.yelp.nrtsearch.server.luceneserver.IndexState;
-import com.yelp.nrtsearch.server.luceneserver.ShardState;
 import com.yelp.nrtsearch.server.luceneserver.doc.LoadedDocValues;
 import com.yelp.nrtsearch.server.luceneserver.field.DoubleFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.FieldDef;
@@ -30,6 +28,8 @@
 import com.yelp.nrtsearch.server.luceneserver.field.LongFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.RuntimeFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.VirtualFieldDef;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
 import com.yelp.nrtsearch.server.luceneserver.script.FacetScript;
 import com.yelp.nrtsearch.server.luceneserver.script.ScriptService;
 import com.yelp.nrtsearch.server.utils.ScriptParamsUtils;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/facet/FacetTopDocs.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/facet/FacetTopDocs.java
@@ -18,10 +18,10 @@
 import com.yelp.nrtsearch.server.grpc.Facet;
 import com.yelp.nrtsearch.server.grpc.FacetResult;
 import com.yelp.nrtsearch.server.grpc.SearchResponse.Diagnostics;
-import com.yelp.nrtsearch.server.luceneserver.IndexState;
 import com.yelp.nrtsearch.server.luceneserver.doc.LoadedDocValues;
 import com.yelp.nrtsearch.server.luceneserver.field.FieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;