Reorganize server classes (#746)

Yelp · Oct 4, 2024 · cdbdfb0 · cdbdfb0
1 parent 1155762
commit cdbdfb0
Show file tree

Hide file tree

Showing 194 changed files with 486 additions and 562 deletions.
diff --git a/src/main/java/com/yelp/nrtsearch/server/config/ThreadPoolConfiguration.java b/src/main/java/com/yelp/nrtsearch/server/config/ThreadPoolConfiguration.java
@@ -16,8 +16,8 @@
 package com.yelp.nrtsearch.server.config;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.yelp.nrtsearch.server.luceneserver.concurrent.ThreadPoolExecutorFactory;
 import com.yelp.nrtsearch.server.utils.JsonUtils;
-import com.yelp.nrtsearch.server.utils.ThreadPoolExecutorFactory;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -56,7 +56,7 @@ public class ThreadPoolConfiguration {
       Math.max(100, 2 * DEFAULT_VECTOR_MERGE_THREADS);
 
   /**
-   * Settings for a {@link com.yelp.nrtsearch.server.utils.ThreadPoolExecutorFactory.ExecutorType}.
+   * Settings for a {@link ThreadPoolExecutorFactory.ExecutorType}.
    *
    * @param maxThreads max number of threads
    * @param maxBufferedItems max number of buffered items

diff --git a/src/main/java/com/yelp/nrtsearch/server/grpc/GrpcServerExecutorSupplier.java b/src/main/java/com/yelp/nrtsearch/server/grpc/GrpcServerExecutorSupplier.java
@@ -15,7 +15,7 @@
  */
 package com.yelp.nrtsearch.server.grpc;
 
-import com.yelp.nrtsearch.server.utils.ThreadPoolExecutorFactory;
+import com.yelp.nrtsearch.server.luceneserver.concurrent.ThreadPoolExecutorFactory;
 import io.grpc.Metadata;
 import io.grpc.ServerCall;
 import io.grpc.ServerCallExecutorSupplier;

diff --git a/src/main/java/com/yelp/nrtsearch/server/grpc/LuceneServer.java b/src/main/java/com/yelp/nrtsearch/server/grpc/LuceneServer.java
@@ -26,20 +26,17 @@
 import com.google.protobuf.ByteString;
 import com.google.protobuf.Empty;
 import com.google.protobuf.util.JsonFormat;
-import com.yelp.nrtsearch.LuceneServerModule;
 import com.yelp.nrtsearch.server.config.LuceneServerConfiguration;
 import com.yelp.nrtsearch.server.config.QueryCacheConfig;
 import com.yelp.nrtsearch.server.luceneserver.AddReplicaHandler;
 import com.yelp.nrtsearch.server.luceneserver.CopyFilesHandler;
 import com.yelp.nrtsearch.server.luceneserver.GetNodesInfoHandler;
-import com.yelp.nrtsearch.server.luceneserver.GlobalState;
-import com.yelp.nrtsearch.server.luceneserver.IndexState;
 import com.yelp.nrtsearch.server.luceneserver.NewNRTPointHandler;
 import com.yelp.nrtsearch.server.luceneserver.RecvCopyStateHandler;
 import com.yelp.nrtsearch.server.luceneserver.ReplicaCurrentSearchingVersionHandler;
-import com.yelp.nrtsearch.server.luceneserver.ShardState;
 import com.yelp.nrtsearch.server.luceneserver.WriteNRTPointHandler;
 import com.yelp.nrtsearch.server.luceneserver.analysis.AnalyzerCreator;
+import com.yelp.nrtsearch.server.luceneserver.concurrent.ThreadPoolExecutorFactory;
 import com.yelp.nrtsearch.server.luceneserver.custom.request.CustomRequestProcessor;
 import com.yelp.nrtsearch.server.luceneserver.field.FieldDefCreator;
 import com.yelp.nrtsearch.server.luceneserver.handler.AddDocumentHandler;
@@ -75,7 +72,9 @@
 import com.yelp.nrtsearch.server.luceneserver.handler.StopIndexHandler;
 import com.yelp.nrtsearch.server.luceneserver.handler.UpdateFieldsHandler;
 import com.yelp.nrtsearch.server.luceneserver.highlights.HighlighterService;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
 import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
 import com.yelp.nrtsearch.server.luceneserver.index.handlers.LiveSettingsV2Handler;
 import com.yelp.nrtsearch.server.luceneserver.index.handlers.SettingsV2Handler;
 import com.yelp.nrtsearch.server.luceneserver.logging.HitsLoggerCreator;
@@ -85,6 +84,8 @@
 import com.yelp.nrtsearch.server.luceneserver.search.cache.NrtQueryCache;
 import com.yelp.nrtsearch.server.luceneserver.search.collectors.CollectorCreator;
 import com.yelp.nrtsearch.server.luceneserver.similarity.SimilarityCreator;
+import com.yelp.nrtsearch.server.luceneserver.state.GlobalState;
+import com.yelp.nrtsearch.server.modules.LuceneServerModule;
 import com.yelp.nrtsearch.server.monitoring.Configuration;
 import com.yelp.nrtsearch.server.monitoring.DeadlineMetrics;
 import com.yelp.nrtsearch.server.monitoring.DirSizeCollector;
@@ -100,7 +101,6 @@
 import com.yelp.nrtsearch.server.plugins.Plugin;
 import com.yelp.nrtsearch.server.plugins.PluginsService;
 import com.yelp.nrtsearch.server.remote.RemoteBackend;
-import com.yelp.nrtsearch.server.utils.ThreadPoolExecutorFactory;
 import com.yelp.nrtsearch.tools.cli.VersionProvider;
 import io.grpc.Server;
 import io.grpc.ServerBuilder;

diff --git a/src/main/java/com/yelp/nrtsearch/server/grpc/ReplicationServerClient.java b/src/main/java/com/yelp/nrtsearch/server/grpc/ReplicationServerClient.java
@@ -20,7 +20,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.yelp.nrtsearch.server.grpc.ReplicationServerGrpc.ReplicationServerBlockingStub;
 import com.yelp.nrtsearch.server.grpc.discovery.PrimaryFileNameResolverProvider;
-import com.yelp.nrtsearch.server.luceneserver.SimpleCopyJob.FileChunkStreamingIterator;
+import com.yelp.nrtsearch.server.luceneserver.nrt.SimpleCopyJob.FileChunkStreamingIterator;
 import io.grpc.Deadline;
 import io.grpc.ManagedChannel;
 import io.grpc.ManagedChannelBuilder;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/AddReplicaHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/AddReplicaHandler.java
@@ -18,6 +18,8 @@
 import com.yelp.nrtsearch.server.grpc.AddReplicaRequest;
 import com.yelp.nrtsearch.server.grpc.AddReplicaResponse;
 import com.yelp.nrtsearch.server.grpc.ReplicationServerClient;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
 import java.io.IOException;
 
 public class AddReplicaHandler implements Handler<AddReplicaRequest, AddReplicaResponse> {

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/CopyFilesHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/CopyFilesHandler.java
@@ -18,6 +18,9 @@
 import com.yelp.nrtsearch.server.grpc.CopyFiles;
 import com.yelp.nrtsearch.server.grpc.TransferStatus;
 import com.yelp.nrtsearch.server.grpc.TransferStatusCode;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
+import com.yelp.nrtsearch.server.luceneserver.nrt.NRTReplicaNode;
 import com.yelp.nrtsearch.server.monitoring.NrtMetrics;
 import io.grpc.stub.StreamObserver;
 import java.io.IOException;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/GetNodesInfoHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/GetNodesInfoHandler.java
@@ -18,6 +18,9 @@
 import com.yelp.nrtsearch.server.grpc.GetNodesRequest;
 import com.yelp.nrtsearch.server.grpc.GetNodesResponse;
 import com.yelp.nrtsearch.server.grpc.NodeInfo;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
+import com.yelp.nrtsearch.server.luceneserver.nrt.NRTPrimaryNode;
 import com.yelp.nrtsearch.server.utils.HostPort;
 import java.util.Collection;
 import org.slf4j.Logger;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/Handler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/Handler.java
@@ -17,6 +17,7 @@
 
 import com.google.protobuf.GeneratedMessageV3;
 import com.yelp.nrtsearch.server.grpc.ReplicationServerClient;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
 import io.grpc.stub.StreamObserver;
 
 /* Interface for handlers that take in an indexState and the protoBuff request and returns the protoBuff response */

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/IndexBackupUtils.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/IndexBackupUtils.java
diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/NewNRTPointHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/NewNRTPointHandler.java
@@ -18,6 +18,8 @@
 import com.yelp.nrtsearch.server.grpc.NewNRTPoint;
 import com.yelp.nrtsearch.server.grpc.TransferStatus;
 import com.yelp.nrtsearch.server.grpc.TransferStatusCode;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
 import java.io.IOException;
 
 public class NewNRTPointHandler implements Handler<NewNRTPoint, TransferStatus> {

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/RecvCopyStateHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/RecvCopyStateHandler.java
@@ -20,6 +20,9 @@
 import com.yelp.nrtsearch.server.grpc.CopyStateRequest;
 import com.yelp.nrtsearch.server.grpc.FileMetadata;
 import com.yelp.nrtsearch.server.grpc.FilesMetadata;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
+import com.yelp.nrtsearch.server.luceneserver.nrt.NRTPrimaryNode;
 import java.io.IOException;
 import java.util.Map;
 import org.apache.lucene.replicator.nrt.FileMetaData;

diff --git a/...in/java/com/yelp/nrtsearch/server/luceneserver/ReplicaCurrentSearchingVersionHandler.java b/...in/java/com/yelp/nrtsearch/server/luceneserver/ReplicaCurrentSearchingVersionHandler.java
@@ -17,6 +17,8 @@
 
 import com.yelp.nrtsearch.server.grpc.IndexName;
 import com.yelp.nrtsearch.server.grpc.SearcherVersion;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
 import java.io.IOException;
 
 public class ReplicaCurrentSearchingVersionHandler implements Handler<IndexName, SearcherVersion> {

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/StartIndexHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/StartIndexHandler.java
@@ -22,7 +22,9 @@
 import com.yelp.nrtsearch.server.grpc.StartIndexRequest;
 import com.yelp.nrtsearch.server.grpc.StartIndexResponse;
 import com.yelp.nrtsearch.server.luceneserver.handler.Handler.HandlerException;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
 import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
 import com.yelp.nrtsearch.server.luceneserver.nrt.NrtDataManager;
 import com.yelp.nrtsearch.server.luceneserver.state.BackendGlobalState;
 import com.yelp.nrtsearch.server.remote.RemoteBackend;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/WriteNRTPointHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/WriteNRTPointHandler.java
@@ -18,11 +18,14 @@
 import com.yelp.nrtsearch.server.grpc.IndexName;
 import com.yelp.nrtsearch.server.grpc.ReplicationServerClient;
 import com.yelp.nrtsearch.server.grpc.SearcherVersion;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
+import com.yelp.nrtsearch.server.luceneserver.nrt.NRTPrimaryNode;
 import io.grpc.Status;
 import io.grpc.StatusRuntimeException;
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Iterator;
-import java.util.Queue;
 import org.apache.lucene.facet.taxonomy.SearcherTaxonomyManager;
 import org.apache.lucene.index.DirectoryReader;
 import org.slf4j.Logger;
@@ -55,8 +58,8 @@ public SearcherVersion handle(IndexState indexState, IndexName protoRequest)
         // before notifying all replicas, at which point we have a newer version index than client
         // knew about?
         long version = shardState.nrtPrimaryNode.getCopyStateVersion();
-        Queue<NRTPrimaryNode.ReplicaDetails> replicasInfos =
-            shardState.nrtPrimaryNode.replicasInfos;
+        Collection<NRTPrimaryNode.ReplicaDetails> replicasInfos =
+            shardState.nrtPrimaryNode.getNodesInfo();
         shardState.nrtPrimaryNode.message(
             "send flushed version=" + version + " replica count " + replicasInfos.size());
 

diff --git a/...arch/server/luceneserver/ServerCodec.java → ...erver/luceneserver/codec/ServerCodec.java b/...arch/server/luceneserver/ServerCodec.java → ...erver/luceneserver/codec/ServerCodec.java
@@ -13,11 +13,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.yelp.nrtsearch.server.luceneserver;
+package com.yelp.nrtsearch.server.luceneserver.codec;
 
 import com.yelp.nrtsearch.server.luceneserver.field.FieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.VectorFieldDef;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
 import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;

diff --git a/...rver/utils/ThreadPoolExecutorFactory.java → ...concurrent/ThreadPoolExecutorFactory.java b/...rver/utils/ThreadPoolExecutorFactory.java → ...concurrent/ThreadPoolExecutorFactory.java
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.yelp.nrtsearch.server.utils;
+package com.yelp.nrtsearch.server.luceneserver.concurrent;
 
 import com.yelp.nrtsearch.server.config.ThreadPoolConfiguration;
 import com.yelp.nrtsearch.server.monitoring.ThreadPoolCollector;

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/doc/LoadedDocValues.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/doc/LoadedDocValues.java
@@ -26,6 +26,8 @@
 import com.yelp.nrtsearch.server.grpc.SearchResponse.Hit.FieldValue.Vector;
 import com.yelp.nrtsearch.server.grpc.SearchResponse.Hit.FieldValue.Vector.Builder;
 import com.yelp.nrtsearch.server.luceneserver.geo.GeoPoint;
+import com.yelp.nrtsearch.server.luceneserver.vector.ByteVectorType;
+import com.yelp.nrtsearch.server.luceneserver.vector.FloatVectorType;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.FloatBuffer;
@@ -951,9 +953,9 @@ public int size() {
     }
   }
 
-  public static final class SingleVector extends LoadedDocValues<VectorType> {
+  public static final class SingleVector extends LoadedDocValues<FloatVectorType> {
     private final BinaryDocValues docValues;
-    private VectorType value;
+    private FloatVectorType value;
 
     public SingleVector(BinaryDocValues docValues) {
       this.docValues = docValues;
@@ -972,18 +974,18 @@ public void setDocId(int docID) throws IOException {
     }
 
     /** Decodes binary doc value to float array and wraps it into a VectorType */
-    private static VectorType decodeBytesRefToVectorType(BytesRef bytesRef) {
+    private static FloatVectorType decodeBytesRefToVectorType(BytesRef bytesRef) {
       float[] floats = new float[bytesRef.length / Float.BYTES];
       FloatBuffer fb =
           ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length).asFloatBuffer();
       fb.get(floats);
-      return new VectorType(floats);
+      return new FloatVectorType(floats);
     }
 
     /** Provide field value containing the doc value data for a given index */
     @Override
     public FieldValue toFieldValue(int index) {
-      VectorType vector = get(index);
+      FloatVectorType vector = get(index);
       Builder vectorBuilder = Vector.newBuilder();
       for (float value : vector.getVectorData()) {
         vectorBuilder.addValue(value);
@@ -994,7 +996,7 @@ public FieldValue toFieldValue(int index) {
     }
 
     @Override
-    public VectorType get(int index) {
+    public FloatVectorType get(int index) {
       if (value == null) {
         throw new IllegalStateException("No doc values for document");
       } else if (index != 0) {
@@ -1008,7 +1010,7 @@ public int size() {
       return value == null ? 0 : 1;
     }
 
-    public VectorType getValue() {
+    public FloatVectorType getValue() {
       return get(0);
     }
   }
@@ -1071,9 +1073,9 @@ public ByteVectorType getValue() {
    * Doc value interface for vector data loaded from index vector values indexed for vector search.
    * Calls to {@link #setDocId(int)} must provide ids in increasing order.
    */
-  public static final class SingleSearchVector extends LoadedDocValues<VectorType> {
+  public static final class SingleSearchVector extends LoadedDocValues<FloatVectorType> {
     private final FloatVectorValues vectorValues;
-    private VectorType value = null;
+    private FloatVectorType value = null;
 
     public SingleSearchVector(FloatVectorValues vectorValues) {
       this.vectorValues = vectorValues;
@@ -1086,7 +1088,7 @@ public void setDocId(int docID) throws IOException {
           vectorValues.advance(docID);
         }
         if (vectorValues.docID() == docID) {
-          value = new VectorType(vectorValues.vectorValue());
+          value = new FloatVectorType(vectorValues.vectorValue());
         } else {
           value = null;
         }
@@ -1095,7 +1097,7 @@ public void setDocId(int docID) throws IOException {
 
     @Override
     public FieldValue toFieldValue(int index) {
-      VectorType vector = get(index);
+      FloatVectorType vector = get(index);
       Builder vectorBuilder = Vector.newBuilder();
       for (float value : vector.getVectorData()) {
         vectorBuilder.addValue(value);
@@ -1106,7 +1108,7 @@ public FieldValue toFieldValue(int index) {
     }
 
     @Override
-    public VectorType get(int index) {
+    public FloatVectorType get(int index) {
       if (value == null) {
         throw new IllegalStateException("No doc values for document");
       } else if (index != 0) {
@@ -1120,7 +1122,7 @@ public int size() {
       return value == null ? 0 : 1;
     }
 
-    public VectorType getValue() {
+    public FloatVectorType getValue() {
       return get(0);
     }
   }

diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/facet/DrillSidewaysImpl.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/facet/DrillSidewaysImpl.java
@@ -19,8 +19,6 @@
 import com.yelp.nrtsearch.server.grpc.Facet;
 import com.yelp.nrtsearch.server.grpc.NumericRangeType;
 import com.yelp.nrtsearch.server.grpc.SearchResponse.Diagnostics;
-import com.yelp.nrtsearch.server.luceneserver.IndexState;
-import com.yelp.nrtsearch.server.luceneserver.ShardState;
 import com.yelp.nrtsearch.server.luceneserver.doc.LoadedDocValues;
 import com.yelp.nrtsearch.server.luceneserver.field.DoubleFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.FieldDef;
@@ -30,6 +28,8 @@
 import com.yelp.nrtsearch.server.luceneserver.field.LongFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.RuntimeFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.VirtualFieldDef;
+import com.yelp.nrtsearch.server.luceneserver.index.IndexState;
+import com.yelp.nrtsearch.server.luceneserver.index.ShardState;
 import com.yelp.nrtsearch.server.luceneserver.script.FacetScript;
 import com.yelp.nrtsearch.server.luceneserver.script.ScriptService;
 import com.yelp.nrtsearch.server.utils.ScriptParamsUtils;