From a243ea5bec54535d75a71fd261eb6397f5fbfa3f Mon Sep 17 00:00:00 2001 From: Sarthak Nandi Date: Thu, 3 Oct 2024 12:12:37 -0700 Subject: [PATCH] Refactor handler interface and LuceneServer class (#743) * Refactor handlers for LuceneServer * Remove static singletons * Spotless apply * Added a javadoc, remove unused method * Passing GlobalState to SearchHandler from Warmer --- .../nrtsearch/server/grpc/LuceneServer.java | 1338 ++--------------- .../luceneserver/AddDocumentHandler.java | 207 ++- .../luceneserver/CreateSnapshotHandler.java | 87 +- .../DeleteAllDocumentsHandler.java | 36 +- .../luceneserver/DeleteByQueryHandler.java | 35 +- .../luceneserver/DeleteDocumentsHandler.java | 35 +- .../luceneserver/DeleteIndexHandler.java | 32 +- .../server/luceneserver/GetStateHandler.java | 33 +- .../luceneserver/LiveSettingsHandler.java | 41 +- .../luceneserver/ReleaseSnapshotHandler.java | 56 +- .../server/luceneserver/SearchHandler.java | 73 +- .../server/luceneserver/SettingsHandler.java | 51 +- .../luceneserver/StartIndexHandler.java | 6 +- .../handler/BackupWarmingQueriesHandler.java | 109 ++ .../luceneserver/handler/CommitHandler.java | 105 ++ .../handler/CreateIndexHandler.java | 73 + .../luceneserver/handler/CustomHandler.java | 48 + .../handler/ForceMergeDeletesHandler.java | 74 + .../handler/ForceMergeHandler.java | 76 + .../GetAllSnapshotIndexGenHandler.java | 56 + .../handler/GlobalStateHandler.java | 49 + .../server/luceneserver/handler/Handler.java | 93 ++ .../luceneserver/handler/IndicesHandler.java | 77 + .../handler/MetricsHandler.java} | 34 +- .../luceneserver/handler/ReadyHandler.java | 102 ++ .../luceneserver/handler/RefreshHandler.java | 76 + .../handler/RegisterFieldsHandler.java | 72 + .../handler/ReloadStateHandler.java | 59 + .../luceneserver/handler/SearchV2Handler.java | 91 ++ .../handler/StartIndexHandler.java | 77 + .../handler/StartIndexV2Handler.java | 66 + .../StatsHandler.java} | 81 +- .../luceneserver/handler/StatusHandler.java | 51 + .../handler/StopIndexHandler.java | 53 + .../handler/UpdateFieldsHandler.java | 73 + .../index/handlers/LiveSettingsV2Handler.java | 47 +- .../index/handlers/SettingsV2Handler.java | 52 +- .../server/luceneserver/warming/Warmer.java | 4 +- .../server/utils/ProtoMessagePrinter.java | 28 + 39 files changed, 2433 insertions(+), 1323 deletions(-) create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/BackupWarmingQueriesHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/CommitHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/CreateIndexHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/CustomHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ForceMergeDeletesHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ForceMergeHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/GetAllSnapshotIndexGenHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/GlobalStateHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/Handler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/IndicesHandler.java rename src/main/java/com/yelp/nrtsearch/server/{MetricsRequestHandler.java => luceneserver/handler/MetricsHandler.java} (55%) create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ReadyHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/RefreshHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/RegisterFieldsHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ReloadStateHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/SearchV2Handler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StartIndexHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StartIndexV2Handler.java rename src/main/java/com/yelp/nrtsearch/server/luceneserver/{StatsRequestHandler.java => handler/StatsHandler.java} (70%) create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StatusHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StopIndexHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/UpdateFieldsHandler.java create mode 100644 src/main/java/com/yelp/nrtsearch/server/utils/ProtoMessagePrinter.java diff --git a/src/main/java/com/yelp/nrtsearch/server/grpc/LuceneServer.java b/src/main/java/com/yelp/nrtsearch/server/grpc/LuceneServer.java index 689c2ef94..85f912853 100644 --- a/src/main/java/com/yelp/nrtsearch/server/grpc/LuceneServer.java +++ b/src/main/java/com/yelp/nrtsearch/server/grpc/LuceneServer.java @@ -19,30 +19,63 @@ import com.google.api.HttpBody; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Splitter; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; -import com.google.common.collect.Sets; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import com.google.protobuf.Any; import com.google.protobuf.ByteString; import com.google.protobuf.Empty; -import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import com.yelp.nrtsearch.LuceneServerModule; -import com.yelp.nrtsearch.server.MetricsRequestHandler; import com.yelp.nrtsearch.server.config.LuceneServerConfiguration; import com.yelp.nrtsearch.server.config.QueryCacheConfig; -import com.yelp.nrtsearch.server.luceneserver.*; -import com.yelp.nrtsearch.server.luceneserver.AddDocumentHandler.DocumentIndexer; +import com.yelp.nrtsearch.server.luceneserver.AddDocumentHandler; +import com.yelp.nrtsearch.server.luceneserver.AddReplicaHandler; +import com.yelp.nrtsearch.server.luceneserver.CopyFilesHandler; +import com.yelp.nrtsearch.server.luceneserver.CreateSnapshotHandler; +import com.yelp.nrtsearch.server.luceneserver.DeleteAllDocumentsHandler; +import com.yelp.nrtsearch.server.luceneserver.DeleteByQueryHandler; +import com.yelp.nrtsearch.server.luceneserver.DeleteDocumentsHandler; +import com.yelp.nrtsearch.server.luceneserver.DeleteIndexHandler; +import com.yelp.nrtsearch.server.luceneserver.GetNodesInfoHandler; +import com.yelp.nrtsearch.server.luceneserver.GetStateHandler; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.IndexState; +import com.yelp.nrtsearch.server.luceneserver.LiveSettingsHandler; +import com.yelp.nrtsearch.server.luceneserver.NewNRTPointHandler; +import com.yelp.nrtsearch.server.luceneserver.RecvCopyStateHandler; +import com.yelp.nrtsearch.server.luceneserver.ReleaseSnapshotHandler; +import com.yelp.nrtsearch.server.luceneserver.ReplicaCurrentSearchingVersionHandler; +import com.yelp.nrtsearch.server.luceneserver.SearchHandler; +import com.yelp.nrtsearch.server.luceneserver.SettingsHandler; +import com.yelp.nrtsearch.server.luceneserver.ShardState; +import com.yelp.nrtsearch.server.luceneserver.WriteNRTPointHandler; import com.yelp.nrtsearch.server.luceneserver.analysis.AnalyzerCreator; import com.yelp.nrtsearch.server.luceneserver.custom.request.CustomRequestProcessor; import com.yelp.nrtsearch.server.luceneserver.field.FieldDefCreator; +import com.yelp.nrtsearch.server.luceneserver.handler.BackupWarmingQueriesHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.CommitHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.CreateIndexHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.CustomHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.ForceMergeDeletesHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.ForceMergeHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.GetAllSnapshotIndexGenHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.GlobalStateHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.IndicesHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.MetricsHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.ReadyHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.RefreshHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.RegisterFieldsHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.ReloadStateHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.SearchV2Handler; +import com.yelp.nrtsearch.server.luceneserver.handler.StartIndexHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.StartIndexV2Handler; +import com.yelp.nrtsearch.server.luceneserver.handler.StatsHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.StatusHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.StopIndexHandler; +import com.yelp.nrtsearch.server.luceneserver.handler.UpdateFieldsHandler; import com.yelp.nrtsearch.server.luceneserver.highlights.HighlighterService; import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager; -import com.yelp.nrtsearch.server.luceneserver.index.handlers.FieldUpdateHandler; import com.yelp.nrtsearch.server.luceneserver.index.handlers.LiveSettingsV2Handler; import com.yelp.nrtsearch.server.luceneserver.index.handlers.SettingsV2Handler; import com.yelp.nrtsearch.server.luceneserver.logging.HitsLoggerCreator; @@ -52,15 +85,23 @@ import com.yelp.nrtsearch.server.luceneserver.search.cache.NrtQueryCache; import com.yelp.nrtsearch.server.luceneserver.search.collectors.CollectorCreator; import com.yelp.nrtsearch.server.luceneserver.similarity.SimilarityCreator; -import com.yelp.nrtsearch.server.luceneserver.warming.Warmer; -import com.yelp.nrtsearch.server.monitoring.*; +import com.yelp.nrtsearch.server.monitoring.Configuration; +import com.yelp.nrtsearch.server.monitoring.DeadlineMetrics; +import com.yelp.nrtsearch.server.monitoring.DirSizeCollector; +import com.yelp.nrtsearch.server.monitoring.IndexMetrics; +import com.yelp.nrtsearch.server.monitoring.LuceneServerMonitoringServerInterceptor; +import com.yelp.nrtsearch.server.monitoring.MergeSchedulerCollector; +import com.yelp.nrtsearch.server.monitoring.NrtMetrics; +import com.yelp.nrtsearch.server.monitoring.ProcStatCollector; +import com.yelp.nrtsearch.server.monitoring.QueryCacheCollector; +import com.yelp.nrtsearch.server.monitoring.SearchResponseCollector; +import com.yelp.nrtsearch.server.monitoring.ThreadPoolCollector; import com.yelp.nrtsearch.server.monitoring.ThreadPoolCollector.RejectionCounterWrapper; import com.yelp.nrtsearch.server.plugins.Plugin; import com.yelp.nrtsearch.server.plugins.PluginsService; import com.yelp.nrtsearch.server.remote.RemoteBackend; import com.yelp.nrtsearch.server.utils.ThreadPoolExecutorFactory; import com.yelp.nrtsearch.tools.cli.VersionProvider; -import io.grpc.Context; import io.grpc.Server; import io.grpc.ServerBuilder; import io.grpc.ServerInterceptors; @@ -68,7 +109,6 @@ import io.grpc.StatusRuntimeException; import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; import io.grpc.protobuf.services.ProtoReflectionService; -import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; import io.prometheus.metrics.instrumentation.jvm.JvmMetrics; import io.prometheus.metrics.model.registry.PrometheusRegistry; @@ -76,10 +116,10 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.lang.management.ManagementFactory; -import java.util.*; -import java.util.concurrent.*; -import java.util.stream.Collectors; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.QueryCache; import org.apache.lucene.search.suggest.document.CompletionPostingsFormatUtil; @@ -92,7 +132,6 @@ /** Server that manages startup/shutdown of a {@code LuceneServer} server. */ public class LuceneServer { private static final Logger logger = LoggerFactory.getLogger(LuceneServer.class.getName()); - private static final Splitter COMMA_SPLITTER = Splitter.on(","); private final RemoteBackend remoteBackend; private final PrometheusRegistry prometheusRegistry; private final PluginsService pluginsService; @@ -286,8 +325,41 @@ static class LuceneServerImpl extends LuceneServerGrpc.LuceneServerImplBase { private final JsonFormat.Printer protoMessagePrinter = JsonFormat.printer().omittingInsignificantWhitespace(); private final GlobalState globalState; - private final PrometheusRegistry prometheusRegistry; - private final ThreadPoolExecutor searchThreadPoolExecutor; + + private final AddDocumentHandler addDocumentHandler; + private final BackupWarmingQueriesHandler backupWarmingQueriesHandler; + private final CommitHandler commitHandler; + private final CreateIndexHandler createIndexHandler; + private final CreateSnapshotHandler createSnapshotHandler; + private final CustomHandler customHandler; + private final DeleteAllDocumentsHandler deleteAllDocumentsHandler; + private final DeleteByQueryHandler deleteByQueryHandler; + private final DeleteDocumentsHandler deleteDocumentsHandler; + private final DeleteIndexHandler deleteIndexHandler; + private final ForceMergeDeletesHandler forceMergeDeletesHandler; + private final ForceMergeHandler forceMergeHandler; + private final GetAllSnapshotIndexGenHandler getAllSnapshotIndexGenHandler; + private final GetStateHandler getStateHandler; + private final GlobalStateHandler globalStateHandler; + private final IndicesHandler indicesHandler; + private final LiveSettingsHandler liveSettingsHandler; + private final LiveSettingsV2Handler liveSettingsV2Handler; + private final MetricsHandler metricsHandler; + private final ReadyHandler readyHandler; + private final RefreshHandler refreshHandler; + private final RegisterFieldsHandler registerFieldsHandler; + private final ReleaseSnapshotHandler releaseSnapshotHandler; + private final ReloadStateHandler reloadStateHandler; + private final SearchHandler searchHandler; + private final SearchV2Handler searchV2Handler; + private final SettingsHandler settingsHandler; + private final SettingsV2Handler settingsV2Handler; + private final StartIndexHandler startIndexHandler; + private final StartIndexV2Handler startIndexV2Handler; + private final StatsHandler statsHandler; + private final StatusHandler statusHandler; + private final StopIndexHandler stopIndexHandler; + private final UpdateFieldsHandler updateFieldsHandler; /** * Constructor used with newer state handling. Defers initialization of global state until after @@ -305,7 +377,6 @@ static class LuceneServerImpl extends LuceneServerGrpc.LuceneServerImplBase { PrometheusRegistry prometheusRegistry, List plugins) throws IOException { - this.prometheusRegistry = prometheusRegistry; DeadlineUtils.setCancellationEnabled(configuration.getDeadlineCancellation()); CompletionPostingsFormatUtil.setCompletionCodecLoadMode( @@ -316,7 +387,42 @@ static class LuceneServerImpl extends LuceneServerGrpc.LuceneServerImplBase { initExtendableComponents(configuration, plugins); this.globalState = GlobalState.createState(configuration, remoteBackend); - this.searchThreadPoolExecutor = globalState.getSearchThreadPoolExecutor(); + + // Initialize handlers + addDocumentHandler = new AddDocumentHandler(globalState); + backupWarmingQueriesHandler = new BackupWarmingQueriesHandler(globalState); + commitHandler = new CommitHandler(globalState); + createIndexHandler = new CreateIndexHandler(globalState); + createSnapshotHandler = new CreateSnapshotHandler(globalState); + customHandler = new CustomHandler(globalState); + deleteAllDocumentsHandler = new DeleteAllDocumentsHandler(globalState); + deleteByQueryHandler = new DeleteByQueryHandler(globalState); + deleteDocumentsHandler = new DeleteDocumentsHandler(globalState); + deleteIndexHandler = new DeleteIndexHandler(globalState); + forceMergeDeletesHandler = new ForceMergeDeletesHandler(globalState); + forceMergeHandler = new ForceMergeHandler(globalState); + getAllSnapshotIndexGenHandler = new GetAllSnapshotIndexGenHandler(globalState); + getStateHandler = new GetStateHandler(globalState); + globalStateHandler = new GlobalStateHandler(globalState); + indicesHandler = new IndicesHandler(globalState); + liveSettingsHandler = new LiveSettingsHandler(globalState); + liveSettingsV2Handler = new LiveSettingsV2Handler(globalState); + metricsHandler = new MetricsHandler(prometheusRegistry); + readyHandler = new ReadyHandler(globalState); + refreshHandler = new RefreshHandler(globalState); + registerFieldsHandler = new RegisterFieldsHandler(globalState); + releaseSnapshotHandler = new ReleaseSnapshotHandler(globalState); + reloadStateHandler = new ReloadStateHandler(globalState); + searchHandler = new SearchHandler(globalState); + searchV2Handler = new SearchV2Handler(globalState, searchHandler); + settingsHandler = new SettingsHandler(globalState); + settingsV2Handler = new SettingsV2Handler(globalState); + startIndexHandler = new StartIndexHandler(globalState); + startIndexV2Handler = new StartIndexV2Handler(globalState); + statsHandler = new StatsHandler(globalState); + statusHandler = new StatusHandler(); + stopIndexHandler = new StopIndexHandler(globalState); + updateFieldsHandler = new UpdateFieldsHandler(globalState); } @VisibleForTesting @@ -354,959 +460,142 @@ public GlobalState getGlobalState() { return globalState; } - /** - * Set response compression on the provided {@link StreamObserver}. Should be a valid - * compression type from the {@link LuceneServerStubBuilder#COMPRESSOR_REGISTRY}, or empty - * string for default. Falls back to uncompressed on any error. - * - * @param compressionType compression type, or empty string - * @param responseObserver observer to set compression on - */ - private void setResponseCompression( - String compressionType, StreamObserver responseObserver) { - if (!compressionType.isEmpty()) { - try { - ServerCallStreamObserver serverCallStreamObserver = - (ServerCallStreamObserver) responseObserver; - serverCallStreamObserver.setCompression(compressionType); - } catch (Exception e) { - logger.warn( - "Unable to set response compression to type '" + compressionType + "' : " + e); - } - } - } - @Override public void createIndex( CreateIndexRequest req, StreamObserver responseObserver) { - logger.info("Received create index request: {}", req); - String indexName = req.getIndexName(); - String validIndexNameRegex = "[A-z0-9_-]+"; - if (!indexName.matches(validIndexNameRegex)) { - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - String.format( - "Index name %s is invalid - must contain only a-z, A-Z or 0-9", indexName)) - .asRuntimeException()); - return; - } - - try { - IndexState indexState = globalState.createIndex(req); - String response = String.format("Created Index name: %s", indexName); - CreateIndexResponse reply = CreateIndexResponse.newBuilder().setResponse(response).build(); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (IllegalArgumentException e) { - logger.warn("invalid IndexName: " + indexName, e); - responseObserver.onError( - Status.ALREADY_EXISTS - .withDescription("invalid indexName: " + indexName) - .augmentDescription("IllegalArgumentException()") - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - logger.warn( - "error while trying to save index state to disk for indexName: " + indexName, e); - responseObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to save index state to disk for indexName: " + indexName) - .augmentDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } + createIndexHandler.handle(req, responseObserver); } @Override public void liveSettings( LiveSettingsRequest req, StreamObserver responseObserver) { - logger.info("Received live settings request: {}", req); - try { - IndexState indexState = globalState.getIndex(req.getIndexName()); - LiveSettingsResponse reply = new LiveSettingsHandler().handle(indexState, req); - logger.info("LiveSettingsHandler returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (IllegalArgumentException e) { - logger.warn("index: " + req.getIndexName() + " was not yet created", e); - responseObserver.onError( - Status.ALREADY_EXISTS - .withDescription("invalid indexName: " + req.getIndexName()) - .augmentDescription("IllegalArgumentException()") - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - logger.warn( - "error while trying to read index state dir for indexName: " + req.getIndexName(), e); - responseObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to read index state dir for indexName: " - + req.getIndexName() - + "at rootDir: ") - .augmentDescription("IOException()") - .withCause(e) - .asRuntimeException()); - } + liveSettingsHandler.handle(req, responseObserver); } @Override public void liveSettingsV2( LiveSettingsV2Request req, StreamObserver responseObserver) { - logger.info("Received live settings V2 request: {}", req); - try { - IndexStateManager indexStateManager = globalState.getIndexStateManager(req.getIndexName()); - LiveSettingsV2Response reply = LiveSettingsV2Handler.handle(indexStateManager, req); - logger.info("LiveSettingsV2Handler returned " + JsonFormat.printer().print(reply)); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (IllegalArgumentException e) { - logger.warn("index: " + req.getIndexName() + " was not yet created", e); - responseObserver.onError( - Status.ALREADY_EXISTS - .withDescription("invalid indexName: " + req.getIndexName()) - .augmentDescription("IllegalArgumentException()") - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - logger.warn( - "error while trying to process live settings for indexName: " + req.getIndexName(), e); - responseObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to process live settings for indexName: " - + req.getIndexName()) - .augmentDescription("Exception()") - .withCause(e) - .asRuntimeException()); - } + liveSettingsV2Handler.handle(req, responseObserver); } @Override public void registerFields( FieldDefRequest fieldDefRequest, StreamObserver responseObserver) { - logger.info("Received register fields request: {}", fieldDefRequest); - try { - IndexStateManager indexStateManager = - globalState.getIndexStateManager(fieldDefRequest.getIndexName()); - FieldDefResponse reply = FieldUpdateHandler.handle(indexStateManager, fieldDefRequest); - logger.info("RegisterFieldsHandler registered fields " + reply); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (IOException e) { - logger.warn( - "error while trying to read index state dir for indexName: " - + fieldDefRequest.getIndexName(), - e); - responseObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to read index state dir for indexName: " - + fieldDefRequest.getIndexName()) - .augmentDescription("IOException()") - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - logger.warn( - "error while trying to RegisterFields for index " + fieldDefRequest.getIndexName(), e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - "error while trying to RegisterFields for index: " - + fieldDefRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + registerFieldsHandler.handle(fieldDefRequest, responseObserver); } @Override public void updateFields( FieldDefRequest fieldDefRequest, StreamObserver responseObserver) { - logger.info("Received update fields request: {}", fieldDefRequest); - try { - IndexStateManager indexStateManager = - globalState.getIndexStateManager(fieldDefRequest.getIndexName()); - FieldDefResponse reply = FieldUpdateHandler.handle(indexStateManager, fieldDefRequest); - logger.info("UpdateFieldsHandler registered fields " + reply); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (IOException e) { - logger.warn( - "error while trying to read index state dir for indexName: " - + fieldDefRequest.getIndexName(), - e); - responseObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to read index state dir for indexName: " - + fieldDefRequest.getIndexName()) - .augmentDescription("IOException()") - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - logger.warn( - "error while trying to UpdateFieldsHandler for index " + fieldDefRequest.getIndexName(), - e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - "error while trying to UpdateFieldsHandler for index: " - + fieldDefRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + updateFieldsHandler.handle(fieldDefRequest, responseObserver); } @Override public void settings( SettingsRequest settingsRequest, StreamObserver responseObserver) { - logger.info("Received settings request: {}", settingsRequest); - try { - IndexState indexState = globalState.getIndex(settingsRequest.getIndexName()); - SettingsResponse reply = new SettingsHandler().handle(indexState, settingsRequest); - logger.info("SettingsHandler returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (IOException e) { - logger.warn( - "error while trying to read index state dir for indexName: " - + settingsRequest.getIndexName(), - e); - responseObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to read index state dir for indexName: " - + settingsRequest.getIndexName()) - .augmentDescription("IOException()") - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - logger.warn( - "error while trying to update/get settings for index " + settingsRequest.getIndexName(), - e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - "error while trying to update/get settings for index: " - + settingsRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + settingsHandler.handle(settingsRequest, responseObserver); } @Override public void settingsV2( SettingsV2Request settingsRequest, StreamObserver responseObserver) { - logger.info("Received settings V2 request: {}", settingsRequest); - try { - IndexStateManager indexStateManager = - globalState.getIndexStateManager(settingsRequest.getIndexName()); - SettingsV2Response reply = SettingsV2Handler.handle(indexStateManager, settingsRequest); - logger.info("SettingsV2Handler returned: " + JsonFormat.printer().print(reply)); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (IOException e) { - logger.warn( - "error while trying to read index state dir for indexName: " - + settingsRequest.getIndexName(), - e); - responseObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to read index state dir for indexName: " - + settingsRequest.getIndexName()) - .augmentDescription("IOException()") - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - logger.warn( - "error while trying to update/get settings for index " + settingsRequest.getIndexName(), - e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - "error while trying to update/get settings for index: " - + settingsRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + settingsV2Handler.handle(settingsRequest, responseObserver); } @Override public void startIndex( StartIndexRequest startIndexRequest, StreamObserver responseObserver) { - logger.info("Received start index request: {}", startIndexRequest); - if (startIndexRequest.getIndexName().isEmpty()) { - logger.warn("error while trying to start index with empty index name."); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - String.format("error while trying to start index since indexName was empty.")) - .asRuntimeException()); - return; - } - try { - StartIndexResponse reply = globalState.startIndex(startIndexRequest); - - logger.info("StartIndexHandler returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - - } catch (IOException e) { - logger.warn( - "error while trying to read index state dir for indexName: " - + startIndexRequest.getIndexName(), - e); - responseObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to read index state dir for indexName: " - + startIndexRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - logger.warn("error while trying to start index " + startIndexRequest.getIndexName(), e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - "error while trying to start index: " + startIndexRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + startIndexHandler.handle(startIndexRequest, responseObserver); } @Override public void startIndexV2( StartIndexV2Request startIndexRequest, StreamObserver responseObserver) { - logger.info("Received start index v2 request: {}", startIndexRequest); - try { - StartIndexResponse reply = globalState.startIndexV2(startIndexRequest); - logger.info("StartIndexV2Handler returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (IOException e) { - logger.warn( - "error while trying to read index state dir for indexName: " - + startIndexRequest.getIndexName(), - e); - responseObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to read index state dir for indexName: " - + startIndexRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - logger.warn("error while trying to start index " + startIndexRequest.getIndexName(), e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - "error while trying to start index: " + startIndexRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + startIndexV2Handler.handle(startIndexRequest, responseObserver); } @Override public StreamObserver addDocuments( StreamObserver responseObserver) { - - return new StreamObserver<>() { - Multimap> futures = HashMultimap.create(); - // Map of {indexName: addDocumentRequestQueue} - Map> addDocumentRequestQueueMap = - new ConcurrentHashMap<>(); - // Map of {indexName: count} - Map countMap = new ConcurrentHashMap<>(); - - private int getAddDocumentsMaxBufferLen(String indexName) { - try { - return globalState.getIndex(indexName).getAddDocumentsMaxBufferLen(); - } catch (Exception e) { - String error = - String.format("Index %s does not exist, unable to add documents", indexName); - logger.error(error, e); - throw Status.INVALID_ARGUMENT.withDescription(error).withCause(e).asRuntimeException(); - } - } - - private ArrayBlockingQueue getAddDocumentRequestQueue( - String indexName) { - if (addDocumentRequestQueueMap.containsKey(indexName)) { - return addDocumentRequestQueueMap.get(indexName); - } else { - int addDocumentsMaxBufferLen = getAddDocumentsMaxBufferLen(indexName); - ArrayBlockingQueue addDocumentRequestQueue = - new ArrayBlockingQueue<>(addDocumentsMaxBufferLen); - addDocumentRequestQueueMap.put(indexName, addDocumentRequestQueue); - return addDocumentRequestQueue; - } - } - - private long getCount(String indexName) { - return countMap.getOrDefault(indexName, 0L); - } - - private void incrementCount(String indexName) { - if (countMap.containsKey(indexName)) { - countMap.put(indexName, countMap.get(indexName) + 1); - } else { - countMap.put(indexName, 1L); - } - } - - @Override - public void onNext(AddDocumentRequest addDocumentRequest) { - String indexName = addDocumentRequest.getIndexName(); - ArrayBlockingQueue addDocumentRequestQueue; - try { - addDocumentRequestQueue = getAddDocumentRequestQueue(indexName); - } catch (Exception e) { - onError(e); - return; - } - logger.debug( - String.format( - "onNext, index: %s, addDocumentRequestQueue size: %s", - indexName, addDocumentRequestQueue.size())); - incrementCount(indexName); - addDocumentRequestQueue.add(addDocumentRequest); - if (addDocumentRequestQueue.remainingCapacity() == 0) { - logger.debug( - String.format( - "indexing addDocumentRequestQueue size: %s, total: %s", - addDocumentRequestQueue.size(), getCount(indexName))); - try { - DeadlineUtils.checkDeadline("addDocuments: onNext", "INDEXING"); - - List addDocRequestList = new ArrayList<>(addDocumentRequestQueue); - Future future = - globalState.submitIndexingTask( - Context.current() - .wrap(new DocumentIndexer(globalState, addDocRequestList, indexName))); - futures.put(indexName, future); - } catch (Exception e) { - responseObserver.onError(e); - } finally { - addDocumentRequestQueue.clear(); - } - } - } - - @Override - public void onError(Throwable t) { - logger.warn("addDocuments Cancelled", t); - responseObserver.onError(t); - } - - private String onCompletedForIndex(String indexName) { - ArrayBlockingQueue addDocumentRequestQueue = - getAddDocumentRequestQueue(indexName); - logger.debug( - String.format( - "onCompleted, addDocumentRequestQueue: %s", addDocumentRequestQueue.size())); - long highestGen = -1; - try { - DeadlineUtils.checkDeadline("addDocuments: onCompletedForIndex", "INDEXING"); - - // index the left over docs - if (!addDocumentRequestQueue.isEmpty()) { - logger.debug( - String.format( - "indexing left over addDocumentRequestQueue of size: %s", - addDocumentRequestQueue.size())); - List addDocRequestList = new ArrayList<>(addDocumentRequestQueue); - // Since we are already running in the indexing threadpool run the indexing job - // for remaining documents directly. This serializes indexing remaining documents for - // multiple indices but avoids deadlocking if there aren't more threads than the - // maximum - // number of parallel addDocuments calls. - long gen = - new DocumentIndexer(globalState, addDocRequestList, indexName).runIndexingJob(); - if (gen > highestGen) { - highestGen = gen; - } - } - // collect futures, block if needed - int numIndexingChunks = futures.size(); - long t0 = System.nanoTime(); - for (Future result : futures.get(indexName)) { - Long gen = result.get(); - logger.debug("Indexing returned sequence-number {}", gen); - if (gen > highestGen) { - highestGen = gen; - } - } - long t1 = System.nanoTime(); - logger.debug( - "Indexing job completed for {} docs, in {} chunks, with latest sequence number: {}, took: {} micro seconds", - getCount(indexName), - numIndexingChunks, - highestGen, - ((t1 - t0) / 1000)); - return String.valueOf(highestGen); - } catch (Exception e) { - logger.warn("error while trying to addDocuments", e); - throw Status.INTERNAL - .withDescription("error while trying to addDocuments ") - .augmentDescription(e.getMessage()) - .withCause(e) - .asRuntimeException(); - } finally { - addDocumentRequestQueue.clear(); - countMap.put(indexName, 0L); - } - } - - @Override - public void onCompleted() { - try { - globalState.submitIndexingTask( - Context.current() - .wrap( - () -> { - try { - // TODO: this should return a map on index to genId in the response - String genId = "-1"; - for (String indexName : addDocumentRequestQueueMap.keySet()) { - genId = onCompletedForIndex(indexName); - } - responseObserver.onNext( - AddDocumentResponse.newBuilder() - .setGenId(genId) - .setPrimaryId(globalState.getEphemeralId()) - .build()); - responseObserver.onCompleted(); - } catch (Throwable t) { - responseObserver.onError(t); - } - return null; - })); - } catch (RejectedExecutionException e) { - logger.error("Threadpool is full, unable to submit indexing completion job"); - responseObserver.onError( - Status.RESOURCE_EXHAUSTED - .withDescription("Threadpool is full, unable to submit indexing completion job") - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } - } - }; + return addDocumentHandler.handle(responseObserver); } @Override public void refresh( RefreshRequest refreshRequest, StreamObserver refreshResponseStreamObserver) { - try { - IndexState indexState = globalState.getIndex(refreshRequest.getIndexName()); - final ShardState shardState = indexState.getShard(0); - long t0 = System.nanoTime(); - shardState.maybeRefreshBlocking(); - long t1 = System.nanoTime(); - double refreshTimeMs = (t1 - t0) / 1000000.0; - RefreshResponse reply = - RefreshResponse.newBuilder().setRefreshTimeMS(refreshTimeMs).build(); - logger.info( - String.format( - "RefreshHandler refreshed index: %s in %f", - refreshRequest.getIndexName(), refreshTimeMs)); - refreshResponseStreamObserver.onNext(reply); - refreshResponseStreamObserver.onCompleted(); - } catch (IOException e) { - logger.warn( - "error while trying to read index state dir for indexName: " - + refreshRequest.getIndexName(), - e); - refreshResponseStreamObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to read index state dir for indexName: " - + refreshRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - logger.warn("error while trying to refresh index " + refreshRequest.getIndexName(), e); - refreshResponseStreamObserver.onError( - Status.UNKNOWN - .withDescription( - "error while trying to refresh index: " + refreshRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + refreshHandler.handle(refreshRequest, refreshResponseStreamObserver); } @Override public void commit( CommitRequest commitRequest, StreamObserver commitResponseStreamObserver) { - try { - globalState.submitIndexingTask( - Context.current() - .wrap( - () -> { - try { - IndexState indexState = globalState.getIndex(commitRequest.getIndexName()); - long gen = indexState.commit(); - CommitResponse reply = - CommitResponse.newBuilder() - .setGen(gen) - .setPrimaryId(globalState.getEphemeralId()) - .build(); - logger.debug( - String.format( - "CommitHandler committed to index: %s for sequenceId: %s", - commitRequest.getIndexName(), gen)); - commitResponseStreamObserver.onNext(reply); - commitResponseStreamObserver.onCompleted(); - } catch (IOException e) { - logger.warn( - "error while trying to read index state dir for indexName: " - + commitRequest.getIndexName(), - e); - commitResponseStreamObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to read index state dir for indexName: " - + commitRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - logger.warn( - "error while trying to commit to index " - + commitRequest.getIndexName(), - e); - if (e instanceof StatusRuntimeException) { - commitResponseStreamObserver.onError(e); - } else { - commitResponseStreamObserver.onError( - Status.UNKNOWN - .withDescription( - "error while trying to commit to index: " - + commitRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } - } - return null; - })); - } catch (RejectedExecutionException e) { - logger.error( - "Threadpool is full, unable to submit commit to index {}", - commitRequest.getIndexName()); - commitResponseStreamObserver.onError( - Status.RESOURCE_EXHAUSTED - .withDescription( - "Threadpool is full, unable to submit commit to index: " - + commitRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + commitHandler.handle(commitRequest, commitResponseStreamObserver); } @Override public void stats( StatsRequest statsRequest, StreamObserver statsResponseStreamObserver) { - try { - IndexState indexState = globalState.getIndex(statsRequest.getIndexName()); - indexState.verifyStarted(); - StatsResponse reply = new StatsRequestHandler().handle(indexState, statsRequest); - logger.debug(String.format("StatsHandler retrieved stats for index: %s ", reply)); - statsResponseStreamObserver.onNext(reply); - statsResponseStreamObserver.onCompleted(); - } catch (IOException e) { - logger.warn( - "error while trying to read index state dir for indexName: " - + statsRequest.getIndexName(), - e); - statsResponseStreamObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to read index state dir for indexName: " - + statsRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - logger.warn( - "error while trying to retrieve stats for index " + statsRequest.getIndexName(), e); - statsResponseStreamObserver.onError( - Status.UNKNOWN - .withDescription( - "error while trying to retrieve stats for index: " - + statsRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + statsHandler.handle(statsRequest, statsResponseStreamObserver); } @Override public void search( - SearchRequest searchRequest, StreamObserver searchResponseStreamObserver) { - try { - IndexState indexState = globalState.getIndex(searchRequest.getIndexName()); - setResponseCompression( - searchRequest.getResponseCompression(), searchResponseStreamObserver); - SearchHandler searchHandler = new SearchHandler(searchThreadPoolExecutor); - SearchResponse reply = searchHandler.handle(indexState, searchRequest); - searchResponseStreamObserver.onNext(reply); - searchResponseStreamObserver.onCompleted(); - } catch (IOException e) { - logger.warn( - "error while trying to read index state dir for indexName: " - + searchRequest.getIndexName(), - e); - searchResponseStreamObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to read index state dir for indexName: " - + searchRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - String searchRequestJson = null; - try { - searchRequestJson = protoMessagePrinter.print(searchRequest); - } catch (InvalidProtocolBufferException ignored) { - // Ignore as invalid proto would have thrown an exception earlier - } - logger.warn( - String.format( - "error while trying to execute search for index %s: request: %s", - searchRequest.getIndexName(), searchRequestJson), - e); - if (e instanceof StatusRuntimeException) { - searchResponseStreamObserver.onError(e); - } else { - searchResponseStreamObserver.onError( - Status.UNKNOWN - .withDescription( - String.format( - "error while trying to execute search for index %s. check logs for full searchRequest.", - searchRequest.getIndexName())) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } - } + SearchRequest searchRequest, StreamObserver responseObserver) { + searchHandler.handle(searchRequest, responseObserver); } @Override public void searchV2( SearchRequest searchRequest, StreamObserver searchResponseStreamObserver) { - try { - IndexState indexState = globalState.getIndex(searchRequest.getIndexName()); - setResponseCompression( - searchRequest.getResponseCompression(), searchResponseStreamObserver); - SearchHandler searchHandler = new SearchHandler(searchThreadPoolExecutor); - SearchResponse reply = searchHandler.handle(indexState, searchRequest); - searchResponseStreamObserver.onNext(Any.pack(reply)); - searchResponseStreamObserver.onCompleted(); - } catch (IOException e) { - logger.warn( - "error while trying to read index state dir for indexName: " - + searchRequest.getIndexName(), - e); - searchResponseStreamObserver.onError( - Status.INTERNAL - .withDescription( - "error while trying to read index state dir for indexName: " - + searchRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - String searchRequestJson = null; - try { - searchRequestJson = protoMessagePrinter.print(searchRequest); - } catch (InvalidProtocolBufferException ignored) { - // Ignore as invalid proto would have thrown an exception earlier - } - logger.warn( - String.format( - "error while trying to execute search for index %s: request: %s", - searchRequest.getIndexName(), searchRequestJson), - e); - if (e instanceof StatusRuntimeException) { - searchResponseStreamObserver.onError(e); - } else { - searchResponseStreamObserver.onError( - Status.UNKNOWN - .withDescription( - String.format( - "error while trying to execute search for index %s. check logs for full searchRequest.", - searchRequest.getIndexName())) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } - } + searchV2Handler.handle(searchRequest, searchResponseStreamObserver); } @Override public void delete( AddDocumentRequest addDocumentRequest, StreamObserver responseObserver) { - try { - IndexState indexState = globalState.getIndex(addDocumentRequest.getIndexName()); - AddDocumentResponse reply = - new DeleteDocumentsHandler().handle(indexState, addDocumentRequest); - logger.debug("DeleteDocumentsHandler returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (Exception e) { - logger.warn( - "error while trying to delete documents for index " + addDocumentRequest.getIndexName(), - e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - "error while trying to delete documents for index: " - + addDocumentRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + deleteDocumentsHandler.handle(addDocumentRequest, responseObserver); } @Override public void deleteByQuery( DeleteByQueryRequest deleteByQueryRequest, StreamObserver responseObserver) { - try { - IndexState indexState = globalState.getIndex(deleteByQueryRequest.getIndexName()); - AddDocumentResponse reply = - new DeleteByQueryHandler().handle(indexState, deleteByQueryRequest); - logger.debug("DeleteDocumentsHandler returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (Exception e) { - logger.warn( - "Error while trying to delete documents from index: {}", - deleteByQueryRequest.getIndexName(), - e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - "Error while trying to delete documents from index: " - + deleteByQueryRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + deleteByQueryHandler.handle(deleteByQueryRequest, responseObserver); } @Override public void deleteAll( DeleteAllDocumentsRequest deleteAllDocumentsRequest, StreamObserver responseObserver) { - logger.info("Received delete all documents request: {}", deleteAllDocumentsRequest); - try { - IndexState indexState = globalState.getIndex(deleteAllDocumentsRequest.getIndexName()); - DeleteAllDocumentsResponse reply = - new DeleteAllDocumentsHandler().handle(indexState, deleteAllDocumentsRequest); - logger.info("DeleteAllDocumentsHandler returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (Exception e) { - logger.warn( - "error while trying to deleteAll for index " + deleteAllDocumentsRequest.getIndexName(), - e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - "error while trying to deleteAll for index: " - + deleteAllDocumentsRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + deleteAllDocumentsHandler.handle(deleteAllDocumentsRequest, responseObserver); } @Override public void deleteIndex( DeleteIndexRequest deleteIndexRequest, StreamObserver responseObserver) { - logger.info("Received delete index request: {}", deleteIndexRequest); - try { - IndexState indexState = globalState.getIndex(deleteIndexRequest.getIndexName()); - DeleteIndexResponse reply = new DeleteIndexHandler().handle(indexState, deleteIndexRequest); - logger.info("DeleteIndexHandler returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (Exception e) { - logger.warn("error while trying to delete index " + deleteIndexRequest.getIndexName(), e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - "error while trying to delete index: " + deleteIndexRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + deleteIndexHandler.handle(deleteIndexRequest, responseObserver); } @Override public void stopIndex( StopIndexRequest stopIndexRequest, StreamObserver responseObserver) { - logger.info("Received stop index request: {}", stopIndexRequest); - try { - DummyResponse reply = globalState.stopIndex(stopIndexRequest); - - logger.info("StopIndexHandler returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (Exception e) { - logger.warn("error while trying to stop index " + stopIndexRequest.getIndexName(), e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - "error while trying to stop index: " + stopIndexRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + stopIndexHandler.handle(stopIndexRequest, responseObserver); } @Override public void reloadState( ReloadStateRequest request, StreamObserver responseObserver) { - try { - if (globalState.getConfiguration().getIndexStartConfig().getMode().equals(Mode.REPLICA)) { - globalState.reloadStateFromBackend(); - } else { - logger.info("Skip reloading state since it is not replica"); - } - ReloadStateResponse reloadStateResponse = ReloadStateResponse.newBuilder().build(); - responseObserver.onNext(reloadStateResponse); - responseObserver.onCompleted(); - } catch (Exception e) { - logger.warn("error while trying to sync the index state", e); - responseObserver.onError( - Status.INTERNAL - .withDescription("error while trying to sync the index state") - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + reloadStateHandler.handle(request, responseObserver); } @Override public void status( HealthCheckRequest request, StreamObserver responseObserver) { - try { - HealthCheckResponse reply = - HealthCheckResponse.newBuilder().setHealth(TransferStatusCode.Done).build(); - logger.debug("HealthCheckResponse returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (Exception e) { - logger.warn("error while trying to get status", e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription("error while trying to get status") - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + statusHandler.handle(request, responseObserver); } /** @@ -1317,375 +606,74 @@ public void status( @Override public void ready( ReadyCheckRequest request, StreamObserver responseObserver) { - Set indexNames; - - // If specific index names are provided we will check only those indices, otherwise check all - if (request.getIndexNames().isEmpty()) { - indexNames = globalState.getIndicesToStart(); - } else { - List indexNamesToCheck = COMMA_SPLITTER.splitToList(request.getIndexNames()); - - Set allIndices = globalState.getIndexNames(); - - Sets.SetView nonExistentIndices = - Sets.difference(Set.copyOf(indexNamesToCheck), allIndices); - if (!nonExistentIndices.isEmpty()) { - logger.warn("Indices: {} do not exist", nonExistentIndices); - responseObserver.onError( - Status.UNAVAILABLE - .withDescription(String.format("Indices do not exist: %s", nonExistentIndices)) - .asRuntimeException()); - return; - } - - indexNames = - allIndices.stream().filter(indexNamesToCheck::contains).collect(Collectors.toSet()); - } - - try { - List indicesNotStarted = new ArrayList<>(); - for (String indexName : indexNames) { - // The ready endpoint should skip loading index state - IndexState indexState = globalState.getIndex(indexName, true); - if (!indexState.isStarted()) { - indicesNotStarted.add(indexName); - } - } - - if (indicesNotStarted.isEmpty()) { - HealthCheckResponse reply = - HealthCheckResponse.newBuilder().setHealth(TransferStatusCode.Done).build(); - logger.debug("Ready check returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } else { - logger.warn("Indices not started: {}", indicesNotStarted); - responseObserver.onError( - Status.UNAVAILABLE - .withDescription(String.format("Indices not started: %s", indicesNotStarted)) - .asRuntimeException()); - } - } catch (Exception e) { - logger.warn("error while trying to check if all required indices are started", e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription("error while trying to check if all required indices are started") - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + readyHandler.handle(request, responseObserver); } @Override public void createSnapshot( CreateSnapshotRequest createSnapshotRequest, StreamObserver responseObserver) { - try { - IndexState indexState = globalState.getIndex(createSnapshotRequest.getIndexName()); - CreateSnapshotHandler createSnapshotHandler = new CreateSnapshotHandler(); - CreateSnapshotResponse reply = - createSnapshotHandler.handle(indexState, createSnapshotRequest); - logger.info(String.format("CreateSnapshotHandler returned results %s", reply.toString())); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (Exception e) { - logger.warn( - String.format( - "error while trying to createSnapshot for index %s", - createSnapshotRequest.getIndexName()), - e); - responseObserver.onError( - Status.UNKNOWN - .withDescription( - String.format( - "error while trying to createSnapshot for index %s", - createSnapshotRequest.getIndexName())) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + createSnapshotHandler.handle(createSnapshotRequest, responseObserver); } @Override public void releaseSnapshot( ReleaseSnapshotRequest releaseSnapshotRequest, StreamObserver responseObserver) { - try { - IndexState indexState = globalState.getIndex(releaseSnapshotRequest.getIndexName()); - ReleaseSnapshotHandler releaseSnapshotHandler = new ReleaseSnapshotHandler(); - ReleaseSnapshotResponse reply = - releaseSnapshotHandler.handle(indexState, releaseSnapshotRequest); - logger.info(String.format("CreateSnapshotHandler returned results %s", reply.toString())); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (Exception e) { - logger.warn( - String.format( - "error while trying to releaseSnapshot for index %s", - releaseSnapshotRequest.getIndexName()), - e); - responseObserver.onError( - Status.UNKNOWN - .withDescription( - String.format( - "error while trying to releaseSnapshot for index %s", - releaseSnapshotRequest.getIndexName())) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + releaseSnapshotHandler.handle(releaseSnapshotRequest, responseObserver); } @Override public void getAllSnapshotIndexGen( GetAllSnapshotGenRequest request, StreamObserver responseObserver) { - try { - Set snapshotGens = - globalState.getIndex(request.getIndexName()).getShard(0).snapshotGenToVersion.keySet(); - GetAllSnapshotGenResponse response = - GetAllSnapshotGenResponse.newBuilder().addAllIndexGens(snapshotGens).build(); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (IOException e) { - logger.error( - "Error getting all snapshotted index gens for index: {}", request.getIndexName(), e); - responseObserver.onError(e); - } + getAllSnapshotIndexGenHandler.handle(request, responseObserver); } @Override public void backupWarmingQueries( BackupWarmingQueriesRequest request, StreamObserver responseObserver) { - logger.info("Received backup warming queries request: {}", request); - String index = request.getIndex(); - try { - IndexState indexState = globalState.getIndex(index); - Warmer warmer = indexState.getWarmer(); - if (warmer == null) { - logger.warn("Unable to backup warming queries as warmer not found for index: {}", index); - responseObserver.onError( - Status.UNKNOWN - .withDescription( - "Unable to backup warming queries as warmer not found for index: " + index) - .asRuntimeException()); - return; - } - int numQueriesThreshold = request.getNumQueriesThreshold(); - int numWarmingRequests = warmer.getNumWarmingRequests(); - if (numQueriesThreshold > 0 && numWarmingRequests < numQueriesThreshold) { - logger.warn( - "Unable to backup warming queries since warmer has {} requests, which is less than threshold {}", - numWarmingRequests, - numQueriesThreshold); - responseObserver.onError( - Status.UNKNOWN - .withDescription( - String.format( - "Unable to backup warming queries since warmer has %s requests, which is less than threshold %s", - numWarmingRequests, numQueriesThreshold)) - .asRuntimeException()); - return; - } - int uptimeMinutesThreshold = request.getUptimeMinutesThreshold(); - int currUptimeMinutes = - (int) (ManagementFactory.getRuntimeMXBean().getUptime() / 1000L / 60L); - if (uptimeMinutesThreshold > 0 && currUptimeMinutes < uptimeMinutesThreshold) { - logger.warn( - "Unable to backup warming queries since uptime is {} minutes, which is less than threshold {}", - currUptimeMinutes, - uptimeMinutesThreshold); - responseObserver.onError( - Status.UNKNOWN - .withDescription( - String.format( - "Unable to backup warming queries since uptime is %s minutes, which is less than threshold %s", - currUptimeMinutes, uptimeMinutesThreshold)) - .asRuntimeException()); - return; - } - warmer.backupWarmingQueriesToS3(request.getServiceName()); - responseObserver.onNext(BackupWarmingQueriesResponse.newBuilder().build()); - responseObserver.onCompleted(); - } catch (IOException e) { - logger.error( - "Unable to backup warming queries for index: {}, service: {}", - index, - request.getServiceName(), - e); - responseObserver.onError( - Status.UNKNOWN - .withCause(e) - .withDescription( - String.format( - "Unable to backup warming queries for index: %s, service: %s", - index, request.getServiceName())) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + backupWarmingQueriesHandler.handle(request, responseObserver); } @Override public void metrics(Empty request, StreamObserver responseObserver) { - try { - HttpBody reply = new MetricsRequestHandler(prometheusRegistry).process(); - logger.debug("MetricsRequestHandler returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (Exception e) { - logger.warn("error while trying to get metrics", e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription("error while trying to get metrics") - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + metricsHandler.handle(request, responseObserver); } @Override public void indices(IndicesRequest request, StreamObserver responseObserver) { - try { - IndicesResponse reply = StatsRequestHandler.getIndicesResponse(globalState); - logger.debug("IndicesRequestHandler returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (Exception e) { - logger.warn("error while trying to get indices stats", e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription("error while trying to get indices stats") - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + indicesHandler.handle(request, responseObserver); } @Override public void globalState( GlobalStateRequest request, StreamObserver responseObserver) { - try { - responseObserver.onNext( - GlobalStateResponse.newBuilder().setGlobalState(globalState.getStateInfo()).build()); - responseObserver.onCompleted(); - } catch (Exception e) { - logger.warn("error while trying to get global state", e); - responseObserver.onError( - Status.UNKNOWN - .withDescription("error while trying to get global state") - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + globalStateHandler.handle(request, responseObserver); } @Override public void state(StateRequest request, StreamObserver responseObserver) { - try { - IndexState indexState = globalState.getIndex(request.getIndexName()); - StateResponse reply = new GetStateHandler().handle(indexState, request); - logger.debug("GetStateHandler returned " + reply.toString()); - responseObserver.onNext(reply); - responseObserver.onCompleted(); - } catch (Exception e) { - logger.warn("error while trying to get state for index " + request.getIndexName(), e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription( - "error while trying to get state for index " + request.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - } + getStateHandler.handle(request, responseObserver); } @Override public void forceMerge( ForceMergeRequest forceMergeRequest, StreamObserver responseObserver) { - logger.info("Received force merge request: {}", forceMergeRequest); - if (forceMergeRequest.getIndexName().isEmpty()) { - responseObserver.onError(new IllegalArgumentException("Index name in request is empty")); - return; - } - if (forceMergeRequest.getMaxNumSegments() == 0) { - responseObserver.onError(new IllegalArgumentException("Cannot have 0 max segments")); - return; - } - - try { - IndexState indexState = globalState.getIndex(forceMergeRequest.getIndexName()); - ShardState shardState = indexState.getShards().get(0); - logger.info("Beginning force merge for index: {}", forceMergeRequest.getIndexName()); - shardState.writer.forceMerge( - forceMergeRequest.getMaxNumSegments(), forceMergeRequest.getDoWait()); - } catch (IOException e) { - logger.warn("Error during force merge for index {} ", forceMergeRequest.getIndexName(), e); - responseObserver.onError( - Status.INTERNAL - .withDescription( - "Error during force merge for index " + forceMergeRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - return; - } - - ForceMergeResponse.Status status = - forceMergeRequest.getDoWait() - ? ForceMergeResponse.Status.FORCE_MERGE_COMPLETED - : ForceMergeResponse.Status.FORCE_MERGE_SUBMITTED; - logger.info("Force merge status: {}", status); - ForceMergeResponse response = ForceMergeResponse.newBuilder().setStatus(status).build(); - responseObserver.onNext(response); - responseObserver.onCompleted(); + forceMergeHandler.handle(forceMergeRequest, responseObserver); } @Override public void forceMergeDeletes( ForceMergeDeletesRequest forceMergeRequest, StreamObserver responseObserver) { - logger.info("Received force merge deletes request: {}", forceMergeRequest); - if (forceMergeRequest.getIndexName().isEmpty()) { - responseObserver.onError(new IllegalArgumentException("Index name in request is empty")); - return; - } - - try { - IndexState indexState = globalState.getIndex(forceMergeRequest.getIndexName()); - ShardState shardState = indexState.getShards().get(0); - logger.info( - "Beginning force merge deletes for index: {}", forceMergeRequest.getIndexName()); - shardState.writer.forceMergeDeletes(forceMergeRequest.getDoWait()); - } catch (IOException e) { - logger.warn( - "Error during force merge deletes for index {} ", forceMergeRequest.getIndexName(), e); - responseObserver.onError( - Status.INTERNAL - .withDescription( - "Error during force merge deletes for index " - + forceMergeRequest.getIndexName()) - .augmentDescription(e.getMessage()) - .asRuntimeException()); - return; - } - - ForceMergeDeletesResponse.Status status = - forceMergeRequest.getDoWait() - ? ForceMergeDeletesResponse.Status.FORCE_MERGE_DELETES_COMPLETED - : ForceMergeDeletesResponse.Status.FORCE_MERGE_DELETES_SUBMITTED; - logger.info("Force merge deletes status: {}", status); - ForceMergeDeletesResponse response = - ForceMergeDeletesResponse.newBuilder().setStatus(status).build(); - responseObserver.onNext(response); - responseObserver.onCompleted(); + forceMergeDeletesHandler.handle(forceMergeRequest, responseObserver); } @Override public void custom(CustomRequest request, StreamObserver responseObserver) { - logger.info("Received custom request: {}", request); - try { - CustomResponse response = CustomRequestProcessor.processCustomRequest(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (Exception e) { - String error = - String.format("Error processing custom request %s, error: %s", request, e.getMessage()); - logger.error(error); - responseObserver.onError(Status.INTERNAL.withDescription(error).withCause(e).asException()); - } + customHandler.handle(request, responseObserver); } } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/AddDocumentHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/AddDocumentHandler.java index acca7c168..646511caf 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/AddDocumentHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/AddDocumentHandler.java @@ -15,13 +15,20 @@ */ package com.yelp.nrtsearch.server.luceneserver; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; import com.google.protobuf.ProtocolStringList; import com.yelp.nrtsearch.server.grpc.AddDocumentRequest; +import com.yelp.nrtsearch.server.grpc.AddDocumentResponse; import com.yelp.nrtsearch.server.grpc.DeadlineUtils; import com.yelp.nrtsearch.server.grpc.FacetHierarchyPath; import com.yelp.nrtsearch.server.luceneserver.field.FieldDef; import com.yelp.nrtsearch.server.luceneserver.field.IdFieldDef; import com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler; +import io.grpc.Context; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -29,17 +36,215 @@ import java.util.List; import java.util.Map; import java.util.Queue; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.RejectedExecutionException; import java.util.stream.Collectors; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexableField; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class AddDocumentHandler { +public class AddDocumentHandler extends Handler { private static final Logger logger = LoggerFactory.getLogger(AddDocumentHandler.class); + public AddDocumentHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public StreamObserver handle( + StreamObserver responseObserver) { + return new StreamObserver<>() { + Multimap> futures = HashMultimap.create(); + // Map of {indexName: addDocumentRequestQueue} + Map> addDocumentRequestQueueMap = + new ConcurrentHashMap<>(); + // Map of {indexName: count} + Map countMap = new ConcurrentHashMap<>(); + + private int getAddDocumentsMaxBufferLen(String indexName) { + try { + return getGlobalState().getIndex(indexName).getAddDocumentsMaxBufferLen(); + } catch (Exception e) { + String error = + String.format("Index %s does not exist, unable to add documents", indexName); + logger.error(error, e); + throw Status.INVALID_ARGUMENT.withDescription(error).withCause(e).asRuntimeException(); + } + } + + private ArrayBlockingQueue getAddDocumentRequestQueue(String indexName) { + if (addDocumentRequestQueueMap.containsKey(indexName)) { + return addDocumentRequestQueueMap.get(indexName); + } else { + int addDocumentsMaxBufferLen = getAddDocumentsMaxBufferLen(indexName); + ArrayBlockingQueue addDocumentRequestQueue = + new ArrayBlockingQueue<>(addDocumentsMaxBufferLen); + addDocumentRequestQueueMap.put(indexName, addDocumentRequestQueue); + return addDocumentRequestQueue; + } + } + + private long getCount(String indexName) { + return countMap.getOrDefault(indexName, 0L); + } + + private void incrementCount(String indexName) { + if (countMap.containsKey(indexName)) { + countMap.put(indexName, countMap.get(indexName) + 1); + } else { + countMap.put(indexName, 1L); + } + } + + @Override + public void onNext(AddDocumentRequest addDocumentRequest) { + String indexName = addDocumentRequest.getIndexName(); + ArrayBlockingQueue addDocumentRequestQueue; + try { + addDocumentRequestQueue = getAddDocumentRequestQueue(indexName); + } catch (Exception e) { + onError(e); + return; + } + logger.debug( + String.format( + "onNext, index: %s, addDocumentRequestQueue size: %s", + indexName, addDocumentRequestQueue.size())); + incrementCount(indexName); + addDocumentRequestQueue.add(addDocumentRequest); + if (addDocumentRequestQueue.remainingCapacity() == 0) { + logger.debug( + String.format( + "indexing addDocumentRequestQueue size: %s, total: %s", + addDocumentRequestQueue.size(), getCount(indexName))); + try { + DeadlineUtils.checkDeadline("addDocuments: onNext", "INDEXING"); + + List addDocRequestList = new ArrayList<>(addDocumentRequestQueue); + Future future = + getGlobalState() + .submitIndexingTask( + Context.current() + .wrap( + new DocumentIndexer( + getGlobalState(), addDocRequestList, indexName))); + futures.put(indexName, future); + } catch (Exception e) { + responseObserver.onError(e); + } finally { + addDocumentRequestQueue.clear(); + } + } + } + + @Override + public void onError(Throwable t) { + logger.warn("addDocuments Cancelled", t); + responseObserver.onError(t); + } + + private String onCompletedForIndex(String indexName) { + ArrayBlockingQueue addDocumentRequestQueue = + getAddDocumentRequestQueue(indexName); + logger.debug( + String.format( + "onCompleted, addDocumentRequestQueue: %s", addDocumentRequestQueue.size())); + long highestGen = -1; + try { + DeadlineUtils.checkDeadline("addDocuments: onCompletedForIndex", "INDEXING"); + + // index the left over docs + if (!addDocumentRequestQueue.isEmpty()) { + logger.debug( + String.format( + "indexing left over addDocumentRequestQueue of size: %s", + addDocumentRequestQueue.size())); + List addDocRequestList = new ArrayList<>(addDocumentRequestQueue); + // Since we are already running in the indexing threadpool run the indexing job + // for remaining documents directly. This serializes indexing remaining documents for + // multiple indices but avoids deadlocking if there aren't more threads than the + // maximum + // number of parallel addDocuments calls. + long gen = + new DocumentIndexer(getGlobalState(), addDocRequestList, indexName) + .runIndexingJob(); + if (gen > highestGen) { + highestGen = gen; + } + } + // collect futures, block if needed + int numIndexingChunks = futures.size(); + long t0 = System.nanoTime(); + for (Future result : futures.get(indexName)) { + Long gen = result.get(); + logger.debug("Indexing returned sequence-number {}", gen); + if (gen > highestGen) { + highestGen = gen; + } + } + long t1 = System.nanoTime(); + logger.debug( + "Indexing job completed for {} docs, in {} chunks, with latest sequence number: {}, took: {} micro seconds", + getCount(indexName), + numIndexingChunks, + highestGen, + ((t1 - t0) / 1000)); + return String.valueOf(highestGen); + } catch (Exception e) { + logger.warn("error while trying to addDocuments", e); + throw Status.INTERNAL + .withDescription("error while trying to addDocuments ") + .augmentDescription(e.getMessage()) + .withCause(e) + .asRuntimeException(); + } finally { + addDocumentRequestQueue.clear(); + countMap.put(indexName, 0L); + } + } + + @Override + public void onCompleted() { + try { + getGlobalState() + .submitIndexingTask( + Context.current() + .wrap( + () -> { + try { + // TODO: this should return a map on index to genId in the response + String genId = "-1"; + for (String indexName : addDocumentRequestQueueMap.keySet()) { + genId = onCompletedForIndex(indexName); + } + responseObserver.onNext( + AddDocumentResponse.newBuilder() + .setGenId(genId) + .setPrimaryId(getGlobalState().getEphemeralId()) + .build()); + responseObserver.onCompleted(); + } catch (Throwable t) { + responseObserver.onError(t); + } + return null; + })); + } catch (RejectedExecutionException e) { + logger.error("Threadpool is full, unable to submit indexing completion job"); + responseObserver.onError( + Status.RESOURCE_EXHAUSTED + .withDescription("Threadpool is full, unable to submit indexing completion job") + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } + }; + } + /** * DocumentsContext is created for each GRPC AddDocumentRequest It hold all lucene documents * context for the AddDocumentRequest including root document and optional child documents if diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/CreateSnapshotHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/CreateSnapshotHandler.java index 96ccf8d70..b17205277 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/CreateSnapshotHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/CreateSnapshotHandler.java @@ -19,29 +19,54 @@ import com.yelp.nrtsearch.server.grpc.CreateSnapshotRequest; import com.yelp.nrtsearch.server.grpc.CreateSnapshotResponse; import com.yelp.nrtsearch.server.grpc.SnapshotId; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import java.io.IOException; -import java.util.Collection; import org.apache.lucene.facet.taxonomy.SearcherTaxonomyManager; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.index.StandardDirectoryReader; import org.apache.lucene.search.IndexSearcher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CreateSnapshotHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(CreateSnapshotHandler.class); + + public CreateSnapshotHandler(GlobalState globalState) { + super(globalState); + } -public class CreateSnapshotHandler - implements Handler { @Override - public CreateSnapshotResponse handle( - IndexState indexState, CreateSnapshotRequest createSnapshotRequest) throws HandlerException { + public void handle( + CreateSnapshotRequest createSnapshotRequest, + StreamObserver responseObserver) { try { - return createSnapshot(indexState, createSnapshotRequest); - } catch (IOException e) { - throw new RuntimeException(e); + IndexState indexState = getGlobalState().getIndex(createSnapshotRequest.getIndexName()); + CreateSnapshotResponse reply = createSnapshot(indexState, createSnapshotRequest); + logger.info(String.format("CreateSnapshotHandler returned results %s", reply.toString())); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.warn( + String.format( + "error while trying to createSnapshot for index %s", + createSnapshotRequest.getIndexName()), + e); + responseObserver.onError( + Status.UNKNOWN + .withDescription( + String.format( + "error while trying to createSnapshot for index %s", + createSnapshotRequest.getIndexName())) + .augmentDescription(e.getMessage()) + .asRuntimeException()); } } - public CreateSnapshotResponse createSnapshot( + private CreateSnapshotResponse createSnapshot( IndexState indexState, CreateSnapshotRequest createSnapshotRequest) throws IOException { indexState.verifyStarted(); @@ -116,46 +141,4 @@ public static String getSnapshotIdAsString(SnapshotId snapshotId) { + ":" + snapshotId.getStateGen(); } - - /** - * Get names of all index files in a given snapshot. - * - * @param indexState index state - * @param snapshotId snapshot id - * @return collection of file names - * @throws IOException - */ - public static Collection getSegmentFilesInSnapshot( - IndexState indexState, SnapshotId snapshotId) throws IOException { - String snapshotIdAsString = CreateSnapshotHandler.getSnapshotIdAsString(snapshotId); - IndexState.Gens snapshot = new IndexState.Gens(snapshotIdAsString); - if (indexState.getShards().size() != 1) { - throw new IllegalStateException( - String.format( - "%s shards found index %s instead of exactly 1", - indexState.getShards().size(), indexState.getName())); - } - ShardState state = indexState.getShards().entrySet().iterator().next().getValue(); - SearcherTaxonomyManager.SearcherAndTaxonomy searcherAndTaxonomy = null; - IndexReader indexReader = null; - try { - searcherAndTaxonomy = state.acquire(); - indexReader = - DirectoryReader.openIfChanged( - (DirectoryReader) searcherAndTaxonomy.searcher.getIndexReader(), - state.snapshots.getIndexCommit(snapshot.indexGen)); - if (!(indexReader instanceof StandardDirectoryReader)) { - throw new IllegalStateException("Unable to find segments to backup"); - } - StandardDirectoryReader standardDirectoryReader = (StandardDirectoryReader) indexReader; - return standardDirectoryReader.getSegmentInfos().files(true); - } finally { - if (searcherAndTaxonomy != null) { - state.release(searcherAndTaxonomy); - } - if (indexReader != null) { - indexReader.close(); - } - } - } } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteAllDocumentsHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteAllDocumentsHandler.java index 05f09a6ae..ec6daba2b 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteAllDocumentsHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteAllDocumentsHandler.java @@ -16,18 +16,48 @@ package com.yelp.nrtsearch.server.luceneserver; import com.yelp.nrtsearch.server.grpc.*; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DeleteAllDocumentsHandler - implements Handler { + extends Handler { private static final Logger logger = LoggerFactory.getLogger(DeleteAllDocumentsHandler.class.getName()); + public DeleteAllDocumentsHandler(GlobalState globalState) { + super(globalState); + } + @Override - public DeleteAllDocumentsResponse handle( - IndexState indexState, DeleteAllDocumentsRequest deleteAllDocumentsRequest) + public void handle( + DeleteAllDocumentsRequest deleteAllDocumentsRequest, + StreamObserver responseObserver) { + logger.info("Received delete all documents request: {}", deleteAllDocumentsRequest); + try { + IndexState indexState = getGlobalState().getIndex(deleteAllDocumentsRequest.getIndexName()); + DeleteAllDocumentsResponse reply = handle(indexState); + logger.info("DeleteAllDocumentsHandler returned " + reply); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.warn( + "error while trying to deleteAll for index " + deleteAllDocumentsRequest.getIndexName(), + e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + "error while trying to deleteAll for index: " + + deleteAllDocumentsRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } + + private DeleteAllDocumentsResponse handle(IndexState indexState) throws DeleteAllDocumentsHandlerException { final ShardState shardState = indexState.getShard(0); indexState.verifyStarted(); diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteByQueryHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteByQueryHandler.java index fea654ab2..491e0357f 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteByQueryHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteByQueryHandler.java @@ -17,6 +17,9 @@ import com.yelp.nrtsearch.server.grpc.AddDocumentResponse; import com.yelp.nrtsearch.server.grpc.DeleteByQueryRequest; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; @@ -24,12 +27,40 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class DeleteByQueryHandler implements Handler { +public class DeleteByQueryHandler extends Handler { private static final Logger logger = LoggerFactory.getLogger(DeleteByQueryHandler.class); private final QueryNodeMapper queryNodeMapper = QueryNodeMapper.getInstance(); + public DeleteByQueryHandler(GlobalState globalState) { + super(globalState); + } + @Override - public AddDocumentResponse handle( + public void handle( + DeleteByQueryRequest deleteByQueryRequest, + StreamObserver responseObserver) { + try { + IndexState indexState = getGlobalState().getIndex(deleteByQueryRequest.getIndexName()); + AddDocumentResponse reply = handle(indexState, deleteByQueryRequest); + logger.debug("DeleteDocumentsHandler returned " + reply.toString()); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.warn( + "Error while trying to delete documents from index: {}", + deleteByQueryRequest.getIndexName(), + e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + "Error while trying to delete documents from index: " + + deleteByQueryRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } + + private AddDocumentResponse handle( IndexState indexState, DeleteByQueryRequest deleteByQueryRequest) throws DeleteByQueryHandlerException { final ShardState shardState = indexState.getShard(0); diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteDocumentsHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteDocumentsHandler.java index 23f8a8f01..69702b81c 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteDocumentsHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteDocumentsHandler.java @@ -18,6 +18,9 @@ import com.google.protobuf.ProtocolStringList; import com.yelp.nrtsearch.server.grpc.AddDocumentRequest; import com.yelp.nrtsearch.server.grpc.AddDocumentResponse; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -26,12 +29,40 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class DeleteDocumentsHandler implements Handler { +public class DeleteDocumentsHandler extends Handler { private static final Logger logger = LoggerFactory.getLogger(DeleteDocumentsHandler.class.getName()); + public DeleteDocumentsHandler(GlobalState globalState) { + super(globalState); + } + @Override - public AddDocumentResponse handle(IndexState indexState, AddDocumentRequest addDocumentRequest) + public void handle( + AddDocumentRequest addDocumentRequest, StreamObserver responseObserver) { + try { + IndexState indexState = getGlobalState().getIndex(addDocumentRequest.getIndexName()); + AddDocumentResponse reply = handleInternal(indexState, addDocumentRequest); + logger.debug("DeleteDocumentsHandler returned {}", reply); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.error( + "error while trying to delete documents for index {}", + addDocumentRequest.getIndexName(), + e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + "error while trying to delete documents for index: " + + addDocumentRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } + + private AddDocumentResponse handleInternal( + IndexState indexState, AddDocumentRequest addDocumentRequest) throws DeleteDocumentsHandlerException { final ShardState shardState = indexState.getShard(0); indexState.verifyStarted(); diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteIndexHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteIndexHandler.java index 4cc9729fe..b05a3a3a0 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteIndexHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/DeleteIndexHandler.java @@ -17,16 +17,42 @@ import com.yelp.nrtsearch.server.grpc.DeleteIndexRequest; import com.yelp.nrtsearch.server.grpc.DeleteIndexResponse; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class DeleteIndexHandler implements Handler { +public class DeleteIndexHandler extends Handler { private static final Logger logger = LoggerFactory.getLogger(DeleteIndexHandler.class.getName()); + public DeleteIndexHandler(GlobalState globalState) { + super(globalState); + } + @Override - public DeleteIndexResponse handle(IndexState indexState, DeleteIndexRequest protoRequest) - throws DeleteIndexHandlerException { + public void handle( + DeleteIndexRequest deleteIndexRequest, StreamObserver responseObserver) { + logger.info("Received delete index request: {}", deleteIndexRequest); + try { + IndexState indexState = getGlobalState().getIndex(deleteIndexRequest.getIndexName()); + DeleteIndexResponse reply = handle(indexState); + logger.info("DeleteIndexHandler returned " + reply); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.warn("error while trying to delete index " + deleteIndexRequest.getIndexName(), e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + "error while trying to delete index: " + deleteIndexRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } + + private DeleteIndexResponse handle(IndexState indexState) throws DeleteIndexHandlerException { try { indexState.getGlobalState().deleteIndex(indexState.getName()); indexState.deleteIndex(); diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/GetStateHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/GetStateHandler.java index fa1cfde80..3c4ffed84 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/GetStateHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/GetStateHandler.java @@ -18,16 +18,41 @@ import com.google.gson.JsonObject; import com.yelp.nrtsearch.server.grpc.StateRequest; import com.yelp.nrtsearch.server.grpc.StateResponse; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class GetStateHandler implements Handler { - Logger logger = LoggerFactory.getLogger(GetStateHandler.class); +// TODO: rename to StateHandler +public class GetStateHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(GetStateHandler.class); + + public GetStateHandler(GlobalState globalState) { + super(globalState); + } @Override - public StateResponse handle(IndexState indexState, StateRequest stateRequest) - throws HandlerException { + public void handle(StateRequest request, StreamObserver responseObserver) { + try { + IndexState indexState = getGlobalState().getIndex(request.getIndexName()); + StateResponse reply = handle(indexState); + logger.debug("GetStateHandler returned " + reply); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.warn("error while trying to get state for index " + request.getIndexName(), e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + "error while trying to get state for index " + request.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } + + private StateResponse handle(IndexState indexState) throws HandlerException { StateResponse.Builder builder = StateResponse.newBuilder(); JsonObject savedState = new JsonObject(); try { diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/LiveSettingsHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/LiveSettingsHandler.java index 014a9ebbb..2ada25cdd 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/LiveSettingsHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/LiveSettingsHandler.java @@ -21,15 +21,54 @@ import com.yelp.nrtsearch.server.grpc.IndexLiveSettings; import com.yelp.nrtsearch.server.grpc.LiveSettingsRequest; import com.yelp.nrtsearch.server.grpc.LiveSettingsResponse; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler; import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class LiveSettingsHandler implements Handler { +public class LiveSettingsHandler extends Handler { private static final Logger logger = LoggerFactory.getLogger(LiveSettingsHandler.class); + public LiveSettingsHandler(GlobalState globalState) { + super(globalState); + } + @Override + public void handle( + LiveSettingsRequest req, StreamObserver responseObserver) { + logger.info("Received live settings request: {}", req); + try { + IndexState indexState = getGlobalState().getIndex(req.getIndexName()); + LiveSettingsResponse reply = handle(indexState, req); + logger.info("LiveSettingsHandler returned {}", reply.toString()); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (IllegalArgumentException e) { + logger.warn("index: {} was not yet created", req.getIndexName(), e); + responseObserver.onError( + Status.ALREADY_EXISTS + .withDescription("invalid indexName: " + req.getIndexName()) + .augmentDescription("IllegalArgumentException()") + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + logger.warn( + "error while trying to read index state dir for indexName: " + req.getIndexName(), e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to read index state dir for indexName: " + + req.getIndexName() + + "at rootDir: ") + .augmentDescription("IOException()") + .withCause(e) + .asRuntimeException()); + } + } + public LiveSettingsResponse handle( IndexState indexStateIn, LiveSettingsRequest liveSettingsRequest) { return handleAsLiveSettingsV2(indexStateIn, liveSettingsRequest); diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/ReleaseSnapshotHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/ReleaseSnapshotHandler.java index 75d1edb9a..19a73331a 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/ReleaseSnapshotHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/ReleaseSnapshotHandler.java @@ -17,17 +17,49 @@ import com.yelp.nrtsearch.server.grpc.ReleaseSnapshotRequest; import com.yelp.nrtsearch.server.grpc.ReleaseSnapshotResponse; -import com.yelp.nrtsearch.server.grpc.SnapshotId; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ReleaseSnapshotHandler - implements Handler { + extends Handler { private static final Logger logger = LoggerFactory.getLogger(ReleaseSnapshotHandler.class); + public ReleaseSnapshotHandler(GlobalState globalState) { + super(globalState); + } + @Override - public ReleaseSnapshotResponse handle( + public void handle( + ReleaseSnapshotRequest releaseSnapshotRequest, + StreamObserver responseObserver) { + try { + IndexState indexState = getGlobalState().getIndex(releaseSnapshotRequest.getIndexName()); + ReleaseSnapshotResponse reply = handle(indexState, releaseSnapshotRequest); + logger.info(String.format("CreateSnapshotHandler returned results %s", reply.toString())); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.warn( + String.format( + "error while trying to releaseSnapshot for index %s", + releaseSnapshotRequest.getIndexName()), + e); + responseObserver.onError( + Status.UNKNOWN + .withDescription( + String.format( + "error while trying to releaseSnapshot for index %s", + releaseSnapshotRequest.getIndexName())) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } + + private ReleaseSnapshotResponse handle( IndexState indexState, ReleaseSnapshotRequest releaseSnapshotRequest) { final ShardState shardState = indexState.getShard(0); final IndexState.Gens gens = @@ -56,22 +88,4 @@ public ReleaseSnapshotResponse handle( } return ReleaseSnapshotResponse.newBuilder().setSuccess(true).build(); } - - /** - * Release resources held by the given snapshot. - * - * @param indexState index state - * @param indexName index name - * @param snapshotId snapshot id - */ - public static void releaseSnapshot( - IndexState indexState, String indexName, SnapshotId snapshotId) { - ReleaseSnapshotRequest releaseSnapshotRequest = - ReleaseSnapshotRequest.newBuilder() - .setIndexName(indexName) - .setSnapshotId(snapshotId) - .build(); - ReleaseSnapshotResponse releaseSnapshotResponse = - new ReleaseSnapshotHandler().handle(indexState, releaseSnapshotRequest); - } } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/SearchHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/SearchHandler.java index 7396be04e..8f1dc5727 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/SearchHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/SearchHandler.java @@ -17,6 +17,8 @@ import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat.Printer; import com.yelp.nrtsearch.server.grpc.DeadlineUtils; import com.yelp.nrtsearch.server.grpc.FacetResult; import com.yelp.nrtsearch.server.grpc.ProfileResult; @@ -36,6 +38,7 @@ import com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef; import com.yelp.nrtsearch.server.luceneserver.field.RuntimeFieldDef; import com.yelp.nrtsearch.server.luceneserver.field.VirtualFieldDef; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler; import com.yelp.nrtsearch.server.luceneserver.innerhit.InnerHitFetchTask; import com.yelp.nrtsearch.server.luceneserver.rescore.RescoreTask; import com.yelp.nrtsearch.server.luceneserver.script.RuntimeScript; @@ -46,6 +49,10 @@ import com.yelp.nrtsearch.server.luceneserver.search.SearcherResult; import com.yelp.nrtsearch.server.monitoring.SearchResponseCollector; import com.yelp.nrtsearch.server.utils.ObjectToCompositeFieldTransformer; +import com.yelp.nrtsearch.server.utils.ProtoMessagePrinter; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; @@ -65,27 +72,85 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class SearchHandler implements Handler { +public class SearchHandler extends Handler { private static final ExecutorService DIRECT_EXECUTOR = MoreExecutors.newDirectExecutorService(); + private static final Printer protoMessagePrinter = + ProtoMessagePrinter.omittingInsignificantWhitespace(); private static final Logger logger = LoggerFactory.getLogger(SearchHandler.class); private final ThreadPoolExecutor threadPoolExecutor; private final boolean warming; - public SearchHandler(ThreadPoolExecutor threadPoolExecutor) { - this(threadPoolExecutor, false); + public SearchHandler(GlobalState globalState) { + super(globalState); + this.threadPoolExecutor = globalState.getSearchThreadPoolExecutor(); + this.warming = false; } /** * @param threadPoolExecutor Threadpool to execute a parallel search * @param warming set to true if we are warming the index right now */ - public SearchHandler(ThreadPoolExecutor threadPoolExecutor, boolean warming) { + public SearchHandler( + GlobalState globalState, ThreadPoolExecutor threadPoolExecutor, boolean warming) { + super(globalState); this.threadPoolExecutor = threadPoolExecutor; this.warming = warming; } @Override + public void handle(SearchRequest searchRequest, StreamObserver responseObserver) { + try { + SearchResponse reply = getSearchResponse(searchRequest); + setResponseCompression(searchRequest.getResponseCompression(), responseObserver); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (IOException e) { + logger.warn( + "error while trying to read index state dir for indexName: {}", + searchRequest.getIndexName(), + e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to read index state dir for indexName: " + + searchRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + String searchRequestJson = null; + try { + searchRequestJson = protoMessagePrinter.print(searchRequest); + } catch (InvalidProtocolBufferException ignored) { + // Ignore as invalid proto would have thrown an exception earlier + } + logger.warn( + "error while trying to execute search for index {}: request: {}", + searchRequest.getIndexName(), + searchRequestJson, + e); + if (e instanceof StatusRuntimeException) { + responseObserver.onError(e); + } else { + responseObserver.onError( + Status.UNKNOWN + .withDescription( + String.format( + "error while trying to execute search for index %s. check logs for full searchRequest.", + searchRequest.getIndexName())) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } + } + + public SearchResponse getSearchResponse(SearchRequest searchRequest) + throws IOException, SearchHandlerException { + IndexState indexState = getGlobalState().getIndex(searchRequest.getIndexName()); + return handle(indexState, searchRequest); + } + public SearchResponse handle(IndexState indexState, SearchRequest searchRequest) throws SearchHandlerException { // this request may have been waiting in the grpc queue too long diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/SettingsHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/SettingsHandler.java index c74e6116d..c0ea4d98a 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/SettingsHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/SettingsHandler.java @@ -15,7 +15,6 @@ */ package com.yelp.nrtsearch.server.luceneserver; -import com.google.gson.JsonParser; import com.google.protobuf.BoolValue; import com.google.protobuf.DoubleValue; import com.google.protobuf.Int32Value; @@ -24,17 +23,59 @@ import com.yelp.nrtsearch.server.grpc.IndexSettings; import com.yelp.nrtsearch.server.grpc.SettingsRequest; import com.yelp.nrtsearch.server.grpc.SettingsResponse; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler; import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class SettingsHandler implements Handler { - Logger logger = LoggerFactory.getLogger(SettingsHandler.class); - private final JsonParser jsonParser = new JsonParser(); +public class SettingsHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(SettingsHandler.class); + + public SettingsHandler(GlobalState globalState) { + super(globalState); + } @Override - public SettingsResponse handle(final IndexState indexStateIn, SettingsRequest settingsRequest) + public void handle( + SettingsRequest settingsRequest, StreamObserver responseObserver) { + logger.info("Received settings request: {}", settingsRequest); + try { + IndexState indexState = getGlobalState().getIndex(settingsRequest.getIndexName()); + SettingsResponse reply = handle(indexState, settingsRequest); + logger.info("SettingsHandler returned " + reply); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (IOException e) { + logger.warn( + "error while trying to read index state dir for indexName: " + + settingsRequest.getIndexName(), + e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to read index state dir for indexName: " + + settingsRequest.getIndexName()) + .augmentDescription("IOException()") + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + logger.warn( + "error while trying to update/get settings for index " + settingsRequest.getIndexName(), + e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + "error while trying to update/get settings for index: " + + settingsRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } + + private SettingsResponse handle(final IndexState indexStateIn, SettingsRequest settingsRequest) throws SettingsHandlerException { return handleAsSettingsV2(indexStateIn, settingsRequest); } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/StartIndexHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/StartIndexHandler.java index 14689fca7..ecabad669 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/StartIndexHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/StartIndexHandler.java @@ -21,6 +21,7 @@ import com.yelp.nrtsearch.server.grpc.RestoreIndex; import com.yelp.nrtsearch.server.grpc.StartIndexRequest; import com.yelp.nrtsearch.server.grpc.StartIndexResponse; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler.HandlerException; import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager; import com.yelp.nrtsearch.server.luceneserver.nrt.NrtDataManager; import com.yelp.nrtsearch.server.luceneserver.state.BackendGlobalState; @@ -33,7 +34,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class StartIndexHandler implements Handler { +// TODO: this class should be called something else to differentiate from the handler for grpc +// request, or alternatively the call structure needs to be changed +public class StartIndexHandler { private static final Set startingIndices = new HashSet<>(); private final String serviceName; @@ -69,7 +72,6 @@ public StartIndexHandler( this.discoveryFileUpdateIntervalMs = discoveryFileUpdateIntervalMs; } - @Override public StartIndexResponse handle(IndexState indexState, StartIndexRequest startIndexRequest) throws StartIndexHandlerException { String indexName = indexState.getName(); diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/BackupWarmingQueriesHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/BackupWarmingQueriesHandler.java new file mode 100644 index 000000000..40f8485f6 --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/BackupWarmingQueriesHandler.java @@ -0,0 +1,109 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.BackupWarmingQueriesRequest; +import com.yelp.nrtsearch.server.grpc.BackupWarmingQueriesResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.IndexState; +import com.yelp.nrtsearch.server.luceneserver.warming.Warmer; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BackupWarmingQueriesHandler + extends Handler { + private static final Logger logger = LoggerFactory.getLogger(BackupWarmingQueriesHandler.class); + + public BackupWarmingQueriesHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + BackupWarmingQueriesRequest request, + StreamObserver responseObserver) { + logger.info("Received backup warming queries request: {}", request); + String index = request.getIndex(); + try { + IndexState indexState = getGlobalState().getIndex(index); + Warmer warmer = indexState.getWarmer(); + if (warmer == null) { + logger.warn("Unable to backup warming queries as warmer not found for index: {}", index); + responseObserver.onError( + Status.UNKNOWN + .withDescription( + "Unable to backup warming queries as warmer not found for index: " + index) + .asRuntimeException()); + return; + } + int numQueriesThreshold = request.getNumQueriesThreshold(); + int numWarmingRequests = warmer.getNumWarmingRequests(); + if (numQueriesThreshold > 0 && numWarmingRequests < numQueriesThreshold) { + logger.warn( + "Unable to backup warming queries since warmer has {} requests, which is less than threshold {}", + numWarmingRequests, + numQueriesThreshold); + responseObserver.onError( + Status.UNKNOWN + .withDescription( + String.format( + "Unable to backup warming queries since warmer has %s requests, which is less than threshold %s", + numWarmingRequests, numQueriesThreshold)) + .asRuntimeException()); + return; + } + int uptimeMinutesThreshold = request.getUptimeMinutesThreshold(); + int currUptimeMinutes = + (int) (ManagementFactory.getRuntimeMXBean().getUptime() / 1000L / 60L); + if (uptimeMinutesThreshold > 0 && currUptimeMinutes < uptimeMinutesThreshold) { + logger.warn( + "Unable to backup warming queries since uptime is {} minutes, which is less than threshold {}", + currUptimeMinutes, + uptimeMinutesThreshold); + responseObserver.onError( + Status.UNKNOWN + .withDescription( + String.format( + "Unable to backup warming queries since uptime is %s minutes, which is less than threshold %s", + currUptimeMinutes, uptimeMinutesThreshold)) + .asRuntimeException()); + return; + } + warmer.backupWarmingQueriesToS3(request.getServiceName()); + responseObserver.onNext(BackupWarmingQueriesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } catch (IOException e) { + logger.error( + "Unable to backup warming queries for index: {}, service: {}", + index, + request.getServiceName(), + e); + responseObserver.onError( + Status.UNKNOWN + .withCause(e) + .withDescription( + String.format( + "Unable to backup warming queries for index: %s, service: %s", + index, request.getServiceName())) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/CommitHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/CommitHandler.java new file mode 100644 index 000000000..63c172ac0 --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/CommitHandler.java @@ -0,0 +1,105 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.CommitRequest; +import com.yelp.nrtsearch.server.grpc.CommitResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.IndexState; +import io.grpc.Context; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.util.concurrent.RejectedExecutionException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CommitHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(CommitHandler.class); + + public CommitHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle(CommitRequest commitRequest, StreamObserver responseObserver) { + try { + getGlobalState() + .submitIndexingTask( + Context.current() + .wrap( + () -> { + try { + IndexState indexState = + getGlobalState().getIndex(commitRequest.getIndexName()); + long gen = indexState.commit(); + CommitResponse reply = + CommitResponse.newBuilder() + .setGen(gen) + .setPrimaryId(getGlobalState().getEphemeralId()) + .build(); + logger.debug( + String.format( + "CommitHandler committed to index: %s for sequenceId: %s", + commitRequest.getIndexName(), gen)); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (IOException e) { + logger.warn( + "error while trying to read index state dir for indexName: " + + commitRequest.getIndexName(), + e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to read index state dir for indexName: " + + commitRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + logger.warn( + "error while trying to commit to index " + + commitRequest.getIndexName(), + e); + if (e instanceof StatusRuntimeException) { + responseObserver.onError(e); + } else { + responseObserver.onError( + Status.UNKNOWN + .withDescription( + "error while trying to commit to index: " + + commitRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } + return null; + })); + } catch (RejectedExecutionException e) { + logger.error( + "Threadpool is full, unable to submit commit to index {}", commitRequest.getIndexName()); + responseObserver.onError( + Status.RESOURCE_EXHAUSTED + .withDescription( + "Threadpool is full, unable to submit commit to index: " + + commitRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/CreateIndexHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/CreateIndexHandler.java new file mode 100644 index 000000000..953693a8d --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/CreateIndexHandler.java @@ -0,0 +1,73 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.CreateIndexRequest; +import com.yelp.nrtsearch.server.grpc.CreateIndexResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CreateIndexHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(CreateIndexHandler.class); + + public CreateIndexHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle(CreateIndexRequest req, StreamObserver responseObserver) { + logger.info("Received create index request: {}", req); + String indexName = req.getIndexName(); + String validIndexNameRegex = "[A-z0-9_-]+"; + if (!indexName.matches(validIndexNameRegex)) { + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + String.format( + "Index name %s is invalid - must contain only a-z, A-Z or 0-9", indexName)) + .asRuntimeException()); + return; + } + + try { + getGlobalState().createIndex(req); + String response = String.format("Created Index name: %s", indexName); + CreateIndexResponse reply = CreateIndexResponse.newBuilder().setResponse(response).build(); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (IllegalArgumentException e) { + logger.warn("invalid IndexName: " + indexName, e); + responseObserver.onError( + Status.ALREADY_EXISTS + .withDescription("invalid indexName: " + indexName) + .augmentDescription("IllegalArgumentException()") + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + logger.warn("error while trying to save index state to disk for indexName: " + indexName, e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to save index state to disk for indexName: " + indexName) + .augmentDescription(e.getMessage()) + .withCause(e) + .asRuntimeException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/CustomHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/CustomHandler.java new file mode 100644 index 000000000..d625d1c7f --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/CustomHandler.java @@ -0,0 +1,48 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.CustomRequest; +import com.yelp.nrtsearch.server.grpc.CustomResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.custom.request.CustomRequestProcessor; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CustomHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(CustomHandler.class); + + public CustomHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle(CustomRequest request, StreamObserver responseObserver) { + logger.info("Received custom request: {}", request); + try { + CustomResponse response = CustomRequestProcessor.processCustomRequest(request); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } catch (Exception e) { + String error = + String.format("Error processing custom request %s, error: %s", request, e.getMessage()); + logger.error(error); + responseObserver.onError(Status.INTERNAL.withDescription(error).withCause(e).asException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ForceMergeDeletesHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ForceMergeDeletesHandler.java new file mode 100644 index 000000000..246b8caec --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ForceMergeDeletesHandler.java @@ -0,0 +1,74 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.ForceMergeDeletesRequest; +import com.yelp.nrtsearch.server.grpc.ForceMergeDeletesResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.IndexState; +import com.yelp.nrtsearch.server.luceneserver.ShardState; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ForceMergeDeletesHandler + extends Handler { + private static final Logger logger = LoggerFactory.getLogger(ForceMergeDeletesHandler.class); + + public ForceMergeDeletesHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + ForceMergeDeletesRequest forceMergeRequest, + StreamObserver responseObserver) { + logger.info("Received force merge deletes request: {}", forceMergeRequest); + if (forceMergeRequest.getIndexName().isEmpty()) { + responseObserver.onError(new IllegalArgumentException("Index name in request is empty")); + return; + } + + try { + IndexState indexState = getGlobalState().getIndex(forceMergeRequest.getIndexName()); + ShardState shardState = indexState.getShards().get(0); + logger.info("Beginning force merge deletes for index: {}", forceMergeRequest.getIndexName()); + shardState.writer.forceMergeDeletes(forceMergeRequest.getDoWait()); + } catch (IOException e) { + logger.warn( + "Error during force merge deletes for index {} ", forceMergeRequest.getIndexName(), e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "Error during force merge deletes for index " + forceMergeRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + return; + } + + ForceMergeDeletesResponse.Status status = + forceMergeRequest.getDoWait() + ? ForceMergeDeletesResponse.Status.FORCE_MERGE_DELETES_COMPLETED + : ForceMergeDeletesResponse.Status.FORCE_MERGE_DELETES_SUBMITTED; + logger.info("Force merge deletes status: {}", status); + ForceMergeDeletesResponse response = + ForceMergeDeletesResponse.newBuilder().setStatus(status).build(); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ForceMergeHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ForceMergeHandler.java new file mode 100644 index 000000000..062a7e3bf --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ForceMergeHandler.java @@ -0,0 +1,76 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.ForceMergeRequest; +import com.yelp.nrtsearch.server.grpc.ForceMergeResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.IndexState; +import com.yelp.nrtsearch.server.luceneserver.ShardState; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ForceMergeHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(ForceMergeHandler.class); + private static ForceMergeHandler instance; + + public ForceMergeHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + ForceMergeRequest forceMergeRequest, StreamObserver responseObserver) { + logger.info("Received force merge request: {}", forceMergeRequest); + if (forceMergeRequest.getIndexName().isEmpty()) { + responseObserver.onError(new IllegalArgumentException("Index name in request is empty")); + return; + } + if (forceMergeRequest.getMaxNumSegments() == 0) { + responseObserver.onError(new IllegalArgumentException("Cannot have 0 max segments")); + return; + } + + try { + IndexState indexState = getGlobalState().getIndex(forceMergeRequest.getIndexName()); + ShardState shardState = indexState.getShards().get(0); + logger.info("Beginning force merge for index: {}", forceMergeRequest.getIndexName()); + shardState.writer.forceMerge( + forceMergeRequest.getMaxNumSegments(), forceMergeRequest.getDoWait()); + } catch (IOException e) { + logger.warn("Error during force merge for index {} ", forceMergeRequest.getIndexName(), e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "Error during force merge for index " + forceMergeRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + return; + } + + ForceMergeResponse.Status status = + forceMergeRequest.getDoWait() + ? ForceMergeResponse.Status.FORCE_MERGE_COMPLETED + : ForceMergeResponse.Status.FORCE_MERGE_SUBMITTED; + logger.info("Force merge status: {}", status); + ForceMergeResponse response = ForceMergeResponse.newBuilder().setStatus(status).build(); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/GetAllSnapshotIndexGenHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/GetAllSnapshotIndexGenHandler.java new file mode 100644 index 000000000..5101a444b --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/GetAllSnapshotIndexGenHandler.java @@ -0,0 +1,56 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.GetAllSnapshotGenRequest; +import com.yelp.nrtsearch.server.grpc.GetAllSnapshotGenResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GetAllSnapshotIndexGenHandler + extends Handler { + private static final Logger logger = LoggerFactory.getLogger(GetAllSnapshotIndexGenHandler.class); + + public GetAllSnapshotIndexGenHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + GetAllSnapshotGenRequest request, + StreamObserver responseObserver) { + try { + Set snapshotGens = + getGlobalState() + .getIndex(request.getIndexName()) + .getShard(0) + .snapshotGenToVersion + .keySet(); + GetAllSnapshotGenResponse response = + GetAllSnapshotGenResponse.newBuilder().addAllIndexGens(snapshotGens).build(); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } catch (IOException e) { + logger.error( + "Error getting all snapshotted index gens for index: {}", request.getIndexName(), e); + responseObserver.onError(e); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/GlobalStateHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/GlobalStateHandler.java new file mode 100644 index 000000000..28a999484 --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/GlobalStateHandler.java @@ -0,0 +1,49 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.GlobalStateRequest; +import com.yelp.nrtsearch.server.grpc.GlobalStateResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GlobalStateHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(GlobalStateHandler.class); + + public GlobalStateHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + GlobalStateRequest request, StreamObserver responseObserver) { + try { + responseObserver.onNext( + GlobalStateResponse.newBuilder().setGlobalState(getGlobalState().getStateInfo()).build()); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.warn("error while trying to get global state", e); + responseObserver.onError( + Status.UNKNOWN + .withDescription("error while trying to get global state") + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/Handler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/Handler.java new file mode 100644 index 000000000..fa7472b66 --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/Handler.java @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.google.protobuf.GeneratedMessageV3; +import com.yelp.nrtsearch.server.grpc.AddDocumentRequest; +import com.yelp.nrtsearch.server.grpc.LuceneServerStubBuilder; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import io.grpc.stub.ServerCallStreamObserver; +import io.grpc.stub.StreamObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Abstract class for handlers that process requests and produce responses or provide a handler for + * streaming responses. For a gRPC method x, create a class xHandler that extends Handler. Override + * the {@link #handle(StreamObserver)} method for streaming responses, or the {@link + * #handle(GeneratedMessageV3, StreamObserver)} method for unary responses. Initialize the handler + * in {@link com.yelp.nrtsearch.server.grpc.LuceneServer.LuceneServerImpl} and call the appropriate + * method on the handler in the gRPC call. + * + * @param Request type + * @param Response type + */ +public abstract class Handler { + private static final Logger logger = LoggerFactory.getLogger(Handler.class); + + private final GlobalState globalState; + + public Handler(GlobalState globalState) { + this.globalState = globalState; + } + + protected GlobalState getGlobalState() { + return globalState; + } + + public void handle(T protoRequest, StreamObserver responseObserver) { + throw new UnsupportedOperationException("This method is not supported"); + } + + public StreamObserver handle(StreamObserver responseObserver) { + throw new UnsupportedOperationException("This method is not supported"); + } + + /** + * Set response compression on the provided {@link StreamObserver}. Should be a valid compression + * type from the {@link LuceneServerStubBuilder#COMPRESSOR_REGISTRY}, or empty string for default. + * Falls back to uncompressed on any error. + * + * @param compressionType compression type, or empty string + * @param responseObserver observer to set compression on + */ + protected void setResponseCompression( + String compressionType, StreamObserver responseObserver) { + if (!compressionType.isEmpty()) { + try { + ServerCallStreamObserver serverCallStreamObserver = + (ServerCallStreamObserver) responseObserver; + serverCallStreamObserver.setCompression(compressionType); + } catch (Exception e) { + logger.warn("Unable to set response compression to type '" + compressionType + "' : " + e); + } + } + } + + public static class HandlerException extends Exception { + public HandlerException(Throwable err) { + super(err); + } + + public HandlerException(String errorMessage) { + super(errorMessage); + } + + public HandlerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/IndicesHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/IndicesHandler.java new file mode 100644 index 000000000..2d75747fe --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/IndicesHandler.java @@ -0,0 +1,77 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.IndexStatsResponse; +import com.yelp.nrtsearch.server.grpc.IndicesRequest; +import com.yelp.nrtsearch.server.grpc.IndicesResponse; +import com.yelp.nrtsearch.server.grpc.StatsResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.IndexState; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class IndicesHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(IndicesHandler.class); + + public IndicesHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + IndicesRequest indicesRequest, StreamObserver responseObserver) { + try { + IndicesResponse reply = getIndicesResponse(getGlobalState()); + logger.debug("IndicesRequestHandler returned " + reply); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.warn("error while trying to get indices stats", e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription("error while trying to get indices stats") + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } + + private static IndicesResponse getIndicesResponse(GlobalState globalState) throws IOException { + Set indexNames = globalState.getIndexNames(); + IndicesResponse.Builder builder = IndicesResponse.newBuilder(); + for (String indexName : indexNames) { + IndexState indexState = globalState.getIndex(indexName); + if (indexState.isStarted()) { + StatsResponse statsResponse = StatsHandler.process(indexState); + builder.addIndicesResponse( + IndexStatsResponse.newBuilder() + .setIndexName(indexName) + .setStatsResponse(statsResponse) + .build()); + } else { + builder.addIndicesResponse( + IndexStatsResponse.newBuilder() + .setIndexName(indexName) + .setStatsResponse(StatsResponse.newBuilder().setState("not_started").build())); + } + } + return builder.build(); + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/MetricsRequestHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/MetricsHandler.java similarity index 55% rename from src/main/java/com/yelp/nrtsearch/server/MetricsRequestHandler.java rename to src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/MetricsHandler.java index b76c79b1e..09141addf 100644 --- a/src/main/java/com/yelp/nrtsearch/server/MetricsRequestHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/MetricsHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Yelp Inc. + * Copyright 2024 Yelp Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,23 +13,47 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.yelp.nrtsearch.server; +package com.yelp.nrtsearch.server.luceneserver.handler; import com.google.api.HttpBody; import com.google.protobuf.ByteString; +import com.google.protobuf.Empty; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter; import io.prometheus.metrics.model.registry.PrometheusRegistry; import java.io.ByteArrayOutputStream; import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class MetricsRequestHandler { +public class MetricsHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(MetricsHandler.class); private final PrometheusRegistry prometheusRegistry; - public MetricsRequestHandler(PrometheusRegistry prometheusRegistry) { + public MetricsHandler(PrometheusRegistry prometheusRegistry) { + super(null); this.prometheusRegistry = prometheusRegistry; } - public HttpBody process() throws IOException { + @Override + public void handle(Empty request, StreamObserver responseObserver) { + try { + HttpBody reply = process(); + logger.debug("MetricsRequestHandler returned " + reply.toString()); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.warn("error while trying to get metrics", e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription("error while trying to get metrics") + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } + + private HttpBody process() throws IOException { try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { PrometheusTextFormatWriter prometheusTextFormat = new PrometheusTextFormatWriter(false); prometheusTextFormat.write(byteArrayOutputStream, prometheusRegistry.scrape()); diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ReadyHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ReadyHandler.java new file mode 100644 index 000000000..180665944 --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ReadyHandler.java @@ -0,0 +1,102 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.google.common.base.Splitter; +import com.google.common.collect.Sets; +import com.yelp.nrtsearch.server.grpc.HealthCheckResponse; +import com.yelp.nrtsearch.server.grpc.ReadyCheckRequest; +import com.yelp.nrtsearch.server.grpc.TransferStatusCode; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.IndexState; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ReadyHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(ReadyHandler.class); + private static final Splitter COMMA_SPLITTER = Splitter.on(","); + + public ReadyHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + ReadyCheckRequest request, StreamObserver responseObserver) { + Set indexNames; + + // If specific index names are provided we will check only those indices, otherwise check all + if (request.getIndexNames().isEmpty()) { + indexNames = getGlobalState().getIndicesToStart(); + } else { + List indexNamesToCheck = COMMA_SPLITTER.splitToList(request.getIndexNames()); + + Set allIndices = getGlobalState().getIndexNames(); + + Sets.SetView nonExistentIndices = + Sets.difference(Set.copyOf(indexNamesToCheck), allIndices); + if (!nonExistentIndices.isEmpty()) { + logger.warn("Indices: {} do not exist", nonExistentIndices); + responseObserver.onError( + Status.UNAVAILABLE + .withDescription(String.format("Indices do not exist: %s", nonExistentIndices)) + .asRuntimeException()); + return; + } + + indexNames = + allIndices.stream().filter(indexNamesToCheck::contains).collect(Collectors.toSet()); + } + + try { + List indicesNotStarted = new ArrayList<>(); + for (String indexName : indexNames) { + // The ready endpoint should skip loading index state + IndexState indexState = getGlobalState().getIndex(indexName, true); + if (!indexState.isStarted()) { + indicesNotStarted.add(indexName); + } + } + + if (indicesNotStarted.isEmpty()) { + HealthCheckResponse reply = + HealthCheckResponse.newBuilder().setHealth(TransferStatusCode.Done).build(); + logger.debug("Ready check returned " + reply); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } else { + logger.warn("Indices not started: {}", indicesNotStarted); + responseObserver.onError( + Status.UNAVAILABLE + .withDescription(String.format("Indices not started: %s", indicesNotStarted)) + .asRuntimeException()); + } + } catch (Exception e) { + logger.warn("error while trying to check if all required indices are started", e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription("error while trying to check if all required indices are started") + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/RefreshHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/RefreshHandler.java new file mode 100644 index 000000000..c8bd6507b --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/RefreshHandler.java @@ -0,0 +1,76 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.RefreshRequest; +import com.yelp.nrtsearch.server.grpc.RefreshResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.IndexState; +import com.yelp.nrtsearch.server.luceneserver.ShardState; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RefreshHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(RefreshHandler.class); + + public RefreshHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + RefreshRequest refreshRequest, StreamObserver responseObserver) { + try { + IndexState indexState = getGlobalState().getIndex(refreshRequest.getIndexName()); + final ShardState shardState = indexState.getShard(0); + long t0 = System.nanoTime(); + shardState.maybeRefreshBlocking(); + long t1 = System.nanoTime(); + double refreshTimeMs = (t1 - t0) / 1000000.0; + RefreshResponse reply = RefreshResponse.newBuilder().setRefreshTimeMS(refreshTimeMs).build(); + logger.info( + String.format( + "RefreshHandler refreshed index: %s in %f", + refreshRequest.getIndexName(), refreshTimeMs)); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (IOException e) { + logger.warn( + "error while trying to read index state dir for indexName: " + + refreshRequest.getIndexName(), + e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to read index state dir for indexName: " + + refreshRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + logger.warn("error while trying to refresh index " + refreshRequest.getIndexName(), e); + responseObserver.onError( + Status.UNKNOWN + .withDescription( + "error while trying to refresh index: " + refreshRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/RegisterFieldsHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/RegisterFieldsHandler.java new file mode 100644 index 000000000..6ebe0c946 --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/RegisterFieldsHandler.java @@ -0,0 +1,72 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.FieldDefRequest; +import com.yelp.nrtsearch.server.grpc.FieldDefResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager; +import com.yelp.nrtsearch.server.luceneserver.index.handlers.FieldUpdateHandler; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RegisterFieldsHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(RegisterFieldsHandler.class); + + public RegisterFieldsHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + FieldDefRequest fieldDefRequest, StreamObserver responseObserver) { + logger.info("Received register fields request: {}", fieldDefRequest); + try { + IndexStateManager indexStateManager = + getGlobalState().getIndexStateManager(fieldDefRequest.getIndexName()); + FieldDefResponse reply = FieldUpdateHandler.handle(indexStateManager, fieldDefRequest); + logger.info("RegisterFieldsHandler registered fields " + reply); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (IOException e) { + logger.warn( + "error while trying to read index state dir for indexName: " + + fieldDefRequest.getIndexName(), + e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to read index state dir for indexName: " + + fieldDefRequest.getIndexName()) + .augmentDescription("IOException()") + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + logger.warn( + "error while trying to RegisterFields for index " + fieldDefRequest.getIndexName(), e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + "error while trying to RegisterFields for index: " + + fieldDefRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ReloadStateHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ReloadStateHandler.java new file mode 100644 index 000000000..9c2466a63 --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/ReloadStateHandler.java @@ -0,0 +1,59 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.Mode; +import com.yelp.nrtsearch.server.grpc.ReloadStateRequest; +import com.yelp.nrtsearch.server.grpc.ReloadStateResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ReloadStateHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(ReloadStateHandler.class); + + public ReloadStateHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + ReloadStateRequest request, StreamObserver responseObserver) { + try { + if (getGlobalState() + .getConfiguration() + .getIndexStartConfig() + .getMode() + .equals(Mode.REPLICA)) { + getGlobalState().reloadStateFromBackend(); + } else { + logger.info("Skip reloading state since it is not replica"); + } + ReloadStateResponse reloadStateResponse = ReloadStateResponse.newBuilder().build(); + responseObserver.onNext(reloadStateResponse); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.warn("error while trying to sync the index state", e); + responseObserver.onError( + Status.INTERNAL + .withDescription("error while trying to sync the index state") + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/SearchV2Handler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/SearchV2Handler.java new file mode 100644 index 000000000..2362c3940 --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/SearchV2Handler.java @@ -0,0 +1,91 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.google.protobuf.Any; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat.Printer; +import com.yelp.nrtsearch.server.grpc.SearchRequest; +import com.yelp.nrtsearch.server.grpc.SearchResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.SearchHandler; +import com.yelp.nrtsearch.server.utils.ProtoMessagePrinter; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SearchV2Handler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(SearchV2Handler.class.getName()); + private static final Printer protoMessagePrinter = + ProtoMessagePrinter.omittingInsignificantWhitespace(); + + private final SearchHandler searchHandler; + + public SearchV2Handler(GlobalState globalState, SearchHandler searchHandler) { + super(globalState); + this.searchHandler = searchHandler; + } + + @Override + public void handle(SearchRequest searchRequest, StreamObserver responseObserver) { + try { + SearchResponse searchResponse = searchHandler.getSearchResponse(searchRequest); + setResponseCompression(searchRequest.getResponseCompression(), responseObserver); + responseObserver.onNext(Any.pack(searchResponse)); + responseObserver.onCompleted(); + } catch (IOException e) { + logger.warn( + "error while trying to read index state dir for indexName: {}", + searchRequest.getIndexName(), + e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to read index state dir for indexName: " + + searchRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + String searchRequestJson = null; + try { + searchRequestJson = protoMessagePrinter.print(searchRequest); + } catch (InvalidProtocolBufferException ignored) { + // Ignore as invalid proto would have thrown an exception earlier + } + logger.warn( + String.format( + "error while trying to execute search for index %s: request: %s", + searchRequest.getIndexName(), searchRequestJson), + e); + if (e instanceof StatusRuntimeException) { + responseObserver.onError(e); + } else { + responseObserver.onError( + Status.UNKNOWN + .withDescription( + String.format( + "error while trying to execute search for index %s. check logs for full searchRequest.", + searchRequest.getIndexName())) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StartIndexHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StartIndexHandler.java new file mode 100644 index 000000000..81707e663 --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StartIndexHandler.java @@ -0,0 +1,77 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.StartIndexRequest; +import com.yelp.nrtsearch.server.grpc.StartIndexResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StartIndexHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(StartIndexHandler.class); + + public StartIndexHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + StartIndexRequest startIndexRequest, StreamObserver responseObserver) { + logger.info("Received start index request: {}", startIndexRequest); + if (startIndexRequest.getIndexName().isEmpty()) { + logger.warn("error while trying to start index with empty index name."); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + String.format("error while trying to start index since indexName was empty.")) + .asRuntimeException()); + return; + } + try { + StartIndexResponse reply = getGlobalState().startIndex(startIndexRequest); + + logger.info("StartIndexHandler returned " + reply.toString()); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + + } catch (IOException e) { + logger.warn( + "error while trying to read index state dir for indexName: " + + startIndexRequest.getIndexName(), + e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to read index state dir for indexName: " + + startIndexRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + logger.warn("error while trying to start index " + startIndexRequest.getIndexName(), e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + "error while trying to start index: " + startIndexRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StartIndexV2Handler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StartIndexV2Handler.java new file mode 100644 index 000000000..0eb80873f --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StartIndexV2Handler.java @@ -0,0 +1,66 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.StartIndexResponse; +import com.yelp.nrtsearch.server.grpc.StartIndexV2Request; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StartIndexV2Handler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(StartIndexV2Handler.class); + + public StartIndexV2Handler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + StartIndexV2Request startIndexRequest, StreamObserver responseObserver) { + logger.info("Received start index v2 request: {}", startIndexRequest); + try { + StartIndexResponse reply = getGlobalState().startIndexV2(startIndexRequest); + logger.info("StartIndexV2Handler returned " + reply.toString()); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (IOException e) { + logger.warn( + "error while trying to read index state dir for indexName: " + + startIndexRequest.getIndexName(), + e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to read index state dir for indexName: " + + startIndexRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + logger.warn("error while trying to start index " + startIndexRequest.getIndexName(), e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + "error while trying to start index: " + startIndexRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/StatsRequestHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StatsHandler.java similarity index 70% rename from src/main/java/com/yelp/nrtsearch/server/luceneserver/StatsRequestHandler.java rename to src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StatsHandler.java index 5c9f16cb0..a53dc7f0f 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/StatsRequestHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StatsHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Yelp Inc. + * Copyright 2024 Yelp Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,19 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.yelp.nrtsearch.server.luceneserver; +package com.yelp.nrtsearch.server.luceneserver.handler; -import com.yelp.nrtsearch.server.grpc.IndexStatsResponse; -import com.yelp.nrtsearch.server.grpc.IndicesResponse; import com.yelp.nrtsearch.server.grpc.Searcher; import com.yelp.nrtsearch.server.grpc.StatsRequest; import com.yelp.nrtsearch.server.grpc.StatsResponse; import com.yelp.nrtsearch.server.grpc.Taxonomy; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.IndexState; +import com.yelp.nrtsearch.server.luceneserver.ShardState; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.lucene.facet.taxonomy.SearcherTaxonomyManager; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; @@ -36,21 +38,49 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class StatsRequestHandler implements Handler { - Logger logger = LoggerFactory.getLogger(StatsRequestHandler.class); +public class StatsHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(StatsHandler.class); + + public StatsHandler(GlobalState globalState) { + super(globalState); + } @Override - public StatsResponse handle(IndexState indexState, StatsRequest statsRequest) - throws HandlerException { + public void handle(StatsRequest statsRequest, StreamObserver responseObserver) { try { - return process(indexState); + IndexState indexState = getGlobalState().getIndex(statsRequest.getIndexName()); + indexState.verifyStarted(); + StatsResponse reply = process(indexState); + logger.debug("StatsHandler retrieved stats for index: {} ", reply); + responseObserver.onNext(reply); + responseObserver.onCompleted(); } catch (IOException e) { - logger.warn(" Failed to generate stats for index: " + indexState.getName(), e); - throw new HandlerException(e); + logger.warn( + "error while trying to read index state dir for indexName: {}", + statsRequest.getIndexName(), + e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to read index state dir for indexName: " + + statsRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + logger.warn( + "error while trying to retrieve stats for index {}", statsRequest.getIndexName(), e); + responseObserver.onError( + Status.UNKNOWN + .withDescription( + "error while trying to retrieve stats for index: " + statsRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); } } - private StatsResponse process(IndexState indexState) throws IOException { + // Public because it's used by IndicesHandler + public static StatsResponse process(IndexState indexState) throws IOException { StatsResponse.Builder statsResponseBuilder = StatsResponse.newBuilder(); if (indexState.getShards().size() > 1) { logger.error( @@ -127,29 +157,4 @@ public boolean doPrune(double ageSec, IndexSearcher indexSearcher) { } return statsResponseBuilder.build(); } - - public static IndicesResponse getIndicesResponse(GlobalState globalState) - throws IOException, HandlerException { - Set indexNames = globalState.getIndexNames(); - IndicesResponse.Builder builder = IndicesResponse.newBuilder(); - for (String indexName : indexNames) { - IndexState indexState = globalState.getIndex(indexName); - if (indexState.isStarted()) { - StatsResponse statsResponse = - new StatsRequestHandler() - .handle(indexState, StatsRequest.newBuilder().setIndexName(indexName).build()); - builder.addIndicesResponse( - IndexStatsResponse.newBuilder() - .setIndexName(indexName) - .setStatsResponse(statsResponse) - .build()); - } else { - builder.addIndicesResponse( - IndexStatsResponse.newBuilder() - .setIndexName(indexName) - .setStatsResponse(StatsResponse.newBuilder().setState("not_started").build())); - } - } - return builder.build(); - } } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StatusHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StatusHandler.java new file mode 100644 index 000000000..c27aeca4c --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StatusHandler.java @@ -0,0 +1,51 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.HealthCheckRequest; +import com.yelp.nrtsearch.server.grpc.HealthCheckResponse; +import com.yelp.nrtsearch.server.grpc.TransferStatusCode; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StatusHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(StatusHandler.class); + + public StatusHandler() { + super(null); + } + + @Override + public void handle( + HealthCheckRequest request, StreamObserver responseObserver) { + try { + HealthCheckResponse reply = + HealthCheckResponse.newBuilder().setHealth(TransferStatusCode.Done).build(); + logger.debug("HealthCheckResponse returned " + reply); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.warn("error while trying to get status", e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription("error while trying to get status") + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StopIndexHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StopIndexHandler.java new file mode 100644 index 000000000..5aa670ed1 --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/StopIndexHandler.java @@ -0,0 +1,53 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.DummyResponse; +import com.yelp.nrtsearch.server.grpc.StopIndexRequest; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StopIndexHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(StopIndexHandler.class); + + public StopIndexHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + StopIndexRequest stopIndexRequest, StreamObserver responseObserver) { + logger.info("Received stop index request: {}", stopIndexRequest); + try { + DummyResponse reply = getGlobalState().stopIndex(stopIndexRequest); + + logger.info("StopIndexHandler returned " + reply); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.warn("error while trying to stop index " + stopIndexRequest.getIndexName(), e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + "error while trying to stop index: " + stopIndexRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/UpdateFieldsHandler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/UpdateFieldsHandler.java new file mode 100644 index 000000000..f88126739 --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/handler/UpdateFieldsHandler.java @@ -0,0 +1,73 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.luceneserver.handler; + +import com.yelp.nrtsearch.server.grpc.FieldDefRequest; +import com.yelp.nrtsearch.server.grpc.FieldDefResponse; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager; +import com.yelp.nrtsearch.server.luceneserver.index.handlers.FieldUpdateHandler; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class UpdateFieldsHandler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(UpdateFieldsHandler.class); + + public UpdateFieldsHandler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + FieldDefRequest fieldDefRequest, StreamObserver responseObserver) { + logger.info("Received update fields request: {}", fieldDefRequest); + try { + IndexStateManager indexStateManager = + getGlobalState().getIndexStateManager(fieldDefRequest.getIndexName()); + FieldDefResponse reply = FieldUpdateHandler.handle(indexStateManager, fieldDefRequest); + logger.info("UpdateFieldsHandler registered fields " + reply); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (IOException e) { + logger.warn( + "error while trying to read index state dir for indexName: " + + fieldDefRequest.getIndexName(), + e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to read index state dir for indexName: " + + fieldDefRequest.getIndexName()) + .augmentDescription("IOException()") + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + logger.warn( + "error while trying to UpdateFieldsHandler for index " + fieldDefRequest.getIndexName(), + e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + "error while trying to UpdateFieldsHandler for index: " + + fieldDefRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } +} diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/index/handlers/LiveSettingsV2Handler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/index/handlers/LiveSettingsV2Handler.java index 302ebf217..1c942943c 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/index/handlers/LiveSettingsV2Handler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/index/handlers/LiveSettingsV2Handler.java @@ -15,16 +15,59 @@ */ package com.yelp.nrtsearch.server.luceneserver.index.handlers; +import com.google.protobuf.util.JsonFormat; import com.yelp.nrtsearch.server.grpc.IndexLiveSettings; import com.yelp.nrtsearch.server.grpc.LiveSettingsV2Request; import com.yelp.nrtsearch.server.grpc.LiveSettingsV2Response; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler; import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Static helper class to handle a LiveSettingsV2 request and produce a response. */ -public class LiveSettingsV2Handler { +public class LiveSettingsV2Handler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(LiveSettingsV2Handler.class); - private LiveSettingsV2Handler() {} + public LiveSettingsV2Handler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + LiveSettingsV2Request req, StreamObserver responseObserver) { + logger.info("Received live settings V2 request: {}", req); + try { + IndexStateManager indexStateManager = + getGlobalState().getIndexStateManager(req.getIndexName()); + LiveSettingsV2Response reply = handle(indexStateManager, req); + logger.info("LiveSettingsV2Handler returned " + JsonFormat.printer().print(reply)); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (IllegalArgumentException e) { + logger.warn("index: " + req.getIndexName() + " was not yet created", e); + responseObserver.onError( + Status.ALREADY_EXISTS + .withDescription("invalid indexName: " + req.getIndexName()) + .augmentDescription("IllegalArgumentException()") + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + logger.warn( + "error while trying to process live settings for indexName: " + req.getIndexName(), e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to process live settings for indexName: " + + req.getIndexName()) + .augmentDescription("Exception()") + .withCause(e) + .asRuntimeException()); + } + } /** * Handle a LiveSettingsV2 request. diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/index/handlers/SettingsV2Handler.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/index/handlers/SettingsV2Handler.java index 236aa22af..5df43f21c 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/index/handlers/SettingsV2Handler.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/index/handlers/SettingsV2Handler.java @@ -15,16 +15,64 @@ */ package com.yelp.nrtsearch.server.luceneserver.index.handlers; +import com.google.protobuf.util.JsonFormat; import com.yelp.nrtsearch.server.grpc.IndexSettings; import com.yelp.nrtsearch.server.grpc.SettingsV2Request; import com.yelp.nrtsearch.server.grpc.SettingsV2Response; +import com.yelp.nrtsearch.server.luceneserver.GlobalState; +import com.yelp.nrtsearch.server.luceneserver.handler.Handler; import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Static helper class to handle a SettingsV2 request and produce a response. */ -public class SettingsV2Handler { +public class SettingsV2Handler extends Handler { + private static final Logger logger = LoggerFactory.getLogger(SettingsV2Handler.class); - private SettingsV2Handler() {} + public SettingsV2Handler(GlobalState globalState) { + super(globalState); + } + + @Override + public void handle( + SettingsV2Request settingsRequest, StreamObserver responseObserver) { + logger.info("Received settings V2 request: {}", settingsRequest); + try { + IndexStateManager indexStateManager = + getGlobalState().getIndexStateManager(settingsRequest.getIndexName()); + SettingsV2Response reply = handle(indexStateManager, settingsRequest); + logger.info("SettingsV2Handler returned: " + JsonFormat.printer().print(reply)); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } catch (IOException e) { + logger.warn( + "error while trying to read index state dir for indexName: " + + settingsRequest.getIndexName(), + e); + responseObserver.onError( + Status.INTERNAL + .withDescription( + "error while trying to read index state dir for indexName: " + + settingsRequest.getIndexName()) + .augmentDescription("IOException()") + .withCause(e) + .asRuntimeException()); + } catch (Exception e) { + logger.warn( + "error while trying to update/get settings for index " + settingsRequest.getIndexName(), + e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription( + "error while trying to update/get settings for index: " + + settingsRequest.getIndexName()) + .augmentDescription(e.getMessage()) + .asRuntimeException()); + } + } /** * Handle a SettingsV2 request. diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/warming/Warmer.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/warming/Warmer.java index b2b6aa879..79b88c44a 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/warming/Warmer.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/warming/Warmer.java @@ -104,7 +104,9 @@ public synchronized void backupWarmingQueriesToS3(String service) throws IOExcep public void warmFromS3(IndexState indexState, int parallelism) throws IOException, SearchHandler.SearchHandlerException, InterruptedException { - SearchHandler searchHandler = new SearchHandler(indexState.getSearchThreadPoolExecutor(), true); + SearchHandler searchHandler = + new SearchHandler( + indexState.getGlobalState(), indexState.getSearchThreadPoolExecutor(), true); warmFromS3(indexState, parallelism, searchHandler); } diff --git a/src/main/java/com/yelp/nrtsearch/server/utils/ProtoMessagePrinter.java b/src/main/java/com/yelp/nrtsearch/server/utils/ProtoMessagePrinter.java new file mode 100644 index 000000000..e9e56411b --- /dev/null +++ b/src/main/java/com/yelp/nrtsearch/server/utils/ProtoMessagePrinter.java @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Yelp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yelp.nrtsearch.server.utils; + +import com.google.protobuf.util.JsonFormat; +import com.google.protobuf.util.JsonFormat.Printer; + +public class ProtoMessagePrinter { + private static final JsonFormat.Printer OMITTING_INSIGNIFICANT_WHITESPACE = + JsonFormat.printer().omittingInsignificantWhitespace(); + + public static Printer omittingInsignificantWhitespace() { + return OMITTING_INSIGNIFICANT_WHITESPACE; + } +}