Cleanup lucene server usage (#783)

* Clean up references to lucene server

* Additional fixes

README.md

@@ -53,7 +53,7 @@ Note: This code has been tested on *Java21*
 # Run gRPC Server
 
 ```
-./build/install/nrtsearch/bin/lucene-server
+./build/install/nrtsearch/bin/nrtsearch_server
 ```
 
 # Build gRPC Gateway
@@ -72,7 +72,7 @@ Note: This code has been tested on *Java21*
 ## Create Index
 
 ```
-./build/install/nrtsearch/bin/lucene-client createIndex --indexName  testIdx
+./build/install/nrtsearch/bin/nrtsearch_client createIndex --indexName  testIdx
 ```
 
 ```
@@ -82,7 +82,7 @@ curl -XPOST localhost:<REST_PORT>/v1/create_index -d '{"indexName": "testIdx"}'
 ## Update Settings
 
 ```
-./build/install/nrtsearch/bin/lucene-client settings -f settings.json
+./build/install/nrtsearch/bin/nrtsearch_client settings -f settings.json
 cat settings.json
 {             "indexName": "testIdx",
               "directory": "MMapDirectory",
@@ -96,7 +96,7 @@ cat settings.json
 ## Start Index
 
 ```
-./build/install/nrtsearch/bin/lucene-client startIndex -f startIndex.json
+./build/install/nrtsearch/bin/nrtsearch_client startIndex -f startIndex.json
 cat startIndex.json
 {
   "indexName" : "testIdx"
@@ -106,7 +106,7 @@ cat startIndex.json
 ## RegisterFields
 
 ```
-./build/install/nrtsearch/bin/lucene-client registerFields -f registerFields.json
+./build/install/nrtsearch/bin/nrtsearch_client registerFields -f registerFields.json
 cat registerFields.json
 {             "indexName": "testIdx",
               "field":
@@ -121,7 +121,7 @@ cat registerFields.json
 ## Add Documents
 
 ```
-./build/install/nrtsearch/bin/lucene-client addDocuments -i testIdx -f docs.csv -t csv
+./build/install/nrtsearch/bin/nrtsearch_client addDocuments -i testIdx -f docs.csv -t csv
 cat docs.csv
 doc_id,vendor_name,license_no
 0,first vendor,100;200
@@ -131,7 +131,7 @@ doc_id,vendor_name,license_no
 ## Search
 
 ```
-./build/install/nrtsearch/bin/lucene-client search -f search.json
+./build/install/nrtsearch/bin/nrtsearch_client search -f search.json
 cat search.json
 {
         "indexName": "testIdx",

build.gradle

@@ -116,6 +116,22 @@ task luceneServerClient(type: CreateStartScripts) {
     classpath = startScripts.classpath
 }
 
+task nrtsearchServer(type: CreateStartScripts) {
+    mainClass = 'com.yelp.nrtsearch.server.grpc.NrtsearchServer'
+    applicationName = 'nrtsearch_server'
+    outputDir = new File(project.buildDir, 'tmp-app')
+    classpath = startScripts.classpath
+    // Add additional dependencies, e.g. custom loggers
+    classpath += files('$APP_HOME/additional_libs')
+}
+
+task nrtsearchServerClient(type: CreateStartScripts) {
+    mainClass = 'com.yelp.nrtsearch.tools.cli.NrtsearchClientCommand'
+    applicationName = 'nrtsearch_client'
+    outputDir = new File(project.buildDir, 'tmp-app')
+    classpath = startScripts.classpath
+}
+
 task nrtUtils(type: CreateStartScripts) {
     mainClass = 'com.yelp.nrtsearch.tools.nrt_utils.NrtUtilsCommand'
     applicationName = 'nrt_utils'
@@ -126,6 +142,8 @@ task nrtUtils(type: CreateStartScripts) {
 applicationDistribution.into('bin') {
     from(luceneServer)
     from(luceneServerClient)
+    from(nrtsearchServer)
+    from(nrtsearchServerClient)
     from(nrtUtils)
     fileMode = 0755
 }
@@ -141,7 +159,7 @@ task buildGrpcGateway(dependsOn: installDist, type: Exec) {
 //e.g. default is to exclude perfTests: ./gradlew test
 test {
     finalizedBy 'spotlessJavaCheck'
-    // Used by LuceneServerConfigurationTest
+    // Used by NrtsearchConfigTest
     environment(Map.of('CUSTOM_HOST', 'my_custom_host', 'VAR1', 'v1', 'VAR2', 'v2', 'VAR3', 'v3'))
     if (project.hasProperty('longRunningTestsOnly')) {
         include '**/IncrementalDataCleanupCommandTest.class'
@@ -209,7 +227,7 @@ publishing {
             artifact tasks.testsJar
             pom {
                 name = 'nrtSearch Server'
-                description = 'GRPC lucene server using near-real-time replication'
+                description = 'GRPC server using near-real-time replication'
                 url = 'https://github.com/Yelp/nrtsearch'
                 licenses {
                     license {

docker-compose-config/entrypoint_replica.sh

@@ -5,10 +5,10 @@ HOSTNAME=$(hostname -i)
 echo "hostname: "$HOSTNAME
 
 echo "replacing nodeName"
-sed -i "s/node-name/$HOSTNAME/g" docker-compose-config/lucene_server_configuration_replica.yaml
+sed -i "s/node-name/$HOSTNAME/g" docker-compose-config/nrtsearch_replica_config.yaml
 
 echo "replacing nostname"
-sed -i "s/host-name-replica/$HOSTNAME/g" docker-compose-config/lucene_server_configuration_replica.yaml
+sed -i "s/host-name-replica/$HOSTNAME/g" docker-compose-config/nrtsearch_replica_config.yaml
 
 echo "starting service"
-/user/app/build/install/nrtsearch/bin/lucene-server /user/app/docker-compose-config/lucene_server_configuration_replica.yaml
+/user/app/build/install/nrtsearch/bin/nrtsearch_server /user/app/docker-compose-config/nrtsearch_replica_config.yaml

docker-compose-config/lucene_server_configuration_primary.yaml → docker-compose-config/nrtsearch_primary_config.yaml

@@ -1,4 +1,4 @@
-nodeName: "lucene_server_primary"
+nodeName: "nrtsearch_primary"
 hostName: "primary-node"
 port: "8000"
 replicationPort: "8001"

docker-compose.yaml

@@ -6,7 +6,7 @@ services:
     build: ./
     container_name: primary-node
     command: >
-      bash -c "/user/app/build/install/nrtsearch/bin/lucene-server /user/app/docker-compose-config/lucene_server_configuration_primary.yaml"
+      bash -c "/user/app/build/install/nrtsearch/bin/nrtsearch_server /user/app/docker-compose-config/nrtsearch_primary_config.yaml"
     hostname: primary-node
 
   replica-node-1:

docs/analysis.rst

@@ -158,7 +158,7 @@ Available character filters:
 
   * patternreplace
 
-  * mappingV2 - Similar to the ``mapping`` filter, except rules are specified directly in the parameters. See `MappingV2CharFilterFactory <https://github.com/Yelp/nrtsearch/blob/master/src/main/java/com/yelp/nrtsearch/server/luceneserver/analysis/MappingV2CharFilterFactory.java>`_.
+  * mappingV2 - Similar to the ``mapping`` filter, except rules are specified directly in the parameters. See `MappingV2CharFilterFactory <https://github.com/Yelp/nrtsearch/blob/master/src/main/java/com/yelp/nrtsearch/server/analysis/MappingV2CharFilterFactory.java>`_.
 
 Available tokenizers:
 
@@ -388,7 +388,7 @@ Available token filters:
 
   * synonym
 
-  * synonymV2 - Similar to the ``synonymGraph`` filter, except rules are specified directly in the parameters. See `SynonymV2GraphFilterFactory <https://github.com/Yelp/nrtsearch/blob/master/src/main/java/com/yelp/nrtsearch/server/luceneserver/analysis/SynonymV2GraphFilterFactory.java>`_.
+  * synonymV2 - Similar to the ``synonymGraph`` filter, except rules are specified directly in the parameters. See `SynonymV2GraphFilterFactory <https://github.com/Yelp/nrtsearch/blob/master/src/main/java/com/yelp/nrtsearch/server/analysis/SynonymV2GraphFilterFactory.java>`_.
 
   * synonymGraph
 

docs/docker_compose.rst

@@ -44,7 +44,7 @@ Note: The S3 bucket must exist before hand, and is named in the server configura
 
 .. code-block::
 
-  shell% cat docker-compose-config/lucene_server_configuration_primary.yaml # for the replica config as well
+  shell% cat docker-compose-config/nrtsearch_primary_config.yaml # for the replica config as well
   ...
   bucketName: "nrtsearch-bucket"
   ...
@@ -74,10 +74,10 @@ Create the index and settings, register the fields, and start the index:
 
   shell% PRIMARY_CONTAINER_ID=$(docker ps | grep nrtsearch_primary-node | awk '{print $1}')
   shell% docker exec -it $PRIMARY_CONTAINER_ID sh
-  # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 createIndex --indexName  testIdx
-  # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 settings -f docker-compose-config/settings_primary.json
-  # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 registerFields -f docker-compose-config/registerFields.json
-  # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 startIndex -f docker-compose-config/startIndex_primary.json
+  # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 createIndex --indexName  testIdx
+  # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 settings -f docker-compose-config/settings_primary.json
+  # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 registerFields -f docker-compose-config/registerFields.json
+  # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 startIndex -f docker-compose-config/startIndex_primary.json
 
 3. Replica: Start Index
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -88,10 +88,10 @@ Next go into any one of the replicas (i.e. nrtsearch_replica-node-1 here), and r
 
   shell% REPLICA_1_CONTAINER_ID=$(docker ps  | grep nrtsearch_replica-node-1_1 | awk '{print $1}')
   shell% docker exec -it $REPLICA_1_CONTAINER_ID sh
-  # ./build/install/nrtsearch/bin/lucene-client -h replica-node-1 -p 8002 createIndex --indexName  testIdx
-  # ./build/install/nrtsearch/bin/lucene-client -h replica-node-1 -p 8002 settings -f docker-compose-config/settings_replica.json
-  # ./build/install/nrtsearch/bin/lucene-client -h replica-node-1 -p 8002 registerFields -f docker-compose-config/registerFields.json
-  # ./build/install/nrtsearch/bin/lucene-client -h replica-node-1 -p 8002 startIndex -f docker-compose-config/startIndex_replica.json
+  # ./build/install/nrtsearch/bin/nrtsearch_client -h replica-node-1 -p 8002 createIndex --indexName  testIdx
+  # ./build/install/nrtsearch/bin/nrtsearch_client -h replica-node-1 -p 8002 settings -f docker-compose-config/settings_replica.json
+  # ./build/install/nrtsearch/bin/nrtsearch_client -h replica-node-1 -p 8002 registerFields -f docker-compose-config/registerFields.json
+  # ./build/install/nrtsearch/bin/nrtsearch_client -h replica-node-1 -p 8002 startIndex -f docker-compose-config/startIndex_replica.json
 
 4. Replication
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -101,7 +101,7 @@ Search will work on replicas soon after documents are added on the primary. Add
 .. code-block::
 
   shell% docker exec -it $PRIMARY_CONTAINER_ID sh
-  # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 addDocuments -i testIdx -f docker-compose-config/docs.csv -t csv
+  # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 addDocuments -i testIdx -f docker-compose-config/docs.csv -t csv
 
 5. Replica: Search Should Work
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -111,7 +111,7 @@ The search should now work on any of the replicas where the the index was starte
 .. code-block::
 
   shell% docker exec -it $REPLICA_1_CONTAINER_ID sh
-  # ./build/install/nrtsearch/bin/lucene-client -h replica-node-1 -p 8002 search -f docker-compose-config/search.json
+  # ./build/install/nrtsearch/bin/nrtsearch_client -h replica-node-1 -p 8002 search -f docker-compose-config/search.json
   ...
   fields {
     key: "license_no"
@@ -134,7 +134,7 @@ In order to backup the index, one can use the 'backupIndex' command with these p
 .. code-block::
 
   shell% docker exec -it $PRIMARY_CONTAINER_ID sh
-  # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 backupIndex  --indexName testIdx --serviceName nrtsearch-service-test --resourceName testIdx
+  # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 backupIndex  --indexName testIdx --serviceName nrtsearch-service-test --resourceName testIdx
 
 Now the S3 bucket 'nrtsearch-bucket' should contain the service 'nrtsearch-service-test' data :
 
@@ -152,14 +152,14 @@ To demonstrate how one can start nrtSearch and restore the index data from S3, o
 
 .. code-block::
 
-  # update the 2 lucene service configs docker-compose-config/lucene_server_configuration_{primary,replica}.yaml to have this line:
-  shell% cat docker-compose-config/lucene_server_configuration_primary.yaml
+  # update the 2 lucene service configs docker-compose-config/nrtsearch_{primary,replica}_config.yaml to have this line:
+  shell% cat docker-compose-config/nrtsearch_primary_config.yaml
   ...
   # previous lines still there, change this line:
   restoreState: True
   ...
   ...
-  shell% cat docker-compose-config/lucene_server_configuration_replica.yaml
+  shell% cat docker-compose-config/nrtsearch_replica_config.yaml
   ...
   # previous lines still there, change this line:
   restoreState: True
@@ -195,8 +195,8 @@ If one then restarts the containers and restarts the index (do not need to regis
   shell% docker-compose -f docker-compose.yaml up
   shell% PRIMARY_CONTAINER_ID=$(docker ps | grep nrtsearch_primary-node | awk '{print $1}')
   shell% docker exec -it $PRIMARY_CONTAINER_ID sh
-  # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 startIndex -f docker-compose-config/startIndex_primary.json
-  # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 search -f docker-compose-config/search.json
+  # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 startIndex -f docker-compose-config/startIndex_primary.json
+  # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 search -f docker-compose-config/search.json
   ...
   fields {
     key: "license_no"
@@ -218,9 +218,9 @@ To view the logs in the containers use docker-compose logs:
 .. code-block::
 
   shell% docker-compose logs
-  replica-node-1_1  | [INFO ] 2021-12-13 18:58:26.527 [main] LuceneServer - Server started, listening on 8003 for replication messages
+  replica-node-1_1  | [INFO ] 2021-12-13 18:58:26.527 [main] NrtsearchServer - Server started, listening on 8003 for replication messages
   replica-node-1_2  | hostname: 172.24.0.2
-  primary-node      | [INFO ] 2021-12-13 18:58:28.530 [main] LuceneServer - Server started, listening on 8001 for replication messages
+  primary-node      | [INFO ] 2021-12-13 18:58:28.530 [main] NrtsearchServer - Server started, listening on 8001 for replication messages
 
 Stop
 """""""""""""""""""""""""""

docs/index_live_settings.rst

@@ -10,7 +10,7 @@ Changes to these settings can be made any time after the index is created. There
 
 * Directly use the `liveSetting/liveSettingV2 <https://github.com/Yelp/nrtsearch/blob/master/clientlib/src/main/proto/yelp/nrtsearch/luceneserver.proto#L35>`_ gRPC server endpoints
 * Use the '/v1/live_settings' or '/v2/live_settings' endpoints with the gRPC gateway
-* Use the lucene-client `liveSettings/liveSettingsV2 <https://github.com/Yelp/nrtsearch/blob/master/src/main/java/com/yelp/nrtsearch/server/cli/LuceneClientCommand.java>`_ commands
+* Use the nrtsearch_client `liveSettings/liveSettingsV2 <https://github.com/Yelp/nrtsearch/blob/main/src/main/java/com/yelp/nrtsearch/tools/cli/NrtsearchClientCommand.java>`_ commands
 
 Properties
 -----------------------------

docs/index_settings.rst

@@ -10,7 +10,7 @@ Changes to these settings can be made after the index is created, but before it
 
 * Directly use the `setting/settingV2 <https://github.com/Yelp/nrtsearch/blob/master/clientlib/src/main/proto/yelp/nrtsearch/luceneserver.proto#L80>`_ gRPC server endpoints
 * Use the '/v1/settings' or '/v2/settings' endpoints with the gRPC gateway
-* Use the lucene-client `settings/settingsV2 <https://github.com/Yelp/nrtsearch/blob/master/src/main/java/com/yelp/nrtsearch/server/cli/LuceneClientCommand.java>`_ commands
+* Use the nrtsearch_client `settings/settingsV2 <https://github.com/Yelp/nrtsearch/blob/main/src/main/java/com/yelp/nrtsearch/tools/cli/NrtsearchClientCommand.java>`_ commands
 
 Properties
 -----------------------------

docs/introduction.rst

@@ -65,7 +65,7 @@ The server can be run locally like this:
 
 .. code-block::
 
-  ./build/install/nrtsearch/bin/lucene-server
+  ./build/install/nrtsearch/bin/nrtsearch_server
 
 Run REST Server
 ---------------------------

docs/server_configuration.rst

@@ -6,13 +6,13 @@ Usage
 
 .. code-block::
 
-  ./build/install/nrtsearch/bin/lucene-server server_configuration.yaml
+  ./build/install/nrtsearch/bin/nrtsearch_server server_configuration.yaml
 
 Example server configuration
 
 .. code-block:: yaml
 
-  nodeName: "lucene_server_primary"
+  nodeName: "nrtsearch_primary"
   hostName: "primary-node"
   port: "8000"
   replicationPort: "8001"
@@ -28,7 +28,7 @@ Example server configuration
   serviceName: "nrtsearch-service-test"
 
 
-.. list-table:: `LuceneServerConfiguration <https://github.com/Yelp/nrtsearch/blob/master/src/main/java/com/yelp/nrtsearch/server/config/LuceneServerConfiguration.java>`_
+.. list-table:: `NrtsearchConfig <https://github.com/Yelp/nrtsearch/blob/master/src/main/java/com/yelp/nrtsearch/server/config/NrtsearchConfig.java>`_
    :widths: 25 10 50 25
    :header-rows: 1
 
@@ -39,17 +39,17 @@ Example server configuration
 
    * - nodeName
      - str
-     - Name of this NrtSearch instance. Currently used for emitting metrics labels.
+     - Name of this NrtSearch instance. Currently used for registering replicas with primary. This property supports `environment variable substitution <https://github.com/Yelp/nrtsearch/blob/6a9049a840fc2da4816e2a6cf1837bd31218ae97/src/main/java/com/yelp/nrtsearch/server/config/NrtsearchConfig.java#L386>`_.
      - main
 
    * - hostName
      - str
-     - Hostname of this NrtSearch instance. Replicas use this property when registering with the primary. This property supports `environment variable substitution <https://github.com/Yelp/nrtsearch/blob/2ae8bae079ae8a8a59bb896fee775919235710aa/src/main/java/com/yelp/nrtsearch/server/config/LuceneServerConfiguration.java#L298>`_.
+     - Hostname of this NrtSearch instance. Replicas use this property when registering with the primary. This property supports `environment variable substitution <https://github.com/Yelp/nrtsearch/blob/6a9049a840fc2da4816e2a6cf1837bd31218ae97/src/main/java/com/yelp/nrtsearch/server/config/NrtsearchConfig.java#L386>`_.
      - localhost
 
    * - port
      - str
-     - Port for LuceneServer gRPC requests
+     - Port for NrtsearchServer gRPC requests
      - 50051
 
    * - replicationPort
@@ -60,12 +60,12 @@ Example server configuration
    * - stateDir
      - str
      - Path of global state directory
-     - `<DEFAULT_USER_DIR> <https://github.com/Yelp/nrtsearch/blob/f612f5d3e14e468ab8c9b45dd4be0ab84231b9de/src/main/java/com/yelp/nrtsearch/server/config/LuceneServerConfiguration.java#L35>`_/default_state
+     - `<DEFAULT_USER_DIR> <https://github.com/Yelp/nrtsearch/blob/6a9049a840fc2da4816e2a6cf1837bd31218ae97/src/main/java/com/yelp/nrtsearch/server/config/NrtsearchConfig.java#L45>`_/default_state
 
    * - indexDir
      - str
      - Path of directory containing index state and segments
-     - `<DEFAULT_USER_DIR> <https://github.com/Yelp/nrtsearch/blob/f612f5d3e14e468ab8c9b45dd4be0ab84231b9de/src/main/java/com/yelp/nrtsearch/server/config/LuceneServerConfiguration.java#L35>`_/default_index
+     - `<DEFAULT_USER_DIR> <https://github.com/Yelp/nrtsearch/blob/6a9049a840fc2da4816e2a6cf1837bd31218ae97/src/main/java/com/yelp/nrtsearch/server/config/NrtsearchConfig.java#L45>`_/default_index
 
    * - bucketName
      - str
@@ -146,20 +146,20 @@ Example server configuration
      - Name prefix for threads created by indexing threadpool executor
      - LuceneIndexingExecutor
 
-   * - luceneserver.maxThreads
+   * - server.maxThreads
      - int
-     - Size of LuceneServer threadpool executor
+     - Size of NrtsearchServer threadpool executor
      - numCPUs + 1
 
-   * - luceneserver.maxBufferedItems
+   * - server.maxBufferedItems
      - int
-     - Max tasks that can be queued by LuceneServer threadpool executor
+     - Max tasks that can be queued by NrtsearchServer threadpool executor
      - max(200, 2 * (numCPUs + 1))
 
-   * - luceneserver.threadNamePrefix
+   * - server.threadNamePrefix
      - string
-     - Name prefix for threads created by LuceneServer threadpool executor
-     - GrpcLuceneServerExecutor
+     - Name prefix for threads created by NrtsearchServer threadpool executor
+     - GrpcServerExecutor
 
    * - replicationserver.maxThreads
      - int
@@ -265,7 +265,7 @@ Example server configuration
      - Offset in threads formula: (numCPUs * multiplier) + offset
      - 0
 
-.. list-table:: `Warmer Configuration <https://github.com/Yelp/nrtsearch/blob/master/src/main/java/com/yelp/nrtsearch/server/luceneserver/warming/WarmerConfig.java>`_ (``warmer.*``)
+.. list-table:: `Warmer Configuration <https://github.com/Yelp/nrtsearch/blob/master/src/main/java/com/yelp/nrtsearch/server/warming/WarmerConfig.java>`_ (``warmer.*``)
    :widths: 25 10 50 25
    :header-rows: 1