Add ability to update local index live settings (#611)

clientlib/src/main/proto/yelp/nrtsearch/luceneserver.proto

@@ -485,6 +485,8 @@ message LiveSettingsV2Request {
     string indexName = 1;
     // Live settings to merge into existing live settings, or unset to get current live settings
     IndexLiveSettings liveSettings = 2;
+    // When set to true, live settings changes are only applied to the local node. These changes are ephemeral, so will not persist through a restart. Also, the live settings returned in the response will contain the local overrides only when this flag is true.
+    bool local = 3;
 }
 
 message LiveSettingsV2Response {

grpc-gateway/luceneserver.swagger.json

@@ -1389,6 +1389,13 @@
             "in": "query",
             "required": false,
             "type": "boolean"
+          },
+          {
+            "name": "local",
+            "description": "When set to true, live settings changes are only applied to the local node. These changes are ephemeral, so will not persist through a restart. Also, the live settings returned in the response will contain the local overrides only when this flag is true.",
+            "in": "query",
+            "required": false,
+            "type": "boolean"
           }
         ],
         "tags": [
@@ -3883,6 +3890,10 @@
         "liveSettings": {
           "$ref": "#/definitions/luceneserverIndexLiveSettings",
           "title": "Live settings to merge into existing live settings, or unset to get current live settings"
+        },
+        "local": {
+          "type": "boolean",
+          "description": "When set to true, live settings changes are only applied to the local node. These changes are ephemeral, so will not persist through a restart. Also, the live settings returned in the response will contain the local overrides only when this flag is true."
         }
       }
     },

src/main/java/com/yelp/nrtsearch/server/cli/LiveSettingsV2Command.java

@@ -117,6 +117,12 @@ public class LiveSettingsV2Command implements Callable<Integer> {
           "If additional index metrics should be collected and published, must be 'true' or 'false'")
   private String verboseMetrics;
 
+  @CommandLine.Option(
+      names = {"--local"},
+      description =
+          "Applies changes ephemerally to local node only. Response contains local settings only when this flag is used.")
+  private boolean local;
+
   @Override
   public Integer call() throws Exception {
     LuceneServerClient client = baseCmd.getClient();
@@ -188,6 +194,7 @@ public Integer call() throws Exception {
       if (!indexLiveSettings.getAllFields().isEmpty()) {
         settingsRequestV2Builder.setLiveSettings(indexLiveSettings);
       }
+      settingsRequestV2Builder.setLocal(local);
       client.liveSettingsV2(settingsRequestV2Builder.build());
     } finally {
       client.shutdown();

src/main/java/com/yelp/nrtsearch/server/luceneserver/LiveSettingsHandler.java

@@ -120,7 +120,7 @@ private LiveSettingsResponse handleAsLiveSettingsV2(
                 .build());
       }
       try {
-        updatedSettings = indexStateManager.updateLiveSettings(settingsBuilder.build());
+        updatedSettings = indexStateManager.updateLiveSettings(settingsBuilder.build(), false);
       } catch (IOException e) {
         throw new RuntimeException("Unable to update index live settings", e);
       }

src/main/java/com/yelp/nrtsearch/server/luceneserver/index/BackendStateManager.java

@@ -48,9 +48,9 @@ public class BackendStateManager implements IndexStateManager {
   private final String indexName;
   private final String indexUniqueName;
   private final String id;
-  private final IndexLiveSettings liveSettingsOverrides;
   private final StateBackend stateBackend;
   private final GlobalState globalState;
+  private IndexLiveSettings liveSettingsOverrides;
 
   // volatile for atomic replacement
   private volatile ImmutableIndexState currentState;
@@ -60,20 +60,20 @@ public class BackendStateManager implements IndexStateManager {
    *
    * @param indexName index name
    * @param id index instance id
-   * @param liveSettingsOverrides local overrides for index live settings
+   * @param initialLiveSettingsOverrides initial local overrides for index live settings
    * @param stateBackend state backend
    * @param globalState global state
    */
   public BackendStateManager(
       String indexName,
       String id,
-      IndexLiveSettings liveSettingsOverrides,
+      IndexLiveSettings initialLiveSettingsOverrides,
       StateBackend stateBackend,
       GlobalState globalState) {
     this.indexName = indexName;
     this.id = id;
     this.indexUniqueName = BackendGlobalState.getUniqueIndexName(indexName, id);
-    this.liveSettingsOverrides = liveSettingsOverrides;
+    this.liveSettingsOverrides = initialLiveSettingsOverrides;
     this.stateBackend = stateBackend;
     this.globalState = globalState;
   }
@@ -166,23 +166,35 @@ public IndexLiveSettings getLiveSettings(boolean withLocal) {
   }
 
   @Override
-  public synchronized IndexLiveSettings updateLiveSettings(IndexLiveSettings liveSettings)
-      throws IOException {
+  public synchronized IndexLiveSettings updateLiveSettings(
+      IndexLiveSettings liveSettings, boolean local) throws IOException {
     logger.info("Updating live settings for index: " + indexName + " : " + liveSettings);
     if (currentState == null) {
       throw new IllegalStateException("No state for index: " + indexName);
     }
-    IndexStateInfo updatedStateInfo =
-        mergeLiveSettings(currentState.getCurrentStateInfo(), liveSettings);
-    ImmutableIndexState updatedIndexState =
-        createIndexState(
-            updatedStateInfo, currentState.getFieldAndFacetState(), liveSettingsOverrides);
-    stateBackend.commitIndexState(indexUniqueName, updatedStateInfo);
+    ImmutableIndexState updatedIndexState;
+    if (local) {
+      IndexLiveSettings updatedLiveSettingsOverrides =
+          ImmutableIndexState.mergeLiveSettings(liveSettingsOverrides, liveSettings);
+      updatedIndexState =
+          createIndexState(
+              currentState.getCurrentStateInfo(),
+              currentState.getFieldAndFacetState(),
+              updatedLiveSettingsOverrides);
+      liveSettingsOverrides = updatedLiveSettingsOverrides;
+    } else {
+      IndexStateInfo updatedStateInfo =
+          mergeLiveSettings(currentState.getCurrentStateInfo(), liveSettings);
+      updatedIndexState =
+          createIndexState(
+              updatedStateInfo, currentState.getFieldAndFacetState(), liveSettingsOverrides);
+      stateBackend.commitIndexState(indexUniqueName, updatedStateInfo);
+    }
     currentState = updatedIndexState;
     for (Map.Entry<Integer, ShardState> entry : currentState.getShards().entrySet()) {
       entry.getValue().updatedLiveSettings(liveSettings);
     }
-    return updatedIndexState.getMergedLiveSettings(false);
+    return updatedIndexState.getMergedLiveSettings(local);
   }
 
   @Override

src/main/java/com/yelp/nrtsearch/server/luceneserver/index/IndexStateManager.java

@@ -75,10 +75,13 @@ public interface IndexStateManager extends Closeable {
    * the existing index settings, replacing any values that exist.
    *
    * @param liveSettings live settings modifications
-   * @return merged index settings, including default values
+   * @param local only apply settings changes to local node
+   * @return merged index settings, including default values. Only if local is true will local
+   *     overrides be included.
    * @throws IOException on error accessing state
    */
-  IndexLiveSettings updateLiveSettings(IndexLiveSettings liveSettings) throws IOException;
+  IndexLiveSettings updateLiveSettings(IndexLiveSettings liveSettings, boolean local)
+      throws IOException;
 
   /**
    * Update index fields with the given {@link Field} messages. Current only supports addition of

src/main/java/com/yelp/nrtsearch/server/luceneserver/index/handlers/LiveSettingsV2Handler.java

@@ -41,9 +41,10 @@ public static LiveSettingsV2Response handle(
     // if there is no settings message in the request, just return the current settings
     if (liveSettingsRequest.hasLiveSettings()) {
       responseSettings =
-          indexStateManager.updateLiveSettings(liveSettingsRequest.getLiveSettings());
+          indexStateManager.updateLiveSettings(
+              liveSettingsRequest.getLiveSettings(), liveSettingsRequest.getLocal());
     } else {
-      responseSettings = indexStateManager.getLiveSettings(false);
+      responseSettings = indexStateManager.getLiveSettings(liveSettingsRequest.getLocal());
     }
     return LiveSettingsV2Response.newBuilder().setLiveSettings(responseSettings).build();
   }

src/main/java/com/yelp/nrtsearch/server/luceneserver/state/BackendGlobalState.java

@@ -297,7 +297,7 @@ public synchronized IndexState createIndex(CreateIndexRequest createIndexRequest
       stateManager.updateSettings(createIndexRequest.getSettings());
     }
     if (createIndexRequest.hasLiveSettings()) {
-      stateManager.updateLiveSettings(createIndexRequest.getLiveSettings());
+      stateManager.updateLiveSettings(createIndexRequest.getLiveSettings(), false);
     }
     if (!createIndexRequest.getFieldsList().isEmpty()) {
       stateManager.updateFields(createIndexRequest.getFieldsList());

src/test/java/com/yelp/nrtsearch/server/cli/LiveSettingsV2CommandTest.java

@@ -109,4 +109,59 @@ public void testVerboseMetrics_invalid() throws IOException {
             "--verboseMetrics=invalid");
     assertEquals(1, exitCode);
   }
+
+  @Test
+  public void testSetsSettings() throws IOException {
+    TestServer server = getTestServer();
+    server.createSimpleIndex("test_index");
+    server.startIndexV2(StartIndexV2Request.newBuilder().setIndexName("test_index").build());
+
+    assertEquals(
+        0.0, server.getGlobalState().getIndex("test_index").getDefaultSearchTimeoutSec(), 0);
+
+    CommandLine cmd = new CommandLine(new LuceneClientCommand());
+    int exitCode =
+        cmd.execute(
+            "--hostname=localhost",
+            "--port=" + server.getPort(),
+            "liveSettingsV2",
+            "--indexName=test_index",
+            "--defaultSearchTimeoutSec=1.0");
+    assertEquals(0, exitCode);
+    assertEquals(
+        1.0, server.getGlobalState().getIndex("test_index").getDefaultSearchTimeoutSec(), 0);
+
+    server.restart();
+
+    assertEquals(
+        1.0, server.getGlobalState().getIndex("test_index").getDefaultSearchTimeoutSec(), 0);
+  }
+
+  @Test
+  public void testSetsLocalSettings() throws IOException {
+    TestServer server = getTestServer();
+    server.createSimpleIndex("test_index");
+    server.startIndexV2(StartIndexV2Request.newBuilder().setIndexName("test_index").build());
+
+    assertEquals(
+        0.0, server.getGlobalState().getIndex("test_index").getDefaultSearchTimeoutSec(), 0);
+
+    CommandLine cmd = new CommandLine(new LuceneClientCommand());
+    int exitCode =
+        cmd.execute(
+            "--hostname=localhost",
+            "--port=" + server.getPort(),
+            "liveSettingsV2",
+            "--indexName=test_index",
+            "--defaultSearchTimeoutSec=1.0",
+            "--local");
+    assertEquals(0, exitCode);
+    assertEquals(
+        1.0, server.getGlobalState().getIndex("test_index").getDefaultSearchTimeoutSec(), 0);
+
+    server.restart();
+
+    assertEquals(
+        0.0, server.getGlobalState().getIndex("test_index").getDefaultSearchTimeoutSec(), 0);
+  }
 }