[PLAT-13984] Change default metric graph points count from 100 to 250…

… + make runtime configurable.

Summary:
Currently YBA returns metric graphs with up to 100 data points by default.
With that, graphs for long time period have very bad resolution.
We're increasing the points count to 250 (same as Promehteus UI default) to make resolution better.
Also, making it runtime configurable.

Test Plan:
Upgrade YBA.
Make sure metric graphs are showing 250 data points now for longer time periods.

Reviewers: vbansal, #yba-api-review!

Reviewed By: vbansal

Subscribers: sanketh, yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D37912

managed/RUNTIME-FLAGS.md

@@ -18,6 +18,7 @@
 | "Enable IMDSv2" | "yb.aws.enable_imdsv2_support" | "CUSTOMER" | "Enable IMDSv2 support for AWS providers" | "Boolean" |
 | "Backup Garbage Collector Number of Retries" | "yb.backupGC.number_of_retries" | "CUSTOMER" | "Number of retries during backup deletion" | "Integer" |
 | "Enable Certificate Config Validation" | "yb.tls.enable_config_validation" | "CUSTOMER" | "Certificate configuration validation during the addition of new certificates." | "Boolean" |
+| "Default Metric Graph Point Count" | "yb.metrics.default_points" | "CUSTOMER" | "Default Metric Graph Point Count, if step is not defined in the query" | "Integer" |
 | "Allow Unsupported Instances" | "yb.internal.allow_unsupported_instances" | "PROVIDER" | "Enabling removes supported instance type filtering on AWS providers." | "Boolean" |
 | "Default AWS Instance Type" | "yb.aws.default_instance_type" | "PROVIDER" | "Default AWS Instance Type" | "String" |
 | "Default GCP Instance Type" | "yb.gcp.default_instance_type" | "PROVIDER" | "Default GCP Instance Type" | "String" |

managed/src/main/java/com/yugabyte/yw/common/config/CustomerConfKeys.java

@@ -217,4 +217,13 @@ public class CustomerConfKeys extends RuntimeConfigKeysModule {
  "Certificate configuration validation during the addition of new certificates.",
  ConfDataType.BooleanType,
  ImmutableList.of(ConfKeyTags.PUBLIC));
+
+ public static final ConfKeyInfo<Integer> MetricsDefaultPoints =
+ new ConfKeyInfo<>(
+ "yb.metrics.default_points",
+ ScopeType.CUSTOMER,
+ "Default Metric Graph Point Count",
+ "Default Metric Graph Point Count, if step is not defined in the query",
+ ConfDataType.IntegerType,
+ ImmutableList.of(ConfKeyTags.PUBLIC));
 }

managed/src/main/java/com/yugabyte/yw/controllers/XClusterConfigController.java

@@ -365,7 +365,7 @@ public Result get(UUID customerUUID, UUID xclusterConfigUUID, boolean syncWithDB
  metricParams.put("filters", Json.stringify(filterJson));
  lagMetricData =
  metricQueryHelper.query(
- Collections.singletonList(metric), metricParams, Collections.emptyMap());
+ customer, Collections.singletonList(metric), metricParams, Collections.emptyMap());
  } catch (Exception e) {
  String errorMsg =
  String.format(

managed/src/main/java/com/yugabyte/yw/metrics/MetricQueryHelper.java

@@ -16,6 +16,7 @@
 import com.yugabyte.yw.commissioner.tasks.UniverseTaskBase;
 import com.yugabyte.yw.commissioner.tasks.XClusterConfigTaskBase;
 import com.yugabyte.yw.common.*;
+import com.yugabyte.yw.common.config.CustomerConfKeys;
 import com.yugabyte.yw.common.config.GlobalConfKeys;
 import com.yugabyte.yw.common.config.ProviderConfKeys;
 import com.yugabyte.yw.common.config.RuntimeConfGetter;
@@ -50,7 +51,6 @@
 public class MetricQueryHelper {
 
  public static final Logger LOG = LoggerFactory.getLogger(MetricQueryHelper.class);
- public static final Integer STEP_SIZE = 100;
  public static final Integer QUERY_EXECUTOR_THREAD_POOL = 5;
 
  public static final String METRICS_QUERY_PATH = "query";
@@ -128,18 +128,19 @@ public MetricQueryHelper() {
  * "cpu_usage_user", "start": <start timestamp>, "end": <end timestamp>}
  * @return MetricQueryResponse Object
  */
- public JsonNode query(List<String> metricKeys, Map<String, String> params) {
+ public JsonNode query(Customer customer, List<String> metricKeys, Map<String, String> params) {
  HashMap<String, Map<String, String>> filterOverrides = new HashMap<>();
  List<MetricSettings> metricSettings = MetricSettings.defaultSettings(metricKeys);
- return query(metricSettings, params, filterOverrides, false);
+ return query(customer, metricSettings, params, filterOverrides, false);
  }
 
  public JsonNode query(
+ Customer customer,
  List<String> metricKeys,
  Map<String, String> params,
  Map<String, Map<String, String>> filterOverrides) {
  List<MetricSettings> metricSettings = MetricSettings.defaultSettings(metricKeys);
- return query(metricSettings, params, filterOverrides, false);
+ return query(customer, metricSettings, params, filterOverrides, false);
  }
 
  public JsonNode query(Customer customer, MetricQueryParams metricQueryParams) {
@@ -170,6 +171,7 @@ public JsonNode query(Customer customer, MetricQueryParams metricQueryParams) {
  // node/universe level filters.
  if (isExclusivelyPlatformLevelMetrics(metricQueryParams)) {
  return query(
+ customer,
  new ArrayList<>(metricSettingsMap.values()),
  params,
  filterOverrides,
@@ -359,6 +361,7 @@ public JsonNode query(Customer customer, MetricQueryParams metricQueryParams) {
  }
  params.put("filters", Json.stringify(filterJson));
  return query(
+ customer,
  new ArrayList<>(metricSettingsMap.values()),
  params,
  filterOverrides,
@@ -373,6 +376,7 @@ public JsonNode query(Customer customer, MetricQueryParams metricQueryParams) {
  * @return MetricQueryResponse Object
  */
  public JsonNode query(
+ Customer customer,
  List<MetricSettings> metricsWithSettings,
  Map<String, String> params,
  Map<String, Map<String, String>> filterOverrides,
@@ -401,7 +405,9 @@ public JsonNode query(
  if (timeDifference <= 0) {
  throw new PlatformServiceException(BAD_REQUEST, "Queried time interval should be positive");
  }
- long resolution = Math.max(scrapeInterval * 3, Math.round(timeDifference / STEP_SIZE));
+ int defaultPoints =
+ confGetter.getConfForScope(customer, CustomerConfKeys.MetricsDefaultPoints);
+ long resolution = Math.max(scrapeInterval * 3, Math.round(timeDifference / defaultPoints));
  params.put("step", String.valueOf(resolution));
  } else {
  try {

managed/src/main/resources/reference.conf

@@ -677,6 +677,7 @@ yb {
  auth = false
  auth_username = ""
  auth_password = ""
+ default_points = 250
  link.use_browser_fqdn = true
  management.enabled = true
  db_read_write_test = true

managed/src/test/java/com/yugabyte/yw/controllers/CustomerControllerTest.java

@@ -715,7 +715,7 @@ public void testCustomerMetricsWithValidMetricsParams() {
 
  doReturn(response)
  .when(mockMetricQueryHelper)
- .query(anyList(), anyMap(), anyMap(), anyBoolean());
+ .query(any(), anyList(), anyMap(), anyMap(), anyBoolean());
  Result result =
  route(
  fakeRequest("POST", baseRoute + customer.getUuid() + "/metrics")
@@ -750,12 +750,12 @@ public void testCustomerMetricsForContainerMetricsMultiAZ() {
  ArgumentCaptor<Map> queryParams = ArgumentCaptor.forClass(Map.class);
  doReturn(response)
  .when(mockMetricQueryHelper)
- .query(anyList(), anyMap(), anyMap(), anyBoolean());
+ .query(any(), anyList(), anyMap(), anyMap(), anyBoolean());
  Result result =
  doRequestWithAuthTokenAndBody(
  "POST", baseRoute + customer.getUuid() + "/metrics", authToken, params);
  verify(mockMetricQueryHelper)
- .query(metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
+ .query(any(), metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
  assertThat(queryParams.getValue(), is(notNullValue()));
  JsonNode filters = Json.parse(queryParams.getValue().get("filters").toString());
  assertValue(filters, "namespace", "yb-tc-demo-az-1|yb-tc-demo-az-2|test-ns-1");
@@ -797,12 +797,12 @@ private void testCustomerMetricsForContainerMetricsSingleAZBase(boolean helmNewN
  ArgumentCaptor<Map> queryParams = ArgumentCaptor.forClass(Map.class);
  doReturn(response)
  .when(mockMetricQueryHelper)
- .query(anyList(), anyMap(), anyMap(), anyBoolean());
+ .query(any(), anyList(), anyMap(), anyMap(), anyBoolean());
  Result result =
  doRequestWithAuthTokenAndBody(
  "POST", baseRoute + customer.getUuid() + "/metrics", authToken, params);
  verify(mockMetricQueryHelper)
- .query(metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
+ .query(any(), metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
  assertThat(queryParams.getValue(), is(notNullValue()));
  JsonNode filters = Json.parse(queryParams.getValue().get("filters").toString());
  assertValue(filters, "namespace", "yb-tc-demo");
@@ -851,12 +851,12 @@ public void testCustomerMetricsForContainerMetricsWithNodeName() {
  ArgumentCaptor<Map> queryParams = ArgumentCaptor.forClass(Map.class);
  doReturn(response)
  .when(mockMetricQueryHelper)
- .query(anyList(), anyMap(), anyMap(), anyBoolean());
+ .query(any(), anyList(), anyMap(), anyMap(), anyBoolean());
  Result result =
  doRequestWithAuthTokenAndBody(
  "POST", baseRoute + customer.getUuid() + "/metrics", authToken, params);
  verify(mockMetricQueryHelper)
- .query(metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
+ .query(any(), metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
  assertThat(queryParams.getValue(), is(notNullValue()));
  JsonNode filters = Json.parse(queryParams.getValue().get("filters").toString());
  assertValue(filters, "namespace", "diff-ns");
@@ -879,7 +879,7 @@ public void testCustomerMetricsWithInValidMetricsParam()
 
  doReturn(response)
  .when(mockMetricQueryHelper)
- .query(anyList(), anyMap(), anyMap(), anyBoolean());
+ .query(any(), anyList(), anyMap(), anyMap(), anyBoolean());
  Result result =
  routeWithYWErrHandler(
  fakeRequest("POST", baseRoute + customer.getUuid() + "/metrics")
@@ -913,7 +913,7 @@ public void testCustomerMetricsWithValidTableNameParams()
  .cookie(validCookie)
  .bodyJson(params));
  verify(mockMetricQueryHelper)
- .query(metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
+ .query(any(), metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
  assertThat(queryParams.getValue(), is(notNullValue()));
  JsonNode filters = Json.parse(queryParams.getValue().get("filters").toString());
  String tableName = filters.get("table_name").asText();
@@ -934,7 +934,7 @@ public void testCustomerMetricsWithoutTableNameParams() {
  response.put("foo", "bar");
  doReturn(response)
  .when(mockMetricQueryHelper)
- .query(anyList(), anyMap(), anyMap(), anyBoolean());
+ .query(any(), anyList(), anyMap(), anyMap(), anyBoolean());
 
  ObjectNode params = Json.newObject();
  params.set("metrics", Json.toJson(ImmutableList.of("metric")));
@@ -948,7 +948,7 @@ public void testCustomerMetricsWithoutTableNameParams() {
  .cookie(validCookie)
  .bodyJson(params));
  verify(mockMetricQueryHelper)
- .query(metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
+ .query(any(), metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
  assertThat(queryParams.getValue(), is(notNullValue()));
  JsonNode filters = Json.parse(queryParams.getValue().get("filters").toString());
  assertThat(filters.get("table_name"), nullValue());
@@ -968,7 +968,7 @@ public void testCustomerMetricsWithTableIdParams()
  response.put("foo", "bar");
  doReturn(response)
  .when(mockMetricQueryHelper)
- .query(anyList(), anyMap(), anyMap(), anyBoolean());
+ .query(any(), anyList(), anyMap(), anyMap(), anyBoolean());
 
  ObjectNode params = Json.newObject();
  params.set("metrics", Json.toJson(ImmutableList.of("metric")));
@@ -982,7 +982,7 @@ public void testCustomerMetricsWithTableIdParams()
  .cookie(validCookie)
  .bodyJson(params));
  verify(mockMetricQueryHelper)
- .query(metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
+ .query(any(), metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
  assertThat(queryParams.getValue(), is(notNullValue()));
  JsonNode filters = Json.parse(queryParams.getValue().get("filters"));
  String tableId = filters.get("table_id").asText();
@@ -1014,7 +1014,7 @@ public void testCustomerMetricsWithXClusterConfigUuidParams()
  response.put("foo", "bar");
  doReturn(response)
  .when(mockMetricQueryHelper)
- .query(anyList(), anyMap(), anyMap(), anyBoolean());
+ .query(any(), anyList(), anyMap(), anyMap(), anyBoolean());
 
  ObjectNode params = Json.newObject();
  params.set("metrics", Json.toJson(ImmutableList.of("metric")));
@@ -1028,7 +1028,7 @@ public void testCustomerMetricsWithXClusterConfigUuidParams()
  .cookie(validCookie)
  .bodyJson(params));
  verify(mockMetricQueryHelper)
- .query(metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
+ .query(any(), metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
  assertThat(queryParams.getValue(), is(notNullValue()));
  JsonNode filters = Json.parse(queryParams.getValue().get("filters"));
  String tableId = filters.get("table_id").asText();
@@ -1061,7 +1061,7 @@ public void testCustomerMetricsExceptionThrown()
 
  doThrow(new PlatformServiceException(BAD_REQUEST, userVisibleMessage))
  .when(mockMetricQueryHelper)
- .query(anyList(), anyMap(), anyMap(), anyBoolean());
+ .query(any(), anyList(), anyMap(), anyMap(), anyBoolean());
  Result result =
  routeWithYWErrHandler(fakeRequest(method, uri).cookie(validCookie).bodyJson(params));
  assertEquals(BAD_REQUEST, result.status());
@@ -1095,7 +1095,7 @@ public void testCustomerMetricsWithMultipleUniverses()
  .cookie(validCookie)
  .bodyJson(params));
  verify(mockMetricQueryHelper)
- .query(metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
+ .query(any(), metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
 
  assertThat(metricKeys.getValue(), is(notNullValue()));
  assertThat(queryParams.getValue(), is(notNullValue()));
@@ -1121,7 +1121,7 @@ public void testCustomerMetricsWithNodePrefixParam() {
  response.put("foo", "bar");
  doReturn(response)
  .when(mockMetricQueryHelper)
- .query(anyList(), anyMap(), anyMap(), anyBoolean());
+ .query(any(), anyList(), anyMap(), anyMap(), anyBoolean());
 
  ObjectNode params = Json.newObject();
  params.set("metrics", Json.toJson(ImmutableList.of("metric")));
@@ -1135,7 +1135,7 @@ public void testCustomerMetricsWithNodePrefixParam() {
  .cookie(validCookie)
  .bodyJson(params));
  verify(mockMetricQueryHelper)
- .query(metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
+ .query(any(), metricKeys.capture(), queryParams.capture(), anyMap(), anyBoolean());
  assertThat(queryParams.getValue(), is(notNullValue()));
  JsonNode filters = Json.parse(queryParams.getValue().get("filters").toString());
  String nodePrefix = filters.get("node_prefix").asText();
@@ -1153,7 +1153,7 @@ public void testCustomerMetricsWithNodeNameParam() {
  response.put("foo", "bar");
  doReturn(response)
  .when(mockMetricQueryHelper)
- .query(anyList(), anyMap(), anyMap(), anyBoolean());
+ .query(any(), anyList(), anyMap(), anyMap(), anyBoolean());
 
  ObjectNode params = Json.newObject();
  params.set("metrics", Json.toJson(ImmutableList.of("metric")));
@@ -1167,7 +1167,7 @@ public void testCustomerMetricsWithNodeNameParam() {
  doRequestWithAuthTokenAndBody(
  "POST", baseRoute + customer.getUuid() + "/metrics", authToken, params);
  verify(mockMetricQueryHelper)
- .query(metricKeys.capture(), queryParams.capture(), any(), anyBoolean());
+ .query(any(), metricKeys.capture(), queryParams.capture(), any(), anyBoolean());
  assertThat(queryParams.getValue(), is(notNullValue()));
  JsonNode filters = Json.parse(queryParams.getValue().get("filters").toString());
  String nodeName = filters.get("exported_instance").asText();

managed/src/test/java/com/yugabyte/yw/controllers/XClusterConfigControllerTest.java

@@ -270,7 +270,7 @@ private void setupMockMetricQueryHelperResponse() {
  JsonNode fakeMetricResponse = Json.newObject().put("value", "0");
  doReturn(fakeMetricResponse)
  .when(mockMetricQueryHelper)
- .query(anyList(), anyMap(), anyMap(), anyBoolean());
+ .query(any(), anyList(), anyMap(), anyMap(), anyBoolean());
  }
 
  public void setupMetricValues() {
@@ -817,7 +817,7 @@ public void testGetMetricFailure() {
  String fakeErrMsg = "failed to fetch metric data";
  doThrow(new PlatformServiceException(INTERNAL_SERVER_ERROR, fakeErrMsg))
  .when(mockMetricQueryHelper)
- .query(any(), any(), any());
+ .query(any(), anyList(), anyMap(), anyMap());
 
  String getAPIEndpoint = apiEndpoint + "/" + xClusterConfig.getUuid();