diff --git a/server/controller/db/metadb/migrator/schema/const.go b/server/controller/db/metadb/migrator/schema/const.go index 8e1dcd1b073..caaa5154e6f 100644 --- a/server/controller/db/metadb/migrator/schema/const.go +++ b/server/controller/db/metadb/migrator/schema/const.go @@ -20,5 +20,5 @@ const ( RAW_SQL_ROOT_DIR = "/etc/metadb/schema/rawsql" DB_VERSION_TABLE = "db_version" - DB_VERSION_EXPECTED = "7.1.0.30" + DB_VERSION_EXPECTED = "7.1.0.31" ) diff --git a/server/controller/db/metadb/migrator/schema/rawsql/mysql/ddl_create_table.sql b/server/controller/db/metadb/migrator/schema/rawsql/mysql/ddl_create_table.sql index 6b3cac94244..b303cc6d997 100644 --- a/server/controller/db/metadb/migrator/schema/rawsql/mysql/ddl_create_table.sql +++ b/server/controller/db/metadb/migrator/schema/rawsql/mysql/ddl_create_table.sql @@ -2412,7 +2412,9 @@ CREATE TABLE IF NOT EXISTS alarm_policy ( auto_service_id_1 INTEGER DEFAULT 0, auto_service_type_1 INTEGER DEFAULT 0, auto_service_1 VARCHAR(256) DEFAULT '', - comb_policy_lcuuids TEXT + comb_policy_lcuuids TEXT, + static_labels TEXT, + dynamic_labels TEXT ) ENGINE=innodb DEFAULT CHARSET=utf8 AUTO_INCREMENT=1; TRUNCATE TABLE alarm_policy; diff --git a/server/controller/db/metadb/migrator/schema/rawsql/mysql/issu/7.1.0.31.sql b/server/controller/db/metadb/migrator/schema/rawsql/mysql/issu/7.1.0.31.sql new file mode 100644 index 00000000000..0d5b4ac17ef --- /dev/null +++ b/server/controller/db/metadb/migrator/schema/rawsql/mysql/issu/7.1.0.31.sql @@ -0,0 +1,44 @@ +-- ColumnExists function +DROP PROCEDURE IF EXISTS ColumnExists; + +CREATE PROCEDURE ColumnExists( + IN p_table_name VARCHAR(255), + IN p_col_name VARCHAR(255), + OUT p_exists TINYINT(1) +) +BEGIN + SELECT COUNT(*) > 0 + INTO p_exists + FROM information_schema.columns + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = p_table_name + AND COLUMN_NAME = p_col_name; +END; + +-- AddColumnIfNotExists procedure +DROP PROCEDURE IF EXISTS AddColumnIfNotExists; + +CREATE PROCEDURE AddColumnIfNotExists( + IN tableName VARCHAR(255), + IN colName VARCHAR(255), + IN colType VARCHAR(255), + IN afterCol VARCHAR(255) +) +BEGIN + CALL ColumnExists(tableName, colName, @exists); + IF NOT @exists THEN + SET @sql = CONCAT('ALTER TABLE ', tableName, ' ADD COLUMN ', colName, ' ', colType, ' AFTER ', afterCol); + PREPARE stmt FROM @sql; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; + END IF; +END; +CALL AddColumnIfNotExists('alarm_policy', 'dynamic_labels', 'TEXT', 'comb_policy_lcuuids'); +CALL AddColumnIfNotExists('alarm_policy', 'static_labels', 'TEXT', 'comb_policy_lcuuids'); + +-- Cleanup +DROP PROCEDURE IF EXISTS ColumnExists; +DROP PROCEDURE IF EXISTS AddColumnIfNotExists; + +-- Update DB version +UPDATE db_version SET version='7.1.0.31'; diff --git a/server/controller/db/metadb/migrator/schema/rawsql/postgres/ddl_create_table.sql b/server/controller/db/metadb/migrator/schema/rawsql/postgres/ddl_create_table.sql index 6fe3a464af6..7b2c6db62d5 100644 --- a/server/controller/db/metadb/migrator/schema/rawsql/postgres/ddl_create_table.sql +++ b/server/controller/db/metadb/migrator/schema/rawsql/postgres/ddl_create_table.sql @@ -2508,7 +2508,9 @@ CREATE TABLE IF NOT EXISTS alarm_policy ( auto_service_id_1 INTEGER DEFAULT 0, auto_service_type_1 INTEGER DEFAULT 0, auto_service_1 VARCHAR(256) DEFAULT '', - comb_policy_lcuuids TEXT + comb_policy_lcuuids TEXT, + static_labels TEXT, + dynamic_labels TEXT ); TRUNCATE TABLE alarm_policy; COMMENT ON COLUMN alarm_policy.level IS '0.low 1.middle 2.high'; diff --git a/server/querier/engine/clickhouse/filter.go b/server/querier/engine/clickhouse/filter.go index abb845f1b9e..9d145494574 100644 --- a/server/querier/engine/clickhouse/filter.go +++ b/server/querier/engine/clickhouse/filter.go @@ -913,6 +913,13 @@ func (t *WhereTag) Trans(expr sqlparser.Expr, w *Where, e *CHEngine) (view.Node, } else { filter = fmt.Sprintf(tagItem.WhereTranslator, nameNoPrefix, op, t.Value) } + } else if strings.HasPrefix(tagName, "custom_tag.") { + tagItem, ok = tag.GetTag("custom_tag.", db, table, "default") + if strings.Contains(op, "match") { + filter = fmt.Sprintf(tagItem.WhereRegexpTranslator, op, tagName, t.Value) + } else { + filter = fmt.Sprintf(tagItem.WhereTranslator, tagName, op, t.Value) + } } else { switch strings.Trim(t.Tag, "`") { case "policy_type", "metric_value", "event_level", "team_id", "user_id", "target_tags", "_query_region", "_target_uid", "1", "_id": diff --git a/server/querier/engine/clickhouse/tag.go b/server/querier/engine/clickhouse/tag.go index 9b59d02fefc..da80d0f3b9b 100644 --- a/server/querier/engine/clickhouse/tag.go +++ b/server/querier/engine/clickhouse/tag.go @@ -111,6 +111,10 @@ func GetTagTranslator(name, alias string, e *CHEngine) ([]Statement, string, err } TagTranslatorStr := fmt.Sprintf(tagItem.TagTranslator, nameNoPrefix, nameNoPrefix) stmts = append(stmts, &SelectTag{Value: TagTranslatorStr, Alias: selectTag}) + } else if strings.HasPrefix(nameNoBackQuote, "custom_tag.") { + tagItem, ok = tag.GetTag("custom_tag.", db, table, "default") + TagTranslatorStr := fmt.Sprintf(tagItem.TagTranslator, nameNoBackQuote, nameNoBackQuote) + stmts = append(stmts, &SelectTag{Value: TagTranslatorStr, Alias: selectTag}) } else { stmts = append(stmts, &SelectTag{Value: name, Alias: alias}) } diff --git a/server/querier/engine/clickhouse/tag/description.go b/server/querier/engine/clickhouse/tag/description.go index f244aa0a6c6..430a6346d16 100644 --- a/server/querier/engine/clickhouse/tag/description.go +++ b/server/querier/engine/clickhouse/tag/description.go @@ -808,9 +808,6 @@ func GetDynamicTagDescriptions(db, table, rawSql, queryCacheTTL, orgID string, u Values: []interface{}{}, } notSupportOperator := []string{} - if table == "alert_event" { - notSupportOperator = []string{"select", "group"} - } // 查询 k8s_label chClient := client.Client{ @@ -957,7 +954,7 @@ func GetDynamicTagDescriptions(db, table, rawSql, queryCacheTTL, orgID string, u } // 查询外部字段 - if !slices.Contains([]string{ckcommon.DB_NAME_EXT_METRICS, ckcommon.DB_NAME_FLOW_LOG, ckcommon.DB_NAME_DEEPFLOW_ADMIN, ckcommon.DB_NAME_DEEPFLOW_TENANT, ckcommon.DB_NAME_EVENT, ckcommon.DB_NAME_PROFILE, ckcommon.DB_NAME_PROMETHEUS, ckcommon.DB_NAME_APPLICATION_LOG, "_prometheus"}, db) || (db == ckcommon.DB_NAME_FLOW_LOG && table != ckcommon.TABLE_NAME_L7_FLOW_LOG) || (db == ckcommon.DB_NAME_PROFILE && table != ckcommon.TABLE_NAME_IN_PROCESS) || (db == ckcommon.DB_NAME_EVENT && !slices.Contains([]string{ckcommon.TABLE_NAME_EVENT, ckcommon.TABLE_NAME_FILE_EVENT}, table)) { + if !slices.Contains([]string{ckcommon.DB_NAME_EXT_METRICS, ckcommon.DB_NAME_FLOW_LOG, ckcommon.DB_NAME_DEEPFLOW_ADMIN, ckcommon.DB_NAME_DEEPFLOW_TENANT, ckcommon.DB_NAME_EVENT, ckcommon.DB_NAME_PROFILE, ckcommon.DB_NAME_PROMETHEUS, ckcommon.DB_NAME_APPLICATION_LOG, "_prometheus"}, db) || (db == ckcommon.DB_NAME_FLOW_LOG && table != ckcommon.TABLE_NAME_L7_FLOW_LOG) || (db == ckcommon.DB_NAME_PROFILE && table != ckcommon.TABLE_NAME_IN_PROCESS) { return response, nil } externalChClient := client.Client{ @@ -1030,7 +1027,7 @@ func GetDynamicTagDescriptions(db, table, rawSql, queryCacheTTL, orgID string, u externalTag := tagName.(string) var categoryValue string // fieltValueType := _tagName.([]interface{})[2] - if strings.HasPrefix(externalTag, "cloud.tag.") || strings.HasPrefix(externalTag, "k8s.label.") || strings.HasPrefix(externalTag, "os.app.") || strings.HasPrefix(externalTag, "k8s.annotation.") || strings.HasPrefix(externalTag, "k8s.env.") { + if strings.HasPrefix(externalTag, "cloud.tag.") || strings.HasPrefix(externalTag, "k8s.label.") || strings.HasPrefix(externalTag, "os.app.") || strings.HasPrefix(externalTag, "k8s.annotation.") || strings.HasPrefix(externalTag, "k8s.env.") || strings.HasPrefix(externalTag, "custom_tag.") { categoryValue = "Custom Tag" response.Values = append(response.Values, []interface{}{ externalTag, externalTag, externalTag, externalTag, externalTag, externalTag, "map_item", @@ -1042,12 +1039,6 @@ func GetDynamicTagDescriptions(db, table, rawSql, queryCacheTTL, orgID string, u externalTag, externalTag, externalTag, externalTag, externalTag, externalTag, "map_item", categoryValue, tagTypeToOperators["string"], []bool{true, true, true}, externalTag, externalTag, externalTag, "", false, notSupportOperator, tableName, }) - } else { - categoryValue = _tagName.([]interface{})[2].(string) - response.Values = append(response.Values, []interface{}{ - externalTag, externalTag, externalTag, externalTag, externalTag, externalTag, categoryValue, - categoryValue, tagTypeToOperators[categoryValue], []bool{true, true, true}, externalTag, externalTag, externalTag, "", false, notSupportOperator, tableName, - }) } } else if slices.Contains(tagNativeTagDB, db) { @@ -1122,7 +1113,6 @@ func GetDynamicMetric(db, table, metric string) (response *common.Result) { metric, metric, metric, metric, metric, metric, "map_item", "Native Tag", tagTypeToOperators["string"], []bool{true, true, true}, "", "", "", "", false, []string{}, "", }) - return } return } @@ -1137,19 +1127,23 @@ func GetAlertEventTagDescriptions(staticTag, dynamicTag *common.Result) (respons } valuesMap := map[string]interface{}{} staticValues := staticTag.Values - dynamiicValues := dynamicTag.Values + dynamicValues := dynamicTag.Values for _, staticItem := range staticValues { valuesMap[staticItem.([]interface{})[0].(string)] = staticItem } // if dynamiic tag_name not in staticvalues, add to result - for _, dynamicItem := range dynamiicValues { - dynamicTagName := strings.TrimSuffix(dynamicItem.([]interface{})[0].(string), "_0") - dynamicTagName = strings.TrimSuffix(dynamicTagName, "_1") - dynamicTagName = strings.TrimSuffix(dynamicTagName, "_id") - if valuesMap[dynamicTagName] == nil && valuesMap[dynamicItem.([]interface{})[0].(string)] == nil { + for _, dynamicItem := range dynamicValues { + dynamicTagName := dynamicItem.([]interface{})[0].(string) + if strings.HasPrefix(dynamicTagName, "custom_tag.") { valuesMap[dynamicTagName] = dynamicItem + } else { + dynamicTagNameNosuffix := strings.TrimSuffix(dynamicTagName, "_0") + dynamicTagNameNosuffix = strings.TrimSuffix(dynamicTagNameNosuffix, "_1") + dynamicTagNameNoID := strings.TrimSuffix(dynamicTagNameNosuffix, "_id") + valuesMap[dynamicTagNameNoID] = dynamicItem } + } for _, value := range valuesMap { diff --git a/server/querier/engine/clickhouse/tag/translation.go b/server/querier/engine/clickhouse/tag/translation.go index 59460245ea8..198d7aee49d 100644 --- a/server/querier/engine/clickhouse/tag/translation.go +++ b/server/querier/engine/clickhouse/tag/translation.go @@ -2312,6 +2312,27 @@ func GenerateAlarmEventTagResoureMap() map[string]map[string]*Tag { ), } + tagResourceMap["custom_tag"] = map[string]*Tag{ + "default": NewTag( + "toJSONString(CAST((custom_tag_names, custom_tag_values), 'Map(String, String)'))", + "", + "", + "", + "", + ), + } + + tagResourceMap["custom_tag."] = map[string]*Tag{ + "default": NewTag( + // null indicates the absence of this key + "if(indexOf(custom_tag_names,'%s') != 0, custom_tag_values[indexOf(custom_tag_names,'%s')], NULL)", + "indexOf(custom_tag_names,'%s') != 0", + "custom_tag_values[indexOf(custom_tag_names,'%s')] %s %v", + "%s(custom_tag_values[indexOf(custom_tag_names,'%s')],%v)", + "", + ), + } + tagResourceMap["tag_int"] = map[string]*Tag{ "default": NewTag( "toJSONString(CAST((tag_int_names, tag_int_values), 'Map(String, Int64)'))",