From c444db90a1cbbb9bda62a500e756375f74213e4e Mon Sep 17 00:00:00 2001 From: Noriko Hosoi Date: Sat, 20 Jul 2019 16:48:31 -0700 Subject: [PATCH] zzz-rsyslog.sh - Set kubernetes to CDM_KEEP_EMPTY_FIELDS to keep "openshift_io/node-selector": "" in kubernetes.namespace_annotations.openshift_io/node-selector. undefined_field_test.go - Check returned undefString from processUndefinedAndMaxNumFields against the expected value. --- rsyslog/README.md | 2 +- rsyslog/undefined_field/undefined_field_test.go | 6 +++++- test/zzz-rsyslog.sh | 17 +++++++++++++++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/rsyslog/README.md b/rsyslog/README.md index 37c0a96e06..c56f50b966 100644 --- a/rsyslog/README.md +++ b/rsyslog/README.md @@ -19,7 +19,7 @@ Following are the environment variables that can be modified to adjust the confi | `CDM_KEEP_EMPTY_FIELDS` | Empty fields are dropped except the fields which names are set to CDM_KEEP_EMPTY_FIELDS in the CSV format. default to "". | `CDM_KEEP_EMPTY_FIELDS="offset"` | `CDM_UNDEFINED_TO_STRING` | If set to "true", when CDM_USE_UNDEFINED is "true" and undefined property with ${CDM_UNDEFINED_NAME} is created, the value is converted to the json string. default to "false". | `CDM_UNDEFINED_TO_STRING=true | `CDM_UNDEFINED_DOT_REPLACE_CHAR` | A dot character '.' in a property name (key) is replaced with the specified character unless the value is not "UNUSED". default to "UNUSED". Effective when MERGE_JSON_LOG is true. | `CDM_UNDEFINED_DOT_REPLACE_CHAR="_"` -| `CDM_UNDEFINED_MAX_NUM_FIELDS` | If a positive value is set, undefined fields are dropped once the number of fields has reached the value. default to "-1". | `CDM_UNDEFINED_MAX_NUM_FIELDS=4` +| `CDM_UNDEFINED_MAX_NUM_FIELDS` | If set to a number greater than -1, and if the number of undefined fields is greater than this number, all of the undefined fields will be converted to their JSON string representation and stored in the undefined_name named field. default to "-1" although not recommended. | `CDM_UNDEFINED_MAX_NUM_FIELDS=2` | `UNDEFINED_DEBUG` | Debug flag used in undefined_field as well as the config file calling the plugin. A debug file /var/log/rsyslog/rsyslog_debug.log is generated. default to "false". | `UNDEFINED_DEBUG=true` ## Rsyslog logging to file diff --git a/rsyslog/undefined_field/undefined_field_test.go b/rsyslog/undefined_field/undefined_field_test.go index ddce316e16..b29ead960e 100644 --- a/rsyslog/undefined_field/undefined_field_test.go +++ b/rsyslog/undefined_field/undefined_field_test.go @@ -188,6 +188,7 @@ func TestUndefinedMaxNumFields(t *testing.T) { var val1 float64 = 1111 var val2 float64 = 2222 undefined2Map := map[string]interface{}{ + "": "", "undefined2": "undefined2", "undefined22": val2, "undefined23": false, @@ -200,7 +201,10 @@ func TestUndefinedMaxNumFields(t *testing.T) { "undefined.6": "undefined6", } fieldlist = []string{"undefined1", "undefined11", "undefined12", "undefined2", "undefined.6"} - if err = checkFieldsEqual(t, undefinedMap, inputMap, fieldlist); err != nil { + if err := json.Unmarshal([]byte(undefString), &undefMap); err != nil { + t.Errorf("json.Unmarshal failed for undefString [%v]: %v", undefString, err) + } + if err = checkFieldsEqual(t, undefinedMap, undefMap, fieldlist); err != nil { t.Error(err) } } diff --git a/test/zzz-rsyslog.sh b/test/zzz-rsyslog.sh index f88c832721..a4bedce510 100755 --- a/test/zzz-rsyslog.sh +++ b/test/zzz-rsyslog.sh @@ -10,6 +10,8 @@ os::util::environment::use_sudo os::test::junit::declare_suite_start "test/zzz-rsyslog" pushd ${OS_O_A_L_DIR}/rsyslog/undefined_field > /dev/null +cp undefined_field.go $ARTIFACT_DIR || : +cp undefined_field_test.go $ARTIFACT_DIR || : go test -v 2>&1 | artifact_out popd > /dev/null @@ -46,6 +48,7 @@ cleanup() { set +e get_all_logging_pod_logs oc get cm rsyslog -o yaml > $ARTIFACT_DIR/rsyslog_configmap.yaml 2>&1 + oc get cm rsyslog-bin -o yaml > $ARTIFACT_DIR/rsyslog.sh 2>&1 if [ "deploy_using_ansible" = "$deployfunc" ] ; then if [ -n "${tmpinv:-}" -a -f "${tmpinv:-}" ] ; then rm -f $tmpinv @@ -61,6 +64,7 @@ cleanup() { start_fluentd true 2>&1 | artifact_out else rpod=$( get_running_pod rsyslog ) + oc exec $rpod -c rsyslog -- env > $ARTIFACT_DIR/rsyslog_env.txt 2>&1 || : oc exec $rpod -c rsyslog -- cat /var/log/rsyslog/rsyslog_debug.log > $ARTIFACT_DIR/rsyslog_debug.log 2>&1 || : oc get clusterlogging instance -o yaml > $ARTIFACT_DIR/clinstance.yaml 2>&1 oc describe deploy cluster-logging-operator > $ARTIFACT_DIR/deploy.clo.yaml 2>&1 @@ -175,6 +179,10 @@ else fi $deployfunc +# Set kubernetes to CDM_KEEP_EMPTY_FIELDS to keep "openshift_io/node-selector": "" in +# kubernetes.namespace_annotations.openshift_io/node-selector +oc set env $rsyslog_ds CDM_KEEP_EMPTY_FIELDS="kubernetes" +start_rsyslog sleep 10 wait_for_fluentd_to_catch_up get_logmessage get_logmessage2 @@ -237,6 +245,10 @@ os::cmd::expect_success_and_not_text "cat $ops | jq -r .hits.hits[0]._source.hos ts=$( cat $ops | jq -r '.hits.hits[0]._source."@timestamp"' ) os::cmd::expect_success "test ${ts} != null" +# Unset CDM_KEEP_EMPTY_FIELDS +oc set env $rsyslog_ds CDM_KEEP_EMPTY_FIELDS- +start_rsyslog + # see if the prometheus exporter is working rpod="$( get_running_pod rsyslog )" rpod_ip="$( oc get pod ${rpod} -o jsonpath='{.status.podIP}' )" @@ -365,8 +377,9 @@ os::cmd::try_until_success "oc exec $rpod -c rsyslog -- grep 'use_undefined:' /v artifact_log "1. default values" rpod=$( get_running_pod rsyslog ) # skipped empty by mmnormalize/parse_json_skip_empty; mmexternal won't be executed. -oc exec $rpod -c rsyslog -- ls -l /var/lib/rsyslog.pod/undefined.json | artifact_out || : -oc exec $rpod -c rsyslog -- cat /var/lib/rsyslog.pod/undefined.json | artifact_out || : +oc exec $rpod -c rsyslog -- env | artifact_out +oc exec $rpod -c rsyslog -- ls -l /var/lib/rsyslog.pod/undefined.json 2>&1 | artifact_out || : +oc exec $rpod -c rsyslog -- cat /var/lib/rsyslog.pod/undefined.json 2>&1 | artifact_out || : wait_for_fluentd_to_catch_up get_logmessage get_logmessage2 os::cmd::expect_success "cat $proj | python $OS_O_A_L_DIR/hack/testing/test-viaq-data-model.py test1" os::cmd::expect_success "cat $ops | python $OS_O_A_L_DIR/hack/testing/test-viaq-data-model.py test1"