nhosoi · richm · Jul 8, 2019 · Jul 12, 2019 · Jul 12, 2019 · Jul 13, 2019
diff --git a/hack/testing/test-viaq-data-model.py b/hack/testing/test-viaq-data-model.py
@@ -41,7 +41,8 @@ def empty(thing):
                   "undefined23": False
               },
               "undefined4": "undefined4",
-              "undefined5": "undefined5"
+              "undefined5": "undefined5",
+              "undefined.6": "undefined6"
     },
     "test2": {"undefined":
               {"undefined1": "undefined1",
@@ -53,7 +54,8 @@ def empty(thing):
                    "undefined23": False
                },
                "undefined4": "undefined4",
-               "undefined5": "undefined5"
+               "undefined5": "undefined5",
+               "undefined.6": "undefined6"
               }
     },
     "test3": {"undefined":
@@ -64,7 +66,8 @@ def empty(thing):
                    "undefined2": "undefined2",
                    "undefined22": 2222,
                    "undefined23": False
-               }
+               },
+               "undefined.6": "undefined6"
               },
               "undefined4": "undefined4",
               "undefined5": "undefined5"
@@ -77,7 +80,8 @@ def empty(thing):
                    "undefined2": "undefined2",
                    "undefined22": 2222,
                    "undefined23": False
-               }
+               },
+               "undefined.6": "undefined6"
               },
               "undefined4": "undefined4",
               "undefined5": "undefined5"
@@ -90,7 +94,34 @@ def empty(thing):
                    "undefined2": "undefined2",
                    "undefined22": 2222,
                    "undefined23": False
-               }
+               },
+               "undefined.6": "undefined6"
+              },
+              "undefined4": "undefined4",
+              "undefined5": "undefined5",
+              "empty1": "",
+              "undefined3": {"emptyvalue": ""}
+    },
+    "test6": {"myname":
+              {"undefined1": "undefined1",
+               "undefined11": 1111,
+               "undefined12": True,
+               "undefined2": {
+                   "undefined2": "undefined2",
+                   "undefined22": 2222,
+                   "undefined23": False
+               },
+               "undefined_6": "undefined6"
+              },
+              "undefined4": "undefined4",
+              "undefined5": "undefined5",
+              "empty1": "",
+              "undefined3": {"emptyvalue": ""}
+    },
+    "test7": {"myname":
+              {"undefined1": "undefined1",
+               "undefined11": 1111,
+               "undefined12": True,
               },
               "undefined4": "undefined4",
               "undefined5": "undefined5",

diff --git a/hack/testing/util.sh b/hack/testing/util.sh
@@ -433,6 +433,10 @@ function flush_fluentd_pos_files() {
     os::cmd::expect_success "oal_sudo rm -f /var/log/journal.pos /var/log/journal_pos.json"
 }
 
+function flush_rsyslog_pos_files() {
+    os::cmd::expect_success "oal_sudo rm -f /var/lib/rsyslog.pod/imfile-state* /var/lib/rsyslog.pod/imjournal.state"
+}
+
 # return 0 if given file has size ge given size, otherwise, return 1
 function file_has_size() {
     local f=$1
@@ -709,25 +713,29 @@ get_fluentd_pod_log() {
     fi
 }
 
-# rsyslog may have pod logs and logs in the file
-get_rsyslog_pod_log() {
-    local pod=${1:-$( get_running_pod rsyslog )}
-    local container=${2:-rsyslog}
-    oc logs -c $container $pod 2>&1
-    if [ $container = rsyslog ] ; then
-        oc exec -c $container $pod -- logs 2>&1 || oal_sudo cat /var/log/rsyslog/rsyslog.log.* /var/log/rsyslog/rsyslog.log || :
-    else
-        oal_sudo cat /var/lib/rsyslog.pod/logrotate.log /var/log/rsyslog/logrotate.log || :
-    fi
-}
-
 get_mux_pod_log() {
     local pod=${1:-$( get_running_pod mux )}
     local logfile=${2:-/var/log/fluentd/fluentd.log}
     oc logs $pod 2>&1
     oc exec $pod -- cat $logfile 2> /dev/null || :
 }
 
+# rsyslog may have pod logs and logs in the file
+get_rsyslog_pod_log() {
+    local pod=${1:-$( get_running_pod rsyslog )}
+    local logfile=${2:-/var/log/rsyslog/rsyslog.log}
+    oc logs $pod -c rsyslog 2>&1
+    if oc exec $pod -c rsyslog -- logs --all 2>&1 ; then
+        : # done
+    elif oal_sudo test -f $logfile ; then
+        # can't read from the pod directly - see if we can get the log file
+        oal_sudo cat ${logfile}.log-* $logfile 2> /dev/null || :
+        oal_sudo cat /var/log/rsyslog/rsyslog_debug.log-* /var/log/rsyslog/rsyslog_debug.log 2> /dev/null || :
+        oal_sudo cat /var/log/rsyslog/logrotate.log-* /var/log/rsyslog/logrotate.log 2> /dev/null || :
+        oal_sudo cat /var/lib/rsyslog.pod/logrotate.log-* /var/lib/rsyslog.pod/logrotate.log 2> /dev/null || :
+    fi
+}
+
 # writes all pod logs to the given outdir or $ARTIFACT_DIR in the form
 # pod_name.container_name.log
 # it will get both the oc logs output and any log files produced by
@@ -745,17 +753,18 @@ get_all_logging_pod_logs() {
         continue
       fi
       if [ $container = logrotate ] ; then
-        get_rsyslog_pod_log $p $container > $ARTIFACT_DIR/$p.$container.log 2>&1
+        # we got the logs from the rsyslog container already
         continue
       fi
       case "$p" in
         logging-fluentd-*|fluentd-*) get_fluentd_pod_log $p > $ARTIFACT_DIR/$p.$container.log 2>&1 ;;
         logging-mux-*) get_mux_pod_log $p > $ARTIFACT_DIR/$p.$container.log 2>&1 ;;
-        *) oc logs -n ${LOGGING_NS} -c $container $p > $ARTIFACT_DIR/$p.$container.log 2>&1
-           oc exec -n ${LOGGING_NS} -c $container $p -- logs >> $ARTIFACT_DIR/$p.$container.log 2>&1 || :
-           ;;
+        logging-es-*|elasticsearch-*) oc logs -n ${LOGGING_NS} -c $container $p > $ARTIFACT_DIR/$p.$container.log 2>&1
+                      oc exec -c elasticsearch -n ${LOGGING_NS} $p -- logs >> $ARTIFACT_DIR/$p.$container.log 2>&1
+                      ;;
+        *) oc logs -n ${LOGGING_NS} -c $container $p > $ARTIFACT_DIR/$p.$container.log 2>&1 ;;
       esac
-	done
+    done
   done
 }
 
@@ -786,6 +795,32 @@ start_fluentd() {
     os::cmd::try_until_text "oc get pods -l component=fluentd" "^(logging-)*fluentd-.* Running " $wait_time
 }
 
+stop_rsyslog() {
+    local rpod=${1:-$( get_running_pod rsyslog )}
+    local wait_time=${2:-$(( 2 * minute ))}
+
+    oc label node -l logging-infra-rsyslog=true --overwrite logging-infra-rsyslog=false
+    os::cmd::try_until_text "oc get $rsyslog_ds -o jsonpath='{ .status.numberReady }'" "0" $wait_time
+    # not sure if it is a bug or a flake, but sometimes .status.numberReady is 0, the rsyslog pod hangs around
+    # in the Terminating state for many seconds, which seems to cause problems with subsequent tests
+    # so, we have to wait for the pod to completely disappear - we cannot rely on .status.numberReady == 0
+    if [ -n "${rpod:-}" ] ; then
+        os::cmd::try_until_failure "oc get pod $rpod > /dev/null 2>&1" $wait_time
+    fi
+}
+
+start_rsyslog() {
+    local cleanfirst=${1:-false}
+    local wait_time=${2:-$(( 2 * minute ))}
+
+    if [ "$cleanfirst" != false ] ; then
+        flush_rsyslog_pos_files
+        oal_sudo rm -f /var/log/rsyslog/* /var/lib/rsyslog.pod/*
+    fi
+    oc label node -l logging-infra-rsyslog=false --overwrite logging-infra-rsyslog=true
+    os::cmd::try_until_text "oc get pods -l component=rsyslog" "^(logging-)*rsyslog-.* Running " $wait_time
+}
+
 get_fluentd_ds_name() {
     if oc -n ${LOGGING_NS} get daemonset fluentd -o name > /dev/null 2>&1 ; then
         echo daemonset/fluentd
@@ -806,6 +841,9 @@ get_fluentd_cm_name() {
 
 fluentd_cm=${fluentd_cm:-$(get_fluentd_cm_name)}
 
+# Hardcode daemonset/rsyslog for the case the rsyslog pod does not exist.
+rsyslog_ds=${rsyslog_ds:-daemonset/rsyslog}
+
 enable_cluster_logging_operator() {
     if oc -n ${LOGGING_NS} get deploy cluster-logging-operator > /dev/null 2>&1 ; then
         oc patch -n ${LOGGING_NS} clusterlogging instance --type=json --patch '[

diff --git a/openshift/ci-operator/build-image/Dockerfile.full b/openshift/ci-operator/build-image/Dockerfile.full
@@ -14,3 +14,4 @@ ADD openshift/ /go/src/github.com/openshift/origin-aggregated-logging/openshift/
 ADD hack/ /go/src/github.com/openshift/origin-aggregated-logging/hack/
 ADD test/ /go/src/github.com/openshift/origin-aggregated-logging/test/
 ADD fluentd/ /go/src/github.com/openshift/origin-aggregated-logging/fluentd/
+ADD rsyslog/ /go/src/github.com/openshift/origin-aggregated-logging/rsyslog/
diff --git a/rsyslog/README.md b/rsyslog/README.md
@@ -10,6 +10,15 @@ Following are the environment variables that can be modified to adjust the confi
 | `LOGGING_FILE_PATH` | The log file absolute path where Rsyslog is writting its logs. If you want rsyslog to output its logs as Rsyslog does by default (`STDOUT`) set this variable to `console` value. Default value is `/var/log/rsyslog/rsyslog.log`. | `LOGGING_FILE_PATH=console` |
 | `LOGGING_FILE_AGE` | Number of log files that Rsyslog keeps before deleting the oldest file. Default value is `10`. | `LOGGING_FILE_AGE=30` |
 | `LOGGING_FILE_SIZE` | Maximum size of a rsyslog log file in bytes. If the size of the log file is bigger, the log file gets rotated. Default is 1MB | `LOGGING_FILE_SIZE=1024000`
+| `CDM_USE_UNDEFINED` | Parameter to switch whether undefined fields are moved to the undefined property or not. default to "false". | `CDM_USE_UNDEFINED=true`
+| `CDM_DEFAULT_KEEP_FIELDS` | Default set of fields to be kept in the top level of json.  default to "CEE,time,@timestamp,aushape,ci_job,collectd,docker,fedora-ci,file,foreman,geoip,hostname,ipaddr4,ipaddr6,kubernetes,level,message,namespace_name,namespace_uuid,offset,openstack,ovirt,pid,pipeline_metadata,rsyslog,service,systemd,tags,testcase,tlog,viaq_msg_id",  | -
+| `CDM_EXTRA_KEEP_FIELDS` | Extra set of fields to be kept in the top level of json.  A field not included in ${CDM_DEFAULT_KEEP_FIELDS} nor ${CDM_EXTRA_KEEP_FIELDS} are moved to ${CDM_UNDEFINED_NAME} if CDM_USE_UNDEFINED is "true". default to "" | `CDM_EXTRA_KEEP_FIELDS="broker"` 
+| `CDM_UNDEFINED_NAME` | Undefined property name used when CDM_USE_UNDEFINED is set to "true". default to "undefined". | `CDM_UNDEFINED_NAME="undef"`
+| `CDM_KEEP_EMPTY_FIELDS` | Empty fields are dropped except the fields which names are set to CDM_KEEP_EMPTY_FIELDS in the CSV format. default to "". | `CDM_KEEP_EMPTY_FIELDS="offset"`
+| `CDM_UNDEFINED_TO_STRING` | If set to "true", when CDM_USE_UNDEFINED is "true" and undefined property with ${CDM_UNDEFINED_NAME} is created, the value is converted to the json string. default to "false". | `CDM_UNDEFINED_TO_STRING=true
+| `CDM_UNDEFINED_DOT_REPLACE_CHAR` | A dot character '.' in a property name (key) is replaced with the specified character unless the value is not "UNUSED". default to "UNUSED". Effective when MERGE_JSON_LOG is true. | `CDM_UNDEFINED_DOT_REPLACE_CHAR="_"`
+| `CDM_UNDEFINED_MAX_NUM_FIELDS` | If a positive value is set, undefined fields are dropped once the number of fields has reached the value. default to "-1". | `CDM_UNDEFINED_MAX_NUM_FIELDS=4`
+| `UNDEFINED_DEBUG` | Debug flag used in undefined_field as well as the config file calling the plugin.  A debug file /var/log/rsyslog/rsyslog_debug.log is generated.  default to "false". | `UNDEFINED_DEBUG=true`
 
 ## Rsyslog logging to file
 Rsyslog by default writes its logs into a file given by `LOGGING_FILE_PATH` environment variable. You can change the maximum size of a single log file or number of log files to keep(age), by setting `LOGGING_FILE_SIZE` and `LOGGING_FILE_AGE` environment variables accordingly.