rsyslog - skip empty field support

libfastjson - Adding "void fjson_object_array_del_idx(struct
 fjson_object *jso, int idx)", which deletes the idx-th element
 in the array type object.

 upstream PR: rsyslog/libfastjson#155

liblognorm - Adding a parameter skipempty to the json field type.
 If skipempty is set as follows, empty json objects are dropped
 from the parsed result.
  %field_name:json:skipempty%

 If any parameter other than "skipempty" is given ("bogus" in this
 example), an error message "invalid flag for JSON parser: bogus"
 is issued.

 Add test for the skipempty parameter.

 upstream PR: rsyslog/liblognorm#305

rsyslog/vendored_src/libfastjson/libfastjson/arraylist.c

@@ -87,6 +87,22 @@ array_list_add(struct array_list *arr, void *data)
 	return array_list_put_idx(arr, arr->length, data);
 }
 
+/*
+ * Deleting the idx-th element in the array_list.
+ */
+void
+array_list_del_idx(struct array_list *const arr, const int idx)
+{
+	if (idx < 0 || idx >= arr->length) {
+		return;
+	}
+	if(arr->array[idx]) arr->free_fn(arr->array[idx]);
+	if (--arr->length > idx) {
+		memmove(arr->array + idx, arr->array + idx + 1, (arr->length - idx) * sizeof(void *));
+	}
+	return;
+}
+
 /* work around wrong compiler message: GCC and clang do
  * not handle sort_fn correctly if -Werror is given.
  */

rsyslog/vendored_src/libfastjson/libfastjson/arraylist.h

@@ -41,6 +41,9 @@ array_list_put_idx(struct array_list *al, int i, void *data);
 extern int
 array_list_add(struct array_list *al, void *data);
 
+extern void
+array_list_del_idx(struct array_list *const arr, const int idx);
+
 extern int
 array_list_length(struct array_list *al);
 

rsyslog/vendored_src/libfastjson/libfastjson/json_object.c

@@ -1018,6 +1018,14 @@ struct fjson_object* fjson_object_array_get_idx(struct fjson_object *jso,
 	return (struct fjson_object*)array_list_get_idx(jso->o.c_array, idx);
 }
 
+/*
+ * Deleting the idx-th element in the array type object.
+ */
+void fjson_object_array_del_idx(struct fjson_object *jso, int idx)
+{
+	array_list_del_idx(jso->o.c_array, idx);
+}
+
 int fjson_object_get_member_count(struct fjson_object *jso)
 {
 	return jso->o.c_obj.nelem;

rsyslog/vendored_src/libfastjson/libfastjson/json_object.h

@@ -489,6 +489,8 @@ extern int fjson_object_array_put_idx(struct fjson_object *obj, int idx,
 extern struct fjson_object* fjson_object_array_get_idx(struct fjson_object *obj,
 							 int idx);
 
+extern void fjson_object_array_del_idx(struct fjson_object *jso, int idx);
+
 /* fjson_bool type methods */
 
 /** Create a new empty fjson_object of type fjson_type_boolean
@@ -731,6 +733,7 @@ typedef struct fjson_tokener fjson_tokener;
 #define json_object_get_int64 fjson_object_get_int64
 #define json_object_get_string_len fjson_object_get_string_len
 #define json_object_get_member_count fjson_object_get_member_count
+#define json_object_array_del_idx fjson_object_array_del_idx
 
 
 #endif

rsyslog/vendored_src/liblognorm/liblognorm/src/parser.c

@@ -2316,6 +2316,85 @@ PARSER_Parse(v2IPTables)
 	return r;
 }
 
+/*
+ * Delete children of the given object if it has children and they are empty.
+ *
+ * return 0 if object is not empty
+ * return 1 if object is empty
+ * return < 0 if error
+ *
+ * Caller should do this:
+ * if (jsonSkipEmpty(obj) > 0) {
+ *     json_object_put(obj);
+ *     obj = NULL;
+ * }
+ * or otherwise not use obj if jsonSkipEmpty returns > 0.
+ */
+static int
+jsonSkipEmpty(struct json_object *__restrict__ json)
+{
+	int rc = 0;
+	struct json_object *val = NULL;
+
+	if(json == NULL) {
+		rc = 1;
+		goto finalize_it;
+	}
+
+	switch (json_object_get_type(json)) {
+	case json_type_string:
+		rc = json_object_get_string_len(json) == 0;
+		break;
+	case json_type_array:
+	{
+		int i;
+		int arrayLen = json_object_array_length(json);
+		for (i = 0 ; i < arrayLen ; ++i) {
+			val = json_object_array_get_idx(json, i);
+			if ((rc = jsonSkipEmpty(val)) > 0) {
+				/* delete the empty item and reset the index and arrayLen */
+				json_object_array_del_idx(json, i--);
+				arrayLen = json_object_array_length(json);
+			} else if (rc < 0) {
+				goto finalize_it;
+			}
+		}
+		rc = json_object_array_length(json) == 0;
+		break;
+	}
+	case json_type_object:
+	{
+		struct json_object_iterator it = json_object_iter_begin(json);
+		struct json_object_iterator itEnd = json_object_iter_end(json);
+		while (!json_object_iter_equal(&it, &itEnd)) {
+			val = json_object_iter_peek_value(&it);
+			if ((rc = jsonSkipEmpty(val)) > 0) {
+				json_object_object_del(json, json_object_iter_peek_name(&it));
+			} else if (rc < 0) {
+				goto finalize_it;
+			}
+			json_object_iter_next(&it);
+		}
+		rc = json_object_object_length(json) == 0;
+	}
+	case json_type_null:
+	case json_type_boolean:
+	case json_type_double:
+	case json_type_int:
+	default: break;
+	}
+finalize_it:
+	return rc;
+}
+
+/* 
+ * Parameters for field type json
+ *   skipempty - skips empty json objects.
+ *             - %field_name:json:skipempty%
+ */
+struct data_JSON {
+	int skipempty;
+};
 /**
  * Parse JSON. This parser tries to find JSON data inside a message.
  * If it finds valid JSON, it will extract it. Extra data after the
@@ -2331,6 +2410,7 @@ PARSER_Parse(v2IPTables)
 PARSER_Parse(JSON)
 	const size_t i = *offs;
 	struct json_tokener *tokener = NULL;
+	struct data_JSON *const data = (struct data_JSON*) pdata;
 
 	if(npb->str[i] != '{' && npb->str[i] != ']') {
 		/* this can't be json, see RFC4627, Sect. 2
@@ -2359,6 +2439,20 @@ PARSER_Parse(JSON)
 	if(value == NULL) {
 		json_object_put(json);
 	} else {
+		if (data && data->skipempty) {
+			int rc = jsonSkipEmpty(json);
+			if (rc < 0) {
+				json_object_put(json);
+				FAIL(LN_WRONGPARSER);
+			} else if (rc > 0) {
+				/* 
+				 * json value is empty.
+				 * E.g., {"message":""}, {"message":[]}, {"message":{}}
+				 */
+				json_object_put(json);
+				FAIL(0);
+			}
+		}
 		*value = json;
 	}
 
@@ -2367,7 +2461,40 @@ PARSER_Parse(JSON)
 		json_tokener_free(tokener);
 	return r;
 }
+PARSER_Construct(JSON)
+{
+	int r = 0;
+	struct json_object *ed;
+	struct data_JSON *data = NULL;
+	char *flag;
 
+	if(json == NULL)
+		goto done;
+
+	if(json_object_object_get_ex(json, "extradata", &ed) == 0) {
+		/* No JSON parameter */
+		goto done;
+	}
+	data = (struct data_JSON*) calloc(1, sizeof(struct data_JSON));
+	flag = json_object_get_string(ed);
+	if (strcasecmp(flag, "skipempty") == 0) {
+		data->skipempty = 1;
+	} else {
+		ln_errprintf(ctx, 0, "invalid flag for JSON parser: %s", flag);
+		r = LN_BADCONFIG;
+		goto done;
+	}
+	*pdata = data;
+done:
+	if(r != 0) {
+		free(data);
+	}
+	return r;
+}
+PARSER_Destruct(JSON)
+{
+	free(pdata);
+}
 
 /* check if a char is valid inside a name of a NameValue list
  * The set of valid characters may be extended if there is good

rsyslog/vendored_src/liblognorm/liblognorm/src/parser.h

@@ -70,7 +70,7 @@ PARSERDEF_NO_DATA(Time24hr);
 PARSERDEF_NO_DATA(Duration);
 PARSERDEF_NO_DATA(IPv4);
 PARSERDEF_NO_DATA(IPv6);
-PARSERDEF_NO_DATA(JSON);
+PARSERDEF(JSON);
 PARSERDEF_NO_DATA(CEESyslog);
 PARSERDEF_NO_DATA(v2IPTables);
 PARSERDEF_NO_DATA(CiscoInterfaceSpec);

rsyslog/vendored_src/liblognorm/liblognorm/src/pdag.c

@@ -90,7 +90,7 @@ static struct ln_parser_info parser_lookup_table[] = {
 	PARSER_ENTRY_NO_DATA("duration", Duration, 16),
 	PARSER_ENTRY_NO_DATA("cisco-interface-spec", CiscoInterfaceSpec, 4),
 	PARSER_ENTRY_NO_DATA("name-value-list", NameValue, 8),
-	PARSER_ENTRY_NO_DATA("json", JSON, 4),
+	PARSER_ENTRY("json", JSON, 4),
 	PARSER_ENTRY_NO_DATA("cee-syslog", CEESyslog, 4),
 	PARSER_ENTRY_NO_DATA("mac48", MAC48, 16),
 	PARSER_ENTRY_NO_DATA("cef", CEF, 4),

rsyslog/vendored_src/liblognorm/liblognorm/tests/Makefile.am

@@ -86,6 +86,7 @@ TESTS_SHELLSCRIPTS = \
 	field_rest_jsoncnf.sh \
 	field_json.sh \
 	field_json_jsoncnf.sh \
+	field_json_skipempty.sh \
 	field_cee-syslog.sh \
 	field_cee-syslog_jsoncnf.sh \
 	field_ipv6.sh \

rsyslog/vendored_src/liblognorm/liblognorm/tests/field_json_skipempty.sh

@@ -0,0 +1,30 @@
+# added 2018-08-27 by Noriko Hosoi
+# This file is part of the liblognorm project, released under ASL 2.0
+. $srcdir/exec.sh
+
+test_def $0 "JSON field"
+add_rule 'version=2'
+add_rule 'rule=:%field:json%'
+
+# default behaviour
+execute '{"f1": "1", "f2": 2, "f3": "", "f4": {}, "f5": []}'
+assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 , "f3": "", "f4": {}, "f5": []} }'
+
+# skip empty json values
+reset_rules
+add_rule 'version=2'
+add_rule 'rule=:%field:json:skipempty%'
+
+execute '{"f1": "1", "f2": 2, "f3": "", "f4": {}, "f5": []}'
+assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }'
+
+# undefined parameter has to be ignored?
+reset_rules
+add_rule 'version=2'
+add_rule 'rule=:%field:json:bogus%'
+
+execute '{"f1": "1", "f2": 2, "f3": "", "f4": {}, "f5": []}'
+assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 , "f3": "", "f4": {}, "f5": []} }'
+
+cleanup_tmp_files
+