diff --git a/rsyslog/vendored_src/libfastjson/libfastjson/arraylist.c b/rsyslog/vendored_src/libfastjson/libfastjson/arraylist.c index c02907287..0ead3482a 100644 --- a/rsyslog/vendored_src/libfastjson/libfastjson/arraylist.c +++ b/rsyslog/vendored_src/libfastjson/libfastjson/arraylist.c @@ -87,6 +87,22 @@ array_list_add(struct array_list *arr, void *data) return array_list_put_idx(arr, arr->length, data); } +/* + * Deleting the idx-th element in the array_list. + */ +void +array_list_del_idx(struct array_list *const arr, const int idx) +{ + if (idx < 0 || idx >= arr->length) { + return; + } + if(arr->array[idx]) arr->free_fn(arr->array[idx]); + if (--arr->length > idx) { + memmove(arr->array + idx, arr->array + idx + 1, (arr->length - idx) * sizeof(void *)); + } + return; +} + /* work around wrong compiler message: GCC and clang do * not handle sort_fn correctly if -Werror is given. */ diff --git a/rsyslog/vendored_src/libfastjson/libfastjson/arraylist.h b/rsyslog/vendored_src/libfastjson/libfastjson/arraylist.h index 4b0f9e88e..54ddad47f 100644 --- a/rsyslog/vendored_src/libfastjson/libfastjson/arraylist.h +++ b/rsyslog/vendored_src/libfastjson/libfastjson/arraylist.h @@ -41,6 +41,9 @@ array_list_put_idx(struct array_list *al, int i, void *data); extern int array_list_add(struct array_list *al, void *data); +extern void +array_list_del_idx(struct array_list *const arr, const int idx); + extern int array_list_length(struct array_list *al); diff --git a/rsyslog/vendored_src/libfastjson/libfastjson/json_object.c b/rsyslog/vendored_src/libfastjson/libfastjson/json_object.c index db2ef36fd..35338e137 100644 --- a/rsyslog/vendored_src/libfastjson/libfastjson/json_object.c +++ b/rsyslog/vendored_src/libfastjson/libfastjson/json_object.c @@ -1018,6 +1018,14 @@ struct fjson_object* fjson_object_array_get_idx(struct fjson_object *jso, return (struct fjson_object*)array_list_get_idx(jso->o.c_array, idx); } +/* + * Deleting the idx-th element in the array type object. + */ +void fjson_object_array_del_idx(struct fjson_object *jso, int idx) +{ + array_list_del_idx(jso->o.c_array, idx); +} + int fjson_object_get_member_count(struct fjson_object *jso) { return jso->o.c_obj.nelem; diff --git a/rsyslog/vendored_src/libfastjson/libfastjson/json_object.h b/rsyslog/vendored_src/libfastjson/libfastjson/json_object.h index 1d354a7f2..43222dff9 100644 --- a/rsyslog/vendored_src/libfastjson/libfastjson/json_object.h +++ b/rsyslog/vendored_src/libfastjson/libfastjson/json_object.h @@ -489,6 +489,8 @@ extern int fjson_object_array_put_idx(struct fjson_object *obj, int idx, extern struct fjson_object* fjson_object_array_get_idx(struct fjson_object *obj, int idx); +extern void fjson_object_array_del_idx(struct fjson_object *jso, int idx); + /* fjson_bool type methods */ /** Create a new empty fjson_object of type fjson_type_boolean @@ -731,6 +733,7 @@ typedef struct fjson_tokener fjson_tokener; #define json_object_get_int64 fjson_object_get_int64 #define json_object_get_string_len fjson_object_get_string_len #define json_object_get_member_count fjson_object_get_member_count +#define json_object_array_del_idx fjson_object_array_del_idx #endif diff --git a/rsyslog/vendored_src/liblognorm/liblognorm/src/parser.c b/rsyslog/vendored_src/liblognorm/liblognorm/src/parser.c index bcb7b3f35..e40eea7c6 100644 --- a/rsyslog/vendored_src/liblognorm/liblognorm/src/parser.c +++ b/rsyslog/vendored_src/liblognorm/liblognorm/src/parser.c @@ -2316,6 +2316,85 @@ PARSER_Parse(v2IPTables) return r; } +/* + * Delete children of the given object if it has children and they are empty. + * + * return 0 if object is not empty + * return 1 if object is empty + * return < 0 if error + * + * Caller should do this: + * if (jsonSkipEmpty(obj) > 0) { + * json_object_put(obj); + * obj = NULL; + * } + * or otherwise not use obj if jsonSkipEmpty returns > 0. + */ +static int +jsonSkipEmpty(struct json_object *__restrict__ json) +{ + int rc = 0; + struct json_object *val = NULL; + + if(json == NULL) { + rc = 1; + goto finalize_it; + } + + switch (json_object_get_type(json)) { + case json_type_string: + rc = json_object_get_string_len(json) == 0; + break; + case json_type_array: + { + int i; + int arrayLen = json_object_array_length(json); + for (i = 0 ; i < arrayLen ; ++i) { + val = json_object_array_get_idx(json, i); + if ((rc = jsonSkipEmpty(val)) > 0) { + /* delete the empty item and reset the index and arrayLen */ + json_object_array_del_idx(json, i--); + arrayLen = json_object_array_length(json); + } else if (rc < 0) { + goto finalize_it; + } + } + rc = json_object_array_length(json) == 0; + break; + } + case json_type_object: + { + struct json_object_iterator it = json_object_iter_begin(json); + struct json_object_iterator itEnd = json_object_iter_end(json); + while (!json_object_iter_equal(&it, &itEnd)) { + val = json_object_iter_peek_value(&it); + if ((rc = jsonSkipEmpty(val)) > 0) { + json_object_object_del(json, json_object_iter_peek_name(&it)); + } else if (rc < 0) { + goto finalize_it; + } + json_object_iter_next(&it); + } + rc = json_object_object_length(json) == 0; + } + case json_type_null: + case json_type_boolean: + case json_type_double: + case json_type_int: + default: break; + } +finalize_it: + return rc; +} + +/* + * Parameters for field type json + * skipempty - skips empty json objects. + * - %field_name:json:skipempty% + */ +struct data_JSON { + int skipempty; +}; /** * Parse JSON. This parser tries to find JSON data inside a message. * If it finds valid JSON, it will extract it. Extra data after the @@ -2331,6 +2410,7 @@ PARSER_Parse(v2IPTables) PARSER_Parse(JSON) const size_t i = *offs; struct json_tokener *tokener = NULL; + struct data_JSON *const data = (struct data_JSON*) pdata; if(npb->str[i] != '{' && npb->str[i] != ']') { /* this can't be json, see RFC4627, Sect. 2 @@ -2359,6 +2439,20 @@ PARSER_Parse(JSON) if(value == NULL) { json_object_put(json); } else { + if (data && data->skipempty) { + int rc = jsonSkipEmpty(json); + if (rc < 0) { + json_object_put(json); + FAIL(LN_WRONGPARSER); + } else if (rc > 0) { + /* + * json value is empty. + * E.g., {"message":""}, {"message":[]}, {"message":{}} + */ + json_object_put(json); + FAIL(0); + } + } *value = json; } @@ -2367,7 +2461,40 @@ PARSER_Parse(JSON) json_tokener_free(tokener); return r; } +PARSER_Construct(JSON) +{ + int r = 0; + struct json_object *ed; + struct data_JSON *data = NULL; + char *flag; + if(json == NULL) + goto done; + + if(json_object_object_get_ex(json, "extradata", &ed) == 0) { + /* No JSON parameter */ + goto done; + } + data = (struct data_JSON*) calloc(1, sizeof(struct data_JSON)); + flag = json_object_get_string(ed); + if (strcasecmp(flag, "skipempty") == 0) { + data->skipempty = 1; + } else { + ln_errprintf(ctx, 0, "invalid flag for JSON parser: %s", flag); + r = LN_BADCONFIG; + goto done; + } + *pdata = data; +done: + if(r != 0) { + free(data); + } + return r; +} +PARSER_Destruct(JSON) +{ + free(pdata); +} /* check if a char is valid inside a name of a NameValue list * The set of valid characters may be extended if there is good diff --git a/rsyslog/vendored_src/liblognorm/liblognorm/src/parser.h b/rsyslog/vendored_src/liblognorm/liblognorm/src/parser.h index 38be62d10..5b4a82148 100644 --- a/rsyslog/vendored_src/liblognorm/liblognorm/src/parser.h +++ b/rsyslog/vendored_src/liblognorm/liblognorm/src/parser.h @@ -70,7 +70,7 @@ PARSERDEF_NO_DATA(Time24hr); PARSERDEF_NO_DATA(Duration); PARSERDEF_NO_DATA(IPv4); PARSERDEF_NO_DATA(IPv6); -PARSERDEF_NO_DATA(JSON); +PARSERDEF(JSON); PARSERDEF_NO_DATA(CEESyslog); PARSERDEF_NO_DATA(v2IPTables); PARSERDEF_NO_DATA(CiscoInterfaceSpec); diff --git a/rsyslog/vendored_src/liblognorm/liblognorm/src/pdag.c b/rsyslog/vendored_src/liblognorm/liblognorm/src/pdag.c index f8ab1ec52..2e6ff7d36 100644 --- a/rsyslog/vendored_src/liblognorm/liblognorm/src/pdag.c +++ b/rsyslog/vendored_src/liblognorm/liblognorm/src/pdag.c @@ -90,7 +90,7 @@ static struct ln_parser_info parser_lookup_table[] = { PARSER_ENTRY_NO_DATA("duration", Duration, 16), PARSER_ENTRY_NO_DATA("cisco-interface-spec", CiscoInterfaceSpec, 4), PARSER_ENTRY_NO_DATA("name-value-list", NameValue, 8), - PARSER_ENTRY_NO_DATA("json", JSON, 4), + PARSER_ENTRY("json", JSON, 4), PARSER_ENTRY_NO_DATA("cee-syslog", CEESyslog, 4), PARSER_ENTRY_NO_DATA("mac48", MAC48, 16), PARSER_ENTRY_NO_DATA("cef", CEF, 4), diff --git a/rsyslog/vendored_src/liblognorm/liblognorm/tests/Makefile.am b/rsyslog/vendored_src/liblognorm/liblognorm/tests/Makefile.am index 0967f3b35..20ab8fe88 100644 --- a/rsyslog/vendored_src/liblognorm/liblognorm/tests/Makefile.am +++ b/rsyslog/vendored_src/liblognorm/liblognorm/tests/Makefile.am @@ -86,6 +86,7 @@ TESTS_SHELLSCRIPTS = \ field_rest_jsoncnf.sh \ field_json.sh \ field_json_jsoncnf.sh \ + field_json_skipempty.sh \ field_cee-syslog.sh \ field_cee-syslog_jsoncnf.sh \ field_ipv6.sh \ diff --git a/rsyslog/vendored_src/liblognorm/liblognorm/tests/field_json_skipempty.sh b/rsyslog/vendored_src/liblognorm/liblognorm/tests/field_json_skipempty.sh new file mode 100755 index 000000000..14dfb44d1 --- /dev/null +++ b/rsyslog/vendored_src/liblognorm/liblognorm/tests/field_json_skipempty.sh @@ -0,0 +1,30 @@ +# added 2018-08-27 by Noriko Hosoi +# This file is part of the liblognorm project, released under ASL 2.0 +. $srcdir/exec.sh + +test_def $0 "JSON field" +add_rule 'version=2' +add_rule 'rule=:%field:json%' + +# default behaviour +execute '{"f1": "1", "f2": 2, "f3": "", "f4": {}, "f5": []}' +assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 , "f3": "", "f4": {}, "f5": []} }' + +# skip empty json values +reset_rules +add_rule 'version=2' +add_rule 'rule=:%field:json:skipempty%' + +execute '{"f1": "1", "f2": 2, "f3": "", "f4": {}, "f5": []}' +assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' + +# undefined parameter has to be ignored? +reset_rules +add_rule 'version=2' +add_rule 'rule=:%field:json:bogus%' + +execute '{"f1": "1", "f2": 2, "f3": "", "f4": {}, "f5": []}' +assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 , "f3": "", "f4": {}, "f5": []} }' + +cleanup_tmp_files +