From 2b2b5ef7726f7203e4d1bf96face6a1250d66ee0 Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Tue, 17 Dec 2024 11:52:23 +0100 Subject: [PATCH 01/13] Reduce rule trees to one rule tree --- examples/exampledata/config/pipeline.yml | 39 ++---- .../amides_generic.yml => rules/amides_1.yml} | 0 .../amides_2.yml} | 0 .../example_rule_1.yml} | 0 .../example_rule_2.yml} | 0 .../{generic => rules}/example_rule.yml | 0 .../rules/dropper/specific/example_rule.yml | 6 - .../{generic => rules}/example_rule.yml | 0 .../rules/labeler/specific/example_rule.yml | 7 - .../example_rule_1.yml} | 0 .../example_rule_2.yml} | 0 .../{generic => rules}/example_rule.yml | 0 .../pseudonymizer/specific/example_rule.yml | 6 - logprep/abc/processor.py | 81 +++--------- logprep/framework/rule_tree/rule_tree.py | 13 -- logprep/processor/amides/processor.py | 6 +- logprep/processor/base/rule.py | 2 +- logprep/processor/calculator/processor.py | 6 +- logprep/processor/clusterer/processor.py | 20 ++- logprep/processor/concatenator/processor.py | 6 +- .../processor/datetime_extractor/processor.py | 6 +- logprep/processor/deleter/processor.py | 6 +- logprep/processor/dissector/processor.py | 6 +- .../domain_label_extractor/processor.py | 6 +- .../processor/domain_resolver/processor.py | 6 +- logprep/processor/dropper/processor.py | 6 +- logprep/processor/field_manager/processor.py | 6 +- logprep/processor/generic_adder/processor.py | 6 +- .../processor/generic_resolver/processor.py | 6 +- logprep/processor/geoip_enricher/processor.py | 6 +- logprep/processor/grokker/processor.py | 6 +- .../processor/hyperscan_resolver/processor.py | 6 +- logprep/processor/ip_informer/processor.py | 6 +- logprep/processor/key_checker/processor.py | 6 +- logprep/processor/labeler/processor.py | 8 +- .../processor/list_comparison/processor.py | 8 +- logprep/processor/pre_detector/processor.py | 6 +- logprep/processor/pseudonymizer/processor.py | 8 +- logprep/processor/requester/processor.py | 6 +- .../selective_extractor/processor.py | 6 +- .../processor/string_splitter/processor.py | 6 +- .../processor/template_replacer/processor.py | 6 +- .../processor/timestamp_differ/processor.py | 6 +- logprep/processor/timestamper/processor.py | 6 +- .../util/auto_rule_tester/auto_rule_tester.py | 84 +++++------- logprep/util/configuration.py | 56 +++----- logprep/util/template_processor.py.j2 | 6 +- logprep/util/template_processor_test.py.j2 | 7 +- tests/acceptance/test_amides.py | 3 +- tests/acceptance/test_file_input.py | 3 +- tests/acceptance/test_full_configuration.py | 9 +- .../test_http_input_with_requests.py | 3 +- tests/acceptance/test_multiple_outputs.py | 22 +--- tests/acceptance/test_pre_detection.py | 3 +- tests/acceptance/test_preprocessing.py | 3 +- ..._selective_extractor_full_pipeline_pass.py | 6 +- .../acceptance/test_wineventlog_processing.py | 33 ++--- .../test_wineventlog_pseudonymization.py | 9 +- tests/acceptance/util.py | 5 +- ...sector_rule.json => dissector_rule_1.json} | 0 ...sector_rule.json => dissector_rule_2.json} | 0 .../labeling/schema.json | 0 .../rules}/id_1_SecurityCenter.json | 0 .../rules}/id_400_PowerShell.json | 0 ...d_50036_Microsoft-Windows-Dhcp-Client.json | 0 ...51047_Microsoft-Windows-DHCPv6-Client.json | 0 .../rules}/id_5615_Microsoft-Windows-WMI.json | 0 .../rules}/id_6005_EventLog.json | 0 .../rules}/id_6006_EventLog.json | 0 .../id_7040_Service_Control_Manager.json | 0 .../rules}/id_8212_System_Restore.json | 0 .../action/event_data_Started_to_execute.json | 0 .../event_data_Stopped_to_terminate.json | 0 .../action/event_data_paused_to_modify.json | 0 .../event_data_power_off_to_terminate.json | 0 .../action/event_data_running_to_execute.json | 0 .../keywords_Audit_Failure_to_failed.json | 0 .../keywords_Audit_Success_to_success.json | 0 .../windows/action/level_Error_to_failed.json | 0 .../event_data_logontype_2_or_7_to_user.json | 0 ...vent_data_logontype_4_or_5_to_service.json | 0 ..._Audit_Policy_Change_to_configuration.json | 0 ...k_Logoff_to_authenticate_and_accounts.json | 0 ...sk_Logon_to_authenticate_and_accounts.json | 0 .../Desktop_Window_Manager_to_system.json | 0 .../windows/reporter/ESENT_to_database.json | 0 .../windows/reporter/EventLog_to_system.json | 0 .../reporter/FreeSSHDService_to_service.json | 0 ...dows-Application-Experience_to_system.json | 0 ...osoft-Windows-DHCPv6-Client_to_system.json | 0 ...crosoft-Windows-Dhcp-Client_to_system.json | 0 ...crosoft-Windows-EventSystem_to_system.json | 0 ...osoft-Windows-FilterManager_to_system.json | 0 ...crosoft-Windows-GroupPolicy_to_system.json | 0 ...soft-Windows-Kernel-General_to_system.json | 0 ...rosoft-Windows-Kernel-Power_to_system.json | 0 ...dows-Kernel-Processor-Power_to_system.json | 0 ...t-Windows-Security-Auditing_to_system.json | 0 ...rosoft-Windows-Security-SPP_to_system.json | 0 ...rosoft-Windows-Time-Service_to_system.json | 0 ...ndows-User-Profiles-Service_to_system.json | 0 .../Microsoft-Windows-UserPnp_to_system.json | 0 .../Microsoft-Windows-WMI_to_system.json | 0 ...soft-Windows-WMPNSS-Service_to_system.json | 0 .../Microsoft-Windows-Winlogon_to_system.json | 0 .../windows/reporter/NETLOGON_to_system.json | 0 .../reporter/PowerShell_to_system.json | 0 .../reporter/SecurityCenter_to_system.json | 0 .../Service_Control_Manager_to_system.json | 0 .../reporter/System_Restore_to_system.json | 0 .../windows/reporter/VSS_to_service.json | 0 .../windows/reporter/volsnap_to_system.json | 0 .../reporter/wineventlog_to_windows.json | 0 .../labeling/schema.json | 0 .../rules}/computer_name_label.json | 0 .../rules}/event_data_Binary_label.json | 0 .../event_data_TargetLogonID_to_label.json | 0 ...vent_data_param1_auto_discovery_label.json | 0 .../event_data_param1_crypto_label.json | 0 .../event_data_param1_flash_player_label.json | 0 ..._data_param1_font_cache_service_label.json | 0 .../rules}/message_to_logon_label.json | 0 .../provider_guid_to_test_guid_label.json | 0 .../rules}/this_is_not_a_rule.not_json | 0 .../rules}/version_to_label.json | 0 .../{rules_static => }/regex_mapping.yml | 0 ..._NewProcessId_New_ProcessName_id_4688.json | 0 ...ubjectUserName_SubjectUserSid_id_4611.json | 0 ...ubjectUserName_SubjectUserSid_id_4672.json | 0 ...event_data_ClientAddress_to_client_ip.json | 0 .../event_data_FromFolder_to_file_path.json | 0 ...vent_data_IpAddress_to_client_address.json | 0 .../event_data_IpAddress_to_client_ip.json | 0 .../event_data_IpPort_to_client_port.json | 0 ...data_LogonProcessName_to_process_name.json | 0 ...ata_ProcessId_NOT_4688_to_process_pid.json | 0 ...ata_ProcessName_to_process_executable.json | 0 ...data_TargetUserName_to_host_user_name.json | 0 ...nt_data_TargetUserSid_to_host_user_id.json | 0 ...ent_data_ToFolder_to_file_target_path.json | 0 .../event_data_UserSid_to_host_user_id.json | 0 .../param1_to_client_address_id_1104.json | 0 .../param1_to_client_address_id_1106.json | 0 .../param1_to_host_user_name_id_8.json | 0 .../param1_to_host_user_name_id_9.json | 0 .../param2_to_host_user_name_id_2000.json | 0 .../param2_to_host_user_name_id_2001.json | 0 .../param3_to_client_address_id_1104.json | 0 .../param3_to_client_address_id_1107.json | 0 .../param4_to_error_code_id_4098.json | 0 .../this_is_not_a_rule.not_json | 0 .../pre_detect_acceptance_one.json | 0 ...ne.json => pre_detect_acceptance_two.json} | 0 .../{rules_static => }/regex_mapping.yml | 0 .../MetaFrameEvents_id_1104.json | 0 .../MetaFrameEvents_id_1106.json | 0 ...minal-RemoteConnectionManager_id_1060.json | 0 .../specific => rules}/TdIca_id_1004.json | 0 .../specific => rules}/TdIca_id_1007.json | 0 .../generic => rules}/client_address.json | 0 .../generic => rules}/client_ip.json | 0 .../event_data_IpAddress.json | 0 .../event_data_SubjectUserName.json | 0 .../event_data_SubjectUserSid.json | 0 .../event_data_TargetUserName.json | 0 .../event_data_TargetUserSid.json | 0 .../event_data_ToFolder.json | 0 .../generic => rules}/event_data_UserSid.json | 0 .../generic => rules}/file_target_path.json | 0 .../generic => rules}/host_user_id.json | 0 .../generic => rules}/host_user_name.json | 0 .../rules}/this_is_not_a_rule.not_json | 0 .../generic => rules}/user_identifier.json | 0 .../generic => rules}/user_name.json | 0 .../specific/this_is_not_a_rule.not_json | 1 - .../{generic/rules.json => rules_1.json} | 0 .../{specific/rules.json => rules_2.json} | 0 ...tests.yml => rule_with_custom_tests_1.yml} | 0 ...tests.yml => rule_with_custom_tests_2.yml} | 0 .../rules/{generic => }/auto_test_match.json | 0 .../{generic => }/auto_test_match_test.json | 0 .../{specific => }/auto_test_mismatch.json | 0 .../auto_test_mismatch_test.json | 0 .../{specific => }/auto_test_no_test_.json | 0 .../drop_field.json => drop_field_1.json} | 0 ...field_test.json => drop_field_1_test.json} | 0 .../drop_field.json => drop_field_2.json} | 0 ...field_test.json => drop_field_2_test.json} | 0 .../auto_test_labeling_match.json | 0 .../auto_test_labeling_match_existing.json | 0 ...uto_test_labeling_match_existing_test.json | 0 .../auto_test_labeling_match_test.json | 0 .../auto_test_labeling_mismatch.json | 0 .../auto_test_labeling_mismatch_test.json | 0 .../auto_test_labeling_no_test_.json | 0 .../auto_test_pre_detector_match.json | 0 .../auto_test_pre_detector_match_test.json | 0 .../auto_test_pre_detector_mismatch.json | 0 .../auto_test_pre_detector_mismatch_test.json | 0 .../auto_test_pre_detector_no_test_.json | 0 .../auto_test_pseudonymizer_dotted_list.json | 0 ...o_test_pseudonymizer_dotted_list_test.json | 0 .../auto_test_pseudonymizer_list.json | 0 .../auto_test_pseudonymizer_list_escaped.json | 0 ..._test_pseudonymizer_list_escaped_test.json | 0 .../auto_test_pseudonymizer_list_test.json | 0 .../auto_test_pseudonymizer_match.json | 0 .../auto_test_pseudonymizer_match_test.json | 0 .../auto_test_pseudonymizer_mismatch.json | 0 ...auto_test_pseudonymizer_mismatch_test.json | 0 .../auto_test_pseudonymizer_no_test_.json | 0 ...replacer.json => template_replacer_1.json} | 0 ...est.json => template_replacer_1_test.json} | 0 ...replacer.json => template_replacer_2.json} | 0 ...est.json => template_replacer_2_test.json} | 0 tests/testdata/config/config-auto-tests.yml | 42 ++---- tests/testdata/config/config-docker.yml | 6 +- tests/testdata/config/config.yml | 26 ++-- tests/testdata/config/config2.yml | 6 +- .../amides_generic.yml => amides_1.yml} | 0 .../amides_specific.yml => amides_2.yml} | 0 .../calculator_1.json} | 0 .../calculator_2.json} | 0 .../clusterer/rules/{generic => }/rules.json | 0 .../unit/clusterer/rules/specific/rules.json | 11 -- .../add_fields.json => add_fields_1.json} | 0 .../add_fields.json => add_fields_2.json} | 0 ...tractor.json => datetime_extractor_1.json} | 0 ...tractor.json => datetime_extractor_2.json} | 0 .../generic_delete.json => delete_1.json} | 0 .../specific_delete.json => delete_2.json} | 2 +- .../rules/{specific => }/delete_test.json | 0 .../deleter/rules/{specific => }/test.json | 0 .../dissector_rule_1.json} | 0 .../dissector_rule_2.json} | 0 ...gen.json => domain_label_extractor_1.json} | 0 ...tor.json => domain_label_extractor_2.json} | 0 ...ain_resolver.yml => domain_resolver_1.yml} | 0 ...n_resolver.json => domain_resolver_2.json} | 0 .../drop_field.json => drop_field_1.json} | 0 .../drop_field.json => drop_field_2.json} | 0 .../field_manager_1.json} | 0 .../field_manager_2.json} | 0 .../rules/{generic/rules.json => rule_1.json} | 0 .../specific_rules.json => rule_2.json} | 0 .../{generic/rule_01.json => rule_1.json} | 0 .../{specific/rule_01.json => rule_2.json} | 0 .../geoip_all.json => geoip_all_1.json} | 0 .../geoip_all.json => geoip_all_2.json} | 0 .../rule.yml => rules/rule_1.yml} | 0 .../rule.yml => rules/rule_2.yml} | 0 .../{generic/rule_01.json => rule_1.json} | 0 .../{specific/rule_01.json => rule_2.json} | 0 .../{generic/rule.json => rules/rule_1.json} | 0 .../{specific/rule.json => rules/rule_2.json} | 0 .../key_checker_rule_1.json} | 0 .../key_checker_rule_2.json} | 0 .../labeler/rules/{specific => }/first.json | 0 .../labeler/rules/{generic => }/rule.json | 0 .../user_check.json => user_check_1.json} | 0 ..._check_specific.json => user_check_2.json} | 0 .../rules/{generic => }/pre_detect_four.yml | 0 .../rules/{generic => }/pre_detect_one.json | 0 .../rules/{generic => }/pre_detect_three.json | 0 .../rules/{generic => }/pre_detect_two.json | 0 .../{generic => }/pre_detect_two_rules.json | 0 .../rules/specific/pre_detect_one.json | 16 --- .../rules/specific/pre_detect_three.json | 16 --- .../rules/specific/pre_detect_two.json | 16 --- .../rules/specific/pre_detect_two_rules.json | 30 ----- .../{rules => }/regex_mapping.yml | 0 .../rules/{specific => }/Test123_id_789.json | 0 .../rules/{specific => }/Test456_id_1234.json | 0 .../{generic => }/event_data_IpAddress.json | 0 .../rules/generic/this_is_not_a_rule.not_json | 1 - .../rules}/this_is_not_a_rule.not_json | 0 .../requester.json => rules/requester_1.json} | 0 .../requester.json => rules/requester_2.json} | 0 .../{generic/rules.json => rules_1.json} | 0 .../{specific/rules.json => rules_2.json} | 0 .../{generic/generic.json => rules/rule.json} | 0 .../string_splitter/specific/specific.json | 11 -- .../rules/specific/template_replacer.json | 7 - .../{generic => }/template_replacer.json | 0 .../timestamp_differ_rule.json | 0 .../specific_rules/timestamp_differ_rule.json | 9 -- .../timestamper_rule.yml | 2 +- .../specific_rules/timestamper_rule.yml | 3 - .../framework/rule_tree/test_rule_tree.py | 3 +- tests/unit/framework/test_pipeline.py | 6 +- tests/unit/processor/amides/test_amides.py | 3 +- tests/unit/processor/base.py | 120 +++++------------- .../processor/calculator/test_calculator.py | 7 +- .../processor/clusterer/test_clusterer.py | 104 ++++++++------- .../concatenator/test_concatenator.py | 15 +-- .../concatenator/test_concatenator_rule.py | 12 +- .../test_datetime_extractor.py | 17 +-- .../test_datetime_extractor_rule.py | 12 +- tests/unit/processor/deleter/test_deleter.py | 3 +- .../processor/deleter/test_deleter_rule.py | 12 +- .../processor/dissector/test_dissector.py | 7 +- .../test_domain_label_extractor.py | 25 ++-- .../test_domain_label_extractor_rule.py | 12 +- .../domain_resolver/test_domain_resolver.py | 23 ++-- .../test_domain_resolver_rule.py | 8 +- tests/unit/processor/dropper/test_dropper.py | 39 +++--- .../processor/dropper/test_dropper_rule.py | 16 +-- .../field_manager/test_field_manager.py | 15 +-- .../generic_adder/test_generic_adder.py | 19 +-- .../generic_adder/test_generic_adder_rule.py | 8 +- .../generic_resolver/test_generic_resolver.py | 64 +++++----- .../test_generic_resolver_rule.py | 8 +- .../geoip_enricher/test_geoip_enricher.py | 27 ++-- .../test_geoip_enricher_rule.py | 8 +- tests/unit/processor/grokker/test_grokker.py | 11 +- .../test_hyperscan_resolver.py | 69 +++++----- .../test_hyperscan_resolver_rule.py | 20 +-- .../processor/ip_informer/test_ip_informer.py | 7 +- tests/unit/processor/key_checker/__init__.py | 0 .../processor/key_checker/test_key_checker.py | 7 +- tests/unit/processor/labeler/test_labeler.py | 44 +++---- .../list_comparison/test_list_comparison.py | 22 ++-- .../test_list_comparison_rule.py | 12 +- .../pre_detector/test_pre_detector.py | 11 +- .../pre_detector/test_pre_detector_rule.py | 24 ++-- .../pseudonymizer/test_pseudonymizer.py | 37 +++--- .../pseudonymizer/test_pseudonymizer_rule.py | 8 +- .../processor/requester/test_requester.py | 7 +- .../test_selective_extractor.py | 28 ++-- .../test_selective_extractor_rule.py | 12 +- .../string_splitter/test_string_splitter.py | 7 +- .../test_template_replacer.py | 11 +- tests/unit/processor/test_process.py | 89 ++----------- .../timestamp_differ/test_timestamp_differ.py | 7 +- .../processor/timestamper/test_timestamper.py | 7 +- tests/unit/test_configuration.py | 36 ++---- tests/unit/test_factory.py | 52 ++------ tests/unit/util/test_auto_rule_tester.py | 55 ++++---- tests/unit/util/test_configuration.py | 111 +++++----------- tests/unit/util/test_rule_dry_runner.py | 28 ++-- 340 files changed, 731 insertions(+), 1383 deletions(-) rename examples/exampledata/rules/amides/{generic/amides_generic.yml => rules/amides_1.yml} (100%) rename examples/exampledata/rules/amides/{specific/amides_specific.yml => rules/amides_2.yml} (100%) rename examples/exampledata/rules/dissector/{generic/example_rule.yml => rules/example_rule_1.yml} (100%) rename examples/exampledata/rules/dissector/{specific/example_rule.yml => rules/example_rule_2.yml} (100%) rename examples/exampledata/rules/dropper/{generic => rules}/example_rule.yml (100%) delete mode 100644 examples/exampledata/rules/dropper/specific/example_rule.yml rename examples/exampledata/rules/labeler/{generic => rules}/example_rule.yml (100%) delete mode 100644 examples/exampledata/rules/labeler/specific/example_rule.yml rename examples/exampledata/rules/pre_detector/{generic/example_rule.yml => rules/example_rule_1.yml} (100%) rename examples/exampledata/rules/pre_detector/{specific/example_rule.yml => rules/example_rule_2.yml} (100%) rename examples/exampledata/rules/pseudonymizer/{generic => rules}/example_rule.yml (100%) delete mode 100644 examples/exampledata/rules/pseudonymizer/specific/example_rule.yml rename tests/testdata/acceptance/dissector/rules/{generic/dissector_rule.json => dissector_rule_1.json} (100%) rename tests/testdata/acceptance/dissector/rules/{specific/dissector_rule.json => dissector_rule_2.json} (100%) rename tests/testdata/acceptance/labeler/{rules_static => no_regex}/labeling/schema.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/specific => no_regex/rules}/id_1_SecurityCenter.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/specific => no_regex/rules}/id_400_PowerShell.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/specific => no_regex/rules}/id_50036_Microsoft-Windows-Dhcp-Client.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/specific => no_regex/rules}/id_51047_Microsoft-Windows-DHCPv6-Client.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/specific => no_regex/rules}/id_5615_Microsoft-Windows-WMI.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/specific => no_regex/rules}/id_6005_EventLog.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/specific => no_regex/rules}/id_6006_EventLog.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/specific => no_regex/rules}/id_7040_Service_Control_Manager.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/specific => no_regex/rules}/id_8212_System_Restore.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/action/event_data_Started_to_execute.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/action/event_data_Stopped_to_terminate.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/action/event_data_paused_to_modify.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/action/event_data_power_off_to_terminate.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/action/event_data_running_to_execute.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/action/keywords_Audit_Failure_to_failed.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/action/keywords_Audit_Success_to_success.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/action/level_Error_to_failed.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/actor/event_data_logontype_2_or_7_to_user.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/actor/event_data_logontype_4_or_5_to_service.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/multiple/task_Audit_Policy_Change_to_configuration.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/multiple/task_Logoff_to_authenticate_and_accounts.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/multiple/task_Logon_to_authenticate_and_accounts.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Desktop_Window_Manager_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/ESENT_to_database.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/EventLog_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/FreeSSHDService_to_service.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-Application-Experience_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-DHCPv6-Client_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-Dhcp-Client_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-EventSystem_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-FilterManager_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-GroupPolicy_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-Kernel-General_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-Kernel-Power_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-Kernel-Processor-Power_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-Security-Auditing_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-Security-SPP_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-Time-Service_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-User-Profiles-Service_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-UserPnp_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-WMI_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-WMPNSS-Service_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Microsoft-Windows-Winlogon_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/NETLOGON_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/PowerShell_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/SecurityCenter_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/Service_Control_Manager_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/System_Restore_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/VSS_to_service.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/volsnap_to_system.json (100%) rename tests/testdata/acceptance/labeler/{rules_static/rules/generic => no_regex/rules}/windows/reporter/wineventlog_to_windows.json (100%) rename tests/testdata/acceptance/labeler/{rules_static_only_regex => only_regex}/labeling/schema.json (100%) rename tests/testdata/acceptance/labeler/{rules_static_only_regex/rules/specific => only_regex/rules}/computer_name_label.json (100%) rename tests/testdata/acceptance/labeler/{rules_static_only_regex/rules/specific => only_regex/rules}/event_data_Binary_label.json (100%) rename tests/testdata/acceptance/labeler/{rules_static_only_regex/rules/specific => only_regex/rules}/event_data_TargetLogonID_to_label.json (100%) rename tests/testdata/acceptance/labeler/{rules_static_only_regex/rules/specific => only_regex/rules}/event_data_param1_auto_discovery_label.json (100%) rename tests/testdata/acceptance/labeler/{rules_static_only_regex/rules/specific => only_regex/rules}/event_data_param1_crypto_label.json (100%) rename tests/testdata/acceptance/labeler/{rules_static_only_regex/rules/specific => only_regex/rules}/event_data_param1_flash_player_label.json (100%) rename tests/testdata/acceptance/labeler/{rules_static_only_regex/rules/specific => only_regex/rules}/event_data_param1_font_cache_service_label.json (100%) rename tests/testdata/acceptance/labeler/{rules_static_only_regex/rules/generic => only_regex/rules}/message_to_logon_label.json (100%) rename tests/testdata/acceptance/labeler/{rules_static_only_regex/rules/specific => only_regex/rules}/provider_guid_to_test_guid_label.json (100%) rename tests/testdata/acceptance/labeler/{rules_static_only_regex/rules/specific => only_regex/rules}/this_is_not_a_rule.not_json (100%) rename tests/testdata/acceptance/labeler/{rules_static_only_regex/rules/specific => only_regex/rules}/version_to_label.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static => }/regex_mapping.yml (100%) rename tests/testdata/acceptance/normalizer/{rules_static/specific => rules}/ProcessId_NewProcessId_New_ProcessName_id_4688.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/specific => rules}/SubjectUserName_SubjectUserSid_id_4611.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/specific => rules}/SubjectUserName_SubjectUserSid_id_4672.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/generic => rules}/event_data_ClientAddress_to_client_ip.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/generic => rules}/event_data_FromFolder_to_file_path.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/generic => rules}/event_data_IpAddress_to_client_address.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/generic => rules}/event_data_IpAddress_to_client_ip.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/generic => rules}/event_data_IpPort_to_client_port.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/generic => rules}/event_data_LogonProcessName_to_process_name.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/generic => rules}/event_data_ProcessId_NOT_4688_to_process_pid.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/generic => rules}/event_data_ProcessName_to_process_executable.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/generic => rules}/event_data_TargetUserName_to_host_user_name.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/generic => rules}/event_data_TargetUserSid_to_host_user_id.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/generic => rules}/event_data_ToFolder_to_file_target_path.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/generic => rules}/event_data_UserSid_to_host_user_id.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/specific => rules}/param1_to_client_address_id_1104.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/specific => rules}/param1_to_client_address_id_1106.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/specific => rules}/param1_to_host_user_name_id_8.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/specific => rules}/param1_to_host_user_name_id_9.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/specific => rules}/param2_to_host_user_name_id_2000.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/specific => rules}/param2_to_host_user_name_id_2001.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/specific => rules}/param3_to_client_address_id_1104.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/specific => rules}/param3_to_client_address_id_1107.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/specific => rules}/param4_to_error_code_id_4098.json (100%) rename tests/testdata/acceptance/normalizer/{rules_static/generic => rules}/this_is_not_a_rule.not_json (100%) rename tests/testdata/acceptance/pre_detector/rules/{generic => }/pre_detect_acceptance_one.json (100%) rename tests/testdata/acceptance/pre_detector/rules/{specific/pre_detect_acceptance_one.json => pre_detect_acceptance_two.json} (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static => }/regex_mapping.yml (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/specific => rules}/MetaFrameEvents_id_1104.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/specific => rules}/MetaFrameEvents_id_1106.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/specific => rules}/Microsoft-Windows-Terminal-RemoteConnectionManager_id_1060.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/specific => rules}/TdIca_id_1004.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/specific => rules}/TdIca_id_1007.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/client_address.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/client_ip.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/event_data_IpAddress.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/event_data_SubjectUserName.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/event_data_SubjectUserSid.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/event_data_TargetUserName.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/event_data_TargetUserSid.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/event_data_ToFolder.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/event_data_UserSid.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/file_target_path.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/host_user_id.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/host_user_name.json (100%) rename tests/testdata/acceptance/{normalizer/rules_static/specific => pseudonymizer/rules}/this_is_not_a_rule.not_json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/user_identifier.json (100%) rename tests/testdata/acceptance/pseudonymizer/{rules_static/generic => rules}/user_name.json (100%) delete mode 100644 tests/testdata/acceptance/pseudonymizer/rules_static/specific/this_is_not_a_rule.not_json rename tests/testdata/acceptance/selective_extractor/rules/{generic/rules.json => rules_1.json} (100%) rename tests/testdata/acceptance/selective_extractor/rules/{specific/rules.json => rules_2.json} (100%) rename tests/testdata/auto_tests/clusterer/rules/{generic/rule_with_custom_tests.yml => rule_with_custom_tests_1.yml} (100%) rename tests/testdata/auto_tests/clusterer/rules/{specific/rule_with_custom_tests.yml => rule_with_custom_tests_2.yml} (100%) rename tests/testdata/auto_tests/dissector/rules/{generic => }/auto_test_match.json (100%) rename tests/testdata/auto_tests/dissector/rules/{generic => }/auto_test_match_test.json (100%) rename tests/testdata/auto_tests/dissector/rules/{specific => }/auto_test_mismatch.json (100%) rename tests/testdata/auto_tests/dissector/rules/{specific => }/auto_test_mismatch_test.json (100%) rename tests/testdata/auto_tests/dissector/rules/{specific => }/auto_test_no_test_.json (100%) rename tests/testdata/auto_tests/dropper/rules/{generic/drop_field.json => drop_field_1.json} (100%) rename tests/testdata/auto_tests/dropper/rules/{generic/drop_field_test.json => drop_field_1_test.json} (100%) rename tests/testdata/auto_tests/dropper/rules/{specific/drop_field.json => drop_field_2.json} (100%) rename tests/testdata/auto_tests/dropper/rules/{specific/drop_field_test.json => drop_field_2_test.json} (100%) rename tests/testdata/auto_tests/labeler/rules/{generic => }/auto_test_labeling_match.json (100%) rename tests/testdata/auto_tests/labeler/rules/{generic => }/auto_test_labeling_match_existing.json (100%) rename tests/testdata/auto_tests/labeler/rules/{generic => }/auto_test_labeling_match_existing_test.json (100%) rename tests/testdata/auto_tests/labeler/rules/{generic => }/auto_test_labeling_match_test.json (100%) rename tests/testdata/auto_tests/labeler/rules/{specific => }/auto_test_labeling_mismatch.json (100%) rename tests/testdata/auto_tests/labeler/rules/{specific => }/auto_test_labeling_mismatch_test.json (100%) rename tests/testdata/auto_tests/labeler/rules/{specific => }/auto_test_labeling_no_test_.json (100%) rename tests/testdata/auto_tests/pre_detector/rules/{generic => }/auto_test_pre_detector_match.json (100%) rename tests/testdata/auto_tests/pre_detector/rules/{generic => }/auto_test_pre_detector_match_test.json (100%) rename tests/testdata/auto_tests/pre_detector/rules/{specific => }/auto_test_pre_detector_mismatch.json (100%) rename tests/testdata/auto_tests/pre_detector/rules/{specific => }/auto_test_pre_detector_mismatch_test.json (100%) rename tests/testdata/auto_tests/pre_detector/rules/{specific => }/auto_test_pre_detector_no_test_.json (100%) rename tests/testdata/auto_tests/pseudonymizer/rules/{specific => }/auto_test_pseudonymizer_dotted_list.json (100%) rename tests/testdata/auto_tests/pseudonymizer/rules/{specific => }/auto_test_pseudonymizer_dotted_list_test.json (100%) rename tests/testdata/auto_tests/pseudonymizer/rules/{specific => }/auto_test_pseudonymizer_list.json (100%) rename tests/testdata/auto_tests/pseudonymizer/rules/{specific => }/auto_test_pseudonymizer_list_escaped.json (100%) rename tests/testdata/auto_tests/pseudonymizer/rules/{specific => }/auto_test_pseudonymizer_list_escaped_test.json (100%) rename tests/testdata/auto_tests/pseudonymizer/rules/{specific => }/auto_test_pseudonymizer_list_test.json (100%) rename tests/testdata/auto_tests/pseudonymizer/rules/{generic => }/auto_test_pseudonymizer_match.json (100%) rename tests/testdata/auto_tests/pseudonymizer/rules/{generic => }/auto_test_pseudonymizer_match_test.json (100%) rename tests/testdata/auto_tests/pseudonymizer/rules/{specific => }/auto_test_pseudonymizer_mismatch.json (100%) rename tests/testdata/auto_tests/pseudonymizer/rules/{specific => }/auto_test_pseudonymizer_mismatch_test.json (100%) rename tests/testdata/auto_tests/pseudonymizer/rules/{specific => }/auto_test_pseudonymizer_no_test_.json (100%) rename tests/testdata/auto_tests/template_replacer/rules/{generic/template_replacer.json => template_replacer_1.json} (100%) rename tests/testdata/auto_tests/template_replacer/rules/{generic/template_replacer_test.json => template_replacer_1_test.json} (100%) rename tests/testdata/auto_tests/template_replacer/rules/{specific/template_replacer.json => template_replacer_2.json} (100%) rename tests/testdata/auto_tests/template_replacer/rules/{specific/template_replacer_test.json => template_replacer_2_test.json} (100%) rename tests/testdata/unit/amides/rules/{generic/amides_generic.yml => amides_1.yml} (100%) rename tests/testdata/unit/amides/rules/{specific/amides_specific.yml => amides_2.yml} (100%) rename tests/testdata/unit/calculator/{generic_rules/calculator.json => rules/calculator_1.json} (100%) rename tests/testdata/unit/calculator/{specific_rules/calculator.json => rules/calculator_2.json} (100%) rename tests/testdata/unit/clusterer/rules/{generic => }/rules.json (100%) delete mode 100644 tests/testdata/unit/clusterer/rules/specific/rules.json rename tests/testdata/unit/concatenator/rules/{generic/add_fields.json => add_fields_1.json} (100%) rename tests/testdata/unit/concatenator/rules/{specific/add_fields.json => add_fields_2.json} (100%) rename tests/testdata/unit/datetime_extractor/rules/{generic/datetime_extractor.json => datetime_extractor_1.json} (100%) rename tests/testdata/unit/datetime_extractor/rules/{specific/datetime_extractor.json => datetime_extractor_2.json} (100%) rename tests/testdata/unit/deleter/rules/{generic/generic_delete.json => delete_1.json} (100%) rename tests/testdata/unit/deleter/rules/{specific/specific_delete.json => delete_2.json} (71%) rename tests/testdata/unit/deleter/rules/{specific => }/delete_test.json (100%) rename tests/testdata/unit/deleter/rules/{specific => }/test.json (100%) rename tests/testdata/unit/dissector/{generic_rules/dissector_rule.json => rules/dissector_rule_1.json} (100%) rename tests/testdata/unit/dissector/{specific_rules/dissector_rule.json => rules/dissector_rule_2.json} (100%) rename tests/testdata/unit/domain_label_extractor/rules/{generic/domain_label_extractor_gen.json => domain_label_extractor_1.json} (100%) rename tests/testdata/unit/domain_label_extractor/rules/{specific/domain_label_extractor.json => domain_label_extractor_2.json} (100%) rename tests/testdata/unit/domain_resolver/rules/{generic/domain_resolver.yml => domain_resolver_1.yml} (100%) rename tests/testdata/unit/domain_resolver/rules/{specific/domain_resolver.json => domain_resolver_2.json} (100%) rename tests/testdata/unit/dropper/rules/{generic/drop_field.json => drop_field_1.json} (100%) rename tests/testdata/unit/dropper/rules/{specific/drop_field.json => drop_field_2.json} (100%) rename tests/testdata/unit/field_manager/{generic_rules/field_manager.json => rules/field_manager_1.json} (100%) rename tests/testdata/unit/field_manager/{specific_rules/field_manager.json => rules/field_manager_2.json} (100%) rename tests/testdata/unit/generic_adder/rules/{generic/rules.json => rule_1.json} (100%) rename tests/testdata/unit/generic_adder/rules/{specific/specific_rules.json => rule_2.json} (100%) rename tests/testdata/unit/generic_resolver/rules/{generic/rule_01.json => rule_1.json} (100%) rename tests/testdata/unit/generic_resolver/rules/{specific/rule_01.json => rule_2.json} (100%) rename tests/testdata/unit/geoip_enricher/rules/{generic/geoip_all.json => geoip_all_1.json} (100%) rename tests/testdata/unit/geoip_enricher/rules/{specific/geoip_all.json => geoip_all_2.json} (100%) rename tests/testdata/unit/grokker/{generic_rules/rule.yml => rules/rule_1.yml} (100%) rename tests/testdata/unit/grokker/{specific_rules/rule.yml => rules/rule_2.yml} (100%) rename tests/testdata/unit/hyperscan_resolver/rules/{generic/rule_01.json => rule_1.json} (100%) rename tests/testdata/unit/hyperscan_resolver/rules/{specific/rule_01.json => rule_2.json} (100%) rename tests/testdata/unit/ip_informer/{generic/rule.json => rules/rule_1.json} (100%) rename tests/testdata/unit/ip_informer/{specific/rule.json => rules/rule_2.json} (100%) rename tests/testdata/unit/key_checker/{generic_rules/key_checker_rule.json => rules/key_checker_rule_1.json} (100%) rename tests/testdata/unit/key_checker/{specific_rules/key_checker_rule.json => rules/key_checker_rule_2.json} (100%) rename tests/testdata/unit/labeler/rules/{specific => }/first.json (100%) rename tests/testdata/unit/labeler/rules/{generic => }/rule.json (100%) rename tests/testdata/unit/list_comparison/rules/{generic/user_check.json => user_check_1.json} (100%) rename tests/testdata/unit/list_comparison/rules/{specific/user_check_specific.json => user_check_2.json} (100%) rename tests/testdata/unit/pre_detector/rules/{generic => }/pre_detect_four.yml (100%) rename tests/testdata/unit/pre_detector/rules/{generic => }/pre_detect_one.json (100%) rename tests/testdata/unit/pre_detector/rules/{generic => }/pre_detect_three.json (100%) rename tests/testdata/unit/pre_detector/rules/{generic => }/pre_detect_two.json (100%) rename tests/testdata/unit/pre_detector/rules/{generic => }/pre_detect_two_rules.json (100%) delete mode 100644 tests/testdata/unit/pre_detector/rules/specific/pre_detect_one.json delete mode 100644 tests/testdata/unit/pre_detector/rules/specific/pre_detect_three.json delete mode 100644 tests/testdata/unit/pre_detector/rules/specific/pre_detect_two.json delete mode 100644 tests/testdata/unit/pre_detector/rules/specific/pre_detect_two_rules.json rename tests/testdata/unit/pseudonymizer/{rules => }/regex_mapping.yml (100%) rename tests/testdata/unit/pseudonymizer/rules/{specific => }/Test123_id_789.json (100%) rename tests/testdata/unit/pseudonymizer/rules/{specific => }/Test456_id_1234.json (100%) rename tests/testdata/unit/pseudonymizer/rules/{generic => }/event_data_IpAddress.json (100%) delete mode 100644 tests/testdata/unit/pseudonymizer/rules/generic/this_is_not_a_rule.not_json rename tests/testdata/{acceptance/pseudonymizer/rules_static/generic => unit/pseudonymizer/rules}/this_is_not_a_rule.not_json (100%) rename tests/testdata/unit/requester/{generic_rules/requester.json => rules/requester_1.json} (100%) rename tests/testdata/unit/requester/{specific_rules/requester.json => rules/requester_2.json} (100%) rename tests/testdata/unit/selective_extractor/rules/{generic/rules.json => rules_1.json} (100%) rename tests/testdata/unit/selective_extractor/rules/{specific/rules.json => rules_2.json} (100%) rename tests/testdata/unit/string_splitter/{generic/generic.json => rules/rule.json} (100%) delete mode 100644 tests/testdata/unit/string_splitter/specific/specific.json delete mode 100644 tests/testdata/unit/template_replacer/rules/specific/template_replacer.json rename tests/testdata/unit/template_replacer/rules/{generic => }/template_replacer.json (100%) rename tests/testdata/unit/timestamp_differ/{generic_rules => rules}/timestamp_differ_rule.json (100%) delete mode 100644 tests/testdata/unit/timestamp_differ/specific_rules/timestamp_differ_rule.json rename tests/testdata/unit/timestamper/{generic_rules => rules}/timestamper_rule.yml (65%) delete mode 100644 tests/testdata/unit/timestamper/specific_rules/timestamper_rule.yml create mode 100644 tests/unit/processor/key_checker/__init__.py diff --git a/examples/exampledata/config/pipeline.yml b/examples/exampledata/config/pipeline.yml index ce91f79b8..2192f9986 100644 --- a/examples/exampledata/config/pipeline.yml +++ b/examples/exampledata/config/pipeline.yml @@ -25,24 +25,18 @@ pipeline: type: labeler schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true - specific_rules: - - examples/exampledata/rules/labeler/specific - generic_rules: - - examples/exampledata/rules/labeler/generic + rules: + - examples/exampledata/rules/labeler/rules - dissector: type: dissector - specific_rules: - - examples/exampledata/rules/dissector/specific/ - generic_rules: - - examples/exampledata/rules/dissector/generic/ + rules: + - examples/exampledata/rules/dissector/rules - dropper: type: dropper - specific_rules: - - examples/exampledata/rules/dropper/specific - generic_rules: - - examples/exampledata/rules/dropper/generic + rules: + - examples/exampledata/rules/dropper/rules - filter: "test_dropper" dropper: drop: @@ -51,10 +45,8 @@ pipeline: - pre_detector: type: pre_detector - specific_rules: - - examples/exampledata/rules/pre_detector/specific - generic_rules: - - examples/exampledata/rules/pre_detector/generic + rules: + - examples/exampledata/rules/pre_detector/rules outputs: - opensearch: sre tree_config: examples/exampledata/rules/pre_detector/tree_config.json @@ -62,10 +54,8 @@ pipeline: - amides: type: amides - specific_rules: - - examples/exampledata/rules/amides/specific - generic_rules: - - examples/exampledata/rules/amides/generic + rules: + - examples/exampledata/rules/amides/rules models_path: examples/exampledata/models/model.zip num_rule_attributions: 10 max_cache_entries: 1000000 @@ -79,20 +69,17 @@ pipeline: hash_salt: a_secret_tasty_ingredient outputs: - opensearch: pseudonyms - specific_rules: - - examples/exampledata/rules/pseudonymizer/specific/ - generic_rules: - - examples/exampledata/rules/pseudonymizer/generic/ + rules: + - examples/exampledata/rules/pseudonymizer/rules/ max_cached_pseudonyms: 1000000 - calculator: type: calculator - specific_rules: + rules: - filter: "test_label: execute" calculator: target_field: "calculation" calc: "1 + 1" - generic_rules: [] input: kafka: diff --git a/examples/exampledata/rules/amides/generic/amides_generic.yml b/examples/exampledata/rules/amides/rules/amides_1.yml similarity index 100% rename from examples/exampledata/rules/amides/generic/amides_generic.yml rename to examples/exampledata/rules/amides/rules/amides_1.yml diff --git a/examples/exampledata/rules/amides/specific/amides_specific.yml b/examples/exampledata/rules/amides/rules/amides_2.yml similarity index 100% rename from examples/exampledata/rules/amides/specific/amides_specific.yml rename to examples/exampledata/rules/amides/rules/amides_2.yml diff --git a/examples/exampledata/rules/dissector/generic/example_rule.yml b/examples/exampledata/rules/dissector/rules/example_rule_1.yml similarity index 100% rename from examples/exampledata/rules/dissector/generic/example_rule.yml rename to examples/exampledata/rules/dissector/rules/example_rule_1.yml diff --git a/examples/exampledata/rules/dissector/specific/example_rule.yml b/examples/exampledata/rules/dissector/rules/example_rule_2.yml similarity index 100% rename from examples/exampledata/rules/dissector/specific/example_rule.yml rename to examples/exampledata/rules/dissector/rules/example_rule_2.yml diff --git a/examples/exampledata/rules/dropper/generic/example_rule.yml b/examples/exampledata/rules/dropper/rules/example_rule.yml similarity index 100% rename from examples/exampledata/rules/dropper/generic/example_rule.yml rename to examples/exampledata/rules/dropper/rules/example_rule.yml diff --git a/examples/exampledata/rules/dropper/specific/example_rule.yml b/examples/exampledata/rules/dropper/specific/example_rule.yml deleted file mode 100644 index f29532c7d..000000000 --- a/examples/exampledata/rules/dropper/specific/example_rule.yml +++ /dev/null @@ -1,6 +0,0 @@ -filter: "test_dropper" -dropper: - id: dropper-1a3c69b2-5d54-4b6b-ab07-c7ddbea7917c - drop: - - drop_me -description: "..." diff --git a/examples/exampledata/rules/labeler/generic/example_rule.yml b/examples/exampledata/rules/labeler/rules/example_rule.yml similarity index 100% rename from examples/exampledata/rules/labeler/generic/example_rule.yml rename to examples/exampledata/rules/labeler/rules/example_rule.yml diff --git a/examples/exampledata/rules/labeler/specific/example_rule.yml b/examples/exampledata/rules/labeler/specific/example_rule.yml deleted file mode 100644 index 116a6b313..000000000 --- a/examples/exampledata/rules/labeler/specific/example_rule.yml +++ /dev/null @@ -1,7 +0,0 @@ -filter: "test_label: specific" -labeler: - id: labeler-1a3c69b2-5d54-4b6b-ab07-c7ddbea7917c - label: - action: - - execute -description: "..." diff --git a/examples/exampledata/rules/pre_detector/generic/example_rule.yml b/examples/exampledata/rules/pre_detector/rules/example_rule_1.yml similarity index 100% rename from examples/exampledata/rules/pre_detector/generic/example_rule.yml rename to examples/exampledata/rules/pre_detector/rules/example_rule_1.yml diff --git a/examples/exampledata/rules/pre_detector/specific/example_rule.yml b/examples/exampledata/rules/pre_detector/rules/example_rule_2.yml similarity index 100% rename from examples/exampledata/rules/pre_detector/specific/example_rule.yml rename to examples/exampledata/rules/pre_detector/rules/example_rule_2.yml diff --git a/examples/exampledata/rules/pseudonymizer/generic/example_rule.yml b/examples/exampledata/rules/pseudonymizer/rules/example_rule.yml similarity index 100% rename from examples/exampledata/rules/pseudonymizer/generic/example_rule.yml rename to examples/exampledata/rules/pseudonymizer/rules/example_rule.yml diff --git a/examples/exampledata/rules/pseudonymizer/specific/example_rule.yml b/examples/exampledata/rules/pseudonymizer/specific/example_rule.yml deleted file mode 100644 index d10aeea44..000000000 --- a/examples/exampledata/rules/pseudonymizer/specific/example_rule.yml +++ /dev/null @@ -1,6 +0,0 @@ -filter: "test_pseudonymizer AND something_special" -pseudonymizer: - id: pseudonymizer-1352bc0a-53ae-4740-bb9e-1e865f63375f - mapping: - something_special: "RE_WHOLE_FIELD" -description: "..." diff --git a/logprep/abc/processor.py b/logprep/abc/processor.py index 802ba31c3..5ebde3a0e 100644 --- a/logprep/abc/processor.py +++ b/logprep/abc/processor.py @@ -9,7 +9,7 @@ from attr import define, field, validators from logprep.abc.component import Component -from logprep.framework.rule_tree.rule_tree import RuleTree, RuleTreeType +from logprep.framework.rule_tree.rule_tree import RuleTree from logprep.metrics.metrics import Metric from logprep.processor.base.exceptions import ( ProcessingCriticalError, @@ -84,18 +84,7 @@ class Processor(Component): class Config(Component.Config): """Common Configurations""" - specific_rules: List[str] = field( - validator=[ - validators.instance_of(list), - validators.deep_iterable(member_validator=validators.instance_of((str, dict))), - ] - ) - """List of rule locations to load rules from. - In addition to paths to file directories it is possible to retrieve rules from a URI. - For valid URI formats see :ref:`getters`. - As last option it is possible to define entire rules with all their configuration parameters as list elements. - """ - generic_rules: List[str] = field( + rules: List[str] = field( validator=[ validators.instance_of(list), validators.deep_iterable(member_validator=validators.instance_of((str, dict))), @@ -120,8 +109,7 @@ class Config(Component.Config): __slots__ = [ "rule_class", "_event", - "_specific_tree", - "_generic_tree", + "_rule_tree", "result", "_bypass_rule_tree", "_rules", @@ -129,8 +117,7 @@ class Config(Component.Config): rule_class: "Rule" _event: dict - _specific_tree: RuleTree - _generic_tree: RuleTree + _rule_tree: RuleTree _strategy = None _bypass_rule_tree: bool _rules: tuple["Rule"] @@ -138,20 +125,8 @@ class Config(Component.Config): def __init__(self, name: str, configuration: "Processor.Config"): super().__init__(name, configuration) - self._specific_tree = RuleTree( - processor_name=self.name, - processor_config=self._config, - rule_tree_type=RuleTreeType.SPECIFIC, - ) - self._generic_tree = RuleTree( - processor_name=self.name, - processor_config=self._config, - rule_tree_type=RuleTreeType.GENERIC, - ) - self.load_rules( - generic_rules_targets=self._config.generic_rules, - specific_rules_targets=self._config.specific_rules, - ) + self._rule_tree = RuleTree(processor_name=self.name, processor_config=self._config) + self.load_rules(rules_targets=self._config.rules) self.result = None self._bypass_rule_tree = False if os.environ.get("LOGPREP_BYPASS_RULE_TREE"): @@ -160,24 +135,14 @@ def __init__(self, name: str, configuration: "Processor.Config"): logger.debug("Bypassing rule tree for processor %s", self.name) @property - def _specific_rules(self): - """Returns all specific rules - - Returns - ------- - specific_rules: list[Rule] - """ - return self._specific_tree.rules - - @property - def _generic_rules(self): - """Returns all generic rules + def _tree_rules(self): + """Returns all rules Returns ------- - generic_rules: list[Rule] + rules: list[Rule] """ - return self._generic_tree.rules + return self._rule_tree.rules @property def rules(self): @@ -187,7 +152,7 @@ def rules(self): ------- rules: list[Rule] """ - return [*self._generic_rules, *self._specific_rules] + return [*self._tree_rules] @property def metric_labels(self) -> dict: @@ -219,8 +184,7 @@ def process(self, event: dict) -> ProcessorResult: if self._bypass_rule_tree: self._process_all_rules(event) return self.result - self._process_rule_tree(event, self._specific_tree) - self._process_rule_tree(event, self._generic_tree) + self._process_rule_tree(event, self._rule_tree) return self.result def _process_all_rules(self, event: dict): @@ -322,23 +286,16 @@ def resolve_directories(rule_sources: list) -> list: resolved_sources.append(rule_source) return resolved_sources - def load_rules(self, specific_rules_targets: List[str], generic_rules_targets: List[str]): + def load_rules(self, rules_targets: List[str]): """method to add rules from directories or urls""" - specific_rules_targets = self.resolve_directories(specific_rules_targets) - generic_rules_targets = self.resolve_directories(generic_rules_targets) - for specific_rules_target in specific_rules_targets: - rules = self.rule_class.create_rules_from_target(specific_rules_target, self.name) - for rule in rules: - self._specific_tree.add_rule(rule) - for generic_rules_target in generic_rules_targets: - rules = self.rule_class.create_rules_from_target(generic_rules_target, self.name) + rules_targets = self.resolve_directories(rules_targets) + for rules_target in rules_targets: + rules = self.rule_class.create_rules_from_target(rules_target, self.name) for rule in rules: - self._generic_tree.add_rule(rule) + self._rule_tree.add_rule(rule) if logger.isEnabledFor(logging.DEBUG): # pragma: no cover - number_specific_rules = self._specific_tree.number_of_rules - logger.debug(f"{self.describe()} loaded {number_specific_rules} specific rules") - number_generic_rules = self._generic_tree.number_of_rules - logger.debug(f"{self.describe()} loaded {number_generic_rules} generic rules") + number_rules = self._rule_tree.number_of_rules + logger.debug(f"{self.describe()} loaded {number_rules} rules") @staticmethod def _field_exists(event: dict, dotted_field: str) -> bool: diff --git a/logprep/framework/rule_tree/rule_tree.py b/logprep/framework/rule_tree/rule_tree.py index 3fb53d6d1..9165a33bb 100644 --- a/logprep/framework/rule_tree/rule_tree.py +++ b/logprep/framework/rule_tree/rule_tree.py @@ -13,22 +13,12 @@ from logprep.processor.base.rule import Rule -class RuleTreeType(Enum): - """Types of rule trees.""" - - SPECIFIC = 1 - """Specific rule tree that is used to match specific rules.""" - GENERIC = 2 - """Generic rule tree that is used to match generic rules.""" - - class RuleTree: """Represent a set of rules using a rule tree model.""" __slots__ = ( "rule_parser", "priority_dict", - "_rule_tree_type", "_rule_mapping", "_processor_config", "_processor_type", @@ -39,7 +29,6 @@ class RuleTree: rule_parser: Optional[RuleParser] priority_dict: dict - _rule_tree_type: Union[RuleTreeType, str] _rule_mapping: dict _processor_name: str _processor_config: "Processor.Config" @@ -51,7 +40,6 @@ def __init__( root: Node = None, processor_name: str = None, processor_config: "Processor.Config" = None, - rule_tree_type: RuleTreeType = None, ): """Rule tree initialization function. @@ -71,7 +59,6 @@ def __init__( self._rule_mapping = {} self._processor_config = processor_config self._processor_name = processor_name if processor_name is not None else "" - self._rule_tree_type = rule_tree_type.name.lower() if rule_tree_type is not None else "" self._processor_type = processor_config.type if processor_name is not None else "" self._setup() diff --git a/logprep/processor/amides/processor.py b/logprep/processor/amides/processor.py index 5a257091a..2351733c8 100644 --- a/logprep/processor/amides/processor.py +++ b/logprep/processor/amides/processor.py @@ -63,10 +63,8 @@ - amides: type: amides - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules max_cache_entries: 10000 decision_threshold: 0.0 num_rule_attributions: 10 diff --git a/logprep/processor/base/rule.py b/logprep/processor/base/rule.py index 6686013a3..08861f844 100644 --- a/logprep/processor/base/rule.py +++ b/logprep/processor/base/rule.py @@ -336,7 +336,7 @@ def create_rules_from_target(cls, rule_target: str, processor_name: str = None) except InvalidRuleDefinitionError as error: raise InvalidRuleDefinitionError(f"{rule_target}: {error}") from error if len(rules) == 0: - raise InvalidRuleDefinitionError("no rules in file") + raise InvalidRuleDefinitionError(f"no rules in file {rule_target}") for rule in rules: rule.file_name = splitext(basename(rule_target))[0] return rules diff --git a/logprep/processor/calculator/processor.py b/logprep/processor/calculator/processor.py index 8c226f54f..a4d9e888d 100644 --- a/logprep/processor/calculator/processor.py +++ b/logprep/processor/calculator/processor.py @@ -11,10 +11,8 @@ - calculatorname: type: calculator - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.calculator.processor.Calculator.Config :members: diff --git a/logprep/processor/clusterer/processor.py b/logprep/processor/clusterer/processor.py index 1bc375c83..8f33a8093 100644 --- a/logprep/processor/clusterer/processor.py +++ b/logprep/processor/clusterer/processor.py @@ -25,10 +25,8 @@ - clusterername: type: clusterer - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules output_field_name: target_field .. autoclass:: logprep.processor.clusterer.processor.Clusterer.Config @@ -147,14 +145,12 @@ def _cluster(self, event: dict, rule: ClustererRule): self._last_non_extracted_signature = sig_text def _is_new_tree_iteration(self, rule: ClustererRule) -> bool: - for tree in (self._specific_tree, self._generic_tree): - rule_id = tree.get_rule_id(rule) - if rule_id is None: - continue - is_new_iteration = rule_id <= self._last_rule_id - self._last_rule_id = rule_id - return is_new_iteration - return True + rule_id = self._rule_tree.get_rule_id(rule) + if rule_id is None: + return True + is_new_iteration = rule_id <= self._last_rule_id + self._last_rule_id = rule_id + return is_new_iteration def _get_text_to_cluster(self, rule: ClustererRule, event: dict) -> Tuple[str, str]: sig_text = None diff --git a/logprep/processor/concatenator/processor.py b/logprep/processor/concatenator/processor.py index 3e91b0b05..8685ecf73 100644 --- a/logprep/processor/concatenator/processor.py +++ b/logprep/processor/concatenator/processor.py @@ -13,10 +13,8 @@ - Concatenatorname: type: concatenator - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.concatenator.processor.Concatenator.Config :members: diff --git a/logprep/processor/datetime_extractor/processor.py b/logprep/processor/datetime_extractor/processor.py index 71d10a531..5155cd234 100644 --- a/logprep/processor/datetime_extractor/processor.py +++ b/logprep/processor/datetime_extractor/processor.py @@ -12,10 +12,8 @@ - datetimeextractorname: type: datetime_extractor - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.datetime_extractor.processor.DatetimeExtractor.Config :members: diff --git a/logprep/processor/deleter/processor.py b/logprep/processor/deleter/processor.py index 37d18b0df..6bfee3c44 100644 --- a/logprep/processor/deleter/processor.py +++ b/logprep/processor/deleter/processor.py @@ -11,10 +11,8 @@ - deletename: type: deleter - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.deleter.processor.Deleter.Config :members: diff --git a/logprep/processor/dissector/processor.py b/logprep/processor/dissector/processor.py index 3cc3bc34a..bc350ffe0 100644 --- a/logprep/processor/dissector/processor.py +++ b/logprep/processor/dissector/processor.py @@ -14,10 +14,8 @@ - dissectorname: type: dissector - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.dissector.processor.Dissector.Config :members: diff --git a/logprep/processor/domain_label_extractor/processor.py b/logprep/processor/domain_label_extractor/processor.py index 3bb40a607..e8087fea0 100644 --- a/logprep/processor/domain_label_extractor/processor.py +++ b/logprep/processor/domain_label_extractor/processor.py @@ -18,10 +18,8 @@ - domainlabelextractorname: type: domain_label_extractor - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules tagging_field_name: resolved .. autoclass:: logprep.processor.domain_label_extractor.processor.DomainLabelExtractor.Config diff --git a/logprep/processor/domain_resolver/processor.py b/logprep/processor/domain_resolver/processor.py index 5872fe08e..098fce281 100644 --- a/logprep/processor/domain_resolver/processor.py +++ b/logprep/processor/domain_resolver/processor.py @@ -11,10 +11,8 @@ - domainresolvername: type: domain_resolver - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules timeout: 0.5 max_cached_domains: 20000 max_caching_days: 1 diff --git a/logprep/processor/dropper/processor.py b/logprep/processor/dropper/processor.py index 71c28c0a0..a5e4e6ae9 100644 --- a/logprep/processor/dropper/processor.py +++ b/logprep/processor/dropper/processor.py @@ -12,10 +12,8 @@ - droppername: type: dropper - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.dropper.processor.Dropper.Config :members: diff --git a/logprep/processor/field_manager/processor.py b/logprep/processor/field_manager/processor.py index 422dccb0f..4c1ed4cd8 100644 --- a/logprep/processor/field_manager/processor.py +++ b/logprep/processor/field_manager/processor.py @@ -15,10 +15,8 @@ - fieldmanagername: type: field_manager - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.field_manager.processor.FieldManager.Config :members: diff --git a/logprep/processor/generic_adder/processor.py b/logprep/processor/generic_adder/processor.py index 6517fd420..9913c1e83 100644 --- a/logprep/processor/generic_adder/processor.py +++ b/logprep/processor/generic_adder/processor.py @@ -12,10 +12,8 @@ - genericaddername: type: generic_adder - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.generic_adder.processor.GenericAdder.Config :members: diff --git a/logprep/processor/generic_resolver/processor.py b/logprep/processor/generic_resolver/processor.py index 2e3449123..b5d4adbb4 100644 --- a/logprep/processor/generic_resolver/processor.py +++ b/logprep/processor/generic_resolver/processor.py @@ -11,10 +11,8 @@ - genericresolvername: type: generic_resolver - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.generic_resolver.processor.GenericResolver.Config :members: diff --git a/logprep/processor/geoip_enricher/processor.py b/logprep/processor/geoip_enricher/processor.py index 9a917f3fa..fa8592b71 100644 --- a/logprep/processor/geoip_enricher/processor.py +++ b/logprep/processor/geoip_enricher/processor.py @@ -11,10 +11,8 @@ - geoipenrichername: type: geoip_enricher - specific_rules: - - tests/testdata/geoip_enricher/rules/ - generic_rules: - - tests/testdata/geoip_enricher/rules/ + rules: + - tests/testdata/geoip_enricher/rules db_path: /path/to/GeoLite2-City.mmdb .. autoclass:: logprep.processor.geoip_enricher.processor.GeoipEnricher.Config diff --git a/logprep/processor/grokker/processor.py b/logprep/processor/grokker/processor.py index 654a1f8f1..77257c982 100644 --- a/logprep/processor/grokker/processor.py +++ b/logprep/processor/grokker/processor.py @@ -16,10 +16,8 @@ - my_grokker: type: grokker - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules custom_patterns_dir: "http://the.patterns.us/patterns.zip" .. autoclass:: logprep.processor.grokker.processor.Grokker.Config diff --git a/logprep/processor/hyperscan_resolver/processor.py b/logprep/processor/hyperscan_resolver/processor.py index e19a32e4e..960824369 100644 --- a/logprep/processor/hyperscan_resolver/processor.py +++ b/logprep/processor/hyperscan_resolver/processor.py @@ -16,10 +16,8 @@ - hyperscanresolvername: type: hyperscan_resolver - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules hyperscan_db_path: tmp/path/scan.db .. autoclass:: logprep.processor.hyperscan_resolver.processor.HyperscanResolver.Config diff --git a/logprep/processor/ip_informer/processor.py b/logprep/processor/ip_informer/processor.py index 1c75bf702..36d0d61d1 100644 --- a/logprep/processor/ip_informer/processor.py +++ b/logprep/processor/ip_informer/processor.py @@ -11,10 +11,8 @@ - myipinformer: type: ip_informer - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.ip_informer.processor.IpInformer.Config :members: diff --git a/logprep/processor/key_checker/processor.py b/logprep/processor/key_checker/processor.py index 231bd313e..4d230e2bf 100644 --- a/logprep/processor/key_checker/processor.py +++ b/logprep/processor/key_checker/processor.py @@ -12,10 +12,8 @@ - keycheckername: type: key_checker - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.key_checker.processor.KeyChecker.Config :members: diff --git a/logprep/processor/labeler/processor.py b/logprep/processor/labeler/processor.py index 7ee44ed0d..d825e8ad7 100644 --- a/logprep/processor/labeler/processor.py +++ b/logprep/processor/labeler/processor.py @@ -12,10 +12,8 @@ type: labeler schema: tests/testdata/labeler_rules/labeling/schema.json include_parent_labels: true - generic_rules: - - tests/testdata/labeler_rules/rules/ - specific_rules: - - tests/testdata/labeler_rules/rules/ + rules: + - tests/testdata/labeler_rules/rules .. autoclass:: logprep.processor.labeler.processor.Labeler.Config :members: @@ -66,7 +64,7 @@ def __init__(self, name: str, configuration: Processor.Config): def setup(self): super().setup() - for rule in self._generic_rules + self._specific_rules: + for rule in self._tree_rules: if self._config.include_parent_labels: rule.add_parent_labels_from_schema(self._schema) rule.conforms_to_schema(self._schema) diff --git a/logprep/processor/list_comparison/processor.py b/logprep/processor/list_comparison/processor.py index d2064362a..2886aa910 100644 --- a/logprep/processor/list_comparison/processor.py +++ b/logprep/processor/list_comparison/processor.py @@ -13,10 +13,8 @@ - listcomparisonname: type: list_comparison - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules list_search_base_path: /path/to/list/dir .. autoclass:: logprep.processor.list_comparison.processor.ListComparison.Config @@ -53,7 +51,7 @@ class Config(Processor.Config): def setup(self): super().setup() - for rule in [*self._specific_rules, *self._generic_rules]: + for rule in [*self._tree_rules]: rule.init_list_comparison(self._config.list_search_base_path) def _apply_rules(self, event, rule): diff --git a/logprep/processor/pre_detector/processor.py b/logprep/processor/pre_detector/processor.py index ad10ea062..121b96387 100644 --- a/logprep/processor/pre_detector/processor.py +++ b/logprep/processor/pre_detector/processor.py @@ -13,10 +13,8 @@ - predetectorname: type: pre_detector - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules outputs: - kafka: sre_topic alert_ip_list_path: /tmp/ip_list.yml diff --git a/logprep/processor/pseudonymizer/processor.py b/logprep/processor/pseudonymizer/processor.py index fc9687d69..1bd31db84 100644 --- a/logprep/processor/pseudonymizer/processor.py +++ b/logprep/processor/pseudonymizer/processor.py @@ -23,10 +23,8 @@ - pseudonymizername: type: pseudonymizer - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules outputs: - kafka: pseudonyms_topic pubkey_analyst: /path/to/analyst_pubkey.pem @@ -218,7 +216,7 @@ def setup(self): self._replace_regex_keywords_by_regex_expression() def _replace_regex_keywords_by_regex_expression(self): - for rule in self._specific_rules + self._generic_rules: + for rule in self._tree_rules: for dotted_field, regex_keyword in rule.pseudonyms.items(): if regex_keyword in self._regex_mapping: rule.pseudonyms[dotted_field] = re.compile(self._regex_mapping[regex_keyword]) diff --git a/logprep/processor/requester/processor.py b/logprep/processor/requester/processor.py index aa2e6edea..86b817466 100644 --- a/logprep/processor/requester/processor.py +++ b/logprep/processor/requester/processor.py @@ -22,10 +22,8 @@ - requestername: type: requester - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.requester.processor.Requester.Config :members: diff --git a/logprep/processor/selective_extractor/processor.py b/logprep/processor/selective_extractor/processor.py index c0bcf2ddd..d0fda63e9 100644 --- a/logprep/processor/selective_extractor/processor.py +++ b/logprep/processor/selective_extractor/processor.py @@ -15,10 +15,8 @@ - selectiveextractorname: type: selective_extractor - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.selective_extractor.processor.SelectiveExtractor.Config :members: diff --git a/logprep/processor/string_splitter/processor.py b/logprep/processor/string_splitter/processor.py index 7d81a1d20..a6a086928 100644 --- a/logprep/processor/string_splitter/processor.py +++ b/logprep/processor/string_splitter/processor.py @@ -12,10 +12,8 @@ - samplename: type: string_splitter - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.string_splitter.processor.StringSplitter.Config :members: diff --git a/logprep/processor/template_replacer/processor.py b/logprep/processor/template_replacer/processor.py index e5101a292..9b036aec9 100644 --- a/logprep/processor/template_replacer/processor.py +++ b/logprep/processor/template_replacer/processor.py @@ -12,10 +12,8 @@ - templatereplacername: type: template_replacer - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules template: /tmp/template.yml pattern: delimiter: "," diff --git a/logprep/processor/timestamp_differ/processor.py b/logprep/processor/timestamp_differ/processor.py index e49d005cc..568759116 100644 --- a/logprep/processor/timestamp_differ/processor.py +++ b/logprep/processor/timestamp_differ/processor.py @@ -11,10 +11,8 @@ - timestampdiffer_name: type: timestamp_differ - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.timestamp_differ.processor.TimestampDiffer.Config :members: diff --git a/logprep/processor/timestamper/processor.py b/logprep/processor/timestamper/processor.py index 71f2bcaa3..16a61a2ef 100644 --- a/logprep/processor/timestamper/processor.py +++ b/logprep/processor/timestamper/processor.py @@ -11,10 +11,8 @@ - myteimestamper: type: timestamper - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/rules .. autoclass:: logprep.processor.timestamper.processor.Timestamper.Config diff --git a/logprep/util/auto_rule_tester/auto_rule_tester.py b/logprep/util/auto_rule_tester/auto_rule_tester.py index 01ab7c71a..ee00811d1 100644 --- a/logprep/util/auto_rule_tester/auto_rule_tester.py +++ b/logprep/util/auto_rule_tester/auto_rule_tester.py @@ -329,14 +329,15 @@ def _run_rule_tests(self, processor: "Processor", rule_test: dict): rule_test : dict the rules to test """ - temp_rule_path = path.join(self._empty_rules_dirs[0], f"{hashlib.sha256()}.json") + temp_rule_path = path.join( + self._empty_rules_dirs[0], f"{hashlib.sha256().hexdigest()}.json" + ) rules = self._get_rules(processor, rule_test) - for rule_type, rules in rules.items(): - for idx, rule_dict in enumerate(rules): - self._prepare_test_eval(processor, rule_dict, rule_type, temp_rule_path) - self._eval_file_rule_test(rule_test, processor, idx) - remove_file_if_exists(temp_rule_path) + for idx, rule_dict in enumerate(rules): + self._prepare_test_eval(processor, rule_dict, temp_rule_path) + self._eval_file_rule_test(rule_test, processor, idx) + remove_file_if_exists(temp_rule_path) def _get_processors(self) -> OrderedDict: """Get processors in k/v-pairs @@ -354,8 +355,8 @@ def _get_processors(self) -> OrderedDict: return processors_without_custom_test @staticmethod - def _get_rules(processor: "Processor", rule_test: dict) -> dict: - """Assign and get each type of rule + def _get_rules(processor: "Processor", rule_test: dict) -> list: + """Assign and get rule Parameters ---------- @@ -366,7 +367,7 @@ def _get_rules(processor: "Processor", rule_test: dict) -> dict: Returns ------- - dict + list ruleset Raises @@ -375,19 +376,12 @@ def _get_rules(processor: "Processor", rule_test: dict) -> dict: empty ruleset """ if rule_test.get("rules"): - return {"rules": rule_test.get("rules", [])} - if rule_test.get("specific_rules") or rule_test.get("generic_rules"): - result = {} - if rule_test.get("specific_rules"): - result["specific_rules"] = rule_test.get("specific_rules", []) - if rule_test.get("generic_rules"): - result["generic_rules"] = rule_test.get("generic_rules", []) - return result + return rule_test.get("rules", []) raise AutoRuleTesterException( f"No rules provided for processor of type {processor.describe()}" ) - def _load_rules(self, processor: "Processor", rule_type: str): + def _load_rules(self, processor: "Processor"): """Load each type of rules for each processor and set it up Parameters @@ -397,18 +391,13 @@ def _load_rules(self, processor: "Processor", rule_type: str): rule_type : str type """ - if rule_type == "rules": - processor.load_rules(self._empty_rules_dirs) - elif rule_type == "specific_rules": - processor.load_rules(self._empty_rules_dirs, []) - elif rule_type == "generic_rules": - processor.load_rules([], self._empty_rules_dirs) + processor.load_rules(self._empty_rules_dirs) if processor._bypass_rule_tree: processor._rules = processor.rules processor.setup() def _prepare_test_eval( - self, processor: "Processor", rule_dict: dict, rule_type: str, temp_rule_path: str + self, processor: "Processor", rule_dict: dict, temp_rule_path: str ) -> None: """Prepare test eval: Create rule file, then reset tree of processor and then load the rules for the processor @@ -419,14 +408,12 @@ def _prepare_test_eval( processor rule_dict : dict rules for proc - rule_type : str - type of rules temp_rule_path : str temporary path to rules """ self._create_rule_file(rule_dict, temp_rule_path) self._reset(processor) - self._load_rules(processor, rule_type) + self._load_rules(processor) def _eval_file_rule_test(self, rule_test: dict, processor: "Processor", r_idx: int): """Main logic to check each rule file, compare and validate it, then print out results. @@ -511,12 +498,8 @@ def _reset(processor: "Processor"): """ if hasattr(processor, "_rules"): processor.rules.clear() - if hasattr(processor, "_tree"): - processor._tree = RuleTree() - if hasattr(processor, "_specific_tree"): - processor._specific_tree = RuleTree() - if hasattr(processor, "_generic_tree"): - processor._generic_tree = RuleTree() + if hasattr(processor, "_rule_tree"): + processor._rule_tree = RuleTree() @staticmethod def _create_rule_file(rule_dict: dict, rule_path: str): @@ -608,9 +591,6 @@ def _set_rules_dirs_to_empty(self) -> None: if processor_cfg.get("rules"): processor_cfg["rules"] = self._empty_rules_dirs - elif processor_cfg.get("generic_rules") and processor_cfg.get("specific_rules"): - processor_cfg["generic_rules"] = self._empty_rules_dirs - processor_cfg["specific_rules"] = self._empty_rules_dirs def _get_rules_per_processor_name(self, rules_dirs: dict) -> defaultdict: rules_pn = defaultdict(dict) @@ -641,17 +621,17 @@ def _get_rules_for_processor(self, processor_name, proc_rules_dirs, rules_pn): rules_pn[processor_name]["rules"] = [] directories = {"Rules Directory": [f"{processor_name} ({processor_type}):"], "Path": []} - for _, (rule_type, rules_dir) in enumerate(proc_rules_dirs["rule_dirs"].items()): - directories["Path"].append(f" - {rule_type}") - for path in Path(rules_dir).rglob("*"): - if path.is_file() and self._is_valid_rule_name(path.name): + for rules_dir in proc_rules_dirs["rule_dirs"]: + directories["Path"].append(f" - {'rule_type'}") + for file_path in Path(rules_dir).rglob("*"): + if file_path.is_file() and self._is_valid_rule_name(file_path.name): self._get_rule_dict( - path.name, str(path.parent), processor_name, rules_pn, rule_type + file_path.name, str(file_path.parent), processor_name, rules_pn ) self._pd_extra.print_rules(directories) - def _get_rule_dict(self, file, root, processor_name, rules_pn, rule_dirs_type) -> None: + def _get_rule_dict(self, file, root, processor_name, rules_pn) -> None: """Read out (multi-)rules and realize mapping via dict for further processing Parameters @@ -664,8 +644,6 @@ def _get_rule_dict(self, file, root, processor_name, rules_pn, rule_dirs_type) - name rules_pn : dict mapping of procs to rules - rule_dirs_type : str - type of rule Raises ------ @@ -702,7 +680,7 @@ def _get_rule_dict(self, file, root, processor_name, rules_pn, rule_dirs_type) - rules_pn[processor_name]["rules"].append( { - rule_dirs_type: multi_rule, + "rules": multi_rule, "tests": rule_tests, "file": file_path, } @@ -730,10 +708,10 @@ def _get_rule_dirs_by_processor_name(self) -> defaultdict: rules_dirs[processor_name]["type"] = processor_cfg["type"] if not rules_dirs[processor_name]["rule_dirs"]: - rules_dirs[processor_name]["rule_dirs"] = defaultdict(str) + rules_dirs[processor_name]["rule_dirs"] = [] for rule_to_add in rules_to_add: - rules_dirs[processor_name]["rule_dirs"][rule_to_add[0]] += rule_to_add[1] + rules_dirs[processor_name]["rule_dirs"].append(rule_to_add) return rules_dirs @@ -753,10 +731,8 @@ def _get_rules_to_add(processor_cfg) -> list: """ rules_to_add = [] - if processor_cfg.get("rules"): - rules_to_add.append(("rules", processor_cfg["rules"])) - elif processor_cfg.get("generic_rules") and processor_cfg.get("specific_rules"): - rules_to_add.append(("generic_rules", processor_cfg["generic_rules"][0])) - rules_to_add.append(("specific_rules", processor_cfg["specific_rules"][0])) - + rule_path_lists = processor_cfg.get("rules") + if rule_path_lists: + for rule_path_list in rule_path_lists: + rules_to_add.append(rule_path_list) return rules_to_add diff --git a/logprep/util/configuration.py b/logprep/util/configuration.py index 451258f38..e25a689bd 100644 --- a/logprep/util/configuration.py +++ b/logprep/util/configuration.py @@ -54,24 +54,18 @@ type: labeler schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true - specific_rules: - - examples/exampledata/rules/labeler/specific - generic_rules: - - examples/exampledata/rules/labeler/generic + rules: + - examples/exampledata/rules/labeler/rules - dissectorname: type: dissector - specific_rules: - - examples/exampledata/rules/dissector/specific/ - generic_rules: - - examples/exampledata/rules/dissector/generic/ + rules: + - examples/exampledata/rules/dissector/rules - dropper: type: dropper - specific_rules: - - examples/exampledata/rules/dropper/specific - generic_rules: - - examples/exampledata/rules/dropper/generic + rules: + - examples/exampledata/rules/dropper/rules - filter: "test_dropper" dropper: drop: @@ -80,10 +74,8 @@ - pre_detector: type: pre_detector - specific_rules: - - examples/exampledata/rules/pre_detector/specific - generic_rules: - - examples/exampledata/rules/pre_detector/generic + rules: + - examples/exampledata/rules/pre_detector/rules outputs: - opensearch: sre tree_config: examples/exampledata/rules/pre_detector/tree_config.json @@ -91,10 +83,8 @@ - amides: type: amides - specific_rules: - - examples/exampledata/rules/amides/specific - generic_rules: - - examples/exampledata/rules/amides/generic + rules: + - examples/exampledata/rules/amides/rules models_path: examples/exampledata/models/model.zip num_rule_attributions: 10 max_cache_entries: 1000000 @@ -108,20 +98,17 @@ hash_salt: a_secret_tasty_ingredient outputs: - opensearch: pseudonyms - specific_rules: - - examples/exampledata/rules/pseudonymizer/specific/ - generic_rules: - - examples/exampledata/rules/pseudonymizer/generic/ + rules: + - examples/exampledata/rules/pseudonymizer/rules max_cached_pseudonyms: 1000000 - calculator: type: calculator - specific_rules: + rules: - filter: "test_label: execute" calculator: target_field: "calculation" calc: "1 + 1" - generic_rules: [] The options under :code:`input`, :code:`output` and :code:`pipeline` are passed @@ -174,10 +161,8 @@ type: labeler schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true - specific_rules: - - examples/exampledata/rules/labeler/specific - generic_rules: - - examples/exampledata/rules/labeler/generic" + rules: + - examples/exampledata/rules/labeler/rules" export LOGPREP_OUTPUT=" output: kafka: @@ -746,12 +731,11 @@ def _load_rule_definitions(self, processor_definition: dict) -> dict: processor_definition = deepcopy(processor_definition) _ = Factory.create(processor_definition) processor_name, processor_config = processor_definition.popitem() - for rule_tree_name in ("specific_rules", "generic_rules"): - rules_targets = self._resolve_directories(processor_config.get(rule_tree_name, [])) - rules_definitions = list( - chain(*[self._get_dict_list_from_target(target) for target in rules_targets]) - ) - processor_config[rule_tree_name] = rules_definitions + rules_targets = self._resolve_directories(processor_config.get("rules", [])) + rules_definitions = list( + chain(*[self._get_dict_list_from_target(target) for target in rules_targets]) + ) + processor_config["rules"] = rules_definitions return {processor_name: processor_config} @staticmethod diff --git a/logprep/util/template_processor.py.j2 b/logprep/util/template_processor.py.j2 index ffd11b003..a574ab9a0 100644 --- a/logprep/util/template_processor.py.j2 +++ b/logprep/util/template_processor.py.j2 @@ -11,10 +11,8 @@ Processor Configuration - samplename: type: {{ processor.name }} - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules/ .. autoclass:: logprep.processor.{{ processor.name }}.processor.{{ processor.class_name }}.Config :members: diff --git a/logprep/util/template_processor_test.py.j2 b/logprep/util/template_processor_test.py.j2 index cb662bdbd..9d796ed24 100644 --- a/logprep/util/template_processor_test.py.j2 +++ b/logprep/util/template_processor_test.py.j2 @@ -17,19 +17,18 @@ class Test{{ processor.class_name }}(BaseProcessorTestCase): CONFIG: dict = { "type": "{{ processor.name }}", - "specific_rules": [], - "generic_rules": [], + "rules": [], } @pytest.mark.parametrize("testcase, rule, event, expected", test_cases) def test_testcases(self, testcase, rule, event, expected): - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(event) assert event == expected, testcase @pytest.mark.parametrize("testcase, rule, event, expected", failure_test_cases) def test_testcases_failure_handling(self, testcase, rule, event, expected): - self._load_specific_rule(rule) + self._load_rule(rule) with pytest.raises(ProcessingWarning): self.object.process(event) assert event == expected, testcase diff --git a/tests/acceptance/test_amides.py b/tests/acceptance/test_amides.py index 57ad2a4a0..a3db7d20d 100644 --- a/tests/acceptance/test_amides.py +++ b/tests/acceptance/test_amides.py @@ -26,8 +26,7 @@ def config(): "amides": { "type": "amides", "models_path": "tests/testdata/unit/amides/model.zip", - "specific_rules": ["tests/testdata/unit/amides/rules/specific"], - "generic_rules": ["tests/testdata/unit/amides/rules/generic"], + "rules": ["tests/testdata/unit/amides/rules"], "max_cache_entries": 1000, "num_rule_attributions": 10, "decision_threshold": 0.32, diff --git a/tests/acceptance/test_file_input.py b/tests/acceptance/test_file_input.py index b1128239e..77579a4d0 100644 --- a/tests/acceptance/test_file_input.py +++ b/tests/acceptance/test_file_input.py @@ -47,8 +47,7 @@ def config_fixture(): { "dissector": { "type": "dissector", - "specific_rules": ["tests/testdata/acceptance/dissector/rules/specific"], - "generic_rules": ["tests/testdata/acceptance/dissector/rules/generic"], + "rules": ["tests/testdata/acceptance/dissector/rules"], } } ] diff --git a/tests/acceptance/test_full_configuration.py b/tests/acceptance/test_full_configuration.py index aee28e09f..1f98a96bd 100644 --- a/tests/acceptance/test_full_configuration.py +++ b/tests/acceptance/test_full_configuration.py @@ -91,10 +91,8 @@ def test_start_of_logprep_from_http_with_templated_url_and_config(): type: labeler schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true - specific_rules: - - examples/exampledata/rules/labeler/specific - generic_rules: - - examples/exampledata/rules/labeler/generic + rules: + - examples/exampledata/rules/labeler/rules """, "LOGPREP_OUTPUT": """ output: @@ -154,8 +152,7 @@ def test_logprep_exposes_prometheus_metrics_and_healthchecks(tmp_path): config.pipeline.append( { "calculator2": { - "generic_rules": ["tests/testdata/unit/calculator/generic_rules"], - "specific_rules": ["tests/testdata/unit/calculator/specific_rules"], + "rules": ["tests/testdata/unit/calculator/rules"], "type": "calculator", } } diff --git a/tests/acceptance/test_http_input_with_requests.py b/tests/acceptance/test_http_input_with_requests.py index 9cd5f42fb..08cf88be6 100644 --- a/tests/acceptance/test_http_input_with_requests.py +++ b/tests/acceptance/test_http_input_with_requests.py @@ -25,8 +25,7 @@ def config_fixture(): { "dissector": { "type": "dissector", - "specific_rules": ["tests/testdata/acceptance/dissector/rules/specific"], - "generic_rules": ["tests/testdata/acceptance/dissector/rules/generic"], + "rules": ["tests/testdata/acceptance/dissector/rules"], } } ] diff --git a/tests/acceptance/test_multiple_outputs.py b/tests/acceptance/test_multiple_outputs.py index dafdd1cd7..86db6c76c 100644 --- a/tests/acceptance/test_multiple_outputs.py +++ b/tests/acceptance/test_multiple_outputs.py @@ -29,19 +29,13 @@ def get_config(): { "dissector": { "type": "dissector", - "specific_rules": ["tests/testdata/acceptance/dissector/rules/specific"], - "generic_rules": ["tests/testdata/acceptance/dissector/rules/generic"], + "rules": ["tests/testdata/acceptance/dissector/rules"], } }, { "selective_extractor": { "type": "selective_extractor", - "specific_rules": [ - "tests/testdata/acceptance/selective_extractor/rules/specific" - ], - "generic_rules": [ - "tests/testdata/acceptance/selective_extractor/rules/generic" - ], + "rules": ["tests/testdata/acceptance/selective_extractor/rules"], } }, { @@ -51,13 +45,8 @@ def get_config(): "pubkey_depseudo": "tests/testdata/acceptance/pseudonymizer/example_depseudo_pub.pem", "hash_salt": "a_secret_tasty_ingredient", "outputs": [{"second_output": "pseudonyms"}], - "specific_rules": [ - "tests/testdata/acceptance/pseudonymizer/rules_static/specific" - ], - "generic_rules": [ - "tests/testdata/acceptance/pseudonymizer/rules_static/generic" - ], - "regex_mapping": "tests/testdata/acceptance/pseudonymizer/rules_static/regex_mapping.yml", + "rules": ["tests/testdata/acceptance/pseudonymizer/rules"], + "regex_mapping": "tests/testdata/acceptance/pseudonymizer/regex_mapping.yml", "max_cached_pseudonyms": 1000000, } }, @@ -65,8 +54,7 @@ def get_config(): "pre_detector": { "type": "pre_detector", "outputs": [{"jsonl": "pre_detector_topic"}], - "generic_rules": ["tests/testdata/acceptance/pre_detector/rules/generic"], - "specific_rules": ["tests/testdata/acceptance/pre_detector/rules/specific"], + "rules": ["tests/testdata/acceptance/pre_detector/rules"], "tree_config": "tests/testdata/acceptance/pre_detector/tree_config.json", } }, diff --git a/tests/acceptance/test_pre_detection.py b/tests/acceptance/test_pre_detection.py index e49355833..f5177a660 100644 --- a/tests/acceptance/test_pre_detection.py +++ b/tests/acceptance/test_pre_detection.py @@ -20,8 +20,7 @@ "pre_detector": { "type": "pre_detector", "outputs": [{"jsonl": "pre_detector_topic"}], - "generic_rules": [], - "specific_rules": ["tests/testdata/acceptance/pre_detector/rules/"], + "rules": ["tests/testdata/acceptance/pre_detector/rules/"], "tree_config": "tests/testdata/acceptance/pre_detector/tree_config.json", } }, diff --git a/tests/acceptance/test_preprocessing.py b/tests/acceptance/test_preprocessing.py index 15c855545..20100387b 100644 --- a/tests/acceptance/test_preprocessing.py +++ b/tests/acceptance/test_preprocessing.py @@ -18,8 +18,7 @@ def get_config() -> Configuration: { "dissector": { "type": "dissector", - "specific_rules": ["tests/testdata/acceptance/dissector/rules/specific"], - "generic_rules": ["tests/testdata/acceptance/dissector/rules/generic"], + "rules": ["tests/testdata/acceptance/dissector/rules"], } } ] diff --git a/tests/acceptance/test_selective_extractor_full_pipeline_pass.py b/tests/acceptance/test_selective_extractor_full_pipeline_pass.py index 707e1418c..ee769fc21 100644 --- a/tests/acceptance/test_selective_extractor_full_pipeline_pass.py +++ b/tests/acceptance/test_selective_extractor_full_pipeline_pass.py @@ -12,15 +12,13 @@ def config_fixture(): { "dissector": { "type": "dissector", - "specific_rules": ["tests/testdata/acceptance/dissector/rules/specific"], - "generic_rules": ["tests/testdata/acceptance/dissector/rules/generic"], + "rules": ["tests/testdata/acceptance/dissector/rules"], } }, { "selective_extractor": { "type": "selective_extractor", - "specific_rules": ["tests/testdata/acceptance/selective_extractor/rules/specific"], - "generic_rules": ["tests/testdata/acceptance/selective_extractor/rules/generic"], + "rules": ["tests/testdata/acceptance/selective_extractor/rules"], } }, ] diff --git a/tests/acceptance/test_wineventlog_processing.py b/tests/acceptance/test_wineventlog_processing.py index 179065b02..72117854f 100644 --- a/tests/acceptance/test_wineventlog_processing.py +++ b/tests/acceptance/test_wineventlog_processing.py @@ -27,8 +27,7 @@ def fixture_config_template(): "type": "labeler", "schema": "", "include_parent_labels": True, - "specific_rules": None, - "generic_rules": None, + "rules": None, } } ] @@ -36,35 +35,30 @@ def fixture_config_template(): @pytest.mark.parametrize( - "specific_rules, generic_rules, schema, expected_output", + "rules, schema, expected_output", [ ( - ["acceptance/labeler/rules_static/rules/specific"], - ["acceptance/labeler/rules_static/rules/generic"], - "acceptance/labeler/rules_static/labeling/schema.json", + ["acceptance/labeler/no_regex/rules"], + "acceptance/labeler/no_regex/labeling/schema.json", "labeled_win_event_log.jsonl", ), ( [ - "acceptance/labeler/rules_static/rules/specific", - "acceptance/labeler/rules_static_only_regex/rules/specific", + "acceptance/labeler/no_regex/rules", + "acceptance/labeler/only_regex/rules", ], - [ - "acceptance/labeler/rules_static/rules/generic", - "acceptance/labeler/rules_static_only_regex/rules/generic", - ], - "acceptance/labeler/rules_static_only_regex/labeling/schema.json", + "acceptance/labeler/only_regex/labeling/schema.json", "labeled_win_event_log_with_regex.jsonl", ), ], ) def test_events_labeled_correctly( - tmp_path, config: Configuration, specific_rules, generic_rules, schema, expected_output + tmp_path, config: Configuration, rules, schema, expected_output ): # pylint: disable=too-many-arguments expected_output_path = os.path.join( "tests/testdata/acceptance/expected_result", expected_output ) - set_config(config, specific_rules, generic_rules, schema) + set_config(config, rules, schema) config.input["jsonl"]["documents_path"] = "tests/testdata/input_logdata/wineventlog_raw.jsonl" config_path = tmp_path / "generated_config.yml" config_path.write_text(config.as_yaml()) @@ -79,11 +73,8 @@ def test_events_labeled_correctly( ), f"Missmatch in event at line {result['event_line_no']}!" -def set_config(config: Configuration, specific_rules, generic_rules, schema): +def set_config(config: Configuration, rules, schema): config.pipeline[0]["labelername"]["schema"] = os.path.join("tests/testdata", schema) - config.pipeline[0]["labelername"]["specific_rules"] = [ - os.path.join("tests/testdata", rule) for rule in specific_rules - ] - config.pipeline[0]["labelername"]["generic_rules"] = [ - os.path.join("tests/testdata", rule) for rule in generic_rules + config.pipeline[0]["labelername"]["rules"] = [ + os.path.join("tests/testdata", rule) for rule in rules ] diff --git a/tests/acceptance/test_wineventlog_pseudonymization.py b/tests/acceptance/test_wineventlog_pseudonymization.py index b8f9909db..7fa305782 100644 --- a/tests/acceptance/test_wineventlog_pseudonymization.py +++ b/tests/acceptance/test_wineventlog_pseudonymization.py @@ -35,13 +35,8 @@ def get_config(): "pubkey_depseudo": "tests/testdata/acceptance/pseudonymizer/example_depseudo_pub.pem", "hash_salt": "a_secret_tasty_ingredient", "outputs": [{"jsonl": "pseudonyms"}], - "specific_rules": [ - "tests/testdata/acceptance/pseudonymizer/rules_static/specific" - ], - "generic_rules": [ - "tests/testdata/acceptance/pseudonymizer/rules_static/generic" - ], - "regex_mapping": "tests/testdata/acceptance/pseudonymizer/rules_static/regex_mapping.yml", + "rules": ["tests/testdata/acceptance/pseudonymizer/rules"], + "regex_mapping": "tests/testdata/acceptance/pseudonymizer/regex_mapping.yml", "max_cached_pseudonyms": 1000000, } } diff --git a/tests/acceptance/util.py b/tests/acceptance/util.py index 04de101df..11feb4148 100644 --- a/tests/acceptance/util.py +++ b/tests/acceptance/util.py @@ -341,9 +341,8 @@ def convert_to_http_config(config: Configuration, endpoint) -> dict: ] for processor_config in config.pipeline: name, value = processor_config.popitem() - for rule_kind in ("specific_rules", "generic_rules"): - rules = Processor.resolve_directories(value.get(rule_kind)) - value[rule_kind] = [f"{endpoint}/{rule}" for rule in rules] + rules = Processor.resolve_directories(value.get("rules")) + value["rules"] = [f"{endpoint}/{rule}" for rule in rules] for config_key, config_value in value.items(): if config_key in http_fields: value.update({config_key: f"{endpoint}/{config_value}"}) diff --git a/tests/testdata/acceptance/dissector/rules/generic/dissector_rule.json b/tests/testdata/acceptance/dissector/rules/dissector_rule_1.json similarity index 100% rename from tests/testdata/acceptance/dissector/rules/generic/dissector_rule.json rename to tests/testdata/acceptance/dissector/rules/dissector_rule_1.json diff --git a/tests/testdata/acceptance/dissector/rules/specific/dissector_rule.json b/tests/testdata/acceptance/dissector/rules/dissector_rule_2.json similarity index 100% rename from tests/testdata/acceptance/dissector/rules/specific/dissector_rule.json rename to tests/testdata/acceptance/dissector/rules/dissector_rule_2.json diff --git a/tests/testdata/acceptance/labeler/rules_static/labeling/schema.json b/tests/testdata/acceptance/labeler/no_regex/labeling/schema.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/labeling/schema.json rename to tests/testdata/acceptance/labeler/no_regex/labeling/schema.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/specific/id_1_SecurityCenter.json b/tests/testdata/acceptance/labeler/no_regex/rules/id_1_SecurityCenter.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/specific/id_1_SecurityCenter.json rename to tests/testdata/acceptance/labeler/no_regex/rules/id_1_SecurityCenter.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/specific/id_400_PowerShell.json b/tests/testdata/acceptance/labeler/no_regex/rules/id_400_PowerShell.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/specific/id_400_PowerShell.json rename to tests/testdata/acceptance/labeler/no_regex/rules/id_400_PowerShell.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/specific/id_50036_Microsoft-Windows-Dhcp-Client.json b/tests/testdata/acceptance/labeler/no_regex/rules/id_50036_Microsoft-Windows-Dhcp-Client.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/specific/id_50036_Microsoft-Windows-Dhcp-Client.json rename to tests/testdata/acceptance/labeler/no_regex/rules/id_50036_Microsoft-Windows-Dhcp-Client.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/specific/id_51047_Microsoft-Windows-DHCPv6-Client.json b/tests/testdata/acceptance/labeler/no_regex/rules/id_51047_Microsoft-Windows-DHCPv6-Client.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/specific/id_51047_Microsoft-Windows-DHCPv6-Client.json rename to tests/testdata/acceptance/labeler/no_regex/rules/id_51047_Microsoft-Windows-DHCPv6-Client.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/specific/id_5615_Microsoft-Windows-WMI.json b/tests/testdata/acceptance/labeler/no_regex/rules/id_5615_Microsoft-Windows-WMI.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/specific/id_5615_Microsoft-Windows-WMI.json rename to tests/testdata/acceptance/labeler/no_regex/rules/id_5615_Microsoft-Windows-WMI.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/specific/id_6005_EventLog.json b/tests/testdata/acceptance/labeler/no_regex/rules/id_6005_EventLog.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/specific/id_6005_EventLog.json rename to tests/testdata/acceptance/labeler/no_regex/rules/id_6005_EventLog.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/specific/id_6006_EventLog.json b/tests/testdata/acceptance/labeler/no_regex/rules/id_6006_EventLog.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/specific/id_6006_EventLog.json rename to tests/testdata/acceptance/labeler/no_regex/rules/id_6006_EventLog.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/specific/id_7040_Service_Control_Manager.json b/tests/testdata/acceptance/labeler/no_regex/rules/id_7040_Service_Control_Manager.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/specific/id_7040_Service_Control_Manager.json rename to tests/testdata/acceptance/labeler/no_regex/rules/id_7040_Service_Control_Manager.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/specific/id_8212_System_Restore.json b/tests/testdata/acceptance/labeler/no_regex/rules/id_8212_System_Restore.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/specific/id_8212_System_Restore.json rename to tests/testdata/acceptance/labeler/no_regex/rules/id_8212_System_Restore.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/event_data_Started_to_execute.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/action/event_data_Started_to_execute.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/event_data_Started_to_execute.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/action/event_data_Started_to_execute.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/event_data_Stopped_to_terminate.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/action/event_data_Stopped_to_terminate.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/event_data_Stopped_to_terminate.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/action/event_data_Stopped_to_terminate.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/event_data_paused_to_modify.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/action/event_data_paused_to_modify.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/event_data_paused_to_modify.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/action/event_data_paused_to_modify.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/event_data_power_off_to_terminate.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/action/event_data_power_off_to_terminate.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/event_data_power_off_to_terminate.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/action/event_data_power_off_to_terminate.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/event_data_running_to_execute.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/action/event_data_running_to_execute.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/event_data_running_to_execute.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/action/event_data_running_to_execute.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/keywords_Audit_Failure_to_failed.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/action/keywords_Audit_Failure_to_failed.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/keywords_Audit_Failure_to_failed.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/action/keywords_Audit_Failure_to_failed.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/keywords_Audit_Success_to_success.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/action/keywords_Audit_Success_to_success.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/keywords_Audit_Success_to_success.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/action/keywords_Audit_Success_to_success.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/level_Error_to_failed.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/action/level_Error_to_failed.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/action/level_Error_to_failed.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/action/level_Error_to_failed.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/actor/event_data_logontype_2_or_7_to_user.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/actor/event_data_logontype_2_or_7_to_user.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/actor/event_data_logontype_2_or_7_to_user.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/actor/event_data_logontype_2_or_7_to_user.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/actor/event_data_logontype_4_or_5_to_service.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/actor/event_data_logontype_4_or_5_to_service.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/actor/event_data_logontype_4_or_5_to_service.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/actor/event_data_logontype_4_or_5_to_service.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/multiple/task_Audit_Policy_Change_to_configuration.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/multiple/task_Audit_Policy_Change_to_configuration.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/multiple/task_Audit_Policy_Change_to_configuration.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/multiple/task_Audit_Policy_Change_to_configuration.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/multiple/task_Logoff_to_authenticate_and_accounts.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/multiple/task_Logoff_to_authenticate_and_accounts.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/multiple/task_Logoff_to_authenticate_and_accounts.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/multiple/task_Logoff_to_authenticate_and_accounts.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/multiple/task_Logon_to_authenticate_and_accounts.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/multiple/task_Logon_to_authenticate_and_accounts.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/multiple/task_Logon_to_authenticate_and_accounts.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/multiple/task_Logon_to_authenticate_and_accounts.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Desktop_Window_Manager_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Desktop_Window_Manager_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Desktop_Window_Manager_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Desktop_Window_Manager_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/ESENT_to_database.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/ESENT_to_database.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/ESENT_to_database.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/ESENT_to_database.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/EventLog_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/EventLog_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/EventLog_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/EventLog_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/FreeSSHDService_to_service.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/FreeSSHDService_to_service.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/FreeSSHDService_to_service.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/FreeSSHDService_to_service.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Application-Experience_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Application-Experience_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Application-Experience_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Application-Experience_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-DHCPv6-Client_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-DHCPv6-Client_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-DHCPv6-Client_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-DHCPv6-Client_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Dhcp-Client_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Dhcp-Client_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Dhcp-Client_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Dhcp-Client_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-EventSystem_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-EventSystem_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-EventSystem_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-EventSystem_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-FilterManager_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-FilterManager_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-FilterManager_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-FilterManager_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-GroupPolicy_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-GroupPolicy_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-GroupPolicy_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-GroupPolicy_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Kernel-General_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Kernel-General_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Kernel-General_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Kernel-General_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Kernel-Power_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Kernel-Power_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Kernel-Power_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Kernel-Power_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Kernel-Processor-Power_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Kernel-Processor-Power_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Kernel-Processor-Power_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Kernel-Processor-Power_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Security-Auditing_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Security-Auditing_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Security-Auditing_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Security-Auditing_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Security-SPP_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Security-SPP_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Security-SPP_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Security-SPP_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Time-Service_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Time-Service_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Time-Service_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Time-Service_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-User-Profiles-Service_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-User-Profiles-Service_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-User-Profiles-Service_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-User-Profiles-Service_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-UserPnp_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-UserPnp_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-UserPnp_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-UserPnp_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-WMI_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-WMI_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-WMI_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-WMI_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-WMPNSS-Service_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-WMPNSS-Service_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-WMPNSS-Service_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-WMPNSS-Service_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Winlogon_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Winlogon_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Microsoft-Windows-Winlogon_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Microsoft-Windows-Winlogon_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/NETLOGON_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/NETLOGON_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/NETLOGON_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/NETLOGON_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/PowerShell_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/PowerShell_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/PowerShell_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/PowerShell_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/SecurityCenter_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/SecurityCenter_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/SecurityCenter_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/SecurityCenter_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Service_Control_Manager_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Service_Control_Manager_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/Service_Control_Manager_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/Service_Control_Manager_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/System_Restore_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/System_Restore_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/System_Restore_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/System_Restore_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/VSS_to_service.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/VSS_to_service.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/VSS_to_service.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/VSS_to_service.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/volsnap_to_system.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/volsnap_to_system.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/volsnap_to_system.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/volsnap_to_system.json diff --git a/tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/wineventlog_to_windows.json b/tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/wineventlog_to_windows.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static/rules/generic/windows/reporter/wineventlog_to_windows.json rename to tests/testdata/acceptance/labeler/no_regex/rules/windows/reporter/wineventlog_to_windows.json diff --git a/tests/testdata/acceptance/labeler/rules_static_only_regex/labeling/schema.json b/tests/testdata/acceptance/labeler/only_regex/labeling/schema.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static_only_regex/labeling/schema.json rename to tests/testdata/acceptance/labeler/only_regex/labeling/schema.json diff --git a/tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/computer_name_label.json b/tests/testdata/acceptance/labeler/only_regex/rules/computer_name_label.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/computer_name_label.json rename to tests/testdata/acceptance/labeler/only_regex/rules/computer_name_label.json diff --git a/tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/event_data_Binary_label.json b/tests/testdata/acceptance/labeler/only_regex/rules/event_data_Binary_label.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/event_data_Binary_label.json rename to tests/testdata/acceptance/labeler/only_regex/rules/event_data_Binary_label.json diff --git a/tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/event_data_TargetLogonID_to_label.json b/tests/testdata/acceptance/labeler/only_regex/rules/event_data_TargetLogonID_to_label.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/event_data_TargetLogonID_to_label.json rename to tests/testdata/acceptance/labeler/only_regex/rules/event_data_TargetLogonID_to_label.json diff --git a/tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/event_data_param1_auto_discovery_label.json b/tests/testdata/acceptance/labeler/only_regex/rules/event_data_param1_auto_discovery_label.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/event_data_param1_auto_discovery_label.json rename to tests/testdata/acceptance/labeler/only_regex/rules/event_data_param1_auto_discovery_label.json diff --git a/tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/event_data_param1_crypto_label.json b/tests/testdata/acceptance/labeler/only_regex/rules/event_data_param1_crypto_label.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/event_data_param1_crypto_label.json rename to tests/testdata/acceptance/labeler/only_regex/rules/event_data_param1_crypto_label.json diff --git a/tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/event_data_param1_flash_player_label.json b/tests/testdata/acceptance/labeler/only_regex/rules/event_data_param1_flash_player_label.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/event_data_param1_flash_player_label.json rename to tests/testdata/acceptance/labeler/only_regex/rules/event_data_param1_flash_player_label.json diff --git a/tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/event_data_param1_font_cache_service_label.json b/tests/testdata/acceptance/labeler/only_regex/rules/event_data_param1_font_cache_service_label.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/event_data_param1_font_cache_service_label.json rename to tests/testdata/acceptance/labeler/only_regex/rules/event_data_param1_font_cache_service_label.json diff --git a/tests/testdata/acceptance/labeler/rules_static_only_regex/rules/generic/message_to_logon_label.json b/tests/testdata/acceptance/labeler/only_regex/rules/message_to_logon_label.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static_only_regex/rules/generic/message_to_logon_label.json rename to tests/testdata/acceptance/labeler/only_regex/rules/message_to_logon_label.json diff --git a/tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/provider_guid_to_test_guid_label.json b/tests/testdata/acceptance/labeler/only_regex/rules/provider_guid_to_test_guid_label.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/provider_guid_to_test_guid_label.json rename to tests/testdata/acceptance/labeler/only_regex/rules/provider_guid_to_test_guid_label.json diff --git a/tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/this_is_not_a_rule.not_json b/tests/testdata/acceptance/labeler/only_regex/rules/this_is_not_a_rule.not_json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/this_is_not_a_rule.not_json rename to tests/testdata/acceptance/labeler/only_regex/rules/this_is_not_a_rule.not_json diff --git a/tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/version_to_label.json b/tests/testdata/acceptance/labeler/only_regex/rules/version_to_label.json similarity index 100% rename from tests/testdata/acceptance/labeler/rules_static_only_regex/rules/specific/version_to_label.json rename to tests/testdata/acceptance/labeler/only_regex/rules/version_to_label.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/regex_mapping.yml b/tests/testdata/acceptance/normalizer/regex_mapping.yml similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/regex_mapping.yml rename to tests/testdata/acceptance/normalizer/regex_mapping.yml diff --git a/tests/testdata/acceptance/normalizer/rules_static/specific/ProcessId_NewProcessId_New_ProcessName_id_4688.json b/tests/testdata/acceptance/normalizer/rules/ProcessId_NewProcessId_New_ProcessName_id_4688.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/specific/ProcessId_NewProcessId_New_ProcessName_id_4688.json rename to tests/testdata/acceptance/normalizer/rules/ProcessId_NewProcessId_New_ProcessName_id_4688.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/specific/SubjectUserName_SubjectUserSid_id_4611.json b/tests/testdata/acceptance/normalizer/rules/SubjectUserName_SubjectUserSid_id_4611.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/specific/SubjectUserName_SubjectUserSid_id_4611.json rename to tests/testdata/acceptance/normalizer/rules/SubjectUserName_SubjectUserSid_id_4611.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/specific/SubjectUserName_SubjectUserSid_id_4672.json b/tests/testdata/acceptance/normalizer/rules/SubjectUserName_SubjectUserSid_id_4672.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/specific/SubjectUserName_SubjectUserSid_id_4672.json rename to tests/testdata/acceptance/normalizer/rules/SubjectUserName_SubjectUserSid_id_4672.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/generic/event_data_ClientAddress_to_client_ip.json b/tests/testdata/acceptance/normalizer/rules/event_data_ClientAddress_to_client_ip.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/generic/event_data_ClientAddress_to_client_ip.json rename to tests/testdata/acceptance/normalizer/rules/event_data_ClientAddress_to_client_ip.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/generic/event_data_FromFolder_to_file_path.json b/tests/testdata/acceptance/normalizer/rules/event_data_FromFolder_to_file_path.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/generic/event_data_FromFolder_to_file_path.json rename to tests/testdata/acceptance/normalizer/rules/event_data_FromFolder_to_file_path.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/generic/event_data_IpAddress_to_client_address.json b/tests/testdata/acceptance/normalizer/rules/event_data_IpAddress_to_client_address.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/generic/event_data_IpAddress_to_client_address.json rename to tests/testdata/acceptance/normalizer/rules/event_data_IpAddress_to_client_address.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/generic/event_data_IpAddress_to_client_ip.json b/tests/testdata/acceptance/normalizer/rules/event_data_IpAddress_to_client_ip.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/generic/event_data_IpAddress_to_client_ip.json rename to tests/testdata/acceptance/normalizer/rules/event_data_IpAddress_to_client_ip.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/generic/event_data_IpPort_to_client_port.json b/tests/testdata/acceptance/normalizer/rules/event_data_IpPort_to_client_port.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/generic/event_data_IpPort_to_client_port.json rename to tests/testdata/acceptance/normalizer/rules/event_data_IpPort_to_client_port.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/generic/event_data_LogonProcessName_to_process_name.json b/tests/testdata/acceptance/normalizer/rules/event_data_LogonProcessName_to_process_name.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/generic/event_data_LogonProcessName_to_process_name.json rename to tests/testdata/acceptance/normalizer/rules/event_data_LogonProcessName_to_process_name.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/generic/event_data_ProcessId_NOT_4688_to_process_pid.json b/tests/testdata/acceptance/normalizer/rules/event_data_ProcessId_NOT_4688_to_process_pid.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/generic/event_data_ProcessId_NOT_4688_to_process_pid.json rename to tests/testdata/acceptance/normalizer/rules/event_data_ProcessId_NOT_4688_to_process_pid.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/generic/event_data_ProcessName_to_process_executable.json b/tests/testdata/acceptance/normalizer/rules/event_data_ProcessName_to_process_executable.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/generic/event_data_ProcessName_to_process_executable.json rename to tests/testdata/acceptance/normalizer/rules/event_data_ProcessName_to_process_executable.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/generic/event_data_TargetUserName_to_host_user_name.json b/tests/testdata/acceptance/normalizer/rules/event_data_TargetUserName_to_host_user_name.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/generic/event_data_TargetUserName_to_host_user_name.json rename to tests/testdata/acceptance/normalizer/rules/event_data_TargetUserName_to_host_user_name.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/generic/event_data_TargetUserSid_to_host_user_id.json b/tests/testdata/acceptance/normalizer/rules/event_data_TargetUserSid_to_host_user_id.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/generic/event_data_TargetUserSid_to_host_user_id.json rename to tests/testdata/acceptance/normalizer/rules/event_data_TargetUserSid_to_host_user_id.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/generic/event_data_ToFolder_to_file_target_path.json b/tests/testdata/acceptance/normalizer/rules/event_data_ToFolder_to_file_target_path.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/generic/event_data_ToFolder_to_file_target_path.json rename to tests/testdata/acceptance/normalizer/rules/event_data_ToFolder_to_file_target_path.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/generic/event_data_UserSid_to_host_user_id.json b/tests/testdata/acceptance/normalizer/rules/event_data_UserSid_to_host_user_id.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/generic/event_data_UserSid_to_host_user_id.json rename to tests/testdata/acceptance/normalizer/rules/event_data_UserSid_to_host_user_id.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/specific/param1_to_client_address_id_1104.json b/tests/testdata/acceptance/normalizer/rules/param1_to_client_address_id_1104.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/specific/param1_to_client_address_id_1104.json rename to tests/testdata/acceptance/normalizer/rules/param1_to_client_address_id_1104.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/specific/param1_to_client_address_id_1106.json b/tests/testdata/acceptance/normalizer/rules/param1_to_client_address_id_1106.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/specific/param1_to_client_address_id_1106.json rename to tests/testdata/acceptance/normalizer/rules/param1_to_client_address_id_1106.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/specific/param1_to_host_user_name_id_8.json b/tests/testdata/acceptance/normalizer/rules/param1_to_host_user_name_id_8.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/specific/param1_to_host_user_name_id_8.json rename to tests/testdata/acceptance/normalizer/rules/param1_to_host_user_name_id_8.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/specific/param1_to_host_user_name_id_9.json b/tests/testdata/acceptance/normalizer/rules/param1_to_host_user_name_id_9.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/specific/param1_to_host_user_name_id_9.json rename to tests/testdata/acceptance/normalizer/rules/param1_to_host_user_name_id_9.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/specific/param2_to_host_user_name_id_2000.json b/tests/testdata/acceptance/normalizer/rules/param2_to_host_user_name_id_2000.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/specific/param2_to_host_user_name_id_2000.json rename to tests/testdata/acceptance/normalizer/rules/param2_to_host_user_name_id_2000.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/specific/param2_to_host_user_name_id_2001.json b/tests/testdata/acceptance/normalizer/rules/param2_to_host_user_name_id_2001.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/specific/param2_to_host_user_name_id_2001.json rename to tests/testdata/acceptance/normalizer/rules/param2_to_host_user_name_id_2001.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/specific/param3_to_client_address_id_1104.json b/tests/testdata/acceptance/normalizer/rules/param3_to_client_address_id_1104.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/specific/param3_to_client_address_id_1104.json rename to tests/testdata/acceptance/normalizer/rules/param3_to_client_address_id_1104.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/specific/param3_to_client_address_id_1107.json b/tests/testdata/acceptance/normalizer/rules/param3_to_client_address_id_1107.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/specific/param3_to_client_address_id_1107.json rename to tests/testdata/acceptance/normalizer/rules/param3_to_client_address_id_1107.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/specific/param4_to_error_code_id_4098.json b/tests/testdata/acceptance/normalizer/rules/param4_to_error_code_id_4098.json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/specific/param4_to_error_code_id_4098.json rename to tests/testdata/acceptance/normalizer/rules/param4_to_error_code_id_4098.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/generic/this_is_not_a_rule.not_json b/tests/testdata/acceptance/normalizer/rules/this_is_not_a_rule.not_json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/generic/this_is_not_a_rule.not_json rename to tests/testdata/acceptance/normalizer/rules/this_is_not_a_rule.not_json diff --git a/tests/testdata/acceptance/pre_detector/rules/generic/pre_detect_acceptance_one.json b/tests/testdata/acceptance/pre_detector/rules/pre_detect_acceptance_one.json similarity index 100% rename from tests/testdata/acceptance/pre_detector/rules/generic/pre_detect_acceptance_one.json rename to tests/testdata/acceptance/pre_detector/rules/pre_detect_acceptance_one.json diff --git a/tests/testdata/acceptance/pre_detector/rules/specific/pre_detect_acceptance_one.json b/tests/testdata/acceptance/pre_detector/rules/pre_detect_acceptance_two.json similarity index 100% rename from tests/testdata/acceptance/pre_detector/rules/specific/pre_detect_acceptance_one.json rename to tests/testdata/acceptance/pre_detector/rules/pre_detect_acceptance_two.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/regex_mapping.yml b/tests/testdata/acceptance/pseudonymizer/regex_mapping.yml similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/regex_mapping.yml rename to tests/testdata/acceptance/pseudonymizer/regex_mapping.yml diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/specific/MetaFrameEvents_id_1104.json b/tests/testdata/acceptance/pseudonymizer/rules/MetaFrameEvents_id_1104.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/specific/MetaFrameEvents_id_1104.json rename to tests/testdata/acceptance/pseudonymizer/rules/MetaFrameEvents_id_1104.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/specific/MetaFrameEvents_id_1106.json b/tests/testdata/acceptance/pseudonymizer/rules/MetaFrameEvents_id_1106.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/specific/MetaFrameEvents_id_1106.json rename to tests/testdata/acceptance/pseudonymizer/rules/MetaFrameEvents_id_1106.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/specific/Microsoft-Windows-Terminal-RemoteConnectionManager_id_1060.json b/tests/testdata/acceptance/pseudonymizer/rules/Microsoft-Windows-Terminal-RemoteConnectionManager_id_1060.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/specific/Microsoft-Windows-Terminal-RemoteConnectionManager_id_1060.json rename to tests/testdata/acceptance/pseudonymizer/rules/Microsoft-Windows-Terminal-RemoteConnectionManager_id_1060.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/specific/TdIca_id_1004.json b/tests/testdata/acceptance/pseudonymizer/rules/TdIca_id_1004.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/specific/TdIca_id_1004.json rename to tests/testdata/acceptance/pseudonymizer/rules/TdIca_id_1004.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/specific/TdIca_id_1007.json b/tests/testdata/acceptance/pseudonymizer/rules/TdIca_id_1007.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/specific/TdIca_id_1007.json rename to tests/testdata/acceptance/pseudonymizer/rules/TdIca_id_1007.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/client_address.json b/tests/testdata/acceptance/pseudonymizer/rules/client_address.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/client_address.json rename to tests/testdata/acceptance/pseudonymizer/rules/client_address.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/client_ip.json b/tests/testdata/acceptance/pseudonymizer/rules/client_ip.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/client_ip.json rename to tests/testdata/acceptance/pseudonymizer/rules/client_ip.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_IpAddress.json b/tests/testdata/acceptance/pseudonymizer/rules/event_data_IpAddress.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_IpAddress.json rename to tests/testdata/acceptance/pseudonymizer/rules/event_data_IpAddress.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_SubjectUserName.json b/tests/testdata/acceptance/pseudonymizer/rules/event_data_SubjectUserName.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_SubjectUserName.json rename to tests/testdata/acceptance/pseudonymizer/rules/event_data_SubjectUserName.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_SubjectUserSid.json b/tests/testdata/acceptance/pseudonymizer/rules/event_data_SubjectUserSid.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_SubjectUserSid.json rename to tests/testdata/acceptance/pseudonymizer/rules/event_data_SubjectUserSid.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_TargetUserName.json b/tests/testdata/acceptance/pseudonymizer/rules/event_data_TargetUserName.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_TargetUserName.json rename to tests/testdata/acceptance/pseudonymizer/rules/event_data_TargetUserName.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_TargetUserSid.json b/tests/testdata/acceptance/pseudonymizer/rules/event_data_TargetUserSid.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_TargetUserSid.json rename to tests/testdata/acceptance/pseudonymizer/rules/event_data_TargetUserSid.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_ToFolder.json b/tests/testdata/acceptance/pseudonymizer/rules/event_data_ToFolder.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_ToFolder.json rename to tests/testdata/acceptance/pseudonymizer/rules/event_data_ToFolder.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_UserSid.json b/tests/testdata/acceptance/pseudonymizer/rules/event_data_UserSid.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/event_data_UserSid.json rename to tests/testdata/acceptance/pseudonymizer/rules/event_data_UserSid.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/file_target_path.json b/tests/testdata/acceptance/pseudonymizer/rules/file_target_path.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/file_target_path.json rename to tests/testdata/acceptance/pseudonymizer/rules/file_target_path.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/host_user_id.json b/tests/testdata/acceptance/pseudonymizer/rules/host_user_id.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/host_user_id.json rename to tests/testdata/acceptance/pseudonymizer/rules/host_user_id.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/host_user_name.json b/tests/testdata/acceptance/pseudonymizer/rules/host_user_name.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/host_user_name.json rename to tests/testdata/acceptance/pseudonymizer/rules/host_user_name.json diff --git a/tests/testdata/acceptance/normalizer/rules_static/specific/this_is_not_a_rule.not_json b/tests/testdata/acceptance/pseudonymizer/rules/this_is_not_a_rule.not_json similarity index 100% rename from tests/testdata/acceptance/normalizer/rules_static/specific/this_is_not_a_rule.not_json rename to tests/testdata/acceptance/pseudonymizer/rules/this_is_not_a_rule.not_json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/user_identifier.json b/tests/testdata/acceptance/pseudonymizer/rules/user_identifier.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/user_identifier.json rename to tests/testdata/acceptance/pseudonymizer/rules/user_identifier.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/user_name.json b/tests/testdata/acceptance/pseudonymizer/rules/user_name.json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/user_name.json rename to tests/testdata/acceptance/pseudonymizer/rules/user_name.json diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/specific/this_is_not_a_rule.not_json b/tests/testdata/acceptance/pseudonymizer/rules_static/specific/this_is_not_a_rule.not_json deleted file mode 100644 index 129b1e3f3..000000000 --- a/tests/testdata/acceptance/pseudonymizer/rules_static/specific/this_is_not_a_rule.not_json +++ /dev/null @@ -1 +0,0 @@ -I'm not a json file and should not be loaded as a rule! diff --git a/tests/testdata/acceptance/selective_extractor/rules/generic/rules.json b/tests/testdata/acceptance/selective_extractor/rules/rules_1.json similarity index 100% rename from tests/testdata/acceptance/selective_extractor/rules/generic/rules.json rename to tests/testdata/acceptance/selective_extractor/rules/rules_1.json diff --git a/tests/testdata/acceptance/selective_extractor/rules/specific/rules.json b/tests/testdata/acceptance/selective_extractor/rules/rules_2.json similarity index 100% rename from tests/testdata/acceptance/selective_extractor/rules/specific/rules.json rename to tests/testdata/acceptance/selective_extractor/rules/rules_2.json diff --git a/tests/testdata/auto_tests/clusterer/rules/generic/rule_with_custom_tests.yml b/tests/testdata/auto_tests/clusterer/rules/rule_with_custom_tests_1.yml similarity index 100% rename from tests/testdata/auto_tests/clusterer/rules/generic/rule_with_custom_tests.yml rename to tests/testdata/auto_tests/clusterer/rules/rule_with_custom_tests_1.yml diff --git a/tests/testdata/auto_tests/clusterer/rules/specific/rule_with_custom_tests.yml b/tests/testdata/auto_tests/clusterer/rules/rule_with_custom_tests_2.yml similarity index 100% rename from tests/testdata/auto_tests/clusterer/rules/specific/rule_with_custom_tests.yml rename to tests/testdata/auto_tests/clusterer/rules/rule_with_custom_tests_2.yml diff --git a/tests/testdata/auto_tests/dissector/rules/generic/auto_test_match.json b/tests/testdata/auto_tests/dissector/rules/auto_test_match.json similarity index 100% rename from tests/testdata/auto_tests/dissector/rules/generic/auto_test_match.json rename to tests/testdata/auto_tests/dissector/rules/auto_test_match.json diff --git a/tests/testdata/auto_tests/dissector/rules/generic/auto_test_match_test.json b/tests/testdata/auto_tests/dissector/rules/auto_test_match_test.json similarity index 100% rename from tests/testdata/auto_tests/dissector/rules/generic/auto_test_match_test.json rename to tests/testdata/auto_tests/dissector/rules/auto_test_match_test.json diff --git a/tests/testdata/auto_tests/dissector/rules/specific/auto_test_mismatch.json b/tests/testdata/auto_tests/dissector/rules/auto_test_mismatch.json similarity index 100% rename from tests/testdata/auto_tests/dissector/rules/specific/auto_test_mismatch.json rename to tests/testdata/auto_tests/dissector/rules/auto_test_mismatch.json diff --git a/tests/testdata/auto_tests/dissector/rules/specific/auto_test_mismatch_test.json b/tests/testdata/auto_tests/dissector/rules/auto_test_mismatch_test.json similarity index 100% rename from tests/testdata/auto_tests/dissector/rules/specific/auto_test_mismatch_test.json rename to tests/testdata/auto_tests/dissector/rules/auto_test_mismatch_test.json diff --git a/tests/testdata/auto_tests/dissector/rules/specific/auto_test_no_test_.json b/tests/testdata/auto_tests/dissector/rules/auto_test_no_test_.json similarity index 100% rename from tests/testdata/auto_tests/dissector/rules/specific/auto_test_no_test_.json rename to tests/testdata/auto_tests/dissector/rules/auto_test_no_test_.json diff --git a/tests/testdata/auto_tests/dropper/rules/generic/drop_field.json b/tests/testdata/auto_tests/dropper/rules/drop_field_1.json similarity index 100% rename from tests/testdata/auto_tests/dropper/rules/generic/drop_field.json rename to tests/testdata/auto_tests/dropper/rules/drop_field_1.json diff --git a/tests/testdata/auto_tests/dropper/rules/generic/drop_field_test.json b/tests/testdata/auto_tests/dropper/rules/drop_field_1_test.json similarity index 100% rename from tests/testdata/auto_tests/dropper/rules/generic/drop_field_test.json rename to tests/testdata/auto_tests/dropper/rules/drop_field_1_test.json diff --git a/tests/testdata/auto_tests/dropper/rules/specific/drop_field.json b/tests/testdata/auto_tests/dropper/rules/drop_field_2.json similarity index 100% rename from tests/testdata/auto_tests/dropper/rules/specific/drop_field.json rename to tests/testdata/auto_tests/dropper/rules/drop_field_2.json diff --git a/tests/testdata/auto_tests/dropper/rules/specific/drop_field_test.json b/tests/testdata/auto_tests/dropper/rules/drop_field_2_test.json similarity index 100% rename from tests/testdata/auto_tests/dropper/rules/specific/drop_field_test.json rename to tests/testdata/auto_tests/dropper/rules/drop_field_2_test.json diff --git a/tests/testdata/auto_tests/labeler/rules/generic/auto_test_labeling_match.json b/tests/testdata/auto_tests/labeler/rules/auto_test_labeling_match.json similarity index 100% rename from tests/testdata/auto_tests/labeler/rules/generic/auto_test_labeling_match.json rename to tests/testdata/auto_tests/labeler/rules/auto_test_labeling_match.json diff --git a/tests/testdata/auto_tests/labeler/rules/generic/auto_test_labeling_match_existing.json b/tests/testdata/auto_tests/labeler/rules/auto_test_labeling_match_existing.json similarity index 100% rename from tests/testdata/auto_tests/labeler/rules/generic/auto_test_labeling_match_existing.json rename to tests/testdata/auto_tests/labeler/rules/auto_test_labeling_match_existing.json diff --git a/tests/testdata/auto_tests/labeler/rules/generic/auto_test_labeling_match_existing_test.json b/tests/testdata/auto_tests/labeler/rules/auto_test_labeling_match_existing_test.json similarity index 100% rename from tests/testdata/auto_tests/labeler/rules/generic/auto_test_labeling_match_existing_test.json rename to tests/testdata/auto_tests/labeler/rules/auto_test_labeling_match_existing_test.json diff --git a/tests/testdata/auto_tests/labeler/rules/generic/auto_test_labeling_match_test.json b/tests/testdata/auto_tests/labeler/rules/auto_test_labeling_match_test.json similarity index 100% rename from tests/testdata/auto_tests/labeler/rules/generic/auto_test_labeling_match_test.json rename to tests/testdata/auto_tests/labeler/rules/auto_test_labeling_match_test.json diff --git a/tests/testdata/auto_tests/labeler/rules/specific/auto_test_labeling_mismatch.json b/tests/testdata/auto_tests/labeler/rules/auto_test_labeling_mismatch.json similarity index 100% rename from tests/testdata/auto_tests/labeler/rules/specific/auto_test_labeling_mismatch.json rename to tests/testdata/auto_tests/labeler/rules/auto_test_labeling_mismatch.json diff --git a/tests/testdata/auto_tests/labeler/rules/specific/auto_test_labeling_mismatch_test.json b/tests/testdata/auto_tests/labeler/rules/auto_test_labeling_mismatch_test.json similarity index 100% rename from tests/testdata/auto_tests/labeler/rules/specific/auto_test_labeling_mismatch_test.json rename to tests/testdata/auto_tests/labeler/rules/auto_test_labeling_mismatch_test.json diff --git a/tests/testdata/auto_tests/labeler/rules/specific/auto_test_labeling_no_test_.json b/tests/testdata/auto_tests/labeler/rules/auto_test_labeling_no_test_.json similarity index 100% rename from tests/testdata/auto_tests/labeler/rules/specific/auto_test_labeling_no_test_.json rename to tests/testdata/auto_tests/labeler/rules/auto_test_labeling_no_test_.json diff --git a/tests/testdata/auto_tests/pre_detector/rules/generic/auto_test_pre_detector_match.json b/tests/testdata/auto_tests/pre_detector/rules/auto_test_pre_detector_match.json similarity index 100% rename from tests/testdata/auto_tests/pre_detector/rules/generic/auto_test_pre_detector_match.json rename to tests/testdata/auto_tests/pre_detector/rules/auto_test_pre_detector_match.json diff --git a/tests/testdata/auto_tests/pre_detector/rules/generic/auto_test_pre_detector_match_test.json b/tests/testdata/auto_tests/pre_detector/rules/auto_test_pre_detector_match_test.json similarity index 100% rename from tests/testdata/auto_tests/pre_detector/rules/generic/auto_test_pre_detector_match_test.json rename to tests/testdata/auto_tests/pre_detector/rules/auto_test_pre_detector_match_test.json diff --git a/tests/testdata/auto_tests/pre_detector/rules/specific/auto_test_pre_detector_mismatch.json b/tests/testdata/auto_tests/pre_detector/rules/auto_test_pre_detector_mismatch.json similarity index 100% rename from tests/testdata/auto_tests/pre_detector/rules/specific/auto_test_pre_detector_mismatch.json rename to tests/testdata/auto_tests/pre_detector/rules/auto_test_pre_detector_mismatch.json diff --git a/tests/testdata/auto_tests/pre_detector/rules/specific/auto_test_pre_detector_mismatch_test.json b/tests/testdata/auto_tests/pre_detector/rules/auto_test_pre_detector_mismatch_test.json similarity index 100% rename from tests/testdata/auto_tests/pre_detector/rules/specific/auto_test_pre_detector_mismatch_test.json rename to tests/testdata/auto_tests/pre_detector/rules/auto_test_pre_detector_mismatch_test.json diff --git a/tests/testdata/auto_tests/pre_detector/rules/specific/auto_test_pre_detector_no_test_.json b/tests/testdata/auto_tests/pre_detector/rules/auto_test_pre_detector_no_test_.json similarity index 100% rename from tests/testdata/auto_tests/pre_detector/rules/specific/auto_test_pre_detector_no_test_.json rename to tests/testdata/auto_tests/pre_detector/rules/auto_test_pre_detector_no_test_.json diff --git a/tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_dotted_list.json b/tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_dotted_list.json similarity index 100% rename from tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_dotted_list.json rename to tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_dotted_list.json diff --git a/tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_dotted_list_test.json b/tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_dotted_list_test.json similarity index 100% rename from tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_dotted_list_test.json rename to tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_dotted_list_test.json diff --git a/tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_list.json b/tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_list.json similarity index 100% rename from tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_list.json rename to tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_list.json diff --git a/tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_list_escaped.json b/tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_list_escaped.json similarity index 100% rename from tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_list_escaped.json rename to tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_list_escaped.json diff --git a/tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_list_escaped_test.json b/tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_list_escaped_test.json similarity index 100% rename from tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_list_escaped_test.json rename to tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_list_escaped_test.json diff --git a/tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_list_test.json b/tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_list_test.json similarity index 100% rename from tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_list_test.json rename to tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_list_test.json diff --git a/tests/testdata/auto_tests/pseudonymizer/rules/generic/auto_test_pseudonymizer_match.json b/tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_match.json similarity index 100% rename from tests/testdata/auto_tests/pseudonymizer/rules/generic/auto_test_pseudonymizer_match.json rename to tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_match.json diff --git a/tests/testdata/auto_tests/pseudonymizer/rules/generic/auto_test_pseudonymizer_match_test.json b/tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_match_test.json similarity index 100% rename from tests/testdata/auto_tests/pseudonymizer/rules/generic/auto_test_pseudonymizer_match_test.json rename to tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_match_test.json diff --git a/tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_mismatch.json b/tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_mismatch.json similarity index 100% rename from tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_mismatch.json rename to tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_mismatch.json diff --git a/tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_mismatch_test.json b/tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_mismatch_test.json similarity index 100% rename from tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_mismatch_test.json rename to tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_mismatch_test.json diff --git a/tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_no_test_.json b/tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_no_test_.json similarity index 100% rename from tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_no_test_.json rename to tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_no_test_.json diff --git a/tests/testdata/auto_tests/template_replacer/rules/generic/template_replacer.json b/tests/testdata/auto_tests/template_replacer/rules/template_replacer_1.json similarity index 100% rename from tests/testdata/auto_tests/template_replacer/rules/generic/template_replacer.json rename to tests/testdata/auto_tests/template_replacer/rules/template_replacer_1.json diff --git a/tests/testdata/auto_tests/template_replacer/rules/generic/template_replacer_test.json b/tests/testdata/auto_tests/template_replacer/rules/template_replacer_1_test.json similarity index 100% rename from tests/testdata/auto_tests/template_replacer/rules/generic/template_replacer_test.json rename to tests/testdata/auto_tests/template_replacer/rules/template_replacer_1_test.json diff --git a/tests/testdata/auto_tests/template_replacer/rules/specific/template_replacer.json b/tests/testdata/auto_tests/template_replacer/rules/template_replacer_2.json similarity index 100% rename from tests/testdata/auto_tests/template_replacer/rules/specific/template_replacer.json rename to tests/testdata/auto_tests/template_replacer/rules/template_replacer_2.json diff --git a/tests/testdata/auto_tests/template_replacer/rules/specific/template_replacer_test.json b/tests/testdata/auto_tests/template_replacer/rules/template_replacer_2_test.json similarity index 100% rename from tests/testdata/auto_tests/template_replacer/rules/specific/template_replacer_test.json rename to tests/testdata/auto_tests/template_replacer/rules/template_replacer_2_test.json diff --git a/tests/testdata/config/config-auto-tests.yml b/tests/testdata/config/config-auto-tests.yml index d228b0a15..fd2bd1c01 100644 --- a/tests/testdata/config/config-auto-tests.yml +++ b/tests/testdata/config/config-auto-tests.yml @@ -15,28 +15,20 @@ pipeline: type: labeler schema: tests/testdata/auto_tests/labeler/schema.json include_parent_labels: True - specific_rules: - - tests/testdata/auto_tests/labeler/rules/specific/ - generic_rules: - - tests/testdata/auto_tests/labeler/rules/generic/ + rules: + - tests/testdata/auto_tests/labeler/rules - dissector: type: dissector - specific_rules: - - tests/testdata/auto_tests/dissector/rules/specific - generic_rules: - - tests/testdata/auto_tests/dissector/rules/generic + rules: + - tests/testdata/auto_tests/dissector/rules - dropper: type: dropper - specific_rules: - - tests/testdata/auto_tests/dropper/rules/specific/ - generic_rules: - - tests/testdata/auto_tests/dropper/rules/generic/ + rules: + - tests/testdata/auto_tests/dropper/rules - pre_detector: type: pre_detector - specific_rules: - - tests/testdata/auto_tests/pre_detector/rules/specific/ - generic_rules: - - tests/testdata/auto_tests/pre_detector/rules/generic/ + rules: + - tests/testdata/auto_tests/pre_detector/rules outputs: - dummy_output: sre - pseudonymizer: @@ -47,17 +39,13 @@ pipeline: hash_salt: a_secret_tasty_ingredient outputs: - dummy_output: pseudonyms - specific_rules: - - tests/testdata/auto_tests/pseudonymizer/rules/generic/ - generic_rules: - - tests/testdata/auto_tests/pseudonymizer/rules/specific/ + rules: + - tests/testdata/auto_tests/pseudonymizer/rules max_cached_pseudonyms: 1000000 - templatereplacername: type: template_replacer - specific_rules: - - tests/testdata/auto_tests/template_replacer/rules/specific/ - generic_rules: - - tests/testdata/auto_tests/template_replacer/rules/generic/ + rules: + - tests/testdata/auto_tests/template_replacer/rules template: tests/testdata/unit/template_replacer/replacer_template.yml pattern: delimiter: "," @@ -68,8 +56,6 @@ pipeline: target_field: target.field - clusterername: type: clusterer - specific_rules: - - tests/testdata/auto_tests/clusterer/rules/specific/ - generic_rules: - - tests/testdata/auto_tests/clusterer/rules/generic/ + rules: + - tests/testdata/auto_tests/clusterer/rules output_field_name: target_field diff --git a/tests/testdata/config/config-docker.yml b/tests/testdata/config/config-docker.yml index 187771905..c3364c5d4 100644 --- a/tests/testdata/config/config-docker.yml +++ b/tests/testdata/config/config-docker.yml @@ -6,10 +6,8 @@ pipeline: type: labeler schema: tests/testdata/acceptance/labeler/rules_static/labeling/schema.json include_parent_labels: true - specific_rules: - - tests/testdata/acceptance/labeler/rules_static/rules/specific/ - generic_rules: - - tests/testdata/acceptance/labeler/rules_static/rules/generic/ + rules: + - tests/testdata/acceptance/labeler/no_regex/rules/ input: kafka: type: confluentkafka_input diff --git a/tests/testdata/config/config.yml b/tests/testdata/config/config.yml index f9e05bbee..a109124f9 100644 --- a/tests/testdata/config/config.yml +++ b/tests/testdata/config/config.yml @@ -5,36 +5,28 @@ timeout: 0.1 pipeline: - dissector: type: dissector - specific_rules: - - tests/testdata/unit/dissector/specific_rules/ - generic_rules: - - tests/testdata/unit/dissector/generic_rules/ + rules: + - tests/testdata/unit/dissector/rules/ - calculatorname: type: calculator - specific_rules: - - tests/testdata/unit/calculator/generic_rules - generic_rules: - - tests/testdata/unit/calculator/specific_rules + rules: + - tests/testdata/unit/calculator/rules - labelername: type: labeler schema: tests/testdata/unit/labeler/schemas/schema3.json include_parent_labels: true - specific_rules: - - tests/testdata/unit/labeler/rules/specific/ - generic_rules: - - tests/testdata/unit/labeler/rules/generic/ + rules: + - tests/testdata/unit/labeler/rules/ - pseudonymizer: type: pseudonymizer pubkey_analyst: tests/testdata/unit/pseudonymizer/example_analyst_pub.pem pubkey_depseudo: tests/testdata/unit/pseudonymizer/example_depseudo_pub.pem - regex_mapping: tests/testdata/unit/pseudonymizer/rules/regex_mapping.yml + regex_mapping: tests/testdata/unit/pseudonymizer/regex_mapping.yml hash_salt: a_secret_tasty_ingredient outputs: - kafka_output: pseudonyms - specific_rules: - - tests/testdata/unit/pseudonymizer/rules/specific/ - generic_rules: - - tests/testdata/unit/pseudonymizer/rules/generic/ + rules: + - tests/testdata/unit/pseudonymizer/rules/ max_cached_pseudonyms: 1000000 input: diff --git a/tests/testdata/config/config2.yml b/tests/testdata/config/config2.yml index 30a89d051..1a33257e7 100644 --- a/tests/testdata/config/config2.yml +++ b/tests/testdata/config/config2.yml @@ -6,10 +6,8 @@ pipeline: type: labeler schema: tests/testdata/unit/labeler/schemas/schema3.json include_parent_labels: true - specific_rules: - - tests/testdata/unit/labeler/rules/specific/ - generic_rules: - - tests/testdata/unit/labeler/rules/generic/ + rules: + - tests/testdata/unit/labeler/rules input: kafka: diff --git a/tests/testdata/unit/amides/rules/generic/amides_generic.yml b/tests/testdata/unit/amides/rules/amides_1.yml similarity index 100% rename from tests/testdata/unit/amides/rules/generic/amides_generic.yml rename to tests/testdata/unit/amides/rules/amides_1.yml diff --git a/tests/testdata/unit/amides/rules/specific/amides_specific.yml b/tests/testdata/unit/amides/rules/amides_2.yml similarity index 100% rename from tests/testdata/unit/amides/rules/specific/amides_specific.yml rename to tests/testdata/unit/amides/rules/amides_2.yml diff --git a/tests/testdata/unit/calculator/generic_rules/calculator.json b/tests/testdata/unit/calculator/rules/calculator_1.json similarity index 100% rename from tests/testdata/unit/calculator/generic_rules/calculator.json rename to tests/testdata/unit/calculator/rules/calculator_1.json diff --git a/tests/testdata/unit/calculator/specific_rules/calculator.json b/tests/testdata/unit/calculator/rules/calculator_2.json similarity index 100% rename from tests/testdata/unit/calculator/specific_rules/calculator.json rename to tests/testdata/unit/calculator/rules/calculator_2.json diff --git a/tests/testdata/unit/clusterer/rules/generic/rules.json b/tests/testdata/unit/clusterer/rules/rules.json similarity index 100% rename from tests/testdata/unit/clusterer/rules/generic/rules.json rename to tests/testdata/unit/clusterer/rules/rules.json diff --git a/tests/testdata/unit/clusterer/rules/specific/rules.json b/tests/testdata/unit/clusterer/rules/specific/rules.json deleted file mode 100644 index dac74f1fb..000000000 --- a/tests/testdata/unit/clusterer/rules/specific/rules.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "filter": "message2", - "clusterer": { - "source_fields": ["message"], - "pattern": "test (signature) test", - "repl": "<+>\\1" - }, - "description": "insert a description text" - } -] \ No newline at end of file diff --git a/tests/testdata/unit/concatenator/rules/generic/add_fields.json b/tests/testdata/unit/concatenator/rules/add_fields_1.json similarity index 100% rename from tests/testdata/unit/concatenator/rules/generic/add_fields.json rename to tests/testdata/unit/concatenator/rules/add_fields_1.json diff --git a/tests/testdata/unit/concatenator/rules/specific/add_fields.json b/tests/testdata/unit/concatenator/rules/add_fields_2.json similarity index 100% rename from tests/testdata/unit/concatenator/rules/specific/add_fields.json rename to tests/testdata/unit/concatenator/rules/add_fields_2.json diff --git a/tests/testdata/unit/datetime_extractor/rules/generic/datetime_extractor.json b/tests/testdata/unit/datetime_extractor/rules/datetime_extractor_1.json similarity index 100% rename from tests/testdata/unit/datetime_extractor/rules/generic/datetime_extractor.json rename to tests/testdata/unit/datetime_extractor/rules/datetime_extractor_1.json diff --git a/tests/testdata/unit/datetime_extractor/rules/specific/datetime_extractor.json b/tests/testdata/unit/datetime_extractor/rules/datetime_extractor_2.json similarity index 100% rename from tests/testdata/unit/datetime_extractor/rules/specific/datetime_extractor.json rename to tests/testdata/unit/datetime_extractor/rules/datetime_extractor_2.json diff --git a/tests/testdata/unit/deleter/rules/generic/generic_delete.json b/tests/testdata/unit/deleter/rules/delete_1.json similarity index 100% rename from tests/testdata/unit/deleter/rules/generic/generic_delete.json rename to tests/testdata/unit/deleter/rules/delete_1.json diff --git a/tests/testdata/unit/deleter/rules/specific/specific_delete.json b/tests/testdata/unit/deleter/rules/delete_2.json similarity index 71% rename from tests/testdata/unit/deleter/rules/specific/specific_delete.json rename to tests/testdata/unit/deleter/rules/delete_2.json index 1184d3450..6ed0ba935 100644 --- a/tests/testdata/unit/deleter/rules/specific/specific_delete.json +++ b/tests/testdata/unit/deleter/rules/delete_2.json @@ -1,6 +1,6 @@ [ { - "filter": "delete_event_specific", + "filter": "delete_event_2", "deleter": { "delete": true }, diff --git a/tests/testdata/unit/deleter/rules/specific/delete_test.json b/tests/testdata/unit/deleter/rules/delete_test.json similarity index 100% rename from tests/testdata/unit/deleter/rules/specific/delete_test.json rename to tests/testdata/unit/deleter/rules/delete_test.json diff --git a/tests/testdata/unit/deleter/rules/specific/test.json b/tests/testdata/unit/deleter/rules/test.json similarity index 100% rename from tests/testdata/unit/deleter/rules/specific/test.json rename to tests/testdata/unit/deleter/rules/test.json diff --git a/tests/testdata/unit/dissector/generic_rules/dissector_rule.json b/tests/testdata/unit/dissector/rules/dissector_rule_1.json similarity index 100% rename from tests/testdata/unit/dissector/generic_rules/dissector_rule.json rename to tests/testdata/unit/dissector/rules/dissector_rule_1.json diff --git a/tests/testdata/unit/dissector/specific_rules/dissector_rule.json b/tests/testdata/unit/dissector/rules/dissector_rule_2.json similarity index 100% rename from tests/testdata/unit/dissector/specific_rules/dissector_rule.json rename to tests/testdata/unit/dissector/rules/dissector_rule_2.json diff --git a/tests/testdata/unit/domain_label_extractor/rules/generic/domain_label_extractor_gen.json b/tests/testdata/unit/domain_label_extractor/rules/domain_label_extractor_1.json similarity index 100% rename from tests/testdata/unit/domain_label_extractor/rules/generic/domain_label_extractor_gen.json rename to tests/testdata/unit/domain_label_extractor/rules/domain_label_extractor_1.json diff --git a/tests/testdata/unit/domain_label_extractor/rules/specific/domain_label_extractor.json b/tests/testdata/unit/domain_label_extractor/rules/domain_label_extractor_2.json similarity index 100% rename from tests/testdata/unit/domain_label_extractor/rules/specific/domain_label_extractor.json rename to tests/testdata/unit/domain_label_extractor/rules/domain_label_extractor_2.json diff --git a/tests/testdata/unit/domain_resolver/rules/generic/domain_resolver.yml b/tests/testdata/unit/domain_resolver/rules/domain_resolver_1.yml similarity index 100% rename from tests/testdata/unit/domain_resolver/rules/generic/domain_resolver.yml rename to tests/testdata/unit/domain_resolver/rules/domain_resolver_1.yml diff --git a/tests/testdata/unit/domain_resolver/rules/specific/domain_resolver.json b/tests/testdata/unit/domain_resolver/rules/domain_resolver_2.json similarity index 100% rename from tests/testdata/unit/domain_resolver/rules/specific/domain_resolver.json rename to tests/testdata/unit/domain_resolver/rules/domain_resolver_2.json diff --git a/tests/testdata/unit/dropper/rules/generic/drop_field.json b/tests/testdata/unit/dropper/rules/drop_field_1.json similarity index 100% rename from tests/testdata/unit/dropper/rules/generic/drop_field.json rename to tests/testdata/unit/dropper/rules/drop_field_1.json diff --git a/tests/testdata/unit/dropper/rules/specific/drop_field.json b/tests/testdata/unit/dropper/rules/drop_field_2.json similarity index 100% rename from tests/testdata/unit/dropper/rules/specific/drop_field.json rename to tests/testdata/unit/dropper/rules/drop_field_2.json diff --git a/tests/testdata/unit/field_manager/generic_rules/field_manager.json b/tests/testdata/unit/field_manager/rules/field_manager_1.json similarity index 100% rename from tests/testdata/unit/field_manager/generic_rules/field_manager.json rename to tests/testdata/unit/field_manager/rules/field_manager_1.json diff --git a/tests/testdata/unit/field_manager/specific_rules/field_manager.json b/tests/testdata/unit/field_manager/rules/field_manager_2.json similarity index 100% rename from tests/testdata/unit/field_manager/specific_rules/field_manager.json rename to tests/testdata/unit/field_manager/rules/field_manager_2.json diff --git a/tests/testdata/unit/generic_adder/rules/generic/rules.json b/tests/testdata/unit/generic_adder/rules/rule_1.json similarity index 100% rename from tests/testdata/unit/generic_adder/rules/generic/rules.json rename to tests/testdata/unit/generic_adder/rules/rule_1.json diff --git a/tests/testdata/unit/generic_adder/rules/specific/specific_rules.json b/tests/testdata/unit/generic_adder/rules/rule_2.json similarity index 100% rename from tests/testdata/unit/generic_adder/rules/specific/specific_rules.json rename to tests/testdata/unit/generic_adder/rules/rule_2.json diff --git a/tests/testdata/unit/generic_resolver/rules/generic/rule_01.json b/tests/testdata/unit/generic_resolver/rules/rule_1.json similarity index 100% rename from tests/testdata/unit/generic_resolver/rules/generic/rule_01.json rename to tests/testdata/unit/generic_resolver/rules/rule_1.json diff --git a/tests/testdata/unit/generic_resolver/rules/specific/rule_01.json b/tests/testdata/unit/generic_resolver/rules/rule_2.json similarity index 100% rename from tests/testdata/unit/generic_resolver/rules/specific/rule_01.json rename to tests/testdata/unit/generic_resolver/rules/rule_2.json diff --git a/tests/testdata/unit/geoip_enricher/rules/generic/geoip_all.json b/tests/testdata/unit/geoip_enricher/rules/geoip_all_1.json similarity index 100% rename from tests/testdata/unit/geoip_enricher/rules/generic/geoip_all.json rename to tests/testdata/unit/geoip_enricher/rules/geoip_all_1.json diff --git a/tests/testdata/unit/geoip_enricher/rules/specific/geoip_all.json b/tests/testdata/unit/geoip_enricher/rules/geoip_all_2.json similarity index 100% rename from tests/testdata/unit/geoip_enricher/rules/specific/geoip_all.json rename to tests/testdata/unit/geoip_enricher/rules/geoip_all_2.json diff --git a/tests/testdata/unit/grokker/generic_rules/rule.yml b/tests/testdata/unit/grokker/rules/rule_1.yml similarity index 100% rename from tests/testdata/unit/grokker/generic_rules/rule.yml rename to tests/testdata/unit/grokker/rules/rule_1.yml diff --git a/tests/testdata/unit/grokker/specific_rules/rule.yml b/tests/testdata/unit/grokker/rules/rule_2.yml similarity index 100% rename from tests/testdata/unit/grokker/specific_rules/rule.yml rename to tests/testdata/unit/grokker/rules/rule_2.yml diff --git a/tests/testdata/unit/hyperscan_resolver/rules/generic/rule_01.json b/tests/testdata/unit/hyperscan_resolver/rules/rule_1.json similarity index 100% rename from tests/testdata/unit/hyperscan_resolver/rules/generic/rule_01.json rename to tests/testdata/unit/hyperscan_resolver/rules/rule_1.json diff --git a/tests/testdata/unit/hyperscan_resolver/rules/specific/rule_01.json b/tests/testdata/unit/hyperscan_resolver/rules/rule_2.json similarity index 100% rename from tests/testdata/unit/hyperscan_resolver/rules/specific/rule_01.json rename to tests/testdata/unit/hyperscan_resolver/rules/rule_2.json diff --git a/tests/testdata/unit/ip_informer/generic/rule.json b/tests/testdata/unit/ip_informer/rules/rule_1.json similarity index 100% rename from tests/testdata/unit/ip_informer/generic/rule.json rename to tests/testdata/unit/ip_informer/rules/rule_1.json diff --git a/tests/testdata/unit/ip_informer/specific/rule.json b/tests/testdata/unit/ip_informer/rules/rule_2.json similarity index 100% rename from tests/testdata/unit/ip_informer/specific/rule.json rename to tests/testdata/unit/ip_informer/rules/rule_2.json diff --git a/tests/testdata/unit/key_checker/generic_rules/key_checker_rule.json b/tests/testdata/unit/key_checker/rules/key_checker_rule_1.json similarity index 100% rename from tests/testdata/unit/key_checker/generic_rules/key_checker_rule.json rename to tests/testdata/unit/key_checker/rules/key_checker_rule_1.json diff --git a/tests/testdata/unit/key_checker/specific_rules/key_checker_rule.json b/tests/testdata/unit/key_checker/rules/key_checker_rule_2.json similarity index 100% rename from tests/testdata/unit/key_checker/specific_rules/key_checker_rule.json rename to tests/testdata/unit/key_checker/rules/key_checker_rule_2.json diff --git a/tests/testdata/unit/labeler/rules/specific/first.json b/tests/testdata/unit/labeler/rules/first.json similarity index 100% rename from tests/testdata/unit/labeler/rules/specific/first.json rename to tests/testdata/unit/labeler/rules/first.json diff --git a/tests/testdata/unit/labeler/rules/generic/rule.json b/tests/testdata/unit/labeler/rules/rule.json similarity index 100% rename from tests/testdata/unit/labeler/rules/generic/rule.json rename to tests/testdata/unit/labeler/rules/rule.json diff --git a/tests/testdata/unit/list_comparison/rules/generic/user_check.json b/tests/testdata/unit/list_comparison/rules/user_check_1.json similarity index 100% rename from tests/testdata/unit/list_comparison/rules/generic/user_check.json rename to tests/testdata/unit/list_comparison/rules/user_check_1.json diff --git a/tests/testdata/unit/list_comparison/rules/specific/user_check_specific.json b/tests/testdata/unit/list_comparison/rules/user_check_2.json similarity index 100% rename from tests/testdata/unit/list_comparison/rules/specific/user_check_specific.json rename to tests/testdata/unit/list_comparison/rules/user_check_2.json diff --git a/tests/testdata/unit/pre_detector/rules/generic/pre_detect_four.yml b/tests/testdata/unit/pre_detector/rules/pre_detect_four.yml similarity index 100% rename from tests/testdata/unit/pre_detector/rules/generic/pre_detect_four.yml rename to tests/testdata/unit/pre_detector/rules/pre_detect_four.yml diff --git a/tests/testdata/unit/pre_detector/rules/generic/pre_detect_one.json b/tests/testdata/unit/pre_detector/rules/pre_detect_one.json similarity index 100% rename from tests/testdata/unit/pre_detector/rules/generic/pre_detect_one.json rename to tests/testdata/unit/pre_detector/rules/pre_detect_one.json diff --git a/tests/testdata/unit/pre_detector/rules/generic/pre_detect_three.json b/tests/testdata/unit/pre_detector/rules/pre_detect_three.json similarity index 100% rename from tests/testdata/unit/pre_detector/rules/generic/pre_detect_three.json rename to tests/testdata/unit/pre_detector/rules/pre_detect_three.json diff --git a/tests/testdata/unit/pre_detector/rules/generic/pre_detect_two.json b/tests/testdata/unit/pre_detector/rules/pre_detect_two.json similarity index 100% rename from tests/testdata/unit/pre_detector/rules/generic/pre_detect_two.json rename to tests/testdata/unit/pre_detector/rules/pre_detect_two.json diff --git a/tests/testdata/unit/pre_detector/rules/generic/pre_detect_two_rules.json b/tests/testdata/unit/pre_detector/rules/pre_detect_two_rules.json similarity index 100% rename from tests/testdata/unit/pre_detector/rules/generic/pre_detect_two_rules.json rename to tests/testdata/unit/pre_detector/rules/pre_detect_two_rules.json diff --git a/tests/testdata/unit/pre_detector/rules/specific/pre_detect_one.json b/tests/testdata/unit/pre_detector/rules/specific/pre_detect_one.json deleted file mode 100644 index 26a14bf0d..000000000 --- a/tests/testdata/unit/pre_detector/rules/specific/pre_detect_one.json +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "filter": "specific.winlog.event_id: 123 AND winlog.event_data.ServiceName: \"VERY BAD\"", - "pre_detector": { - "id": "de2daca7-95d5-426b-96bf-7ba0e63cc808", - "title": "RULE_ONE", - "severity": "critical", - "mitre": [ - "attack.test1", - "attack.test2" - ], - "case_condition": "directly" - }, - "description": "Test rule one" - } -] \ No newline at end of file diff --git a/tests/testdata/unit/pre_detector/rules/specific/pre_detect_three.json b/tests/testdata/unit/pre_detector/rules/specific/pre_detect_three.json deleted file mode 100644 index 1093c4f48..000000000 --- a/tests/testdata/unit/pre_detector/rules/specific/pre_detect_three.json +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "filter": "tags: \"specifictest2\" AND process.program: \"test\" AND (message: \"test1*xyz\" OR message: \"test2?xyz\")", - "pre_detector": { - "id": "cfef5779-fad7-495b-bc60-338d283c46f4", - "title": "RULE_THREE", - "severity": "critical", - "mitre": [], - "case_condition": "directly" - }, - "sigma_fields": [ - "message" - ], - "description": "Test rule three" - } -] \ No newline at end of file diff --git a/tests/testdata/unit/pre_detector/rules/specific/pre_detect_two.json b/tests/testdata/unit/pre_detector/rules/specific/pre_detect_two.json deleted file mode 100644 index cb5254169..000000000 --- a/tests/testdata/unit/pre_detector/rules/specific/pre_detect_two.json +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "filter": "tags: \"specific\" AND process.program: \"test\" AND (message: \"test1*xyz\" OR message:\"test2*xyz\")", - "pre_detector": { - "id": "d58ebf8e-21eb-480b-bdc0-0ce8eeae82c0", - "title": "RULE_TWO", - "severity": "critical", - "mitre": [], - "case_condition": "directly" - }, - "sigma_fields": [ - "message" - ], - "description": "Test rule two" - } -] \ No newline at end of file diff --git a/tests/testdata/unit/pre_detector/rules/specific/pre_detect_two_rules.json b/tests/testdata/unit/pre_detector/rules/specific/pre_detect_two_rules.json deleted file mode 100644 index a142ab77b..000000000 --- a/tests/testdata/unit/pre_detector/rules/specific/pre_detect_two_rules.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "filter": "other_match", - "pre_detector": { - "id": "a6e68f5b-fa60-4a12-ac93-3a211afd6d47", - "title": "RULE_ONE", - "severity": "critical", - "mitre": [ - "attack.test1", - "attack.test2" - ], - "case_condition": "directly" - }, - "description": "Test two rules one" - }, - { - "filter": "another_match", - "pre_detector": { - "id": "69e0f582-0727-4e9c-853f-1e6313492672", - "title": "RULE_TWO", - "severity": "suspicious", - "mitre": [ - "attack.test2", - "attack.test4" - ], - "case_condition": "directly" - }, - "description": "Test two rules two" - } -] \ No newline at end of file diff --git a/tests/testdata/unit/pseudonymizer/rules/regex_mapping.yml b/tests/testdata/unit/pseudonymizer/regex_mapping.yml similarity index 100% rename from tests/testdata/unit/pseudonymizer/rules/regex_mapping.yml rename to tests/testdata/unit/pseudonymizer/regex_mapping.yml diff --git a/tests/testdata/unit/pseudonymizer/rules/specific/Test123_id_789.json b/tests/testdata/unit/pseudonymizer/rules/Test123_id_789.json similarity index 100% rename from tests/testdata/unit/pseudonymizer/rules/specific/Test123_id_789.json rename to tests/testdata/unit/pseudonymizer/rules/Test123_id_789.json diff --git a/tests/testdata/unit/pseudonymizer/rules/specific/Test456_id_1234.json b/tests/testdata/unit/pseudonymizer/rules/Test456_id_1234.json similarity index 100% rename from tests/testdata/unit/pseudonymizer/rules/specific/Test456_id_1234.json rename to tests/testdata/unit/pseudonymizer/rules/Test456_id_1234.json diff --git a/tests/testdata/unit/pseudonymizer/rules/generic/event_data_IpAddress.json b/tests/testdata/unit/pseudonymizer/rules/event_data_IpAddress.json similarity index 100% rename from tests/testdata/unit/pseudonymizer/rules/generic/event_data_IpAddress.json rename to tests/testdata/unit/pseudonymizer/rules/event_data_IpAddress.json diff --git a/tests/testdata/unit/pseudonymizer/rules/generic/this_is_not_a_rule.not_json b/tests/testdata/unit/pseudonymizer/rules/generic/this_is_not_a_rule.not_json deleted file mode 100644 index 129b1e3f3..000000000 --- a/tests/testdata/unit/pseudonymizer/rules/generic/this_is_not_a_rule.not_json +++ /dev/null @@ -1 +0,0 @@ -I'm not a json file and should not be loaded as a rule! diff --git a/tests/testdata/acceptance/pseudonymizer/rules_static/generic/this_is_not_a_rule.not_json b/tests/testdata/unit/pseudonymizer/rules/this_is_not_a_rule.not_json similarity index 100% rename from tests/testdata/acceptance/pseudonymizer/rules_static/generic/this_is_not_a_rule.not_json rename to tests/testdata/unit/pseudonymizer/rules/this_is_not_a_rule.not_json diff --git a/tests/testdata/unit/requester/generic_rules/requester.json b/tests/testdata/unit/requester/rules/requester_1.json similarity index 100% rename from tests/testdata/unit/requester/generic_rules/requester.json rename to tests/testdata/unit/requester/rules/requester_1.json diff --git a/tests/testdata/unit/requester/specific_rules/requester.json b/tests/testdata/unit/requester/rules/requester_2.json similarity index 100% rename from tests/testdata/unit/requester/specific_rules/requester.json rename to tests/testdata/unit/requester/rules/requester_2.json diff --git a/tests/testdata/unit/selective_extractor/rules/generic/rules.json b/tests/testdata/unit/selective_extractor/rules/rules_1.json similarity index 100% rename from tests/testdata/unit/selective_extractor/rules/generic/rules.json rename to tests/testdata/unit/selective_extractor/rules/rules_1.json diff --git a/tests/testdata/unit/selective_extractor/rules/specific/rules.json b/tests/testdata/unit/selective_extractor/rules/rules_2.json similarity index 100% rename from tests/testdata/unit/selective_extractor/rules/specific/rules.json rename to tests/testdata/unit/selective_extractor/rules/rules_2.json diff --git a/tests/testdata/unit/string_splitter/generic/generic.json b/tests/testdata/unit/string_splitter/rules/rule.json similarity index 100% rename from tests/testdata/unit/string_splitter/generic/generic.json rename to tests/testdata/unit/string_splitter/rules/rule.json diff --git a/tests/testdata/unit/string_splitter/specific/specific.json b/tests/testdata/unit/string_splitter/specific/specific.json deleted file mode 100644 index 909caad7e..000000000 --- a/tests/testdata/unit/string_splitter/specific/specific.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "filter": "message1", - "string_splitter": { - "source_fields": [ - "message" - ], - "target_field": "result" - } - } -] \ No newline at end of file diff --git a/tests/testdata/unit/template_replacer/rules/specific/template_replacer.json b/tests/testdata/unit/template_replacer/rules/specific/template_replacer.json deleted file mode 100644 index 97cf27ec8..000000000 --- a/tests/testdata/unit/template_replacer/rules/specific/template_replacer.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - { - "filter": "winlog.provider_name_specific AND winlog.event_id", - "template_replacer": {}, - "description": "" - } -] \ No newline at end of file diff --git a/tests/testdata/unit/template_replacer/rules/generic/template_replacer.json b/tests/testdata/unit/template_replacer/rules/template_replacer.json similarity index 100% rename from tests/testdata/unit/template_replacer/rules/generic/template_replacer.json rename to tests/testdata/unit/template_replacer/rules/template_replacer.json diff --git a/tests/testdata/unit/timestamp_differ/generic_rules/timestamp_differ_rule.json b/tests/testdata/unit/timestamp_differ/rules/timestamp_differ_rule.json similarity index 100% rename from tests/testdata/unit/timestamp_differ/generic_rules/timestamp_differ_rule.json rename to tests/testdata/unit/timestamp_differ/rules/timestamp_differ_rule.json diff --git a/tests/testdata/unit/timestamp_differ/specific_rules/timestamp_differ_rule.json b/tests/testdata/unit/timestamp_differ/specific_rules/timestamp_differ_rule.json deleted file mode 100644 index dc0ceac93..000000000 --- a/tests/testdata/unit/timestamp_differ/specific_rules/timestamp_differ_rule.json +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "filter": "field1 AND field3", - "timestamp_differ": { - "diff": "${field1:YYYYMdd-HH:MM:SS} - ${field3:YYYYMdd-HH:MM:SS}", - "target_field": "new_field" - } - } -] diff --git a/tests/testdata/unit/timestamper/generic_rules/timestamper_rule.yml b/tests/testdata/unit/timestamper/rules/timestamper_rule.yml similarity index 65% rename from tests/testdata/unit/timestamper/generic_rules/timestamper_rule.yml rename to tests/testdata/unit/timestamper/rules/timestamper_rule.yml index 4ca7b7405..c34d4d92d 100644 --- a/tests/testdata/unit/timestamper/generic_rules/timestamper_rule.yml +++ b/tests/testdata/unit/timestamper/rules/timestamper_rule.yml @@ -1,3 +1,3 @@ -filter: "@timestamp1" +filter: "@timestamp" timestamper: source_fields: ["message"] diff --git a/tests/testdata/unit/timestamper/specific_rules/timestamper_rule.yml b/tests/testdata/unit/timestamper/specific_rules/timestamper_rule.yml deleted file mode 100644 index c193b1a20..000000000 --- a/tests/testdata/unit/timestamper/specific_rules/timestamper_rule.yml +++ /dev/null @@ -1,3 +0,0 @@ -filter: "@timestamp" -timestamper: - source_fields: ["message1"] diff --git a/tests/unit/framework/rule_tree/test_rule_tree.py b/tests/unit/framework/rule_tree/test_rule_tree.py index 1c3df2eb3..3b5a7a7cd 100644 --- a/tests/unit/framework/rule_tree/test_rule_tree.py +++ b/tests/unit/framework/rule_tree/test_rule_tree.py @@ -44,8 +44,7 @@ def test_init_with_specifying_config(self): { "processor": { "type": "dissector", - "generic_rules": [], - "specific_rules": [], + "rules": [], "tree_config": "tests/testdata/unit/tree_config.json", } } diff --git a/tests/unit/framework/test_pipeline.py b/tests/unit/framework/test_pipeline.py index 3dec94242..849ac08ee 100644 --- a/tests/unit/framework/test_pipeline.py +++ b/tests/unit/framework/test_pipeline.py @@ -903,8 +903,7 @@ class TestPipelineResult: { "dummy": { "type": "dropper", - "specific_rules": [], - "generic_rules": [], + "rules": [], } } ) @@ -922,8 +921,7 @@ class TestPipelineResult: { "dummy": { "type": "dropper", - "specific_rules": [], - "generic_rules": [], + "rules": [], } } ) diff --git a/tests/unit/processor/amides/test_amides.py b/tests/unit/processor/amides/test_amides.py index 8ef9e27b0..30a29a0a2 100644 --- a/tests/unit/processor/amides/test_amides.py +++ b/tests/unit/processor/amides/test_amides.py @@ -16,8 +16,7 @@ class TestAmides(BaseProcessorTestCase): CONFIG = { "type": "amides", - "generic_rules": ["tests/testdata/unit/amides/rules/generic"], - "specific_rules": ["tests/testdata/unit/amides/rules/specific"], + "rules": ["tests/testdata/unit/amides/rules"], "models_path": "tests/testdata/unit/amides/model.zip", "max_cache_entries": 5, "decision_threshold": 0.32, diff --git a/tests/unit/processor/base.py b/tests/unit/processor/base.py index a67ac640e..adc15507a 100644 --- a/tests/unit/processor/base.py +++ b/tests/unit/processor/base.py @@ -38,23 +38,14 @@ class BaseProcessorTestCase(BaseComponentTestCase): patchers: list = None - specific_rules: list - - generic_rules: list - - @property - def specific_rules_dirs(self): - """ - gets the specific rules_dirs for the processor from CONFIG - """ - return self.CONFIG.get("specific_rules") + rules: list @property - def generic_rules_dirs(self): + def rules_dirs(self): """ - gets the generic rules_dirs for the processor from CONFIG + gets the rules_dirs for the processor from CONFIG """ - return self.CONFIG.get("generic_rules") + return self.CONFIG.get("rules") @staticmethod def set_rules(rules_dirs): @@ -77,13 +68,10 @@ def set_rules(rules_dirs): rules.append(rule) return rules - def _load_specific_rule(self, rule: dict | Rule): - self.object._generic_tree = RuleTree() - self.object._specific_tree = RuleTree() - specific_rule = ( - self.object.rule_class._create_from_dict(rule) if isinstance(rule, dict) else rule - ) - self.object._specific_tree.add_rule(specific_rule, self.logger) + def _load_rule(self, rule: dict | Rule): + self.object._rule_tree = RuleTree() + rule = self.object.rule_class._create_from_dict(rule) if isinstance(rule, dict) else rule + self.object._rule_tree.add_rule(rule, self.logger) def setup_method(self) -> None: """ @@ -95,8 +83,7 @@ def setup_method(self) -> None: patcher.start() self.patchers.append(patcher) super().setup_method() - self.specific_rules = self.set_rules(self.specific_rules_dirs) - self.generic_rules = self.set_rules(self.generic_rules_dirs) + self.rules = self.set_rules(self.rules_dirs) self.match_all_event = { "message": "event", "winlog": { @@ -129,13 +116,11 @@ def teardown_method(self) -> None: def test_is_a_processor_implementation(self): assert isinstance(self.object, Processor) - def test_generic_specific_rule_trees(self): - assert isinstance(self.object._generic_tree, RuleTree) - assert isinstance(self.object._specific_tree, RuleTree) + def test_rule_tree(self): + assert isinstance(self.object._rule_tree, RuleTree) - def test_generic_specific_rule_trees_not_empty(self): - assert self.object._generic_tree.get_size() > 0 - assert self.object._specific_tree.get_size() > 0 + def test_rule_tree_not_empty(self): + assert self.object._rule_tree.get_size() > 0 def test_field_exists(self): event = {"a": {"b": "I do not matter"}} @@ -145,34 +130,23 @@ def test_field_exists(self): @mock.patch("logging.Logger.debug") def test_load_rules_with_debug(self, mock_debug, _): self.object.load_rules( - specific_rules_targets=self.specific_rules_dirs, - generic_rules_targets=self.generic_rules_dirs, + rules_targets=self.rules_dirs, ) mock_debug.assert_called() def test_load_rules(self): - self.object._generic_tree = RuleTree() - self.object._specific_tree = RuleTree() - generic_rules_size = self.object._generic_tree.get_size() - specific_rules_size = self.object._specific_tree.get_size() - self.object.load_rules( - specific_rules_targets=self.specific_rules_dirs, - generic_rules_targets=self.generic_rules_dirs, - ) - new_generic_rules_size = self.object._generic_tree.get_size() - new_specific_rules_size = self.object._specific_tree.get_size() - assert new_generic_rules_size > generic_rules_size - assert new_specific_rules_size > specific_rules_size + self.object._rule_tree = RuleTree() + rules_size = self.object._rule_tree.get_size() + self.object.load_rules(self.rules_dirs) + new_rules_size = self.object._rule_tree.get_size() + assert new_rules_size > rules_size def test_load_rules_calls_getter_factory(self): with mock.patch("logprep.util.getter.GetterFactory.from_string") as getter_factory: with pytest.raises( TypeError, match="must be str, bytes or bytearray, not .*MagicMock.*" ): - self.object.load_rules( - specific_rules_targets=self.specific_rules_dirs, - generic_rules_targets=self.generic_rules_dirs, - ) + self.object.load_rules(rules_targets=self.rules_dirs) getter_factory.assert_called() @responses.activate @@ -181,10 +155,7 @@ def test_accepts_http_in_rules_config(self): responses.add(responses.GET, "http://does.not.matter", mock.MagicMock()) myconfig = deepcopy(self.CONFIG) myconfig.update( - {"specific_rules": ["http://does.not.matter", "https://this.is.not.existent/bla.yml"]} - ) - myconfig.update( - {"generic_rules": ["http://does.not.matter", "https://this.is.not.existent/bla.yml"]} + {"rules": ["http://does.not.matter", "https://this.is.not.existent/bla.yml"]} ) with pytest.raises(TypeError, match="not .*MagicMock.*"): Factory.create({"http_rule_processor": myconfig}) @@ -195,37 +166,16 @@ def test_no_redundant_rules_are_added_to_rule_tree(self): in the rules directories ensures that every rule in rule tree is unique """ - self.object.load_rules( - specific_rules_targets=self.specific_rules_dirs, - generic_rules_targets=self.generic_rules_dirs, - ) - generic_rules_size = self.object._generic_tree.get_size() - specific_rules_size = self.object._specific_tree.get_size() - self.object.load_rules( - specific_rules_targets=self.specific_rules_dirs, - generic_rules_targets=self.generic_rules_dirs, - ) - new_generic_rules_size = self.object._generic_tree.get_size() - new_specific_rules_size = self.object._specific_tree.get_size() - assert new_generic_rules_size == generic_rules_size - assert new_specific_rules_size == specific_rules_size - - def test_specific_rules_returns_all_specific_rules(self): - specific_rules = self.specific_rules - object_specific_rules = self.object._specific_rules - assert len(specific_rules) == len(object_specific_rules) - - def test_generic_rules_returns_all_generic_rules(self): - generic_rules = self.generic_rules - object_generic_rules = self.object._generic_rules - assert len(generic_rules) == len(object_generic_rules) - - def test_rules_returns_all_specific_and_generic_rules(self): - generic_rules = self.generic_rules - specific_rules = self.specific_rules - all_rules_count = len(generic_rules) + len(specific_rules) - object_rules_count = len(self.object.rules) - assert all_rules_count == object_rules_count + self.object.load_rules(rules_targets=self.rules_dirs) + rules_size = self.object._rule_tree.get_size() + self.object.load_rules(rules_targets=self.rules_dirs) + new_rules_size = self.object._rule_tree.get_size() + assert new_rules_size == rules_size + + def test_rules_returns_all_rules(self): + rules = self.rules + object_rules = self.object._tree_rules + assert len(rules) == len(object_rules) @mock.patch("logging.Logger.debug") def test_process_writes_debug_messages(self, mock_debug): @@ -241,14 +191,14 @@ def test_config_object_is_kw_only(self): for attr in attr_attributes: assert attr.kw_only - @pytest.mark.parametrize("rule_list", ["specific_rules", "generic_rules"]) + @pytest.mark.parametrize("rule_list", ["rules"]) def test_validation_raises_if_not_a_list(self, rule_list): config = deepcopy(self.CONFIG) config.update({rule_list: "i am not a list"}) with pytest.raises(TypeError, match=r"must be "): Factory.create({"test instance": config}) - @pytest.mark.parametrize("rule_list", ["specific_rules", "generic_rules"]) + @pytest.mark.parametrize("rule_list", ["rules"]) def test_validation_raises_if_elements_does_not_exist(self, rule_list): config = deepcopy(self.CONFIG) config.update({rule_list: ["/i/do/not/exist"]}) @@ -275,11 +225,11 @@ def test_accepts_tree_config_from_http(self): responses.add(responses.GET, "http://does.not.matter.bla/tree_config.yml", tree_config) processor = Factory.create({"test instance": config}) assert ( - processor._specific_tree._processor_config.tree_config + processor._rule_tree._processor_config.tree_config == "http://does.not.matter.bla/tree_config.yml" ) tree_config = json.loads(tree_config) - assert processor._specific_tree.priority_dict == tree_config.get("priority_dict") + assert processor._rule_tree.priority_dict == tree_config.get("priority_dict") @responses.activate def test_raises_http_error(self): diff --git a/tests/unit/processor/calculator/test_calculator.py b/tests/unit/processor/calculator/test_calculator.py index be49fac5e..e3cbd6cbc 100644 --- a/tests/unit/processor/calculator/test_calculator.py +++ b/tests/unit/processor/calculator/test_calculator.py @@ -340,19 +340,18 @@ class TestCalculator(BaseProcessorTestCase): CONFIG: dict = { "type": "calculator", - "specific_rules": ["tests/testdata/unit/calculator/specific_rules"], - "generic_rules": ["tests/testdata/unit/calculator/generic_rules"], + "rules": ["tests/testdata/unit/calculator/rules"], } @pytest.mark.parametrize("testcase, rule, event, expected", test_cases) def test_testcases(self, testcase, rule, event, expected): # pylint: disable=unused-argument - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(event) assert event == expected @pytest.mark.parametrize("testcase, rule, event, expected, error_message", failure_test_cases) def test_testcases_failure_handling(self, testcase, rule, event, expected, error_message): - self._load_specific_rule(rule) + self._load_rule(rule) result = self.object.process(event) assert len(result.warnings) == 1 diff --git a/tests/unit/processor/clusterer/test_clusterer.py b/tests/unit/processor/clusterer/test_clusterer.py index de03329b7..fc2b42684 100644 --- a/tests/unit/processor/clusterer/test_clusterer.py +++ b/tests/unit/processor/clusterer/test_clusterer.py @@ -12,8 +12,7 @@ class TestClusterer(BaseProcessorTestCase): CONFIG = { "type": "clusterer", "output_field_name": "cluster_signature", - "generic_rules": ["tests/testdata/unit/clusterer/rules/generic"], - "specific_rules": ["tests/testdata/unit/clusterer/rules/specific"], + "rules": ["tests/testdata/unit/clusterer/rules"], } def test_has_tag_clusterable(self): @@ -168,17 +167,16 @@ def test_cluster(self): document = {"message": "test signature test"} rule = ClustererRule._create_from_dict(rule_definition) - self.object._generic_tree.add_rule(rule, None) + self.object._rule_tree.add_rule(rule, None) self.object._cluster(document, rule) assert document == expected - def test_rule_dependency(self, tmp_path): + def test_rule_dependency_one(self, tmp_path): config = deepcopy(self.CONFIG) empty_rules_path = tmp_path / "empty" empty_rules_path.mkdir() - config.update({"generic_rules": [empty_rules_path.as_posix()]}) - config.update({"specific_rules": [empty_rules_path.as_posix()]}) + config.update({"rules": [empty_rules_path.as_posix()]}) clusterer = Factory.create({"test instance": config}) rule_0 = { @@ -190,7 +188,6 @@ def test_rule_dependency(self, tmp_path): }, "description": "", } - rule_1 = { "filter": "message", "clusterer": { @@ -227,14 +224,51 @@ def test_rule_dependency(self, tmp_path): }, "description": "", } - rules = [rule_0, rule_1, rule_2, rule_3, rule_4] - specific_rules = [] - for idx, rule in enumerate(rules): + rules_to_add = [rule_0, rule_1, rule_2, rule_3, rule_4] + rules = [] + for idx, rule in enumerate(rules_to_add): new_rule = ClustererRule._create_from_dict(rule) new_rule.file_name = str(idx) - specific_rules.append(new_rule) - clusterer._specific_tree.add_rule(new_rule, None) - rule_5 = { + rules.append(new_rule) + clusterer._rule_tree.add_rule(new_rule, None) + + expected = { + "message": "test some signature xyz-foo", + "cluster_signature": "signature baz", + } + + document = {"message": "test some signature xyz-foo"} + for rule in rules: + clusterer._cluster(document, rule) + assert document == expected + + document = {"message": "test some signature xyz-foo"} + for rule in rules: + clusterer._cluster(document, rule) + assert document == expected + + document = {"message": "test some signature xyz-foo"} + for rule in rules[1:]: + clusterer._cluster(document, rule) + assert document == expected + + def test_rule_dependency_two(self, tmp_path): + config = deepcopy(self.CONFIG) + empty_rules_path = tmp_path / "empty" + empty_rules_path.mkdir() + config.update({"rules": [empty_rules_path.as_posix()]}) + clusterer = Factory.create({"test instance": config}) + + expected = { + "message": "test some signature xyz-foo", + "cluster_signature": "test SIGN", + } + document = { + "message": "test some signature xyz-foo", + "cluster_signature": "signature baz", + } + + rule_0 = { "filter": "no_match", "clusterer": { "source_fields": ["message"], @@ -243,7 +277,7 @@ def test_rule_dependency(self, tmp_path): }, "description": "", } - rule_6 = { + rule_1 = { "filter": "message", "clusterer": { "source_fields": ["message"], @@ -252,7 +286,7 @@ def test_rule_dependency(self, tmp_path): }, "description": "", } - rule_7 = { + rule_2 = { "filter": "message", "clusterer": { "source_fields": ["message"], @@ -261,7 +295,7 @@ def test_rule_dependency(self, tmp_path): }, "description": "", } - rule_8 = { + rule_3 = { "filter": "message", "clusterer": { "source_fields": ["message"], @@ -270,40 +304,14 @@ def test_rule_dependency(self, tmp_path): }, "description": "", } - rules = [rule_5, rule_6, rule_7, rule_8] - generic_rules = [] - - for idx, rule in enumerate(rules): + rules_to_add = [rule_0, rule_1, rule_2, rule_3] + rules = [] + for idx, rule in enumerate(rules_to_add): new_rule = ClustererRule._create_from_dict(rule) new_rule.file_name = str(idx) - generic_rules.append(new_rule) - clusterer._generic_tree.add_rule(new_rule, None) + rules.append(new_rule) + clusterer._rule_tree.add_rule(new_rule, None) - expected = { - "message": "test some signature xyz-foo", - "cluster_signature": "signature baz", - } - - document = {"message": "test some signature xyz-foo"} - for rule in specific_rules: - clusterer._cluster(document, rule) - assert document == expected - - document = {"message": "test some signature xyz-foo"} - for rule in specific_rules: - clusterer._cluster(document, rule) - assert document == expected - - document = {"message": "test some signature xyz-foo"} - for rule in specific_rules[1:]: - clusterer._cluster(document, rule) - assert document == expected - - expected = { - "message": "test some signature xyz-foo", - "cluster_signature": "test SIGN", - } - document = {"message": "test some signature xyz-foo"} - for rule in specific_rules + generic_rules: + for rule in rules: clusterer._cluster(document, rule) assert document == expected diff --git a/tests/unit/processor/concatenator/test_concatenator.py b/tests/unit/processor/concatenator/test_concatenator.py index e29f6a8e5..1f7c69873 100644 --- a/tests/unit/processor/concatenator/test_concatenator.py +++ b/tests/unit/processor/concatenator/test_concatenator.py @@ -10,18 +10,13 @@ class TestConcatenator(BaseProcessorTestCase): CONFIG = { "type": "concatenator", - "specific_rules": ["tests/testdata/unit/concatenator/rules/specific"], - "generic_rules": ["tests/testdata/unit/concatenator/rules/generic"], + "rules": ["tests/testdata/unit/concatenator/rules"], "tree_config": "tests/testdata/unit/shared_data/tree_config.json", } @property - def generic_rules_dirs(self): - return self.CONFIG["generic_rules"] - - @property - def specific_rules_dirs(self): - return self.CONFIG["specific_rules"] + def rules_dirs(self): + return self.CONFIG["rules"] @pytest.mark.parametrize( ["test_case", "rule", "document", "expected_output"], @@ -166,7 +161,7 @@ def specific_rules_dirs(self): ], ) def test_for_expected_output(self, test_case, rule, document, expected_output): - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected_output, test_case @@ -183,7 +178,7 @@ def test_process_raises_field_exists_warning_if_target_field_exists_and_should_n "delete_source_fields": False, }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"field": {"a": "first", "b": "second"}, "target_field": "has already content"} result = self.object.process(document) assert len(result.warnings) == 1 diff --git a/tests/unit/processor/concatenator/test_concatenator_rule.py b/tests/unit/processor/concatenator/test_concatenator_rule.py index 710679b51..380167822 100644 --- a/tests/unit/processor/concatenator/test_concatenator_rule.py +++ b/tests/unit/processor/concatenator/test_concatenator_rule.py @@ -9,8 +9,8 @@ from logprep.processor.concatenator.rule import ConcatenatorRule -@pytest.fixture(name="specific_rule_definition") -def fixture_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def fixture_rule_definition(): return { "filter": "field.a", "concatenator": { @@ -129,9 +129,9 @@ class TestConcatenatorRule: ], ) def test_rules_equality( - self, specific_rule_definition, testcase, other_rule_definition, is_equal + self, rule_definition, testcase, other_rule_definition, is_equal ): - rule_1 = ConcatenatorRule._create_from_dict(specific_rule_definition) + rule_1 = ConcatenatorRule._create_from_dict(rule_definition) rule_2 = ConcatenatorRule._create_from_dict(other_rule_definition) assert (rule_1 == rule_2) == is_equal, testcase @@ -296,6 +296,6 @@ def test_rule_create_from_dict(self, rule_definition, raised, message): extractor_rule = ConcatenatorRule._create_from_dict(rule_definition) assert isinstance(extractor_rule, ConcatenatorRule) - def test_rule_is_hashable(self, specific_rule_definition): - rule = ConcatenatorRule._create_from_dict(specific_rule_definition) + def test_rule_is_hashable(self, rule_definition): + rule = ConcatenatorRule._create_from_dict(rule_definition) assert isinstance(rule, Hashable) diff --git a/tests/unit/processor/datetime_extractor/test_datetime_extractor.py b/tests/unit/processor/datetime_extractor/test_datetime_extractor.py index 2521194eb..d64329681 100644 --- a/tests/unit/processor/datetime_extractor/test_datetime_extractor.py +++ b/tests/unit/processor/datetime_extractor/test_datetime_extractor.py @@ -13,17 +13,12 @@ class TestDatetimeExtractor(BaseProcessorTestCase): CONFIG = { "type": "datetime_extractor", - "specific_rules": ["tests/testdata/unit/datetime_extractor/rules/specific"], - "generic_rules": ["tests/testdata/unit/datetime_extractor/rules/generic"], + "rules": ["tests/testdata/unit/datetime_extractor/rules"], } @property - def specific_rules_dirs(self): - return self.CONFIG.get("specific_rules") - - @property - def generic_rules_dirs(self): - return self.CONFIG.get("generic_rules") + def rules_dirs(self): + return self.CONFIG.get("rules") def test_an_event_extracted_datetime_utc(self): timestamp = "2019-07-30T14:37:42.861Z" @@ -113,7 +108,7 @@ def test_deletes_source_field(self): }, "description": "", } - self._load_specific_rule(rule) + self._load_rule(rule) self.object._local_timezone = tzutc() self.object._local_timezone_name = DatetimeExtractor._get_timezone_name( self.object._local_timezone @@ -146,7 +141,7 @@ def test_overwrite_target(self): }, "description": "", } - self._load_specific_rule(rule) + self._load_rule(rule) self.object._local_timezone = tzutc() self.object._local_timezone_name = DatetimeExtractor._get_timezone_name( self.object._local_timezone @@ -179,7 +174,7 @@ def test_existing_target_raises_if_not_overwrite_target(self): }, "description": "", } - self._load_specific_rule(rule) + self._load_rule(rule) result = self.object.process(document) assert len(result.warnings) == 1 assert isinstance(result.warnings[0], FieldExistsWarning) diff --git a/tests/unit/processor/datetime_extractor/test_datetime_extractor_rule.py b/tests/unit/processor/datetime_extractor/test_datetime_extractor_rule.py index 1a84ef4c6..fcabbda65 100644 --- a/tests/unit/processor/datetime_extractor/test_datetime_extractor_rule.py +++ b/tests/unit/processor/datetime_extractor/test_datetime_extractor_rule.py @@ -7,8 +7,8 @@ from logprep.processor.datetime_extractor.rule import DatetimeExtractorRule -@pytest.fixture(name="specific_rule_definition") -def fixture_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def fixture_rule_definition(): return { "filter": "field.a", "datetime_extractor": {"source_fields": ["field.a"], "target_field": "datetime"}, @@ -83,9 +83,9 @@ class TestDatetimeExtractorRule: ], ) def test_rules_equality( - self, specific_rule_definition, testcase, other_rule_definition, is_equal + self, rule_definition, testcase, other_rule_definition, is_equal ): - rule_1 = DatetimeExtractorRule._create_from_dict(specific_rule_definition) + rule_1 = DatetimeExtractorRule._create_from_dict(rule_definition) rule_2 = DatetimeExtractorRule._create_from_dict(other_rule_definition) assert (rule_1 == rule_2) == is_equal, testcase @@ -172,6 +172,6 @@ def test_rule_create_from_dict(self, rule_definition, raised, message): extractor_rule = DatetimeExtractorRule._create_from_dict(rule_definition) assert isinstance(extractor_rule, DatetimeExtractorRule) - def test_rule_is_hashable(self, specific_rule_definition): - rule = DatetimeExtractorRule._create_from_dict(specific_rule_definition) + def test_rule_is_hashable(self, rule_definition): + rule = DatetimeExtractorRule._create_from_dict(rule_definition) assert isinstance(rule, Hashable) diff --git a/tests/unit/processor/deleter/test_deleter.py b/tests/unit/processor/deleter/test_deleter.py index 1ed1044f0..00eb617ee 100644 --- a/tests/unit/processor/deleter/test_deleter.py +++ b/tests/unit/processor/deleter/test_deleter.py @@ -6,8 +6,7 @@ class TestDeleter(BaseProcessorTestCase): CONFIG = { "type": "deleter", - "specific_rules": ["tests/testdata/unit/deleter/rules/specific/"], - "generic_rules": ["tests/testdata/unit/deleter/rules/generic/"], + "rules": ["tests/testdata/unit/deleter/rules"], } @pytest.mark.parametrize( diff --git a/tests/unit/processor/deleter/test_deleter_rule.py b/tests/unit/processor/deleter/test_deleter_rule.py index b228cdd69..4d8a7f05c 100644 --- a/tests/unit/processor/deleter/test_deleter_rule.py +++ b/tests/unit/processor/deleter/test_deleter_rule.py @@ -7,8 +7,8 @@ from logprep.processor.deleter.rule import DeleterRule -@pytest.fixture(name="specific_rule_definition") -def fixture_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def fixture_rule_definition(): return { "filter": "test", "deleter": {"delete": True}, @@ -55,10 +55,10 @@ class TestDeleterRule: ], ) def test_rules_equality( - self, specific_rule_definition, testcase, other_rule_definition, is_equal + self, rule_definition, testcase, other_rule_definition, is_equal ): rule1 = DeleterRule._create_from_dict( - specific_rule_definition, + rule_definition, ) print(other_rule_definition) @@ -102,6 +102,6 @@ def test_rule_create_from_dict(self, rule_definition, raised, message): deleter_rule = DeleterRule._create_from_dict(rule_definition) assert isinstance(deleter_rule, DeleterRule) - def test_rule_is_hashable(self, specific_rule_definition): - rule = DeleterRule._create_from_dict(specific_rule_definition) + def test_rule_is_hashable(self, rule_definition): + rule = DeleterRule._create_from_dict(rule_definition) assert isinstance(rule, Hashable) diff --git a/tests/unit/processor/dissector/test_dissector.py b/tests/unit/processor/dissector/test_dissector.py index 92c41fd96..3b7b93003 100644 --- a/tests/unit/processor/dissector/test_dissector.py +++ b/tests/unit/processor/dissector/test_dissector.py @@ -717,19 +717,18 @@ class TestDissector(BaseProcessorTestCase): CONFIG: dict = { "type": "dissector", - "generic_rules": ["tests/testdata/unit/dissector/generic_rules"], - "specific_rules": ["tests/testdata/unit/dissector/specific_rules"], + "rules": ["tests/testdata/unit/dissector/rules"], } @pytest.mark.parametrize("testcase, rule, event, expected", test_cases) def test_testcases(self, testcase, rule, event, expected): # pylint: disable=unused-argument - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(event) assert event == expected @pytest.mark.parametrize("testcase, rule, event, expected", failure_test_cases) def test_testcases_failure_handling(self, testcase, rule, event, expected): - self._load_specific_rule(rule) + self._load_rule(rule) result = self.object.process(event) assert len(result.warnings) == 1 assert isinstance(result.warnings[0], ProcessingWarning) diff --git a/tests/unit/processor/domain_label_extractor/test_domain_label_extractor.py b/tests/unit/processor/domain_label_extractor/test_domain_label_extractor.py index a650f8123..fd8358e32 100644 --- a/tests/unit/processor/domain_label_extractor/test_domain_label_extractor.py +++ b/tests/unit/processor/domain_label_extractor/test_domain_label_extractor.py @@ -10,18 +10,13 @@ class TestDomainLabelExtractor(BaseProcessorTestCase): CONFIG = { "type": "domain_label_extractor", - "generic_rules": ["tests/testdata/unit/domain_label_extractor/rules/generic"], - "specific_rules": ["tests/testdata/unit/domain_label_extractor/rules/specific"], + "rules": ["tests/testdata/unit/domain_label_extractor/rules"], "tree_config": "tests/testdata/unit/shared_data/tree_config.json", } @property - def generic_rules_dirs(self): - return self.CONFIG.get("generic_rules") - - @property - def specific_rules_dirs(self): - return self.CONFIG.get("specific_rules") + def rules_dirs(self): + return self.CONFIG.get("rules") def test_domain_extraction_from_full_url(self): document = {"url": {"domain": "https://url.full.domain.de/path/file?param=1"}} @@ -166,8 +161,7 @@ def test_new_non_default_tagging_field(self): config = { "Test DomainLabelExtractor Name": { "type": "domain_label_extractor", - "generic_rules": ["tests/testdata/unit/domain_label_extractor/rules/generic"], - "specific_rules": ["tests/testdata/unit/domain_label_extractor/rules/specific"], + "rules": ["tests/testdata/unit/domain_label_extractor/rules"], "tagging_field_name": "special_tags", "tree_config": "tests/testdata/unit/shared_data/tree_config.json", } @@ -187,8 +181,7 @@ def test_append_to_non_default_tagging_field(self): config = { "Test DomainLabelExtractor Name": { "type": "domain_label_extractor", - "generic_rules": ["tests/testdata/unit/domain_label_extractor/rules/generic"], - "specific_rules": ["tests/testdata/unit/domain_label_extractor/rules/specific"], + "rules": ["tests/testdata/unit/domain_label_extractor/rules"], "tagging_field_name": "special_tags", "tree_config": "tests/testdata/unit/shared_data/tree_config.json", } @@ -260,7 +253,7 @@ def test_domain_extraction_overwrites_target_field(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.process(document) assert document == expected @@ -283,7 +276,7 @@ def test_domain_extraction_delete_source_fields(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.process(document) assert document == expected @@ -303,7 +296,7 @@ def test_does_nothing_if_source_field_not_exits(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.process(document) assert document == expected @@ -327,7 +320,7 @@ def test_raises_field_exists_warning_if_target_field_exits(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) result = self.object.process(document) assert len(result.warnings) == 1 assert isinstance(result.warnings[0], FieldExistsWarning) diff --git a/tests/unit/processor/domain_label_extractor/test_domain_label_extractor_rule.py b/tests/unit/processor/domain_label_extractor/test_domain_label_extractor_rule.py index 100214f83..127c381f5 100644 --- a/tests/unit/processor/domain_label_extractor/test_domain_label_extractor_rule.py +++ b/tests/unit/processor/domain_label_extractor/test_domain_label_extractor_rule.py @@ -6,8 +6,8 @@ from logprep.processor.domain_label_extractor.rule import DomainLabelExtractorRule -@pytest.fixture(name="specific_rule_definition") -def fixture_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def fixture_rule_definition(): return { "filter": "field.a", "domain_label_extractor": { @@ -85,9 +85,9 @@ class TestDomainLabelExtractorRule: ], ) def test_rules_equality( - self, specific_rule_definition, testcase, other_rule_definition, is_equal + self, rule_definition, testcase, other_rule_definition, is_equal ): - rule_1 = DomainLabelExtractorRule._create_from_dict(specific_rule_definition) + rule_1 = DomainLabelExtractorRule._create_from_dict(rule_definition) rule_2 = DomainLabelExtractorRule._create_from_dict(other_rule_definition) assert (rule_1 == rule_2) == is_equal, testcase @@ -174,6 +174,6 @@ def test_rule_create_from_dict(self, rule_definition, raised, message): extractor_rule = DomainLabelExtractorRule._create_from_dict(rule_definition) assert isinstance(extractor_rule, DomainLabelExtractorRule) - def test_rule_is_hashable(self, specific_rule_definition): - rule = DomainLabelExtractorRule._create_from_dict(specific_rule_definition) + def test_rule_is_hashable(self, rule_definition): + rule = DomainLabelExtractorRule._create_from_dict(rule_definition) assert isinstance(rule, Hashable) diff --git a/tests/unit/processor/domain_resolver/test_domain_resolver.py b/tests/unit/processor/domain_resolver/test_domain_resolver.py index 27cfec8dd..a30175d65 100644 --- a/tests/unit/processor/domain_resolver/test_domain_resolver.py +++ b/tests/unit/processor/domain_resolver/test_domain_resolver.py @@ -15,8 +15,7 @@ class TestDomainResolver(BaseProcessorTestCase): CONFIG = { "type": "domain_resolver", - "generic_rules": ["tests/testdata/unit/domain_resolver/rules/generic"], - "specific_rules": ["tests/testdata/unit/domain_resolver/rules/specific"], + "rules": ["tests/testdata/unit/domain_resolver/rules"], "timeout": 0.25, "max_cached_domains": 1000000, "max_caching_days": 1, @@ -38,7 +37,7 @@ def test_domain_to_ip_resolved_and_added(self, mock_gethostbyname): "domain_resolver": {"source_fields": ["fqdn"]}, "description": "", } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"fqdn": "google.de"} expected = {"fqdn": "google.de", "resolved_ip": "1.2.3.4"} self.object.process(document) @@ -52,7 +51,7 @@ def test_domain_to_ip_resolved_and_added_from_cache(self, mock_gethostbyname): "domain_resolver": {"source_fields": ["fqdn"]}, "description": "", } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"fqdn": "google.de"} self.object.process(document) document = {"fqdn": "google.de"} @@ -68,7 +67,7 @@ def test_url_to_ip_resolved_and_added(self, _): "domain_resolver": {"source_fields": ["url"]}, "description": "", } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"url": "https://www.google.de/something"} expected = {"url": "https://www.google.de/something", "resolved_ip": "1.2.3.4"} self.object.process(document) @@ -83,7 +82,7 @@ def test_domain_ip_map_greater_cache(self): "domain_resolver": {"source_fields": ["url"]}, "description": "", } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"url": "https://www.google.de/something"} with mock.patch("socket.gethostbyname", return_value="1.2.3.4"): self.object.process(document) @@ -99,7 +98,7 @@ def test_do_nothing_if_source_not_in_event(self): "domain_resolver": {"source_fields": ["not_available"]}, "description": "", } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"url": "https://www.google.de/something"} expected = {"url": "https://www.google.de/something"} self.object.process(document) @@ -115,7 +114,7 @@ def test_url_to_ip_resolved_and_added_with_debug_cache(self, _): "domain_resolver": {"source_fields": ["url"]}, "description": "", } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"url": "https://www.google.de/something"} expected = { "url": "https://www.google.de/something", @@ -135,7 +134,7 @@ def test_url_to_ip_resolved_from_cache_and_added_with_debug_cache(self, _): "domain_resolver": {"source_fields": ["url"]}, "description": "", } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"url": "https://www.google.de/something"} self.object.process(document) document = {"url": "https://www.google.de/something_else"} @@ -157,7 +156,7 @@ def test_url_to_ip_resolved_and_added_with_cache_disabled(self, _): "domain_resolver": {"source_fields": ["url"]}, "description": "", } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"url": "https://www.google.de/something"} expected = {"url": "https://www.google.de/something", "resolved_ip": "1.2.3.4"} self.object.process(document) @@ -211,7 +210,7 @@ def test_overwrite_target_field(self, _): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.process(document) assert document == expected @@ -229,6 +228,6 @@ def test_delete_source_field(self, _): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.process(document) assert document == expected diff --git a/tests/unit/processor/domain_resolver/test_domain_resolver_rule.py b/tests/unit/processor/domain_resolver/test_domain_resolver_rule.py index b3a08e373..db4582ebc 100644 --- a/tests/unit/processor/domain_resolver/test_domain_resolver_rule.py +++ b/tests/unit/processor/domain_resolver/test_domain_resolver_rule.py @@ -9,8 +9,8 @@ pytest.importorskip("logprep.processor.domain_resolver") -@pytest.fixture(name="specific_rule_definition") -def fixture_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def fixture_rule_definition(): return { "filter": "message", "domain_resolver": { @@ -79,8 +79,8 @@ def fixture_specific_rule_definition(): ), ], ) -def test_rules_equality(specific_rule_definition, testcase, other_rule_definition, is_equal): - rule1 = DomainResolverRule._create_from_dict(specific_rule_definition) +def test_rules_equality(rule_definition, testcase, other_rule_definition, is_equal): + rule1 = DomainResolverRule._create_from_dict(rule_definition) rule2 = DomainResolverRule._create_from_dict(other_rule_definition) assert (rule1 == rule2) == is_equal, testcase diff --git a/tests/unit/processor/dropper/test_dropper.py b/tests/unit/processor/dropper/test_dropper.py index 53c5d0af5..7fed2fa1b 100644 --- a/tests/unit/processor/dropper/test_dropper.py +++ b/tests/unit/processor/dropper/test_dropper.py @@ -9,22 +9,17 @@ class TestDropper(BaseProcessorTestCase): CONFIG = { "type": "dropper", - "specific_rules": ["tests/testdata/unit/dropper/rules/specific/"], - "generic_rules": ["tests/testdata/unit/dropper/rules/generic/"], + "rules": ["tests/testdata/unit/dropper/rules"], "tree_config": "tests/testdata/unit/shared_data/tree_config.json", } @property - def specific_rules_dirs(self): - return self.CONFIG["specific_rules"] - - @property - def generic_rules_dirs(self): - return self.CONFIG["generic_rules"] + def rules_dirs(self): + return self.CONFIG["rules"] def test_dropper_instantiates(self): rule = {"filter": "drop_me", "dropper": {"drop": ["drop_me"]}} - self._load_specific_rule(rule) + self._load_rule(rule) assert isinstance(self.object, Dropper) def test_not_nested_field_gets_dropped_with_rule_loaded_from_file(self): @@ -38,7 +33,7 @@ def test_nested_field_gets_dropped(self): rule = {"filter": "drop.me", "dropper": {"drop": ["drop.me"]}} expected = {} document = {"drop": {"me": "something"}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -47,7 +42,7 @@ def test_nested_field_with_neighbour_gets_dropped(self): rule = {"filter": "keep_me.drop_me", "dropper": {"drop": ["keep_me.drop_me"]}} expected = {"keep_me": {"keep_me_too": "something"}} document = {"keep_me": {"drop_me": "something", "keep_me_too": "something"}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -59,7 +54,7 @@ def test_deep_nested_field_gets_dropped(self): } expected = {"keep_me": {"drop": {}}} document = {"keep_me": {"drop": {"me": "something"}}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -68,7 +63,7 @@ def test_deep_nested_field_gets_dropped_fully(self): rule = {"filter": "please.drop.me.fully", "dropper": {"drop": ["please.drop.me.fully"]}} expected = {} document = {"please": {"drop": {"me": {"fully": "something"}}}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -80,7 +75,7 @@ def test_deep_nested_field_with_neighbour_gets_dropped(self): } expected = {"keep_me": {"drop": {}, "keep_me_too": "something"}} document = {"keep_me": {"drop": {"me": "something"}, "keep_me_too": "something"}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -89,7 +84,7 @@ def test_nested_field_with_child_gets_dropped(self): rule = {"filter": "drop.child", "dropper": {"drop": ["drop"]}} expected = {} document = {"drop": {"child": "something"}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -98,7 +93,7 @@ def test_deep_nested_field_with_child_gets_dropped(self): rule = {"filter": "drop.me", "dropper": {"drop": ["drop.me"]}} expected = {} document = {"drop": {"me": {"child": "foo"}}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -107,7 +102,7 @@ def test_deep_nested_field_with_child_and_neighbour_gets_dropped(self): rule = {"filter": "drop.me", "dropper": {"drop": ["drop.me"]}} expected = {"drop": {"neighbour": "bar"}} document = {"drop": {"me": {"child": "foo"}, "neighbour": "bar"}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -116,7 +111,7 @@ def test_deep_nested_field_with_child_and_not_drop_full_gets_partially_dropped(s rule = {"filter": "drop.me", "dropper": {"drop": ["drop.me"], "drop_full": False}} expected = {"drop": {}} document = {"drop": {"me": {"child": "foo"}}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -125,7 +120,7 @@ def test_deep_nested_field_with_child_neighbour_and_not_drop_full_gets_partially rule = {"filter": "drop.child", "dropper": {"drop": ["drop.child"], "drop_full": False}} expected = {"drop": {"neighbour": "bar"}} document = {"drop": {"child": {"grand_child": "foo"}, "neighbour": "bar"}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -133,7 +128,7 @@ def test_deep_nested_field_with_child_neighbour_and_not_drop_full_gets_partially def test_apply_rules_is_called(self): rule = {"filter": "drop.child", "dropper": {"drop": ["drop.child"], "drop_full": False}} document = {"drop": {"child": {"grand_child": "foo"}, "neighbour": "bar"}} - self._load_specific_rule(rule) + self._load_rule(rule) with mock.patch( f"{self.object.__module__}.{self.object.__class__.__name__}._apply_rules" ) as mock_apply_rules: @@ -154,7 +149,7 @@ def test_subkey_not_in_event(self): "drop_full": False, }, } - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) def test_key_not_in_event(self): @@ -168,5 +163,5 @@ def test_key_not_in_event(self): "drop_full": False, }, } - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) diff --git a/tests/unit/processor/dropper/test_dropper_rule.py b/tests/unit/processor/dropper/test_dropper_rule.py index 46cd5e20c..9a992e3e2 100644 --- a/tests/unit/processor/dropper/test_dropper_rule.py +++ b/tests/unit/processor/dropper/test_dropper_rule.py @@ -9,8 +9,8 @@ from logprep.processor.dropper.rule import DropperRule -@pytest.fixture(name="specific_rule_definition") -def fixture_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def fixture_rule_definition(): return { "filter": "test", "dropper": {"drop": ["field1", "field2"]}, @@ -19,8 +19,8 @@ def fixture_specific_rule_definition(): class TestDropperRule: - def test_rule_has_fields_to_drop(self, specific_rule_definition): - rule = DropperRule._create_from_dict(specific_rule_definition) + def test_rule_has_fields_to_drop(self, rule_definition): + rule = DropperRule._create_from_dict(rule_definition) fields_to_drop = rule.fields_to_drop assert isinstance(fields_to_drop, list) assert "field1" in fields_to_drop @@ -61,10 +61,10 @@ def test_rule_has_fields_to_drop(self, specific_rule_definition): ], ) def test_rules_equality( - self, specific_rule_definition, testcase, other_rule_definition, is_equal + self, rule_definition, testcase, other_rule_definition, is_equal ): rule1 = DropperRule._create_from_dict( - specific_rule_definition, + rule_definition, ) print(other_rule_definition) @@ -119,6 +119,6 @@ def test_rule_create_from_dict(self, rule_definition, raised, message): dropper_rule = DropperRule._create_from_dict(rule_definition) assert isinstance(dropper_rule, DropperRule) - def test_rule_is_hashable(self, specific_rule_definition): - rule = DropperRule._create_from_dict(specific_rule_definition) + def test_rule_is_hashable(self, rule_definition): + rule = DropperRule._create_from_dict(rule_definition) assert isinstance(rule, Hashable) diff --git a/tests/unit/processor/field_manager/test_field_manager.py b/tests/unit/processor/field_manager/test_field_manager.py index 5a71fb849..9938f47fb 100644 --- a/tests/unit/processor/field_manager/test_field_manager.py +++ b/tests/unit/processor/field_manager/test_field_manager.py @@ -577,19 +577,18 @@ class TestFieldManager(BaseProcessorTestCase): CONFIG: dict = { "type": "field_manager", - "specific_rules": ["tests/testdata/unit/field_manager/specific_rules"], - "generic_rules": ["tests/testdata/unit/field_manager/generic_rules"], + "rules": ["tests/testdata/unit/field_manager/rules"], } @pytest.mark.parametrize("testcase, rule, event, expected", test_cases) def test_testcases(self, testcase, rule, event, expected): # pylint: disable=unused-argument - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(event) assert event == expected @pytest.mark.parametrize("testcase, rule, event, expected, error", failure_test_cases) def test_testcases_failure_handling(self, testcase, rule, event, expected, error): - self._load_specific_rule(rule) + self._load_rule(rule) result = self.object.process(event) assert len(result.warnings) == 1 assert re.match(error, str(result.warnings[0])) @@ -607,7 +606,7 @@ def test_process_raises_field_exists_warning_if_target_field_exists_and_should_n "delete_source_fields": False, }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"field": {"a": "first", "b": "second"}, "target_field": "has already content"} result = self.object.process(document) assert isinstance(result.warnings[0], FieldExistsWarning) @@ -623,7 +622,7 @@ def test_process_raises_processing_warning_with_missing_fields(self): "target_field": "target_field", }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"field": {"a": "first", "b": "second"}} result = self.object.process(document) assert len(result.warnings) == 1 @@ -642,7 +641,7 @@ def test_process_raises_processing_warning_with_missing_fields_but_event_is_proc } }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"field": {"a": "first", "b": "second"}} expected = { "field": {"a": "first", "b": "second"}, @@ -670,7 +669,7 @@ def test_process_dos_not_raises_processing_warning_with_missing_fields_and_event "ignore_missing_fields": True, }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"field": {"a": "first", "b": "second"}} expected = { "field": {"a": "first", "b": "second"}, diff --git a/tests/unit/processor/generic_adder/test_generic_adder.py b/tests/unit/processor/generic_adder/test_generic_adder.py index b25bd541d..6625bece8 100644 --- a/tests/unit/processor/generic_adder/test_generic_adder.py +++ b/tests/unit/processor/generic_adder/test_generic_adder.py @@ -310,23 +310,18 @@ class TestGenericAdder(BaseProcessorTestCase): CONFIG = { "type": "generic_adder", - "generic_rules": ["tests/testdata/unit/generic_adder/rules/generic"], - "specific_rules": ["tests/testdata/unit/generic_adder/rules/specific"], + "rules": ["tests/testdata/unit/generic_adder/rules"], } @property - def generic_rules_dirs(self): - return self.CONFIG.get("generic_rules") - - @property - def specific_rules_dirs(self): - return self.CONFIG.get("specific_rules") + def rules_dirs(self): + return self.CONFIG.get("rules") @pytest.mark.parametrize("testcase, rule, event, expected", test_cases) def test_generic_adder_testcases( self, testcase, rule, event, expected ): # pylint: disable=unused-argument - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(event) assert event == expected @@ -334,7 +329,7 @@ def test_generic_adder_testcases( def test_generic_adder_testcases_failure_handling( self, testcase, rule, event, expected, error_message ): - self._load_specific_rule(rule) + self._load_rule(rule) result = self.object.process(event) assert len(result.warnings) == 1 assert re.match(rf".*FieldExistsWarning.*{error_message}", str(result.warnings[0])) @@ -343,7 +338,7 @@ def test_generic_adder_testcases_failure_handling( def test_add_generic_fields_from_file_missing_and_existing_with_all_required(self): with pytest.raises(InvalidRuleDefinitionError, match=r"files do not exist"): config = deepcopy(self.CONFIG) - config["specific_rules"] = [RULES_DIR_MISSING] + config["rules"] = [RULES_DIR_MISSING] configuration = {"test_instance_name": config} Factory.create(configuration) @@ -353,6 +348,6 @@ def test_add_generic_fields_from_file_invalid(self): match=r"must be a dictionary with string values", ): config = deepcopy(self.CONFIG) - config["generic_rules"] = [RULES_DIR_INVALID] + config["rules"] = [RULES_DIR_INVALID] configuration = {"test processor": config} Factory.create(configuration) diff --git a/tests/unit/processor/generic_adder/test_generic_adder_rule.py b/tests/unit/processor/generic_adder/test_generic_adder_rule.py index daf7298db..b0dea6c27 100644 --- a/tests/unit/processor/generic_adder/test_generic_adder_rule.py +++ b/tests/unit/processor/generic_adder/test_generic_adder_rule.py @@ -5,8 +5,8 @@ from logprep.processor.generic_adder.rule import GenericAdderRule -@pytest.fixture(name="specific_rule_definition") -def fixture_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def fixture_rule_definition(): return { "filter": "add_generic_test", "generic_adder": { @@ -79,12 +79,12 @@ class TestGenericAdderRule: ) def test_rules_equality( self, - specific_rule_definition, + rule_definition, testcase, other_rule_definition, is_equal, ): - rule1 = GenericAdderRule._create_from_dict(specific_rule_definition) + rule1 = GenericAdderRule._create_from_dict(rule_definition) rule2 = GenericAdderRule._create_from_dict(other_rule_definition) assert (rule1 == rule2) == is_equal, testcase diff --git a/tests/unit/processor/generic_resolver/test_generic_resolver.py b/tests/unit/processor/generic_resolver/test_generic_resolver.py index 09e5d2e6e..ca6324902 100644 --- a/tests/unit/processor/generic_resolver/test_generic_resolver.py +++ b/tests/unit/processor/generic_resolver/test_generic_resolver.py @@ -14,8 +14,7 @@ class TestGenericResolver(BaseProcessorTestCase): CONFIG = { "type": "generic_resolver", - "specific_rules": ["tests/testdata/unit/generic_resolver/rules/specific/"], - "generic_rules": ["tests/testdata/unit/generic_resolver/rules/generic/"], + "rules": ["tests/testdata/unit/generic_resolver/rules"], "tree_config": "tests/testdata/unit/shared_data/tree_config.json", } @@ -27,18 +26,13 @@ class TestGenericResolver(BaseProcessorTestCase): ] @property - def specific_rules_dirs(self): - """Return the paths of the specific rules""" - return self.CONFIG["specific_rules"] - - @property - def generic_rules_dirs(self): - """Return the paths of the generic rules""" - return self.CONFIG["generic_rules"] + def rules_dirs(self): + """Return the paths of the rules""" + return self.CONFIG["rules"] def test_resolve_generic_instantiates(self): rule = {"filter": "anything", "generic_resolver": {"field_mapping": {}}} - self._load_specific_rule(rule) + self._load_rule(rule) assert isinstance(self.object, GenericResolver) def test_resolve_not_dotted_field_no_conflict_match(self): @@ -50,7 +44,7 @@ def test_resolve_not_dotted_field_no_conflict_match(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "something HELLO1", "resolved": "Greeting"} @@ -69,7 +63,7 @@ def test_resolve_with_dict_value(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "something HELLO1", "resolved": {"Greeting": "Hello"}} @@ -89,7 +83,7 @@ def test_resolve_from_mapping_with_ignore_case(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "something HELLO1", "resolved": "Greeting"} document = {"to_resolve": "something HELLO1"} @@ -112,7 +106,7 @@ def test_resolve_not_dotted_field_no_conflict_and_to_list_entries_match( }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "something HELLO1", "resolved": "Greeting"} @@ -135,7 +129,7 @@ def test_resolve_not_dotted_field_no_conflict_no_match(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "something no"} document = {"to_resolve": "something no"} @@ -152,7 +146,7 @@ def test_resolve_dotted_field_no_conflict_match(self): "resolve_list": {".*HELLO\\d": "Greeting"}, }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to": {"resolve": "something HELLO1"}, "resolved": "Greeting"} @@ -176,7 +170,7 @@ def test_resolve_dotted_field_no_conflict_match_from_file( "resolve_list": {"FOO": "BAR"}, }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "ab", "resolved": "ab_server_type"} @@ -201,7 +195,7 @@ def test_resolve_from_file_with_ignore_case( "resolve_list": {"FOO": "BAR"}, }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "ab", "resolved": "ab_server_type"} document = {"to_resolve": "ab"} @@ -229,7 +223,7 @@ def test_resolve_from_file_and_from_list(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = { "to_resolve_1": "ab", @@ -258,7 +252,7 @@ def test_resolve_dotted_field_no_conflict_no_match_from_file( "resolve_list": {"FOO": "BAR"}, }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = { "to_resolve": "not_in_list", @@ -284,7 +278,7 @@ def test_resolve_dotted_field_no_conflict_match_from_file_and_list( "extend_target_list": True, }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "12ab34", "resolved": ["ab_server_type"]} @@ -308,7 +302,7 @@ def test_resolve_dotted_field_no_conflict_match_from_file_and_list_has_conflict( "extend_target_list": True, }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "12ab34", "resolved": ["ab_server_type"]} @@ -336,7 +330,7 @@ def test_resolve_dotted_field_no_conflict_match_from_file_and_list_has_conflict_ "extend_target_list": True, }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = { "to_resolve": "12ab34", @@ -359,7 +353,7 @@ def test_resolve_dotted_field_no_conflict_no_match(self): "resolve_list": {".*HELLO\\d": "Greeting"}, }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to": {"resolve": "something no"}} document = {"to": {"resolve": "something no"}} @@ -376,7 +370,7 @@ def test_resolve_dotted_field_is_missing(self): "resolve_list": {".*HELLO\\d": "Greeting"}, }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = { "to": {"other_field": "something no"}, @@ -396,7 +390,7 @@ def test_resolve_dotted_dest_field_no_conflict_match(self): "resolve_list": {".*HELLO\\d": "Greeting"}, }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "something HELLO1", "re": {"solved": "Greeting"}} document = {"to_resolve": "something HELLO1"} @@ -413,7 +407,7 @@ def test_resolve_dotted_dest_field_no_conflict_no_match(self): "resolve_list": {".*HELLO\\d": "Greeting"}, }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "something no"} document = {"to_resolve": "something no"} @@ -432,7 +426,7 @@ def test_resolve_dotted_src_and_dest_field_no_conflict_match( "resolve_list": {".*HELLO\\d": "Greeting"}, }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to": {"resolve": "something HELLO1"}, "re": {"solved": "Greeting"}} document = {"to": {"resolve": "something HELLO1"}} @@ -449,7 +443,7 @@ def test_resolve_dotted_src_and_dest_field_and_conflict_match(self, caplog): "resolve_list": {".*HELLO\\d": "Greeting"}, }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = { "to": {"resolve": "something HELLO1"}, "re": {"solved": "I already exist!"}, @@ -479,7 +473,7 @@ def test_resolve_generic_and_multiple_match_first_only(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to": {"resolve": "something HELLO1"}, "re": {"solved": "Greeting"}} document = {"to": {"resolve": "something HELLO1"}} @@ -502,7 +496,7 @@ def test_resolve_from_cache_with_large_enough_cache(self): }, } event = {"to_resolve": "foo"} - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.setup() self.object.metrics.new_results = 0 @@ -542,7 +536,7 @@ def test_resolve_from_cache_with_cache_smaller_than_results(self): }, } event = {"to_resolve": "foo"} - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.setup() self.object.metrics.new_results = 0 @@ -580,7 +574,7 @@ def test_resolve_without_cache(self): }, } event = {"to_resolve": "foo"} - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.setup() self.object.metrics.new_results = 0 @@ -622,7 +616,7 @@ def test_resolve_from_cache_with_update_interval_2(self): } event = {"to_resolve": "foo"} other_event = {"to_resolve": "bar"} - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.setup() self.object.metrics.new_results = 0 diff --git a/tests/unit/processor/generic_resolver/test_generic_resolver_rule.py b/tests/unit/processor/generic_resolver/test_generic_resolver_rule.py index 547fdf3d4..e055996d0 100644 --- a/tests/unit/processor/generic_resolver/test_generic_resolver_rule.py +++ b/tests/unit/processor/generic_resolver/test_generic_resolver_rule.py @@ -8,8 +8,8 @@ from logprep.processor.generic_resolver.rule import GenericResolverRule -@pytest.fixture(name="specific_rule_definition") -def fixture_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def fixture_rule_definition(): return { "filter": "message", "generic_resolver": { @@ -154,9 +154,9 @@ class TestGenericResolverRule: ], ) def test_rules_equality( - self, specific_rule_definition, testcase, other_rule_definition, is_equal + self, rule_definition, testcase, other_rule_definition, is_equal ): - rule1 = GenericResolverRule._create_from_dict(specific_rule_definition) + rule1 = GenericResolverRule._create_from_dict(rule_definition) rule2 = GenericResolverRule._create_from_dict(other_rule_definition) assert (rule1 == rule2) == is_equal, testcase diff --git a/tests/unit/processor/geoip_enricher/test_geoip_enricher.py b/tests/unit/processor/geoip_enricher/test_geoip_enricher.py index fbf17b9e0..ec55b49f8 100644 --- a/tests/unit/processor/geoip_enricher/test_geoip_enricher.py +++ b/tests/unit/processor/geoip_enricher/test_geoip_enricher.py @@ -88,19 +88,14 @@ class TestGeoipEnricher(BaseProcessorTestCase): CONFIG = { "type": "geoip_enricher", - "specific_rules": ["tests/testdata/unit/geoip_enricher/rules/specific"], - "generic_rules": ["tests/testdata/unit/geoip_enricher/rules/generic"], + "rules": ["tests/testdata/unit/geoip_enricher/rules"], "db_path": "tests/testdata/mock_external/MockGeoLite2-City.mmdb", "tree_config": "tests/testdata/unit/shared_data/tree_config.json", } @property - def generic_rules_dirs(self): - return self.CONFIG["generic_rules"] - - @property - def specific_rules_dirs(self): - return self.CONFIG["specific_rules"] + def rules_dirs(self): + return self.CONFIG["rules"] def test_geoip_data_added(self): document = {"client": {"ip": "1.2.3.4"}} @@ -124,7 +119,7 @@ def test_no_geoip_data_added_if_source_field_is_none(self): def test_source_field_is_none_emits_missing_fields_warning(self): document = {"client": {"ip": None}} expected = {"client": {"ip": None}, "tags": ["_geoip_enricher_missing_field_warning"]} - self._load_specific_rule(self.object.rules[0]) + self._load_rule(self.object.rules[0]) self.object.process(document) assert len(self.object.result.warnings) == 1 assert re.match( @@ -188,7 +183,7 @@ def test_delete_source_field(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.process(document) assert "client" in document assert "ip" not in document.get("client") @@ -204,7 +199,7 @@ def test_overwrite_target_field(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.process(document) assert "client" in document assert document.get("client").get("ip").get("type") is not None @@ -233,7 +228,7 @@ def test_specify_all_target_sub_fields(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.process(document) expected_event = { "client": { @@ -273,7 +268,7 @@ def test_specify_some_target_sub_fields(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.process(document) expected_event = { "client": { @@ -313,7 +308,7 @@ def test_specify_unknown_target_sub_fields(self): "description": "", } with pytest.raises(ValueError, match=r"\'customize_target_subfields\' must be in"): - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) def test_geoip_db_returns_only_limited_data_without_missing_coordinates(self): document = {"client": {"ip": "13.21.21.37"}} @@ -324,7 +319,7 @@ def test_geoip_db_returns_only_limited_data_without_missing_coordinates(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.process(document) expected_event = { "client": {"ip": "13.21.21.37"}, @@ -349,7 +344,7 @@ def test_geoip_db_returns_only_limited_data_with_missing_coordinates(self, sourc }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.process(document) expected_event = { "client": {"ip": source_ip}, diff --git a/tests/unit/processor/geoip_enricher/test_geoip_enricher_rule.py b/tests/unit/processor/geoip_enricher/test_geoip_enricher_rule.py index 0576b7a11..f2acff83a 100644 --- a/tests/unit/processor/geoip_enricher/test_geoip_enricher_rule.py +++ b/tests/unit/processor/geoip_enricher/test_geoip_enricher_rule.py @@ -6,8 +6,8 @@ from logprep.processor.geoip_enricher.rule import GeoipEnricherRule -@pytest.fixture(name="specific_rule_definition") -def fixture_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def fixture_rule_definition(): return { "filter": "message", "geoip_enricher": {"source_fields": ["source"], "target_field": "geoip"}, @@ -68,8 +68,8 @@ class TestListComparisonRule: ], ) def test_rules_equality( - self, specific_rule_definition, testcase, other_rule_definition, is_equal + self, rule_definition, testcase, other_rule_definition, is_equal ): - rule1 = GeoipEnricherRule._create_from_dict(specific_rule_definition) + rule1 = GeoipEnricherRule._create_from_dict(rule_definition) rule2 = GeoipEnricherRule._create_from_dict(other_rule_definition) assert (rule1 == rule2) == is_equal, testcase diff --git a/tests/unit/processor/grokker/test_grokker.py b/tests/unit/processor/grokker/test_grokker.py index a0c6cc258..dcdd59191 100644 --- a/tests/unit/processor/grokker/test_grokker.py +++ b/tests/unit/processor/grokker/test_grokker.py @@ -416,20 +416,19 @@ class TestGrokker(BaseProcessorTestCase): CONFIG: dict = { "type": "grokker", - "specific_rules": ["tests/testdata/unit/grokker/specific_rules"], - "generic_rules": ["tests/testdata/unit/grokker/generic_rules"], + "rules": ["tests/testdata/unit/grokker/rules"], } @pytest.mark.parametrize("testcase, rule, event, expected", test_cases) def test_testcases(self, testcase, rule, event, expected): - self._load_specific_rule(rule) + self._load_rule(rule) self.object.setup() self.object.process(event) assert event == expected, testcase @pytest.mark.parametrize("testcase, rule, event, expected, error", failure_test_cases) def test_testcases_failure_handling(self, testcase, rule, event, expected, error): - self._load_specific_rule(rule) + self._load_rule(rule) self.object.setup() if isinstance(error, str): result = self.object.process(event) @@ -458,7 +457,7 @@ def test_load_custom_patterns_from_http_as_zip_file(self): "http://localhost:8000/tests/testdata/unit/grokker/patterns.zip" ) self.object = Factory.create({"grokker": config}) - self._load_specific_rule(rule) + self._load_rule(rule) self.object.setup() self.object.process(event) assert event == expected @@ -494,7 +493,7 @@ def test_loads_custom_patterns(self): config = deepcopy(self.CONFIG) config["custom_patterns_dir"] = "tests/testdata/unit/grokker/patterns/" self.object = Factory.create({"grokker": config}) - self._load_specific_rule(rule) + self._load_rule(rule) self.object.setup() self.object.process(event) assert event == expected diff --git a/tests/unit/processor/hyperscan_resolver/test_hyperscan_resolver.py b/tests/unit/processor/hyperscan_resolver/test_hyperscan_resolver.py index f644c7cf5..83bb92ae9 100644 --- a/tests/unit/processor/hyperscan_resolver/test_hyperscan_resolver.py +++ b/tests/unit/processor/hyperscan_resolver/test_hyperscan_resolver.py @@ -28,8 +28,7 @@ class TestHyperscanResolverProcessor(BaseProcessorTestCase): CONFIG = { "type": "hyperscan_resolver", - "specific_rules": ["tests/testdata/unit/hyperscan_resolver/rules/specific/"], - "generic_rules": ["tests/testdata/unit/hyperscan_resolver/rules/generic/"], + "rules": ["tests/testdata/unit/hyperscan_resolver/rules"], "tree_config": "tests/testdata/unit/shared_data/tree_config.json", "hyperscan_db_path": "/tmp", } @@ -37,7 +36,7 @@ class TestHyperscanResolverProcessor(BaseProcessorTestCase): def test_resolve_instantiates(self): rule = {"filter": "anything", "hyperscan_resolver": {"field_mapping": {}}} - self._load_specific_rule(rule) + self._load_rule(rule) assert isinstance(self.object, HyperscanResolver) @@ -50,7 +49,7 @@ def test_resolve_not_dotted_field_no_conflict_match(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "something HELLO1", "resolved": "Greeting"} document = {"to_resolve": "something HELLO1"} @@ -70,7 +69,7 @@ def test_resolve_not_dotted_field_no_conflict_and_to_list_entries_match( }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "something HELLO1", "resolved": "Greeting"} document = {"to_resolve": "something HELLO1"} @@ -92,7 +91,7 @@ def test_resolve_not_dotted_field_no_conflict_no_match(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "something no"} document = {"to_resolve": "something no"} @@ -110,7 +109,7 @@ def test_resolve_dotted_no_conflict_match(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to": {"resolve": "something HELLO1"}, "resolved": "Greeting"} document = {"to": {"resolve": "something HELLO1"}} @@ -130,7 +129,7 @@ def test_resolve_dotted_no_conflict_from_file(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "ab", "resolved": "ab_resolved"} document = {"to_resolve": "ab"} @@ -150,7 +149,7 @@ def test_resolve_from_file_and_from_list(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = { "to_resolve_1": "ab", @@ -175,7 +174,7 @@ def test_resolve_dotted_no_conflict_no_from_file(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = { "to_resolve": "not_in_list", @@ -199,7 +198,7 @@ def test_resolve_dotted_no_conflict_from_file_and_list( }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "12ab34", "resolved": ["ab_resolved"]} document = {"to_resolve": "12ab34"} @@ -221,7 +220,7 @@ def test_resolve_dotted_no_conflict_from_file_and_list_has_conflict( }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "12ab34", "resolved": ["ab_resolved"]} document = {"to_resolve": "12ab34"} @@ -244,7 +243,7 @@ def test_resolve_dotted_no_conflict_from_file_and_list_has_conflict_and_diff_inp }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = { "to_resolve": "12ab34", @@ -268,7 +267,7 @@ def test_resolve_from_file_and_file_does_not_exist(self): } with pytest.raises(InvalidHyperscanResolverDefinition): - self._load_specific_rule(rule) + self._load_rule(rule) def test_resolve_dotted_no_conflict_no_match(self): rule = { @@ -279,7 +278,7 @@ def test_resolve_dotted_no_conflict_no_match(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to": {"resolve": "something no"}} document = {"to": {"resolve": "something no"}} @@ -296,7 +295,7 @@ def test_resolve_dotted_field_is_missing(self): "resolve_list": {".*HELLO\\d": "Greeting"}, }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = { "to": {"other_field": "something no"}, @@ -317,7 +316,7 @@ def test_resolve_dotted_dest_field_no_conflict_match(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "something HELLO1", "re": {"solved": "Greeting"}} document = {"to_resolve": "something HELLO1"} @@ -335,7 +334,7 @@ def test_resolve_dotted_dest_field_no_conflict_no_match(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "something no"} document = {"to_resolve": "something no"} @@ -353,7 +352,7 @@ def test_resolve_dotted_and_dest_field_no_conflict_match(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to": {"resolve": "something HELLO1"}, "re": {"solved": "Greeting"}} document = {"to": {"resolve": "something HELLO1"}} @@ -370,7 +369,7 @@ def test_resolve_dotted_and_dest_field_with_conflict_match(self): "resolve_list": {".*HELLO\\d": "Greeting"}, }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"to": {"resolve": "something HELLO1"}, "re": {"solved": "I already exist!"}} expected = { "to": {"resolve": "something HELLO1"}, @@ -397,7 +396,7 @@ def test_resolve_with_multiple_match_first_only(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to": {"resolve": "something HELLO1"}, "re": {"solved": "Greeting"}} document = {"to": {"resolve": "something HELLO1"}} @@ -424,7 +423,7 @@ def test_resolve_no_conflict_from_file(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "ab", "resolved": "ab_resolved"} document = {"to_resolve": "ab"} @@ -448,7 +447,7 @@ def test_resolve_no_conflict_from_file_and_escaped_parenthesis( }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "ab)c", "resolved": "ab)c_resolved"} document = {"to_resolve": "ab)c"} @@ -472,7 +471,7 @@ def test_resolve_dotted_no_conflict_from_file_and_escaped_parenthesis_and_backsl }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": r"ab\)c", "resolved": r"ab\)c_resolved"} document = {"to_resolve": r"ab\)c"} @@ -497,7 +496,7 @@ def test_resolve_dotted_no_conflict_from_file_and_escaped_to_unbalanced_parenthe } with pytest.raises(Exception, match="unbalanced parenthesis"): - self._load_specific_rule(rule) + self._load_rule(rule) def test_resolve_from_file_and_from_list(self): rule = { @@ -513,7 +512,7 @@ def test_resolve_from_file_and_from_list(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = { "to_resolve_1": "ab", @@ -541,7 +540,7 @@ def test_resolve_no_conflict_no_from_file(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = { "to_resolve": "not_in_list", @@ -568,7 +567,7 @@ def test_resolve_no_conflict_from_file_and_list( }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "12ab34", "resolved": ["ab_resolved"]} document = {"to_resolve": "12ab34"} @@ -591,7 +590,7 @@ def test_resolve_with_parenthesis_in_mapping(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "12ab34", "resolved": ["ab_resolved"]} document = {"to_resolve": "12ab34"} @@ -614,7 +613,7 @@ def test_resolve_with_partially_matching_mapping(self): }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "gh", "resolved": ["gh_resolved"]} document = {"to_resolve": "gh"} @@ -636,7 +635,7 @@ def test_resolve_no_matching_pattern(self): "extend_target_list": True, }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"to_resolve": "12ab34"} result = self.object.process(document) assert isinstance(result.errors[0], ProcessingCriticalError) @@ -657,7 +656,7 @@ def test_resolve_no_conflict_from_file_and_list_has_conflict( }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = {"to_resolve": "12ab34", "resolved": ["ab_resolved"]} document = {"to_resolve": "12ab34"} @@ -683,7 +682,7 @@ def test_resolve_no_conflict_from_file_and_list_has_conflict_and_diff_inputs( }, } - self._load_specific_rule(rule) + self._load_rule(rule) expected = { "to_resolve": "12ab34", @@ -712,7 +711,7 @@ def test_resolve_dotted_no_conflict_from_file_group_mapping_does_not_exist( }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"to_resolve": "ab"} @@ -732,4 +731,4 @@ def test_resolve_from_file_and_file_does_not_exist(self): match=r"The following HyperscanResolver definition is invalid: Additions file '{" r"'path': 'i/do/not/exist', 'pattern': 'bar'}' not found!", ): - self._load_specific_rule(rule) + self._load_rule(rule) diff --git a/tests/unit/processor/hyperscan_resolver/test_hyperscan_resolver_rule.py b/tests/unit/processor/hyperscan_resolver/test_hyperscan_resolver_rule.py index 55df70f1f..3982c5560 100644 --- a/tests/unit/processor/hyperscan_resolver/test_hyperscan_resolver_rule.py +++ b/tests/unit/processor/hyperscan_resolver/test_hyperscan_resolver_rule.py @@ -9,8 +9,8 @@ pytest.importorskip("hyperscan") -@pytest.fixture(name="specific_rule_definition") -def fixture_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def fixture_rule_definition(): return { "filter": "some_filter", "hyperscan_resolver": { @@ -22,8 +22,8 @@ def fixture_specific_rule_definition(): } -@pytest.fixture(name="specific_rule_with_resolve_file_definition") -def fixture_specific_rule_with_resolve_file_definition(): +@pytest.fixture(name="rule_with_resolve_file_definition") +def fixture_rule_with_resolve_file_definition(): return { "filter": "some_filter", "hyperscan_resolver": { @@ -143,13 +143,13 @@ def fixture_specific_rule_with_resolve_file_definition(): ], ) def test_rules_equality( - specific_rule_definition, + rule_definition, testcase, other_rule_definition, is_equal, ): rule1 = HyperscanResolverRule._create_from_dict( - specific_rule_definition, + rule_definition, ) rule2 = HyperscanResolverRule._create_from_dict( @@ -160,17 +160,17 @@ def test_rules_equality( def test_rules_with_differently_defined_but_equivalent_regex_pattern_definition_types_are_equal( - specific_rule_with_resolve_file_definition, + rule_with_resolve_file_definition, ): rule_no_regex = HyperscanResolverRule._create_from_dict( - specific_rule_with_resolve_file_definition, + rule_with_resolve_file_definition, ) - specific_rule_with_resolve_file_definition["hyperscan_resolver"][ + rule_with_resolve_file_definition["hyperscan_resolver"][ "resolve_from_file" ] = "tests/testdata/unit/hyperscan_resolver/resolve_mapping_regex.yml" rule_regex = HyperscanResolverRule._create_from_dict( - specific_rule_with_resolve_file_definition, + rule_with_resolve_file_definition, ) assert rule_no_regex == rule_regex diff --git a/tests/unit/processor/ip_informer/test_ip_informer.py b/tests/unit/processor/ip_informer/test_ip_informer.py index 0ef5e25b1..de06142a0 100644 --- a/tests/unit/processor/ip_informer/test_ip_informer.py +++ b/tests/unit/processor/ip_informer/test_ip_informer.py @@ -411,19 +411,18 @@ class TestIpInformer(BaseProcessorTestCase): CONFIG: dict = { "type": "ip_informer", - "specific_rules": ["tests/testdata/unit/ip_informer/specific/"], - "generic_rules": ["tests/testdata/unit/ip_informer/generic/"], + "rules": ["tests/testdata/unit/ip_informer/rules/"], } @pytest.mark.parametrize("testcase, rule, event, expected", test_cases) def test_testcases(self, testcase, rule, event, expected): - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(event) assert event == expected, testcase @pytest.mark.parametrize("testcase, rule, event, expected", failure_test_cases) def test_testcases_failure_handling(self, testcase, rule, event, expected): - self._load_specific_rule(rule) + self._load_rule(rule) result = self.object.process(event) assert len(result.warnings) == 1 assert isinstance(result.warnings[0], ProcessingWarning) diff --git a/tests/unit/processor/key_checker/__init__.py b/tests/unit/processor/key_checker/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/processor/key_checker/test_key_checker.py b/tests/unit/processor/key_checker/test_key_checker.py index 0391c3e82..886b61f4f 100644 --- a/tests/unit/processor/key_checker/test_key_checker.py +++ b/tests/unit/processor/key_checker/test_key_checker.py @@ -243,15 +243,14 @@ class TestKeyChecker(BaseProcessorTestCase): CONFIG = { "type": "key_checker", - "specific_rules": ["tests/testdata/unit/key_checker/specific_rules/"], - "generic_rules": ["tests/testdata/unit/key_checker/generic_rules/"], + "rules": ["tests/testdata/unit/key_checker/rules"], } @pytest.mark.parametrize("testcase, rule, event, expected", test_cases) def test_testcases_positiv( self, testcase, rule, event, expected ): # pylint: disable=unused-argument - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(event) assert event == expected @@ -263,7 +262,7 @@ def test_field_exists_warning(self): "target_field": "missing_fields", }, } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) document = { "key1": { "key2": {"key3": {"key3": "key3_value"}, "random_key": "random_key_value"}, diff --git a/tests/unit/processor/labeler/test_labeler.py b/tests/unit/processor/labeler/test_labeler.py index cae9105b5..a1e9f1681 100644 --- a/tests/unit/processor/labeler/test_labeler.py +++ b/tests/unit/processor/labeler/test_labeler.py @@ -61,32 +61,26 @@ class TestLabeler(BaseProcessorTestCase): CONFIG = { "type": "labeler", "schema": "tests/testdata/unit/labeler/schemas/schema.json", - "specific_rules": ["tests/testdata/unit/labeler/rules/specific/"], - "generic_rules": ["tests/testdata/unit/labeler/rules/generic/"], + "rules": ["tests/testdata/unit/labeler/rules"], } @property - def specific_rules_dirs(self): - """Return path to specific rule directories""" - return self.CONFIG["specific_rules"] + def rules_dirs(self): + """Return path to rule directories""" + return self.CONFIG["rules"] - @property - def generic_rules_dirs(self): - """Return path to generic rule directories""" - return self.CONFIG["generic_rules"] - - def _load_specific_rule(self, rule, schema=None): # pylint: disable=arguments-differ - specific_rule = LabelerRule._create_from_dict(rule) + def _load_rule(self, rule, schema=None): # pylint: disable=arguments-differ + rule = LabelerRule._create_from_dict(rule) if schema: - specific_rule.add_parent_labels_from_schema(schema) - self.object._specific_tree.add_rule(specific_rule, self.logger) + rule.add_parent_labels_from_schema(schema) + self.object._rule_tree.add_rule(rule, self.logger) def test_process_adds_labels_to_event(self): rule = {"filter": "applyrule", "labeler": {"label": {"reporter": ["windows"]}}} document = {"applyrule": "yes"} expected = {"applyrule": "yes", "label": {"reporter": ["windows"]}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -101,7 +95,7 @@ def test_process_adds_labels_to_event_with_umlauts(self): document = {"äpplyrüle": "nö"} expected = {"äpplyrüle": "nö", "label": {"räpörter": ["windöws"]}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -114,7 +108,7 @@ def test_process_adds_labels_including_parents_when_flag_was_set( document = {"applyrule": "yes"} expected = {"applyrule": "yes", "label": {"reporter": ["parentlabel", "windows"]}} - self._load_specific_rule(rule, reporter_schema_expanded) + self._load_rule(rule, reporter_schema_expanded) self.object.process(document) assert document == expected @@ -130,7 +124,7 @@ def test_process_adds_more_than_one_label(self): "label": {"reporter": ["client", "windows"], "object": ["file"]}, } - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -140,7 +134,7 @@ def test_process_does_not_overwrite_existing_values(self): document = {"applyrule": "yes", "label": {"reporter": ["windows"]}} expected = {"applyrule": "yes", "label": {"reporter": ["windows"]}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -149,7 +143,7 @@ def test_process_returns_labels_in_alphabetical_order(self, reporter_schema_expa event = {"applyrule": "yes"} rule = {"filter": "applyrule", "labeler": {"label": {"reporter": ["windows"]}}} - self._load_specific_rule(rule, reporter_schema_expanded) + self._load_rule(rule, reporter_schema_expanded) self.object.process(event) assert event["label"]["reporter"] == ["parentlabel", "windows"] @@ -167,7 +161,7 @@ def test_process_matches_event_with_array_with_one_element(self): "description": "This does even match with arrays!", } - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -185,7 +179,7 @@ def test_process_matches_event_with_array_if_at_least_one_element_matches(self): "description": "This does even match with arrays!", } - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -204,7 +198,7 @@ def test_process_matches_event_with_array_with_one_element_with_regex(self): "description": "This does even match with arrays!", } - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected @@ -226,7 +220,7 @@ def test_process_matches_event_with_array_with_one_element_with_regex_one_withou "description": "This does even match with arrays!", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.process(document) @@ -262,6 +256,6 @@ def test_extend_list_of_existing_labels(self): rule = {"filter": "applyrule", "labeler": {"label": {"reporter": ["windows", "foo"]}}} document = {"applyrule": "yes", "label": {"reporter": ["windows"]}} expected = {"applyrule": "yes", "label": {"reporter": ["foo", "windows"]}} - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(document) assert document == expected diff --git a/tests/unit/processor/list_comparison/test_list_comparison.py b/tests/unit/processor/list_comparison/test_list_comparison.py index 385227ea7..4bfabeb5c 100644 --- a/tests/unit/processor/list_comparison/test_list_comparison.py +++ b/tests/unit/processor/list_comparison/test_list_comparison.py @@ -11,8 +11,7 @@ class TestListComparison(BaseProcessorTestCase): CONFIG = { "type": "list_comparison", - "specific_rules": ["tests/testdata/unit/list_comparison/rules/specific"], - "generic_rules": ["tests/testdata/unit/list_comparison/rules/generic"], + "rules": ["tests/testdata/unit/list_comparison/rules"], "tree_config": "tests/testdata/unit/shared_data/tree_config.json", "list_search_base_path": "tests/testdata/unit/list_comparison/rules", } @@ -105,7 +104,7 @@ def test_extend_dotted_output_field(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.setup() self.object.process(document) @@ -128,7 +127,7 @@ def test_dotted_parent_field_exists_but_subfield_doesnt(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.setup() self.object.process(document) @@ -156,7 +155,7 @@ def test_target_field_exists_and_cant_be_extended(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.setup() result = self.object.process(document) assert len(result.warnings) == 1 @@ -185,7 +184,7 @@ def test_intermediate_output_field_is_wrong_type(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.setup() result = self.object.process(document) assert len(result.warnings) == 1 @@ -223,7 +222,7 @@ def test_delete_source_field(self): "description": "", } expected = {"user_results": {"in_list": ["user_list.txt"]}} - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.setup() self.object.process(document) assert document == expected @@ -241,7 +240,7 @@ def test_overwrite_target_field(self): }, "description": "", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.setup() result = self.object.process(document) assert len(result.warnings) == 1 @@ -269,14 +268,13 @@ def test_list_comparison_loads_rule_with_http_template_in_list_search_base_path( } config = { "type": "list_comparison", - "specific_rules": [], - "generic_rules": [], + "rules": [], "list_search_base_path": "http://localhost/tests/testdata/${LOGPREP_LIST}?ref=bla", } processor = Factory.create({"custom_lister": config}) rule = processor.rule_class._create_from_dict(rule_dict) - processor._specific_tree.add_rule(rule) + processor._rule_tree.add_rule(rule) processor.setup() - assert processor._specific_rules[0].compare_sets == { + assert processor._tree_rules[0].compare_sets == { "bad_users.list": {"Franz", "Heinz", "Hans"} } diff --git a/tests/unit/processor/list_comparison/test_list_comparison_rule.py b/tests/unit/processor/list_comparison/test_list_comparison_rule.py index 5a93e9899..a9baa9936 100644 --- a/tests/unit/processor/list_comparison/test_list_comparison_rule.py +++ b/tests/unit/processor/list_comparison/test_list_comparison_rule.py @@ -6,8 +6,8 @@ from logprep.processor.list_comparison.rule import ListComparisonRule -@pytest.fixture(name="specific_rule_definition") -def fixture_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def fixture_rule_definition(): return { "filter": "user", "list_comparison": { @@ -86,16 +86,16 @@ class TestListComparisonRule: ], ) def test_rules_equality( - self, specific_rule_definition, testcase, other_rule_definition, is_equal + self, rule_definition, testcase, other_rule_definition, is_equal ): - rule1 = ListComparisonRule._create_from_dict(specific_rule_definition) + rule1 = ListComparisonRule._create_from_dict(rule_definition) rule2 = ListComparisonRule._create_from_dict(other_rule_definition) assert (rule1 == rule2) == is_equal, testcase def test_compare_set_not_empty_for_valid_rule_def_after_init_list_comparison( - self, specific_rule_definition + self, rule_definition ): - rule = ListComparisonRule._create_from_dict(specific_rule_definition) + rule = ListComparisonRule._create_from_dict(rule_definition) rule.init_list_comparison("tests/testdata/unit/list_comparison/rules") diff --git a/tests/unit/processor/pre_detector/test_pre_detector.py b/tests/unit/processor/pre_detector/test_pre_detector.py index 213878c9e..cdb06fa21 100644 --- a/tests/unit/processor/pre_detector/test_pre_detector.py +++ b/tests/unit/processor/pre_detector/test_pre_detector.py @@ -11,8 +11,7 @@ class TestPreDetector(BaseProcessorTestCase): CONFIG = { "type": "pre_detector", - "generic_rules": ["tests/testdata/unit/pre_detector/rules/generic"], - "specific_rules": ["tests/testdata/unit/pre_detector/rules/specific"], + "rules": ["tests/testdata/unit/pre_detector/rules"], "outputs": [{"kafka": "pre_detector_alerts"}], "alert_ip_list_path": "tests/testdata/unit/pre_detector/alert_ips.yml", } @@ -345,7 +344,7 @@ def test_adds_timestamp_to_extra_data_if_provided_by_event(self): "@timestamp": "2024-08-12T12:13:04+00:00", "winlog": {"event_id": 123, "event_data": {"ServiceName": "VERY BAD"}}, } - self._load_specific_rule(rule) + self._load_rule(rule) detection_results = self.object.process(document) assert detection_results.data[0][0].get("@timestamp") == "2024-08-12T12:13:04Z" @@ -405,7 +404,7 @@ def test_adds_timestamp_to_extra_data_if_provided_by_event(self): ], ) def test_timestamp_is_normalized(self, testcase, rule, timestamp, expected): - self._load_specific_rule(rule) + self._load_rule(rule) document = { "@timestamp": timestamp, "winlog": {"event_id": 123, "event_data": {"ServiceName": "VERY BAD"}}, @@ -433,7 +432,7 @@ def test_custom_timestamp_field_can_be_used(self): "second_match": "something", "@timestamp": "19960531153655", } - self._load_specific_rule(rule) + self._load_rule(rule) detection_results = self.object.process(document) assert detection_results.data[0][0].get("custom_timestamp") == "2024-08-11T02:11:45Z" assert ( @@ -455,7 +454,7 @@ def test_appends_processing_warning_if_timestamp_could_not_be_parsed(self): document = { "@timestamp": "this is not a timestamp", } - self._load_specific_rule(rule) + self._load_rule(rule) detection_results = self.object.process(document) assert detection_results.warnings assert len(detection_results.warnings) == 1 diff --git a/tests/unit/processor/pre_detector/test_pre_detector_rule.py b/tests/unit/processor/pre_detector/test_pre_detector_rule.py index d32edf25e..3b40db8e9 100644 --- a/tests/unit/processor/pre_detector/test_pre_detector_rule.py +++ b/tests/unit/processor/pre_detector/test_pre_detector_rule.py @@ -6,8 +6,8 @@ from logprep.processor.pre_detector.rule import PreDetectorRule -@pytest.fixture(name="specific_rule_definition") -def fixture_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def fixture_rule_definition(): return { "filter": "message", "pre_detector": { @@ -164,25 +164,25 @@ class TestPreDetectorRule: ], ) def test_rules_equality( - self, specific_rule_definition, testcase, other_rule_definition, is_equal + self, rule_definition, testcase, other_rule_definition, is_equal ): - rule1 = PreDetectorRule._create_from_dict(specific_rule_definition) + rule1 = PreDetectorRule._create_from_dict(rule_definition) rule2 = PreDetectorRule._create_from_dict(other_rule_definition) assert (rule1 == rule2) == is_equal, testcase - specific_rule_definition["pre_detector"]["link"] = "some_link" + rule_definition["pre_detector"]["link"] = "some_link" other_rule_definition["pre_detector"]["link"] = "some_link" - rule1 = PreDetectorRule._create_from_dict(specific_rule_definition) + rule1 = PreDetectorRule._create_from_dict(rule_definition) rule2 = PreDetectorRule._create_from_dict(other_rule_definition) assert (rule1 == rule2) == is_equal, f"{testcase} (with link)" - def test_detection_data_link_is_not_none_does_exists(self, specific_rule_definition): - specific_rule_definition["pre_detector"]["link"] = "some_link" - rule = PreDetectorRule._create_from_dict(specific_rule_definition) + def test_detection_data_link_is_not_none_does_exists(self, rule_definition): + rule_definition["pre_detector"]["link"] = "some_link" + rule = PreDetectorRule._create_from_dict(rule_definition) assert "link" in rule.detection_data assert rule.detection_data["link"] == "some_link" - def test_detection_data_link_is_none_does_not_exist(self, specific_rule_definition): - specific_rule_definition["pre_detector"]["link"] = None - rule = PreDetectorRule._create_from_dict(specific_rule_definition) + def test_detection_data_link_is_none_does_not_exist(self, rule_definition): + rule_definition["pre_detector"]["link"] = None + rule = PreDetectorRule._create_from_dict(rule_definition) assert "link" not in rule.detection_data diff --git a/tests/unit/processor/pseudonymizer/test_pseudonymizer.py b/tests/unit/processor/pseudonymizer/test_pseudonymizer.py index 6b21e68de..39c3e4382 100644 --- a/tests/unit/processor/pseudonymizer/test_pseudonymizer.py +++ b/tests/unit/processor/pseudonymizer/test_pseudonymizer.py @@ -702,9 +702,8 @@ class TestPseudonymizer(BaseProcessorTestCase): "pubkey_analyst": "tests/testdata/unit/pseudonymizer/example_analyst_pub.pem", "pubkey_depseudo": "tests/testdata/unit/pseudonymizer/example_depseudo_pub.pem", "hash_salt": "a_secret_tasty_ingredient", - "specific_rules": ["tests/testdata/unit/pseudonymizer/rules/specific/"], - "generic_rules": ["tests/testdata/unit/pseudonymizer/rules/generic/"], - "regex_mapping": "tests/testdata/unit/pseudonymizer/rules/regex_mapping.yml", + "rules": ["tests/testdata/unit/pseudonymizer/rules"], + "regex_mapping": "tests/testdata/unit/pseudonymizer/regex_mapping.yml", "max_cached_pseudonyms": 1000000, } @@ -755,15 +754,15 @@ def test_config_validation(self, config_change, error, msg): def test_testcases(self, testcase, rule, event, expected, regex_mapping): if regex_mapping is not None: self.regex_mapping = regex_mapping - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(event) assert event == expected, testcase - def _load_specific_rule(self, rule): + def _load_rule(self, rule): config = deepcopy(self.CONFIG) config["regex_mapping"] = self.regex_mapping self.object = Factory.create({"pseudonymizer": config}) - super()._load_specific_rule(rule) + super()._load_rule(rule) self.object.setup() def test_pseudonymize_url_fields_not_in_pseudonymize(self): @@ -782,7 +781,7 @@ def test_pseudonymize_url_fields_not_in_pseudonymize(self): "url_fields": ["do_not_pseudo_this"], } self.regex_mapping = "tests/testdata/unit/pseudonymizer/pseudonymizer_regex_mapping.yml" - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(event) assert event["do_not_pseudo_this"] == url @@ -794,11 +793,11 @@ def test_replace_regex_keywords_by_regex_expression_is_idempotent(self): "pseudonymizer": {"mapping": {"something": "RE_WHOLE_FIELD"}}, "description": "description content irrelevant for these tests", } - self._load_specific_rule(rule_dict) # First call + self._load_rule(rule_dict) # First call expected_pattern = re.compile("(.*)") - assert self.object._specific_tree.rules[0].pseudonyms == {"something": expected_pattern} + assert self.object._rule_tree.rules[0].pseudonyms == {"something": expected_pattern} self.object._replace_regex_keywords_by_regex_expression() # Second Call - assert self.object._specific_tree.rules[0].pseudonyms == {"something": expected_pattern} + assert self.object._rule_tree.rules[0].pseudonyms == {"something": expected_pattern} def test_pseudonymize_string_adds_pseudonyms(self): self.object.result = ProcessorResult(processor_name="test") @@ -825,7 +824,7 @@ def test_resolve_from_cache_pseudonym(self): }, } } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.metrics.new_results = 0 self.object.metrics.cached_results = 0 self.object.metrics.num_cache_entries = 0 @@ -850,7 +849,7 @@ def test_resolve_from_cache_pseudonymize_urls(self): "pseudo_this": "https://www.pseudo.this.de", "and_pseudo_this": "https://www.pseudo.this.de", } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.metrics.new_results = 0 self.object.metrics.cached_results = 0 self.object.metrics.num_cache_entries = 0 @@ -917,7 +916,7 @@ def test_process_returns_extra_output(self): }, }, } - self._load_specific_rule(rule_dict) # First call + self._load_rule(rule_dict) # First call extra_output = self.object.process(event) assert extra_output.data assert isinstance(extra_output.data, list) @@ -954,7 +953,7 @@ def test_extra_output_contains_only_one_pseudonym_even_if_pseudonym_appears_mult }, }, } - self._load_specific_rule(rule_dict) # First call + self._load_rule(rule_dict) # First call extra_output = self.object.process(event) assert extra_output assert isinstance(extra_output.data, list) @@ -992,7 +991,7 @@ def test_extra_output_contains_different_pseudonyms_for_different_values(self): }, }, } - self._load_specific_rule(rule_dict) # First call + self._load_rule(rule_dict) # First call extra_output = self.object.process(event) assert extra_output.data assert isinstance(extra_output.data, list) @@ -1040,7 +1039,7 @@ def test_ignores_missing_field_but_add_warning(self): }, }, } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) extra_output = self.object.process(event) assert extra_output.data[0][0].get("pseudonym"), "pseudonym is set" assert "_pseudonymizer_missing_field_warning" in event.get("tags", []) @@ -1065,8 +1064,8 @@ def test_setup_raises_invalid_configuration_on_missing_regex_mapping(self): } }, } - self._load_specific_rule(rule_dict) - self.object._specific_rules[0].mapping["winlog.event_data.param2"] = "RE_DOES_NOT_EXIST" + self._load_rule(rule_dict) + self.object._tree_rules[0].mapping["winlog.event_data.param2"] = "RE_DOES_NOT_EXIST" error_message = ( r"Regex keyword 'RE_DOES_NOT_EXIST' not found in regex_mapping '.*\/regex_mapping.yml'" ) @@ -1092,7 +1091,7 @@ def test_cache_metrics_updated(self): }, }, } - self._load_specific_rule(rule_dict) + self._load_rule(rule_dict) self.object.metrics.new_results = 0 self.object.metrics.cached_results = 0 diff --git a/tests/unit/processor/pseudonymizer/test_pseudonymizer_rule.py b/tests/unit/processor/pseudonymizer/test_pseudonymizer_rule.py index eb094c820..82584ab30 100644 --- a/tests/unit/processor/pseudonymizer/test_pseudonymizer_rule.py +++ b/tests/unit/processor/pseudonymizer/test_pseudonymizer_rule.py @@ -6,8 +6,8 @@ from logprep.processor.pseudonymizer.rule import PseudonymizerRule -@pytest.fixture(name="specific_rule_definition") -def get_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def get_rule_definition(): return { "filter": 'winlog.event_id: 123 AND source_name: "Test123"', "pseudonymizer": { @@ -108,8 +108,8 @@ def test_create_from_dict_validates_config(self, rule, error, message): ], ) def test_rules_equality( - self, specific_rule_definition, testcase, other_rule_definition, is_equal + self, rule_definition, testcase, other_rule_definition, is_equal ): - rule_1 = PseudonymizerRule._create_from_dict(specific_rule_definition) + rule_1 = PseudonymizerRule._create_from_dict(rule_definition) rule_2 = PseudonymizerRule._create_from_dict(other_rule_definition) assert (rule_1 == rule_2) == is_equal, testcase diff --git a/tests/unit/processor/requester/test_requester.py b/tests/unit/processor/requester/test_requester.py index 17decf03a..04157ebbb 100644 --- a/tests/unit/processor/requester/test_requester.py +++ b/tests/unit/processor/requester/test_requester.py @@ -349,15 +349,14 @@ class TestRequester(BaseProcessorTestCase): CONFIG: dict = { "type": "requester", - "specific_rules": ["tests/testdata/unit/requester/specific_rules"], - "generic_rules": ["tests/testdata/unit/requester/generic_rules"], + "rules": ["tests/testdata/unit/requester/rules"], } @responses.activate @pytest.mark.parametrize("testcase, rule, event, expected, response_kwargs", test_cases) def test_testcases(self, testcase, rule, event, expected, response_kwargs): responses.add(responses.Response(**response_kwargs)) - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(event) assert event == expected, testcase @@ -370,7 +369,7 @@ def test_requester_testcases_failure_handling( ): if response_kwargs: responses.add(responses.Response(**response_kwargs)) - self._load_specific_rule(rule) + self._load_rule(rule) result = self.object.process(event) assert len(result.warnings) == 1 assert re.match(error_message, str(result.warnings[0])) diff --git a/tests/unit/processor/selective_extractor/test_selective_extractor.py b/tests/unit/processor/selective_extractor/test_selective_extractor.py index c10eb3a2d..5d50fc1be 100644 --- a/tests/unit/processor/selective_extractor/test_selective_extractor.py +++ b/tests/unit/processor/selective_extractor/test_selective_extractor.py @@ -11,8 +11,7 @@ class TestSelectiveExtractor(BaseProcessorTestCase): CONFIG = { "type": "selective_extractor", - "specific_rules": ["tests/testdata/unit/selective_extractor/rules/specific"], - "generic_rules": ["tests/testdata/unit/selective_extractor/rules/generic"], + "rules": ["tests/testdata/unit/selective_extractor/rules"], } def test_selective_extractor_does_not_change_orig_doc(self): @@ -40,7 +39,7 @@ def test_process_returns_tuple_list_with_extraction_fields_from_rule(self): }, } ) - self.object._specific_tree.add_rule(rule) + self.object._rule_tree.add_rule(rule) document = {field_name: "the value"} tuple_list = self.object.process(document) for filtered_event, _ in tuple_list.data: @@ -58,7 +57,7 @@ def test_process_returns_selective_extractor_target_topic(self): "outputs": [{"opensearch": "my topic"}], }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = {field_name: "test_message", "other": "field"} result = self.object.process(document) output = result.data[0][1][0] @@ -73,7 +72,7 @@ def test_process_returns_selective_extractor_target_output(self): "outputs": [{"opensearch": "index"}], }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = {field_name: "test_message", "other": "field"} result = self.object.process(document) output = result.data[0][1][0] @@ -88,7 +87,7 @@ def test_process_returns_extracted_fields(self): "outputs": [{"opensearch": "index"}], }, } - self._load_specific_rule(rule) + self._load_rule(rule) result = self.object.process(document) for filtered_event, *_ in result.data: if filtered_event == {"message": "test_message"}: @@ -104,13 +103,10 @@ def test_process_returns_none_when_no_extraction_field_matches(self): assert result.errors == [] assert result.processor_name == "Test Instance Name" - def test_gets_matching_rules_from_rules_trees(self): - rule_trees = [self.object._generic_tree, self.object._specific_tree] - assert len(rule_trees) > 0 - for tree in rule_trees: - matching_rules = tree.get_matching_rules({"message": "the message"}) - assert isinstance(matching_rules, list) - assert len(matching_rules) > 0 + def test_gets_matching_rules_from_rules_tree(self): + matching_rules = self.object._rule_tree.get_matching_rules({"message": "the message"}) + assert isinstance(matching_rules, list) + assert len(matching_rules) > 0 def test_apply_rules_is_called(self): with mock.patch( @@ -127,7 +123,7 @@ def test_process_extracts_dotted_fields(self): "outputs": [{"opensearch": "index"}], }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"message": "test_message", "other": {"message": "my message value"}} result = self.object.process(document) @@ -153,7 +149,7 @@ def test_process_extracts_dotted_fields_complains_on_missing_fields(self): "ignore_missing_fields": False, }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"message": "test_message", "other": {"message": "my message value"}} expected = { "message": "test_message", @@ -172,7 +168,7 @@ def test_process_extracts_dotted_fields_and_ignores_missing_fields(self): "ignore_missing_fields": True, }, } - self._load_specific_rule(rule) + self._load_rule(rule) document = {"message": "test_message", "other": {"message": "my message value"}} expected = { "message": "test_message", diff --git a/tests/unit/processor/selective_extractor/test_selective_extractor_rule.py b/tests/unit/processor/selective_extractor/test_selective_extractor_rule.py index 86417d685..ddf7fcb15 100644 --- a/tests/unit/processor/selective_extractor/test_selective_extractor_rule.py +++ b/tests/unit/processor/selective_extractor/test_selective_extractor_rule.py @@ -14,8 +14,8 @@ ) -@pytest.fixture(name="specific_rule_definition") -def fixture_specific_rule_definition(): +@pytest.fixture(name="rule_definition") +def fixture_rule_definition(): return { "filter": "test", "selective_extractor": { @@ -174,7 +174,7 @@ def test_rule_has_fields_from_directory_path(self, _): ], ) def test_rules_equality( - self, specific_rule_definition, testcase, other_rule_definition, is_equal + self, rule_definition, testcase, other_rule_definition, is_equal ): with mock.patch("pathlib.Path.is_file", return_value=True): read_lines = other_rule_definition.get("selective_extractor").get("extract_from_file") @@ -182,7 +182,7 @@ def test_rules_equality( read_lines = read_lines.encode("utf8") with mock.patch("pathlib.Path.read_bytes", return_value=read_lines): - rule1 = SelectiveExtractorRule._create_from_dict(specific_rule_definition) + rule1 = SelectiveExtractorRule._create_from_dict(rule_definition) rule2 = SelectiveExtractorRule._create_from_dict(other_rule_definition) assert (rule1 == rule2) == is_equal, testcase @@ -301,6 +301,6 @@ def test_rule_create_from_dict(self, rule_definition, read_lines, raised, messag extractor_rule = SelectiveExtractorRule._create_from_dict(rule_definition) assert isinstance(extractor_rule, SelectiveExtractorRule) - def test_rule_is_hashable(self, specific_rule_definition): - rule = SelectiveExtractorRule._create_from_dict(specific_rule_definition) + def test_rule_is_hashable(self, rule_definition): + rule = SelectiveExtractorRule._create_from_dict(rule_definition) assert isinstance(rule, Hashable) diff --git a/tests/unit/processor/string_splitter/test_string_splitter.py b/tests/unit/processor/string_splitter/test_string_splitter.py index 692a8c649..1b07eb131 100644 --- a/tests/unit/processor/string_splitter/test_string_splitter.py +++ b/tests/unit/processor/string_splitter/test_string_splitter.py @@ -57,19 +57,18 @@ class TestStringSplitter(BaseProcessorTestCase): CONFIG: dict = { "type": "string_splitter", - "specific_rules": ["tests/testdata/unit/string_splitter/specific/"], - "generic_rules": ["tests/testdata/unit/string_splitter/generic/"], + "rules": ["tests/testdata/unit/string_splitter/rules"], } @pytest.mark.parametrize("testcase, rule, event, expected", test_cases) def test_testcases(self, testcase, rule, event, expected): # pylint: disable=unused-argument - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(event) assert event == expected @pytest.mark.parametrize("testcase, rule, event, expected, error_message", failure_test_cases) def test_testcases_failure_handling(self, testcase, rule, event, expected, error_message): - self._load_specific_rule(rule) + self._load_rule(rule) result = self.object.process(event) assert len(result.warnings) == 1 assert re.match(error_message, str(result.warnings[0])) diff --git a/tests/unit/processor/template_replacer/test_template_replacer.py b/tests/unit/processor/template_replacer/test_template_replacer.py index 2eca9f0fd..8b8ed1a6c 100644 --- a/tests/unit/processor/template_replacer/test_template_replacer.py +++ b/tests/unit/processor/template_replacer/test_template_replacer.py @@ -12,8 +12,7 @@ class TestTemplateReplacer(BaseProcessorTestCase): CONFIG = { "type": "template_replacer", - "generic_rules": ["tests/testdata/unit/template_replacer/rules/generic"], - "specific_rules": ["tests/testdata/unit/template_replacer/rules/specific"], + "rules": ["tests/testdata/unit/template_replacer/rules"], "template": "tests/testdata/unit/template_replacer/replacer_template.yml", "pattern": { "delimiter": "-", @@ -25,12 +24,8 @@ class TestTemplateReplacer(BaseProcessorTestCase): } @property - def generic_rules_dirs(self): - return self.CONFIG.get("generic_rules") - - @property - def specific_rules_dirs(self): - return self.CONFIG.get("specific_rules") + def rules_dirs(self): + return self.CONFIG.get("rules") def setup_method(self): super().setup_method() diff --git a/tests/unit/processor/test_process.py b/tests/unit/processor/test_process.py index 704622751..600d10298 100644 --- a/tests/unit/processor/test_process.py +++ b/tests/unit/processor/test_process.py @@ -21,39 +21,18 @@ def test_process(self, mock_process_rule_tree): { "dummy": { "type": "calculator", - "generic_rules": [], - "specific_rules": [], + "rules": [], } } ) processor.process({}) mock_process_rule_tree.assert_called() - assert mock_process_rule_tree.call_count == 2 - - @mock.patch("logprep.abc.processor.Processor._process_rule_tree") - def test_process_specific_before_generic(self, mock_process_rule_tree): - processor = Factory.create( - { - "dummy": { - "type": "calculator", - "generic_rules": [], - "specific_rules": [], - } - } - ) - processor.process({}) - assert mock_process_rule_tree.call_count == 2 - mock_calls = [ - call({}, processor._specific_tree), - call({}, processor._generic_tree), - ] - mock_process_rule_tree.assert_has_calls(mock_calls, any_order=False) + assert mock_process_rule_tree.call_count == 1 def test_apply_processor_multiple_times_until_no_new_rule_matches(self): config = { "type": "dissector", - "specific_rules": [], - "generic_rules": [], + "rules": [], "apply_multiple_times": True, } processor = Factory.create({"custom_lister": config}) @@ -67,8 +46,8 @@ def test_apply_processor_multiple_times_until_no_new_rule_matches(self): } rule_one = DissectorRule._create_from_dict(rule_one_dict) rule_two = DissectorRule._create_from_dict(rule_two_dict) - processor._specific_tree.add_rule(rule_one) - processor._specific_tree.add_rule(rule_two) + processor._rule_tree.add_rule(rule_one) + processor._rule_tree.add_rule(rule_two) event = {"message": "time [proto col] url"} expected_event = { "message": "time [proto col] url", @@ -82,7 +61,7 @@ def test_apply_processor_multiple_times_until_no_new_rule_matches(self): assert event == expected_event def test_apply_processor_multiple_times_not_enabled(self): - config = {"type": "dissector", "specific_rules": [], "generic_rules": []} + config = {"type": "dissector", "rules": []} processor = Factory.create({"custom_lister": config}) rule_one_dict = { "filter": "message", @@ -94,8 +73,8 @@ def test_apply_processor_multiple_times_not_enabled(self): } rule_one = DissectorRule._create_from_dict(rule_one_dict) rule_two = DissectorRule._create_from_dict(rule_two_dict) - processor._specific_tree.add_rule(rule_one) - processor._specific_tree.add_rule(rule_two) + processor._rule_tree.add_rule(rule_one) + processor._rule_tree.add_rule(rule_two) event = {"message": "time [proto col] url"} expected_event = { "message": "time [proto col] url", @@ -108,62 +87,16 @@ def test_apply_processor_multiple_times_not_enabled(self): @pytest.mark.parametrize("execution_number", range(5)) # repeat test to ensure determinism def test_applies_rules_in_deterministic_order(self, execution_number): - config = {"type": "generic_adder", "specific_rules": [], "generic_rules": []} + config = {"type": "generic_adder", "rules": []} processor = Factory.create({"custom_lister": config}) rule_one_dict = {"filter": "val", "generic_adder": {"add": {"some": "value"}}} rule_two_dict = {"filter": "NOT something", "generic_adder": {"add": {"something": "else"}}} rule_one = GenericAdderRule._create_from_dict(rule_one_dict) rule_two = GenericAdderRule._create_from_dict(rule_two_dict) - processor._specific_tree.add_rule(rule_one) - processor._specific_tree.add_rule(rule_two) + processor._rule_tree.add_rule(rule_one) + processor._rule_tree.add_rule(rule_two) event = {"val": "content"} with mock.patch("logprep.abc.processor.Processor._apply_rules_wrapper") as mock_callback: expected_call_order = [call(event, rule_one), call(event, rule_two)] processor.process(event=event) mock_callback.assert_has_calls(expected_call_order, any_order=False) - - def test_processes_generic_rules_after_processor_error_in_specific_rules(self): - config = Configuration() - config.pipeline = [ - {"adder": {"type": "generic_adder", "specific_rules": [], "generic_rules": []}} - ] - specific_rule_one_dict = { - "filter": "val", - "generic_adder": {"add": {"first": "value", "second": "value"}}, - } - specific_rule_two_dict = { - "filter": "val", - "generic_adder": {"add": {"third": "value", "fourth": "value"}}, - } - generic_rule_dict = { - "filter": "val", - "generic_adder": {"add": {"fifth": "value", "sixth": "value"}}, - } - specific_rule_one = GenericAdderRule._create_from_dict(specific_rule_one_dict) - specific_rule_two = GenericAdderRule._create_from_dict(specific_rule_two_dict) - generic_rule = GenericAdderRule._create_from_dict(generic_rule_dict) - event = {"val": "content", "first": "exists already"} - expected_event = { - "val": "content", - "first": "exists already", - "second": "value", - "third": "value", - "fourth": "value", - "fifth": "value", - "sixth": "value", - "tags": ["_generic_adder_failure"], - } - pipeline = Pipeline(config=config) - pipeline._pipeline[0]._generic_tree.add_rule(generic_rule) - pipeline._pipeline[0]._specific_tree.add_rule(specific_rule_two) - pipeline._pipeline[0]._specific_tree.add_rule(specific_rule_one) - res = pipeline.process_event(event) - assert len(res.results[0].warnings) == 1 - assert isinstance(res.results[0].warnings[0], FieldExistsWarning) - re.match( - "The following fields could not be written, " - "because one or more subfields existed and could not be extended: first", - str(res.results[0].warnings[0]), - ) - - assert event == expected_event diff --git a/tests/unit/processor/timestamp_differ/test_timestamp_differ.py b/tests/unit/processor/timestamp_differ/test_timestamp_differ.py index dd082562a..cd8943772 100644 --- a/tests/unit/processor/timestamp_differ/test_timestamp_differ.py +++ b/tests/unit/processor/timestamp_differ/test_timestamp_differ.py @@ -435,19 +435,18 @@ class TestTimestampDiffer(BaseProcessorTestCase): CONFIG: dict = { "type": "timestamp_differ", - "specific_rules": ["tests/testdata/unit/timestamp_differ/specific_rules"], - "generic_rules": ["tests/testdata/unit/timestamp_differ/generic_rules"], + "rules": ["tests/testdata/unit/timestamp_differ/rules"], } @pytest.mark.parametrize("testcase, rule, event, expected", test_cases) def test_testcases(self, testcase, rule, event, expected): - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(event) assert event == expected, testcase @pytest.mark.parametrize("testcase, rule, event, expected, error_message", failure_test_cases) def test_testcases_failure_handling(self, testcase, rule, event, expected, error_message): - self._load_specific_rule(rule) + self._load_rule(rule) result = self.object.process(event) assert len(result.warnings) == 1 assert re.match(error_message, str(result.warnings[0])) diff --git a/tests/unit/processor/timestamper/test_timestamper.py b/tests/unit/processor/timestamper/test_timestamper.py index 228d5ffc9..c48078ab0 100644 --- a/tests/unit/processor/timestamper/test_timestamper.py +++ b/tests/unit/processor/timestamper/test_timestamper.py @@ -302,8 +302,7 @@ class TestTimestamper(BaseProcessorTestCase): CONFIG: dict = { "type": "timestamper", - "specific_rules": ["tests/testdata/unit/timestamper/specific_rules"], - "generic_rules": ["tests/testdata/unit/timestamper/generic_rules"], + "rules": ["tests/testdata/unit/timestamper/rules"], } def test_is_field_manager_implementation(self): @@ -312,13 +311,13 @@ def test_is_field_manager_implementation(self): @pytest.mark.parametrize("testcase, rule, event, expected", test_cases) def test_testcases(self, testcase, rule, event, expected): - self._load_specific_rule(rule) + self._load_rule(rule) self.object.process(event) assert event == expected, testcase @pytest.mark.parametrize("testcase, rule, event, expected, error_message", failure_test_cases) def test_testcases_failure_handling(self, testcase, rule, event, expected, error_message): - self._load_specific_rule(rule) + self._load_rule(rule) result = self.object.process(event) assert len(result.warnings) == 1 assert re.match(rf".*{error_message}", str(result.warnings[0])) diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py index e31e00605..4fd26e50a 100644 --- a/tests/unit/test_configuration.py +++ b/tests/unit/test_configuration.py @@ -38,20 +38,18 @@ def teardown_method(self): def test_reads_test_config(self): test_config = { "type": "mock_processor", - "specific_rules": ["tests/testdata/unit/dissector/rules/specific/"], - "generic_rules": ["tests/testdata/unit/dissector/rules/generic/"], + "rules": ["tests/testdata/unit/dissector/rules"], "mandatory_attribute": "I am mandatory", "optional_attribute": "I am optional", } config = Configuration.create("dummy name", test_config) assert config.type == "mock_processor" assert config.mandatory_attribute == "I am mandatory" - assert config.generic_rules == ["tests/testdata/unit/dissector/rules/generic/"] + assert config.rules == ["tests/testdata/unit/dissector/rules"] def test_raises_on_missing_type(self): test_config = { - "specific_rules": ["tests/testdata/unit/dissector/rules/specific/"], - "generic_rules": ["tests/testdata/unit/dissector/rules/generic/"], + "rules": ["tests/testdata/unit/dissector/rules"], "mandatory_attribute": "I am mandatory", "optional_attribute": "I am optional", } @@ -61,8 +59,7 @@ def test_raises_on_missing_type(self): def test_raises_on_unknown_processor(self): test_config = { "type": "unknown_processor", - "specific_rules": ["tests/testdata/unit/dissector/rules/specific/"], - "generic_rules": ["tests/testdata/unit/dissector/rules/generic/"], + "rules": ["tests/testdata/unit/dissector/rules"], "mandatory_attribute": "I am mandatory", "optional_attribute": "I am optional", } @@ -72,8 +69,7 @@ def test_raises_on_unknown_processor(self): def test_raises_if_one_mandatory_field_is_missing(self): test_config = { "type": "mock_processor", - "specific_rules": ["tests/testdata/unit/dissector/rules/specific/"], - "generic_rules": ["tests/testdata/unit/dissector/rules/generic/"], + "rules": ["tests/testdata/unit/dissector/rules"], "optional_attribute": "I am optional", } with pytest.raises( @@ -84,31 +80,26 @@ def test_raises_if_one_mandatory_field_is_missing(self): def test_raises_if_mandatory_attribute_from_base_is_missing(self): test_config = { "type": "mock_processor", - "generic_rules": ["tests/testdata/unit/dissector/rules/generic/"], "mandatory_attribute": "does not matter", } with pytest.raises( TypeError, - match=r"missing 1 required .* argument: 'specific_rules'", + match=r"missing 1 required .* argument: 'rules'", ): Configuration.create("dummy name", test_config) def test_raises_if_multiple_mandatory_field_are_missing(self): - test_config = { - "type": "mock_processor", - "generic_rules": ["tests/testdata/unit/dissector/rules/generic/"], - } + test_config = {"type": "mock_processor"} with pytest.raises( TypeError, - match=r"missing 2 required .* arguments: .*'specific_rules' and 'mandatory_attribute'", + match=r"missing 2 required .* arguments: .*'rules' and 'mandatory_attribute'", ): Configuration.create("dummy name", test_config) def test_raises_on_unknown_field(self): test_config = { "type": "mock_processor", - "specific_rules": ["tests/testdata/unit/dissector/rules/specific/"], - "generic_rules": ["tests/testdata/unit/dissector/rules/generic/"], + "rules": ["tests/testdata/unit/dissector/rules"], "mandatory_attribute": "I am mandatory", "optional_attribute": "I am optional", "i_shoul_not_be_here": "does not matter", @@ -119,8 +110,7 @@ def test_raises_on_unknown_field(self): def test_init_non_mandatory_fields_with_default(self): test_config = { "type": "mock_processor", - "specific_rules": ["tests/testdata/unit/dissector/rules/specific/"], - "generic_rules": ["tests/testdata/unit/dissector/rules/generic/"], + "rules": ["tests/testdata/unit/dissector/rules"], "mandatory_attribute": "I am mandatory", } config = Configuration.create("dummy name", test_config) @@ -130,8 +120,7 @@ def test_init_non_mandatory_fields_with_default(self): def test_init_optional_field_in_sub_class(self): test_config = { "type": "mock_processor", - "specific_rules": ["tests/testdata/unit/dissector/rules/specific/"], - "generic_rules": ["tests/testdata/unit/dissector/rules/generic/"], + "rules": ["tests/testdata/unit/dissector/rules"], "mandatory_attribute": "I am mandatory", "optional_attribute": "I am optional", } @@ -141,8 +130,7 @@ def test_init_optional_field_in_sub_class(self): def test_init_optional_field_in_base_class(self): test_config = { "type": "mock_processor", - "specific_rules": ["tests/testdata/unit/dissector/rules/specific/"], - "generic_rules": ["tests/testdata/unit/dissector/rules/generic/"], + "rules": ["tests/testdata/unit/dissector/rules"], "mandatory_attribute": "I am mandatory", "tree_config": "tests/testdata/unit/tree_config.json", } diff --git a/tests/unit/test_factory.py b/tests/unit/test_factory.py index f1e6ec6b4..56ad306ce 100644 --- a/tests/unit/test_factory.py +++ b/tests/unit/test_factory.py @@ -77,8 +77,7 @@ def test_create_pseudonymizer_returns_pseudonymizer_processor(): "pubkey_analyst": "tests/testdata/unit/pseudonymizer/example_analyst_pub.pem", "pubkey_depseudo": "tests/testdata/unit/pseudonymizer/example_depseudo_pub.pem", "hash_salt": "a_secret_tasty_ingredient", - "specific_rules": ["tests/testdata/unit/pseudonymizer/rules/specific"], - "generic_rules": ["tests/testdata/unit/pseudonymizer/rules/generic"], + "rules": ["tests/testdata/unit/pseudonymizer/rules"], "regex_mapping": "tests/testdata/unit/pseudonymizer/rules/regex_mapping.yml", "outputs": [{"kafka": "topic"}], "max_cached_pseudonyms": 1000000, @@ -95,8 +94,7 @@ def test_create_clusterer_returns_clusterer_processor(): "clusterer": { "type": "clusterer", "output_field_name": "cluster_signature", - "specific_rules": ["tests/testdata/unit/clusterer/rules/specific"], - "generic_rules": ["tests/testdata/unit/clusterer/rules/generic"], + "rules": ["tests/testdata/unit/clusterer/rules"], } } ) @@ -119,8 +117,7 @@ def test_create_labeler_creates_labeler_processor(): "labelername": { "type": "labeler", "schema": path_to_schema, - "generic_rules": [path_to_single_rule], - "specific_rules": [path_to_single_rule], + "rules": [path_to_single_rule], } } ) @@ -133,23 +130,16 @@ def test_creates_calculator_with_inline_rules(): { "calculator": { "type": "calculator", - "generic_rules": [ + "rules": [ { "filter": "message", "calculator": {"target_field": "target", "calc": "1 + 1"}, }, ], - "specific_rules": [ - { - "filter": "message", - "calculator": {"target_field": "target", "calc": "1 + 3"}, - }, - ], } } ) - assert len(processor._generic_rules) == 1 - assert len(processor._specific_rules) == 1 + assert len(processor._tree_rules) == 1 def test_creates_calculator_with_inline_rules_and_files(): @@ -157,27 +147,19 @@ def test_creates_calculator_with_inline_rules_and_files(): { "calculator": { "type": "calculator", - "generic_rules": [ + "rules": [ { "filter": "message1", "calculator": {"target_field": "target", "calc": "1 + 1"}, }, - "tests/testdata/unit/calculator/generic_rules/calculator.json", - ], - "specific_rules": [ - { - "filter": "message", - "calculator": {"target_field": "target", "calc": "1 + 3"}, - }, - "tests/testdata/unit/calculator/specific_rules/calculator.json", + "tests/testdata/unit/calculator/rules/calculator_1.json", ], } } ) - assert len(processor._generic_rules) == 2 - assert len(processor._specific_rules) == 2 - assert processor._generic_rules[0].filter_str == "message1: *" - assert processor._generic_rules[1].filter_str == "(field1: * AND field2: *)" + assert len(processor._tree_rules) == 2 + assert processor._tree_rules[0].filter_str == "message1: *" + assert processor._tree_rules[1].filter_str == "(field1: * AND field2: *)" def test_creates_calculator_with_inline_rules_and_file_and_directory(): @@ -185,25 +167,17 @@ def test_creates_calculator_with_inline_rules_and_file_and_directory(): { "calculator": { "type": "calculator", - "generic_rules": [ + "rules": [ { "filter": "message", "calculator": {"target_field": "target", "calc": "1 + 1"}, }, - "tests/testdata/unit/calculator/generic_rules/", - ], - "specific_rules": [ - { - "filter": "message", - "calculator": {"target_field": "target", "calc": "1 + 3"}, - }, - "tests/testdata/unit/calculator/specific_rules/calculator.json", + "tests/testdata/unit/calculator/rules/", ], } } ) - assert len(processor._generic_rules) == 2 - assert len(processor._specific_rules) == 2 + assert len(processor._tree_rules) == 3 def test_dummy_input_creates_dummy_input_connector(): diff --git a/tests/unit/util/test_auto_rule_tester.py b/tests/unit/util/test_auto_rule_tester.py index 6424a0557..d8d37b186 100644 --- a/tests/unit/util/test_auto_rule_tester.py +++ b/tests/unit/util/test_auto_rule_tester.py @@ -26,14 +26,13 @@ def test_get_rule_dict_valid_file(self, auto_rule_tester): rules_pn = {"dummy": {"type": "dummy", "rules": []}} file = "rule.yml" root = "tests/testdata/auto_tests/dummy" - rule_dirs_type = "doesnt_matter" - auto_rule_tester._get_rule_dict(file, root, processor_name, rules_pn, rule_dirs_type) + auto_rule_tester._get_rule_dict(file, root, processor_name, rules_pn) # raw literal expected_rule_dict = [ { - "doesnt_matter": [ + "rules": [ { "filter": 'winlog.event_data.param2: "pause"', "labeler": {"label": {"action": ["terminate"]}}, @@ -81,9 +80,8 @@ def test_get_rule_dict_target_rule_idx_not_found(self, auto_rule_tester): rules_pn = {"dummy": {"type": "dummy", "rules": []}} file = "rule.yml" root = "tests/testdata/auto_tests/dummy" - rule_dirs_type = "doesnt_matter" - auto_rule_tester._get_rule_dict(file, root, processor_name, rules_pn, rule_dirs_type) + auto_rule_tester._get_rule_dict(file, root, processor_name, rules_pn) def remove_dict_with_target_rule_idx(list_of_dicts): for idx, d in enumerate(list_of_dicts): @@ -189,8 +187,7 @@ def test_does_run_if_rules_exist(self, auto_rule_tester): { "dissector": { "type": "dissector", - "specific_rules": ["tests/testdata/auto_tests/dissector/rules/specific/"], - "generic_rules": ["tests/testdata/auto_tests/dissector/rules/generic/"], + "rules": ["tests/testdata/auto_tests/dissector/rules"], } } ] @@ -224,15 +221,14 @@ def test_pseudonymizer_specific_setup_called_on_load_rules( "pubkey_analyst": "tests/testdata/unit/pseudonymizer/example_analyst_pub.pem", "pubkey_depseudo": "tests/testdata/unit/pseudonymizer/example_depseudo_pub.pem", "hash_salt": "a_secret_tasty_ingredient", - "specific_rules": ["tests/testdata/unit/pseudonymizer/rules/specific/"], - "generic_rules": ["tests/testdata/unit/pseudonymizer/rules/generic/"], - "regex_mapping": "tests/testdata/unit/pseudonymizer/rules/regex_mapping.yml", + "rules": ["tests/testdata/unit/pseudonymizer/rules"], + "regex_mapping": "tests/testdata/unit/pseudonymizer/regex_mapping.yml", "max_cached_pseudonyms": 1000000, } mock_replace_regex_keywords_by_regex_expression.assert_not_called() processor = auto_rule_tester._get_processor_instance("pseudonymizer", pseudonymizer_cfg) auto_rule_tester._reset(processor) # Called every time by auto tester before adding rules - auto_rule_tester._load_rules(processor, "specific_rules") + auto_rule_tester._load_rules(processor) assert mock_replace_regex_keywords_by_regex_expression.call_count == 1 @mock.patch("logprep.processor.list_comparison.processor.ListComparison.setup") @@ -241,8 +237,7 @@ def test_list_comparison_specific_setup_called_on_load_rules( ): list_comparison_cfg = { "type": "list_comparison", - "specific_rules": ["tests/testdata/unit/list_comparison/rules/specific"], - "generic_rules": ["tests/testdata/unit/list_comparison/rules/generic"], + "rules": ["tests/testdata/unit/list_comparison/rules"], "tree_config": "tests/testdata/unit/shared_data/tree_config.json", "list_search_base_path": "tests/testdata/unit/list_comparison/rules", } @@ -251,7 +246,7 @@ def test_list_comparison_specific_setup_called_on_load_rules( auto_rule_tester._reset( processor ) # Called every time by auto tester before adding rules instead - auto_rule_tester._load_rules(processor, "specific_rules") + auto_rule_tester._load_rules(processor) mock_setup.assert_called_once() def test_full_auto_rule_test_run(self, auto_rule_tester, capsys): @@ -259,25 +254,25 @@ def test_full_auto_rule_test_run(self, auto_rule_tester, capsys): auto_rule_tester.run() expected_rules_with_tests = [ "with tests", - "tests/testdata/auto_tests/labeler/rules/generic/auto_test_labeling_match.json", - "tests/testdata/auto_tests/labeler/rules/specific/auto_test_labeling_mismatch.json", - "tests/testdata/auto_tests/dissector/rules/generic/auto_test_match.json", - "tests/testdata/auto_tests/dissector/rules/specific/auto_test_mismatch.json", - "tests/testdata/auto_tests/dropper/rules/generic/drop_field.json", - "tests/testdata/auto_tests/dropper/rules/specific/drop_field.json", - "tests/testdata/auto_tests/pre_detector/rules/generic/auto_test_pre_detector_match.json", - "tests/testdata/auto_tests/pre_detector/rules/specific/auto_test_pre_detector_mismatch.json", - "tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_mismatch.json", - "tests/testdata/auto_tests/pseudonymizer/rules/generic/auto_test_pseudonymizer_match.json", - "tests/testdata/auto_tests/template_replacer/rules/generic/template_replacer.json", - "tests/testdata/auto_tests/template_replacer/rules/specific/template_replacer.json", + "tests/testdata/auto_tests/labeler/rules/auto_test_labeling_match.json", + "tests/testdata/auto_tests/labeler/rules/auto_test_labeling_mismatch.json", + "tests/testdata/auto_tests/dissector/rules/auto_test_match.json", + "tests/testdata/auto_tests/dissector/rules/auto_test_mismatch.json", + "tests/testdata/auto_tests/dropper/rules/drop_field_1.json", + "tests/testdata/auto_tests/dropper/rules/drop_field_2.json", + "tests/testdata/auto_tests/pre_detector/rules/auto_test_pre_detector_match.json", + "tests/testdata/auto_tests/pre_detector/rules/auto_test_pre_detector_mismatch.json", + "tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_mismatch.json", + "tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_match.json", + "tests/testdata/auto_tests/template_replacer/rules/template_replacer_1.json", + "tests/testdata/auto_tests/template_replacer/rules/template_replacer_2.json", ] expected_rules_without_tests = [ "without tests", - "tests/testdata/auto_tests/labeler/rules/specific/auto_test_labeling_no_test_.json", - "tests/testdata/auto_tests/dissector/rules/specific/auto_test_no_test_.json", - "tests/testdata/auto_tests/pre_detector/rules/specific/auto_test_pre_detector_no_test_.json", - "tests/testdata/auto_tests/pseudonymizer/rules/specific/auto_test_pseudonymizer_no_test_.json", + "tests/testdata/auto_tests/labeler/rules/auto_test_labeling_no_test_.json", + "tests/testdata/auto_tests/dissector/rules/auto_test_no_test_.json", + "tests/testdata/auto_tests/pre_detector/rules/auto_test_pre_detector_no_test_.json", + "tests/testdata/auto_tests/pseudonymizer/rules/auto_test_pseudonymizer_no_test_.json", ] expected_overall_results = [ diff --git a/tests/unit/util/test_configuration.py b/tests/unit/util/test_configuration.py index 35cb2dca1..a432e27a2 100644 --- a/tests/unit/util/test_configuration.py +++ b/tests/unit/util/test_configuration.py @@ -201,8 +201,7 @@ def test_pipeline_property_is_merged_from_configs(self, tmp_path): type: labeler schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true - specific_rules: [] - generic_rules: [] + rules: [] """ ) second_config = tmp_path / "pipeline2.yml" @@ -211,8 +210,7 @@ def test_pipeline_property_is_merged_from_configs(self, tmp_path): pipeline: - dissectorname: type: dissector - specific_rules: [] - generic_rules: [] + rules: [] """ ) config = Configuration.from_sources([str(first_config), str(second_config)]) @@ -245,10 +243,8 @@ def test_create_from_sources_loads_rules(self): labeler = config.pipeline[2] assert isinstance(labeler, dict) assert isinstance(labeler["labelername"], dict) - assert isinstance(labeler["labelername"]["specific_rules"], list) - assert isinstance(labeler["labelername"]["generic_rules"], list) - assert isinstance(labeler["labelername"]["specific_rules"][0], dict) - assert isinstance(labeler["labelername"]["generic_rules"][0], dict) + assert isinstance(labeler["labelername"]["rules"], list) + assert isinstance(labeler["labelername"]["rules"][0], dict) def test_verify_passes_for_valid_configuration(self): try: @@ -281,21 +277,19 @@ def test_verify_passes_for_valid_configuration(self): { "processor_name": { "type": "dissector", - "specific_rules": [ + "rules": [ { "filter": "message", "dissector": { "mapping": {"message": "%{source} %{target}"} }, "description": "do nothing rule for dissector", - } - ], - "generic_rules": [ + }, { "filter": "message", "dissector": "THIS SHOULD BE A DICT", "description": "do nothing rule for dissector", - } + }, ], } } @@ -313,21 +307,19 @@ def test_verify_passes_for_valid_configuration(self): { "processor_name": { "type": "dissector", - "specific_rules": [ + "rules": [ { "filter": "message", "dissector": { "mapping": {"message": "%{source} %{target}"} }, "description": "do nothing rule for dissector", - } - ], - "generic_rules": [ + }, { "filter": "message", "dissector": "THIS SHOULD BE A DICT", "description": "do nothing rule for dissector", - } + }, ], }, }, @@ -373,22 +365,6 @@ def test_verify_passes_for_valid_configuration(self): }, 1, ), - ( - "generic_rules missing from processor", - { - "pipeline": [ - { - "labelername": { - "type": "labeler", - "schema": "examples/exampledata/rules/labeler/schema.json", - "include_parent_labels": "on", - "specific_rules": ["examples/exampledata/rules/labeler/specific"], - } - } - ] - }, - 1, - ), ( "unknown option without spaces in processor", { @@ -398,8 +374,7 @@ def test_verify_passes_for_valid_configuration(self): "type": "labeler", "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["examples/exampledata/rules/labeler/specific"], - "generic_rules": ["examples/exampledata/rules/labeler/generic"], + "rules": ["examples/exampledata/rules/labeler/rules"], "SOME_UNKNOWN_OPTION": "FOO", } } @@ -416,8 +391,7 @@ def test_verify_passes_for_valid_configuration(self): "type": "labeler", "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["examples/exampledata/rules/labeler/specific"], - "generic_rules": ["examples/exampledata/rules/labeler/generic"], + "rules": ["examples/exampledata/rules/labeler/rules"], "SOME UNKNOWN OPTION": "FOO", } } @@ -449,8 +423,7 @@ def test_verify_passes_for_valid_configuration(self): "type": "labeler", "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["examples/exampledata/rules/labeler/specific"], - "generic_rules": ["examples/exampledata/rules/labeler/generic"], + "rules": ["examples/exampledata/rules/labeler/rules"], "SOME UNKNOWN OPTION": "FOO", } }, @@ -461,19 +434,14 @@ def test_verify_passes_for_valid_configuration(self): "pubkey_analyst": "tests/testdata/unit/pseudonymizer/example_analyst_pub.pem", "pubkey_depseudo": "tests/testdata/unit/pseudonymizer/example_depseudo_pub.pem", "hash_salt": "a_secret_tasty_ingredient", - "specific_rules": [ - "tests/testdata/unit/pseudonymizer/rules/specific/" - ], - "generic_rules": [ - "tests/testdata/unit/pseudonymizer/rules/generic/" - ], + "rules": ["tests/testdata/unit/pseudonymizer/rules"], "regex_mapping": "tests/testdata/unit/pseudonymizer/rules/regex_mapping.yml", "max_cached_pseudonyms": 1000000, } }, ], }, - 2, + 3, ), ( "rule with not existent output", @@ -483,8 +451,7 @@ def test_verify_passes_for_valid_configuration(self): { "selective_extractor": { "type": "selective_extractor", - "generic_rules": [], - "specific_rules": [ + "rules": [ { "filter": "message", "selective_extractor": { @@ -535,10 +502,8 @@ def test_verify_verifies_config(self, tmp_path, test_case, test_config, error_co type: labeler schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true - specific_rules: - - examples/exampledata/rules/labeler/specific - generic_rules: - - examples/exampledata/rules/labeler/generic + rules: + - examples/exampledata/rules/labeler/rules """, "LOGPREP_OUTPUT": """ output: @@ -615,7 +580,7 @@ def test_duplicate_rule_id_per_processor_raises(self, tmp_path): pipeline: - my dissector: type: dissector - specific_rules: + rules: - filter: message dissector: id: same id @@ -626,7 +591,6 @@ def test_duplicate_rule_id_per_processor_raises(self, tmp_path): id: same id mapping: message: "%{other_field} %{next_field}" - generic_rules: [] """ ) with pytest.raises(InvalidConfigurationErrors) as raised: @@ -649,13 +613,12 @@ def test_duplicate_rule_id_in_different_rule_trees_per_processor_raises(self, tm pipeline: - my dissector: type: dissector - specific_rules: + rules: - filter: message dissector: id: same id mapping: message: "%{new_field} %{next_field}" - generic_rules: - filter: message dissector: id: same id @@ -791,8 +754,7 @@ def test_reload_raises_on_invalid_processor_config(self, tmp_path): type: labeler schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true - specific_rules: [] - generic_rules: [] + rules: [] input: dummy: type: dummy_input @@ -816,8 +778,7 @@ def test_reload_raises_on_invalid_processor_config(self, tmp_path): type: labeler schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true - specific_rules: [] - generic_rules: [] + rules: [] - new_processor: type: THIS SHOULD BE A VALID PROCESSOR input: @@ -865,8 +826,7 @@ def test_reload_raises_on_same_version(self, tmp_path): type: labeler schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true - specific_rules: [] - generic_rules: [] + rules: [] input: dummy: type: dummy_input @@ -888,8 +848,8 @@ def test_as_dict_returns_config(self): assert len(config_dict["output"]) == 1, "only last output should be in config" assert len(config_dict["pipeline"]) == 4, "all processors should be in config" labeler = config_dict["pipeline"][2]["labelername"] - assert len(labeler["specific_rules"]) == 1 - assert isinstance(labeler["specific_rules"][0], dict) + assert len(labeler["rules"]) == 2 + assert isinstance(labeler["rules"][0], dict) def test_as_json_returns_json(self): config = Configuration.from_sources([path_to_config, path_to_only_output_config]) @@ -951,8 +911,7 @@ def test_reload_sets_new_pipline(self, config_path): { "new_processor": { "type": "field_manager", - "generic_rules": [], - "specific_rules": [], + "rules": [], } } ) @@ -1040,8 +999,7 @@ def test_config_with_single_json_rule(self, config_path): { "my dissector": { "type": "dissector", - "specific_rules": [], - "generic_rules": [ + "rules": [ { "filter": "message", "dissector": { @@ -1069,8 +1027,7 @@ def test_config_with_missing_environment_variable_and_other_failure_raises(self, pipeline: - labelername: type: DOES_NOT_EXIST - generic_rules: [] - specific_rules: [] + rules: [] input: dummy: type: dummy_input @@ -1090,9 +1047,8 @@ def test_processor_config_with_file_path(self, config_path): pipeline: - the almighty dissector: type: dissector - generic_rules: - - tests/testdata/unit/dissector/generic_rules/dissector_rule.json - specific_rules: [] + rules: + - tests/testdata/unit/dissector/rules/dissector_rule_1.json input: dummy: type: dummy_input @@ -1104,7 +1060,7 @@ def test_processor_config_with_file_path(self, config_path): ) config = Configuration.from_sources([str(config_path)]) assert len(config.pipeline) == 1 - assert len(config.pipeline[0]["the almighty dissector"]["generic_rules"]) == 1 + assert len(config.pipeline[0]["the almighty dissector"]["rules"]) == 1 @responses.activate def test_processor_config_with_url_path(self, tmp_path): @@ -1114,9 +1070,8 @@ def test_processor_config_with_url_path(self, tmp_path): pipeline: - the almighty dissector: type: dissector - generic_rules: + rules: - http://localhost/dissector_rule.json - specific_rules: [] input: dummy: type: dummy_input @@ -1144,7 +1099,7 @@ def test_processor_config_with_url_path(self, tmp_path): ) config = Configuration.from_sources([str(config_path)]) assert len(config.pipeline) == 1 - assert len(config.pipeline[0]["the almighty dissector"]["generic_rules"]) == 1 + assert len(config.pipeline[0]["the almighty dissector"]["rules"]) == 1 def test_verify_environment_raises_if_metrics_enabled_but_prometheus_multiproc_dir_not_set( self, config_path diff --git a/tests/unit/util/test_rule_dry_runner.py b/tests/unit/util/test_rule_dry_runner.py index 85802fd9d..6a393c9e7 100644 --- a/tests/unit/util/test_rule_dry_runner.py +++ b/tests/unit/util/test_rule_dry_runner.py @@ -18,48 +18,40 @@ def setup_method(self): pipeline: - dissector: type: dissector - specific_rules: - - tests/testdata/unit/dissector/ - generic_rules: [] + rules: + - tests/testdata/unit/dissector - labelername: type: labeler schema: tests/testdata/unit/labeler/schemas/schema3.json include_parent_labels: true - specific_rules: - - tests/testdata/unit/labeler/rules/specific/ - generic_rules: - - tests/testdata/unit/labeler/rules/generic/ + rules: + - tests/testdata/unit/labeler/rules - pseudonymizer: type: pseudonymizer pubkey_analyst: tests/testdata/unit/pseudonymizer/example_analyst_pub.pem pubkey_depseudo: tests/testdata/unit/pseudonymizer/example_depseudo_pub.pem - regex_mapping: tests/testdata/unit/pseudonymizer/rules/regex_mapping.yml + regex_mapping: tests/testdata/unit/pseudonymizer/regex_mapping.yml hash_salt: a_secret_tasty_ingredient outputs: - kafka_output: pseudonyms - specific_rules: - - tests/testdata/unit/pseudonymizer/rules/specific/ - generic_rules: - - tests/testdata/unit/pseudonymizer/rules/generic/ + rules: + - tests/testdata/unit/pseudonymizer/rules max_cached_pseudonyms: 1000000 - predetectorname: type: pre_detector - specific_rules: - - tests/testdata/unit/pre_detector/rules/specific/ - generic_rules: - - tests/testdata/unit/pre_detector/rules/generic/ + rules: + - tests/testdata/unit/pre_detector/rules outputs: - kafka_output: sre_topic - selective_extractor: type: selective_extractor - specific_rules: + rules: - filter: message selective_extractor: source_fields: ["field1", "field2"] outputs: - kafka_output: topic description: my reference rule - generic_rules: [] input: kafka_output: type: dummy_input From 10a6cfba55ff3edaffd96ffbb7c53d30530187a4 Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Tue, 17 Dec 2024 11:52:35 +0100 Subject: [PATCH 02/13] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72ea1f9b2..335882387 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * removed the configuration `tld_lists` in `domain_resolver`, `domain_label_extractor` and `pseudonymizer` as the list is now fixed inside the packaged logprep * remove SQL feature from `generic_adder`, fields can only be added from rule config or from file +* use a single rule tree instead of a generic and a specific rule tree ### Features From 6824d12916fa08604d62dad5bacfaa5bd39bdc5f Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Tue, 17 Dec 2024 11:52:42 +0100 Subject: [PATCH 03/13] Update readme --- README.md | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 67d9ae108..5465bc434 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ and secondly they specify how to process the message. For example which fields should be deleted or to which IP-address the geolocation should be retrieved. -For performance reasons on startup all rules per processor are aggregated to a generic and a specific rule tree, respectively. +For performance reasons on startup all rules per processor are aggregated to a rule tree. Instead of evaluating all rules independently for each log message the message is checked against the rule tree. Each node in the rule tree represents a condition that has to be meet, @@ -130,11 +130,6 @@ This configuration will lead to the prioritization of `tags` and `message` in th } ``` -Instead of writing very specific rules that apply to single log messages, it is also possible -to define generic rules that apply to multiple messages. -It is possible to define a set of generic and specific rules for each processor, resulting -in two rule trees. - ### Connectors Connectors are responsible for reading the input and writing the result to a desired output. @@ -168,24 +163,20 @@ timeout: 0.1 pipeline: - dissector: type: dissector - specific_rules: + rules: - https://your-api/dissector/ - generic_rules: - - rules/01_dissector/generic/ + - rules/01_dissector/rules/ - geoip_enricher: type: geoip_enricher - specific_rules: + rules: - https://your-api/geoip/ - generic_rules: - - rules/02_geoip_enricher/generic/ + - rules/02_geoip_enricher/rules/ tree_config: artifacts/tree_config.json db_path: artifacts/GeoDB.mmdb - dropper: type: dropper - specific_rules: - - rules/03_dropper/specific/ - generic_rules: - - rules/03_dropper/generic/ + rules: + - rules/03_dropper/rules/ input: mykafka: @@ -212,7 +203,7 @@ output: ``` The following yaml represents a dropper rule which according to the previous configuration -should be in the `rules/03_dropper/generic/` directory. +should be in the `rules/03_dropper/rules/` directory. ```yaml filter: "message" From c16801b0f1f6ee84d7eaa08dcd8c7a305afb8df2 Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Tue, 17 Dec 2024 11:53:26 +0100 Subject: [PATCH 04/13] Update notebooks for one rule tree --- .../notebooks/processor_examples/calculator.ipynb | 5 ++--- .../notebooks/processor_examples/concatenator.ipynb | 5 ++--- .../notebooks/processor_examples/dissector.ipynb | 5 ++--- .../notebooks/processor_examples/field_manager.ipynb | 9 ++++----- .../notebooks/processor_examples/generic_adder.ipynb | 5 ++--- .../geo_ip_enricher_custom_outputfields.ipynb | 7 +++---- .../notebooks/processor_examples/grokker.ipynb | 5 ++--- .../notebooks/processor_examples/ip_informer.ipynb | 7 +++---- .../notebooks/processor_examples/key_checker.ipynb | 5 ++--- .../development/notebooks/processor_examples/regex.ipynb | 3 +-- .../notebooks/processor_examples/requester.ipynb | 5 ++--- .../notebooks/processor_examples/string_splitter.ipynb | 9 ++++----- .../notebooks/processor_examples/timestamp_differ.ipynb | 5 ++--- .../notebooks/processor_examples/timestamper.ipynb | 5 ++--- 14 files changed, 33 insertions(+), 47 deletions(-) diff --git a/doc/source/development/notebooks/processor_examples/calculator.ipynb b/doc/source/development/notebooks/processor_examples/calculator.ipynb index 8822d2501..0a2553bd9 100644 --- a/doc/source/development/notebooks/processor_examples/calculator.ipynb +++ b/doc/source/development/notebooks/processor_examples/calculator.ipynb @@ -124,8 +124,7 @@ "processor_config = {\n", " \"mycalculator\":{ \n", " \"type\": \"calculator\",\n", - " \"specific_rules\": [str(rule_path)],\n", - " \"generic_rules\": [\"/dev\"],\n", + " \"rules\": [str(rule_path), \"/dev\"],\n", " }\n", " }" ] @@ -223,4 +222,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/doc/source/development/notebooks/processor_examples/concatenator.ipynb b/doc/source/development/notebooks/processor_examples/concatenator.ipynb index 5e70fe707..0dcfbfc8f 100644 --- a/doc/source/development/notebooks/processor_examples/concatenator.ipynb +++ b/doc/source/development/notebooks/processor_examples/concatenator.ipynb @@ -136,8 +136,7 @@ "processor_config = {\n", " \"myconcatenator\":{ \n", " \"type\": \"concatenator\",\n", - " \"specific_rules\": [str(rule_path)],\n", - " \"generic_rules\": [\"/dev\"],\n", + " \"rules\": [str(rule_path), \"/dev\"],\n", " }\n", " }" ] @@ -235,4 +234,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/doc/source/development/notebooks/processor_examples/dissector.ipynb b/doc/source/development/notebooks/processor_examples/dissector.ipynb index 58fc70bda..2555de884 100644 --- a/doc/source/development/notebooks/processor_examples/dissector.ipynb +++ b/doc/source/development/notebooks/processor_examples/dissector.ipynb @@ -135,8 +135,7 @@ "processor_config = {\n", " \"thealmightydissector\":{ \n", " \"type\": \"dissector\",\n", - " \"specific_rules\": [str(rule_path)],\n", - " \"generic_rules\": [\"/dev\"],\n", + " \"rules\": [str(rule_path), \"/dev\"],\n", " }\n", " }" ] @@ -234,4 +233,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/doc/source/development/notebooks/processor_examples/field_manager.ipynb b/doc/source/development/notebooks/processor_examples/field_manager.ipynb index 2d28de547..97ba0f2de 100644 --- a/doc/source/development/notebooks/processor_examples/field_manager.ipynb +++ b/doc/source/development/notebooks/processor_examples/field_manager.ipynb @@ -112,8 +112,7 @@ "processor_config = {\n", " \"the_field_manager\": {\n", " \"type\": \"field_manager\",\n", - " \"specific_rules\": [\"/dev\"],\n", - " \"generic_rules\": [\"/dev\"],\n", + " \"rules\": [\"/dev\"],\n", " }\n", "}\n" ] @@ -176,9 +175,9 @@ ], "source": [ "for rule in rules:\n", - " processor._specific_tree.add_rule(rule)\n", + " processor._rule_tree.add_rule(rule)\n", " \n", - "processor._specific_rules" + "processor._tree_rules" ] }, { @@ -288,4 +287,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/doc/source/development/notebooks/processor_examples/generic_adder.ipynb b/doc/source/development/notebooks/processor_examples/generic_adder.ipynb index d85d93170..06b3d8f2a 100644 --- a/doc/source/development/notebooks/processor_examples/generic_adder.ipynb +++ b/doc/source/development/notebooks/processor_examples/generic_adder.ipynb @@ -97,8 +97,7 @@ "processor_config = {\n", " \"almighty generic adder\":{ \n", " \"type\": \"generic_adder\",\n", - " \"specific_rules\": [{\"filter\": \"*\", \"generic_adder\": {\"extend_target_list\": True, \"add\": {\"message.tags\": \"New\"}} }],\n", - " \"generic_rules\": [],\n", + " \"rules\": [{\"filter\": \"*\", \"generic_adder\": {\"extend_target_list\": True, \"add\": {\"message.tags\": \"New\"}} }],\n", " }\n", " }" ] @@ -196,4 +195,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/doc/source/development/notebooks/processor_examples/geo_ip_enricher_custom_outputfields.ipynb b/doc/source/development/notebooks/processor_examples/geo_ip_enricher_custom_outputfields.ipynb index f5cebaa2d..c282c784c 100644 --- a/doc/source/development/notebooks/processor_examples/geo_ip_enricher_custom_outputfields.ipynb +++ b/doc/source/development/notebooks/processor_examples/geo_ip_enricher_custom_outputfields.ipynb @@ -162,8 +162,7 @@ "processor_config = {\n", " \"geoip_enricher\": {\n", " \"type\": \"geoip_enricher\",\n", - " \"specific_rules\": [str(rule_path)],\n", - " \"generic_rules\": [\"/dev\"],\n", + " \"rules\": [str(rule_path), \"/dev\"],\n", " \"db_path\": \"\"\n", " }\n", "}\n" @@ -191,7 +190,7 @@ "Cell \u001b[0;32mIn[24], line 5\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mlogprep\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mfactory\u001b[39;00m \u001b[39mimport\u001b[39;00m Factory\n\u001b[1;32m 4\u001b[0m mock_logger \u001b[39m=\u001b[39m mock\u001b[39m.\u001b[39mMagicMock()\n\u001b[0;32m----> 5\u001b[0m geoip_enricher \u001b[39m=\u001b[39m Factory\u001b[39m.\u001b[39;49mcreate(processor_config, mock_logger)\n\u001b[1;32m 6\u001b[0m geoip_enricher\n", "File \u001b[0;32m~/external_work/Logprep/doc/source/development/notebooks/processor_examples/../../../../../logprep/factory.py:36\u001b[0m, in \u001b[0;36mFactory.create\u001b[0;34m(cls, configuration, logger)\u001b[0m\n\u001b[1;32m 34\u001b[0m metric_labels \u001b[39m=\u001b[39m configuration[connector_name]\u001b[39m.\u001b[39mpop(\u001b[39m\"\u001b[39m\u001b[39mmetric_labels\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 35\u001b[0m connector \u001b[39m=\u001b[39m Configuration\u001b[39m.\u001b[39mget_class(connector_name, connector_configuration_dict)\n\u001b[0;32m---> 36\u001b[0m connector_configuration \u001b[39m=\u001b[39m Configuration\u001b[39m.\u001b[39;49mcreate(\n\u001b[1;32m 37\u001b[0m connector_name, connector_configuration_dict\n\u001b[1;32m 38\u001b[0m )\n\u001b[1;32m 39\u001b[0m connector_configuration\u001b[39m.\u001b[39mmetric_labels \u001b[39m=\u001b[39m copy\u001b[39m.\u001b[39mdeepcopy(metric_labels)\n\u001b[1;32m 40\u001b[0m \u001b[39mreturn\u001b[39;00m connector(connector_name, connector_configuration, logger)\n", "File \u001b[0;32m~/external_work/Logprep/doc/source/development/notebooks/processor_examples/../../../../../logprep/configuration.py:34\u001b[0m, in \u001b[0;36mConfiguration.create\u001b[0;34m(cls, name, config_)\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[39m\"\"\"factory method to create component configuration\u001b[39;00m\n\u001b[1;32m 20\u001b[0m \n\u001b[1;32m 21\u001b[0m \u001b[39mParameters\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[39m the pipeline component configuration\u001b[39;00m\n\u001b[1;32m 32\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 33\u001b[0m class_ \u001b[39m=\u001b[39m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39mget_class(name, config_)\n\u001b[0;32m---> 34\u001b[0m \u001b[39mreturn\u001b[39;00m class_\u001b[39m.\u001b[39;49mConfig(\u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mconfig_)\n", - "File \u001b[0;32m:13\u001b[0m, in \u001b[0;36m__init__\u001b[0;34m(self, type, specific_rules, generic_rules, tree_config, db_path)\u001b[0m\n\u001b[1;32m 11\u001b[0m __attr_validator_generic_rules(\u001b[39mself\u001b[39m, __attr_generic_rules, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mgeneric_rules)\n\u001b[1;32m 12\u001b[0m __attr_validator_tree_config(\u001b[39mself\u001b[39m, __attr_tree_config, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtree_config)\n\u001b[0;32m---> 13\u001b[0m __attr_validator_db_path(\u001b[39mself\u001b[39;49m, __attr_db_path, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdb_path)\n", + "File \u001b[0;32m:13\u001b[0m, in \u001b[0;36m__init__\u001b[0;34m(self, type, rules, tree_config, db_path)\u001b[0m\n\u001b[1;32m 11\u001b[0m __attr_validator_generic_rules(\u001b[39mself\u001b[39m, __attr_generic_rules, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mgeneric_rules)\n\u001b[1;32m 12\u001b[0m __attr_validator_tree_config(\u001b[39mself\u001b[39m, __attr_tree_config, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtree_config)\n\u001b[0;32m---> 13\u001b[0m __attr_validator_db_path(\u001b[39mself\u001b[39;49m, __attr_db_path, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdb_path)\n", "File \u001b[0;32m~/external_work/Logprep/doc/source/development/notebooks/processor_examples/../../../../../logprep/util/validators.py:53\u001b[0m, in \u001b[0;36murl_validator\u001b[0;34m(_, attribute, value)\u001b[0m\n\u001b[1;32m 51\u001b[0m \u001b[39mraise\u001b[39;00m InvalidConfigurationError(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mattribute\u001b[39m.\u001b[39mname\u001b[39m}\u001b[39;00m\u001b[39m has no schema, net location and path\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 52\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m parsed_url\u001b[39m.\u001b[39mscheme \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m parsed_url\u001b[39m.\u001b[39mnetloc \u001b[39mand\u001b[39;00m parsed_url\u001b[39m.\u001b[39mpath:\n\u001b[0;32m---> 53\u001b[0m file_validator(_, attribute, value)\n\u001b[1;32m 54\u001b[0m \u001b[39mif\u001b[39;00m parsed_url\u001b[39m.\u001b[39mscheme \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mfile\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m 55\u001b[0m \u001b[39mif\u001b[39;00m parsed_url\u001b[39m.\u001b[39mparams \u001b[39mor\u001b[39;00m parsed_url\u001b[39m.\u001b[39mquery \u001b[39mor\u001b[39;00m parsed_url\u001b[39m.\u001b[39mfragment:\n", "File \u001b[0;32m~/external_work/Logprep/doc/source/development/notebooks/processor_examples/../../../../../logprep/util/validators.py:23\u001b[0m, in \u001b[0;36mfile_validator\u001b[0;34m(_, attribute, value)\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[39mraise\u001b[39;00m InvalidConfigurationError(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mattribute\u001b[39m.\u001b[39mname\u001b[39m}\u001b[39;00m\u001b[39m is not a str\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 22\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mexists(value):\n\u001b[0;32m---> 23\u001b[0m \u001b[39mraise\u001b[39;00m InvalidConfigurationError(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mattribute\u001b[39m.\u001b[39mname\u001b[39m}\u001b[39;00m\u001b[39m file \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mvalue\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m does not exist\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 24\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39misfile(value):\n\u001b[1;32m 25\u001b[0m \u001b[39mraise\u001b[39;00m InvalidConfigurationError(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mattribute\u001b[39m.\u001b[39mname\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mvalue\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m is not a file\u001b[39m\u001b[39m\"\u001b[39m)\n", "\u001b[0;31mInvalidConfigurationError\u001b[0m: db_path file 'tests/testdata/mock_external/MockGeoLite2-City.mmdb' does not exist" @@ -267,4 +266,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/doc/source/development/notebooks/processor_examples/grokker.ipynb b/doc/source/development/notebooks/processor_examples/grokker.ipynb index 6f75a9fa1..3fafe2b6d 100644 --- a/doc/source/development/notebooks/processor_examples/grokker.ipynb +++ b/doc/source/development/notebooks/processor_examples/grokker.ipynb @@ -122,8 +122,7 @@ "processor_config = {\n", " \"mygrokker\":{ \n", " \"type\": \"grokker\",\n", - " \"specific_rules\": [str(rule_path)],\n", - " \"generic_rules\": [\"/dev\"],\n", + " \"rules\": [str(rule_path), \"/dev\"],\n", " }\n", " }" ] @@ -215,4 +214,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/doc/source/development/notebooks/processor_examples/ip_informer.ipynb b/doc/source/development/notebooks/processor_examples/ip_informer.ipynb index d1c827944..33c47abcd 100644 --- a/doc/source/development/notebooks/processor_examples/ip_informer.ipynb +++ b/doc/source/development/notebooks/processor_examples/ip_informer.ipynb @@ -195,8 +195,7 @@ "processor_config = {\n", " \"the_ip_informer_name\":{ \n", " \"type\": \"ip_informer\",\n", - " \"specific_rules\": [],\n", - " \"generic_rules\": [],\n", + " \"rules\": [],\n", " }\n", " }" ] @@ -247,7 +246,7 @@ "metadata": {}, "outputs": [], "source": [ - "ip_informer._specific_tree.add_rule(rule)" + "ip_informer._rule_tree.add_rule(rule)" ] }, { @@ -403,4 +402,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/doc/source/development/notebooks/processor_examples/key_checker.ipynb b/doc/source/development/notebooks/processor_examples/key_checker.ipynb index 8a915732a..6201c6a2f 100644 --- a/doc/source/development/notebooks/processor_examples/key_checker.ipynb +++ b/doc/source/development/notebooks/processor_examples/key_checker.ipynb @@ -208,8 +208,7 @@ "processor_config = {\n", " \"almighty_keychecker\": {\n", " \"type\": \"key_checker\",\n", - " \"specific_rules\": [str(rule_path)],\n", - " \"generic_rules\": [\"/dev\"],\n", + " \"rules\": [str(rule_path), \"/dev\"],\n", " }\n", "}\n" ] @@ -336,4 +335,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/doc/source/development/notebooks/processor_examples/regex.ipynb b/doc/source/development/notebooks/processor_examples/regex.ipynb index f933f9e62..2c05d28bb 100644 --- a/doc/source/development/notebooks/processor_examples/regex.ipynb +++ b/doc/source/development/notebooks/processor_examples/regex.ipynb @@ -69,8 +69,7 @@ "processor_config = {\n", " \"myconcatenator\":{ \n", " \"type\": \"concatenator\",\n", - " \"specific_rules\": [str(rule_path)],\n", - " \"generic_rules\": [\"/dev\"],\n", + " \"rules\": [str(rule_path), \"/dev\"],\n", " }\n", " }\n", "\n", diff --git a/doc/source/development/notebooks/processor_examples/requester.ipynb b/doc/source/development/notebooks/processor_examples/requester.ipynb index ff5549d70..a86604ccc 100644 --- a/doc/source/development/notebooks/processor_examples/requester.ipynb +++ b/doc/source/development/notebooks/processor_examples/requester.ipynb @@ -134,8 +134,7 @@ "processor_config = {\n", " \"cmdbrequests\":{ \n", " \"type\": \"requester\",\n", - " \"specific_rules\": [str(rule_path)],\n", - " \"generic_rules\": [],\n", + " \"rules\": [str(rule_path)],\n", " }\n", " }" ] @@ -243,4 +242,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/doc/source/development/notebooks/processor_examples/string_splitter.ipynb b/doc/source/development/notebooks/processor_examples/string_splitter.ipynb index a29da9874..b89fd52ba 100644 --- a/doc/source/development/notebooks/processor_examples/string_splitter.ipynb +++ b/doc/source/development/notebooks/processor_examples/string_splitter.ipynb @@ -103,8 +103,7 @@ "processor_config = {\n", " \"allmighty_string_splitter\": {\n", " \"type\": \"string_splitter\",\n", - " \"specific_rules\": [\"/dev\"],\n", - " \"generic_rules\": [\"/dev\"],\n", + " \"rules\": [\"/dev\"],\n", " }\n", "}\n" ] @@ -167,9 +166,9 @@ ], "source": [ "for rule in rules:\n", - " processor._specific_tree.add_rule(rule)\n", + " processor._rule_tree.add_rule(rule)\n", " \n", - "processor._specific_rules" + "processor._tree_rules" ] }, { @@ -266,4 +265,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/doc/source/development/notebooks/processor_examples/timestamp_differ.ipynb b/doc/source/development/notebooks/processor_examples/timestamp_differ.ipynb index 35b7745d6..ffce12d36 100644 --- a/doc/source/development/notebooks/processor_examples/timestamp_differ.ipynb +++ b/doc/source/development/notebooks/processor_examples/timestamp_differ.ipynb @@ -119,8 +119,7 @@ "processor_config = {\n", " \"my_timestampdiffer\":{ \n", " \"type\": \"timestamp_differ\",\n", - " \"specific_rules\": [str(rule_path)],\n", - " \"generic_rules\": [\"/dev\"],\n", + " \"rules\": [str(rule_path), \"/dev\"],\n", " }\n", " }" ] @@ -192,4 +191,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/doc/source/development/notebooks/processor_examples/timestamper.ipynb b/doc/source/development/notebooks/processor_examples/timestamper.ipynb index 4ba960dfd..f9e5ddf4a 100644 --- a/doc/source/development/notebooks/processor_examples/timestamper.ipynb +++ b/doc/source/development/notebooks/processor_examples/timestamper.ipynb @@ -123,8 +123,7 @@ "processor_config = {\n", " \"my_timestamper\":{ \n", " \"type\": \"timestamper\",\n", - " \"specific_rules\": [str(rule_path)],\n", - " \"generic_rules\": [\"/dev\"],\n", + " \"rules\": [str(rule_path), \"/dev\"],\n", " }\n", " }" ] @@ -196,4 +195,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} From d4f3304853416412a9532a3bdfb5470774905a41 Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Tue, 17 Dec 2024 11:59:18 +0100 Subject: [PATCH 05/13] Update documentation for rule tree --- doc/source/development/processor_how_to.rst | 10 ++++------ .../development/programaticly_start_logprep.rst | 14 ++++---------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/doc/source/development/processor_how_to.rst b/doc/source/development/processor_how_to.rst index 193ad5570..8800dc825 100644 --- a/doc/source/development/processor_how_to.rst +++ b/doc/source/development/processor_how_to.rst @@ -49,10 +49,9 @@ This :py:class:`Config` class has to inherit from :py:class:`Processor.Config` a - newprocessorname: type: new_processor - specific_rules: - - tests/testdata/rules/specific/ - generic_rules: - - tests/testdata/rules/generic/ + rules: + - tests/testdata/rules_1/ + - tests/testdata/rules_2/ new_config_parameter: config_value """ @@ -170,8 +169,7 @@ the general implementation of a new processor seen in :ref:`implementing_a_new_p self.processor_attribute = [] self.metrics = self.NewProcessorMetrics( labels=self.metric_labels, - generic_rule_tree=self._generic_tree.metrics, - specific_rule_tree=self._specific_tree.metrics, + rule_tree=self._rule_tree.metrics, ) def _apply_rules(self, event, rule): diff --git a/doc/source/development/programaticly_start_logprep.rst b/doc/source/development/programaticly_start_logprep.rst index 51de7c9a4..89a0864d2 100644 --- a/doc/source/development/programaticly_start_logprep.rst +++ b/doc/source/development/programaticly_start_logprep.rst @@ -22,11 +22,8 @@ An example with input connector and preprocessors could look like this: { "predetector": { "type": "pre_detector", - "specific_rules": [ - "examples/exampledata/rules/pre_detector/specific" - ], - "generic_rules": [ - "examples/exampledata/rules/pre_detector/generic" + "rules": [ + "examples/exampledata/rules/pre_detector/rules" ], "pre_detector_topic": "output_topic" } @@ -60,11 +57,8 @@ An example without input connector and preprocessors could look like this: { "predetector": { "type": "pre_detector", - "specific_rules": [ - "examples/exampledata/rules/pre_detector/specific" - ], - "generic_rules": [ - "examples/exampledata/rules/pre_detector/generic" + "rules": [ + "examples/exampledata/rules/pre_detector/rules" ], "pre_detector_topic": "output_topic" } From 91082e7a8a1277e1b593487886143f9a8728df78 Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Tue, 17 Dec 2024 12:56:58 +0100 Subject: [PATCH 06/13] Fix black --- tests/unit/processor/concatenator/test_concatenator_rule.py | 4 +--- .../datetime_extractor/test_datetime_extractor_rule.py | 4 +--- tests/unit/processor/deleter/test_deleter_rule.py | 4 +--- .../test_domain_label_extractor_rule.py | 4 +--- tests/unit/processor/dropper/test_dropper_rule.py | 4 +--- .../processor/generic_resolver/test_generic_resolver_rule.py | 4 +--- .../unit/processor/geoip_enricher/test_geoip_enricher_rule.py | 4 +--- .../processor/list_comparison/test_list_comparison_rule.py | 4 +--- tests/unit/processor/pre_detector/test_pre_detector_rule.py | 4 +--- tests/unit/processor/pseudonymizer/test_pseudonymizer_rule.py | 4 +--- .../selective_extractor/test_selective_extractor_rule.py | 4 +--- 11 files changed, 11 insertions(+), 33 deletions(-) diff --git a/tests/unit/processor/concatenator/test_concatenator_rule.py b/tests/unit/processor/concatenator/test_concatenator_rule.py index 380167822..576d94728 100644 --- a/tests/unit/processor/concatenator/test_concatenator_rule.py +++ b/tests/unit/processor/concatenator/test_concatenator_rule.py @@ -128,9 +128,7 @@ class TestConcatenatorRule: ), ], ) - def test_rules_equality( - self, rule_definition, testcase, other_rule_definition, is_equal - ): + def test_rules_equality(self, rule_definition, testcase, other_rule_definition, is_equal): rule_1 = ConcatenatorRule._create_from_dict(rule_definition) rule_2 = ConcatenatorRule._create_from_dict(other_rule_definition) assert (rule_1 == rule_2) == is_equal, testcase diff --git a/tests/unit/processor/datetime_extractor/test_datetime_extractor_rule.py b/tests/unit/processor/datetime_extractor/test_datetime_extractor_rule.py index fcabbda65..98f1f405d 100644 --- a/tests/unit/processor/datetime_extractor/test_datetime_extractor_rule.py +++ b/tests/unit/processor/datetime_extractor/test_datetime_extractor_rule.py @@ -82,9 +82,7 @@ class TestDatetimeExtractorRule: ), ], ) - def test_rules_equality( - self, rule_definition, testcase, other_rule_definition, is_equal - ): + def test_rules_equality(self, rule_definition, testcase, other_rule_definition, is_equal): rule_1 = DatetimeExtractorRule._create_from_dict(rule_definition) rule_2 = DatetimeExtractorRule._create_from_dict(other_rule_definition) assert (rule_1 == rule_2) == is_equal, testcase diff --git a/tests/unit/processor/deleter/test_deleter_rule.py b/tests/unit/processor/deleter/test_deleter_rule.py index 4d8a7f05c..6bd1a2a71 100644 --- a/tests/unit/processor/deleter/test_deleter_rule.py +++ b/tests/unit/processor/deleter/test_deleter_rule.py @@ -54,9 +54,7 @@ class TestDeleterRule: ), ], ) - def test_rules_equality( - self, rule_definition, testcase, other_rule_definition, is_equal - ): + def test_rules_equality(self, rule_definition, testcase, other_rule_definition, is_equal): rule1 = DeleterRule._create_from_dict( rule_definition, ) diff --git a/tests/unit/processor/domain_label_extractor/test_domain_label_extractor_rule.py b/tests/unit/processor/domain_label_extractor/test_domain_label_extractor_rule.py index 127c381f5..7dde658d1 100644 --- a/tests/unit/processor/domain_label_extractor/test_domain_label_extractor_rule.py +++ b/tests/unit/processor/domain_label_extractor/test_domain_label_extractor_rule.py @@ -84,9 +84,7 @@ class TestDomainLabelExtractorRule: ), ], ) - def test_rules_equality( - self, rule_definition, testcase, other_rule_definition, is_equal - ): + def test_rules_equality(self, rule_definition, testcase, other_rule_definition, is_equal): rule_1 = DomainLabelExtractorRule._create_from_dict(rule_definition) rule_2 = DomainLabelExtractorRule._create_from_dict(other_rule_definition) assert (rule_1 == rule_2) == is_equal, testcase diff --git a/tests/unit/processor/dropper/test_dropper_rule.py b/tests/unit/processor/dropper/test_dropper_rule.py index 9a992e3e2..b0dc0bd16 100644 --- a/tests/unit/processor/dropper/test_dropper_rule.py +++ b/tests/unit/processor/dropper/test_dropper_rule.py @@ -60,9 +60,7 @@ def test_rule_has_fields_to_drop(self, rule_definition): ), ], ) - def test_rules_equality( - self, rule_definition, testcase, other_rule_definition, is_equal - ): + def test_rules_equality(self, rule_definition, testcase, other_rule_definition, is_equal): rule1 = DropperRule._create_from_dict( rule_definition, ) diff --git a/tests/unit/processor/generic_resolver/test_generic_resolver_rule.py b/tests/unit/processor/generic_resolver/test_generic_resolver_rule.py index e055996d0..411702a91 100644 --- a/tests/unit/processor/generic_resolver/test_generic_resolver_rule.py +++ b/tests/unit/processor/generic_resolver/test_generic_resolver_rule.py @@ -153,9 +153,7 @@ class TestGenericResolverRule: ), ], ) - def test_rules_equality( - self, rule_definition, testcase, other_rule_definition, is_equal - ): + def test_rules_equality(self, rule_definition, testcase, other_rule_definition, is_equal): rule1 = GenericResolverRule._create_from_dict(rule_definition) rule2 = GenericResolverRule._create_from_dict(other_rule_definition) assert (rule1 == rule2) == is_equal, testcase diff --git a/tests/unit/processor/geoip_enricher/test_geoip_enricher_rule.py b/tests/unit/processor/geoip_enricher/test_geoip_enricher_rule.py index f2acff83a..d37569449 100644 --- a/tests/unit/processor/geoip_enricher/test_geoip_enricher_rule.py +++ b/tests/unit/processor/geoip_enricher/test_geoip_enricher_rule.py @@ -67,9 +67,7 @@ class TestListComparisonRule: ), ], ) - def test_rules_equality( - self, rule_definition, testcase, other_rule_definition, is_equal - ): + def test_rules_equality(self, rule_definition, testcase, other_rule_definition, is_equal): rule1 = GeoipEnricherRule._create_from_dict(rule_definition) rule2 = GeoipEnricherRule._create_from_dict(other_rule_definition) assert (rule1 == rule2) == is_equal, testcase diff --git a/tests/unit/processor/list_comparison/test_list_comparison_rule.py b/tests/unit/processor/list_comparison/test_list_comparison_rule.py index a9baa9936..677415878 100644 --- a/tests/unit/processor/list_comparison/test_list_comparison_rule.py +++ b/tests/unit/processor/list_comparison/test_list_comparison_rule.py @@ -85,9 +85,7 @@ class TestListComparisonRule: ), ], ) - def test_rules_equality( - self, rule_definition, testcase, other_rule_definition, is_equal - ): + def test_rules_equality(self, rule_definition, testcase, other_rule_definition, is_equal): rule1 = ListComparisonRule._create_from_dict(rule_definition) rule2 = ListComparisonRule._create_from_dict(other_rule_definition) assert (rule1 == rule2) == is_equal, testcase diff --git a/tests/unit/processor/pre_detector/test_pre_detector_rule.py b/tests/unit/processor/pre_detector/test_pre_detector_rule.py index 3b40db8e9..ae972fc92 100644 --- a/tests/unit/processor/pre_detector/test_pre_detector_rule.py +++ b/tests/unit/processor/pre_detector/test_pre_detector_rule.py @@ -163,9 +163,7 @@ class TestPreDetectorRule: ), ], ) - def test_rules_equality( - self, rule_definition, testcase, other_rule_definition, is_equal - ): + def test_rules_equality(self, rule_definition, testcase, other_rule_definition, is_equal): rule1 = PreDetectorRule._create_from_dict(rule_definition) rule2 = PreDetectorRule._create_from_dict(other_rule_definition) assert (rule1 == rule2) == is_equal, testcase diff --git a/tests/unit/processor/pseudonymizer/test_pseudonymizer_rule.py b/tests/unit/processor/pseudonymizer/test_pseudonymizer_rule.py index 82584ab30..5fc0796eb 100644 --- a/tests/unit/processor/pseudonymizer/test_pseudonymizer_rule.py +++ b/tests/unit/processor/pseudonymizer/test_pseudonymizer_rule.py @@ -107,9 +107,7 @@ def test_create_from_dict_validates_config(self, rule, error, message): ), ], ) - def test_rules_equality( - self, rule_definition, testcase, other_rule_definition, is_equal - ): + def test_rules_equality(self, rule_definition, testcase, other_rule_definition, is_equal): rule_1 = PseudonymizerRule._create_from_dict(rule_definition) rule_2 = PseudonymizerRule._create_from_dict(other_rule_definition) assert (rule_1 == rule_2) == is_equal, testcase diff --git a/tests/unit/processor/selective_extractor/test_selective_extractor_rule.py b/tests/unit/processor/selective_extractor/test_selective_extractor_rule.py index ddf7fcb15..e6e7c7b4e 100644 --- a/tests/unit/processor/selective_extractor/test_selective_extractor_rule.py +++ b/tests/unit/processor/selective_extractor/test_selective_extractor_rule.py @@ -173,9 +173,7 @@ def test_rule_has_fields_from_directory_path(self, _): ), ], ) - def test_rules_equality( - self, rule_definition, testcase, other_rule_definition, is_equal - ): + def test_rules_equality(self, rule_definition, testcase, other_rule_definition, is_equal): with mock.patch("pathlib.Path.is_file", return_value=True): read_lines = other_rule_definition.get("selective_extractor").get("extract_from_file") if read_lines is not None: From 8a9bab05f14ec87754f764b2f0b6b34e04ee9978 Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Tue, 17 Dec 2024 14:32:05 +0100 Subject: [PATCH 07/13] Refactor geoip enricher test --- tests/unit/processor/geoip_enricher/test_geoip_enricher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/processor/geoip_enricher/test_geoip_enricher.py b/tests/unit/processor/geoip_enricher/test_geoip_enricher.py index ec55b49f8..9ab1cea26 100644 --- a/tests/unit/processor/geoip_enricher/test_geoip_enricher.py +++ b/tests/unit/processor/geoip_enricher/test_geoip_enricher.py @@ -121,11 +121,11 @@ def test_source_field_is_none_emits_missing_fields_warning(self): expected = {"client": {"ip": None}, "tags": ["_geoip_enricher_missing_field_warning"]} self._load_rule(self.object.rules[0]) self.object.process(document) + assert document == expected assert len(self.object.result.warnings) == 1 assert re.match( r".*missing source_fields: \['client\.ip'].*", str(self.object.result.warnings[0]) ) - assert document == expected def test_nothing_to_enrich(self): document = {"something": {"something": "1.2.3.4"}} From dc89f9926885724000d3ae12ee0516e9a5cea22c Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Wed, 18 Dec 2024 09:24:57 +0100 Subject: [PATCH 08/13] Fix geoip enricher unit test --- .../{geoip_all_1.json => geoip_all.json} | 0 .../geoip_enricher/rules/geoip_all_2.json | 21 ------------------- .../geoip_enricher/test_geoip_enricher.py | 1 - 3 files changed, 22 deletions(-) rename tests/testdata/unit/geoip_enricher/rules/{geoip_all_1.json => geoip_all.json} (100%) delete mode 100644 tests/testdata/unit/geoip_enricher/rules/geoip_all_2.json diff --git a/tests/testdata/unit/geoip_enricher/rules/geoip_all_1.json b/tests/testdata/unit/geoip_enricher/rules/geoip_all.json similarity index 100% rename from tests/testdata/unit/geoip_enricher/rules/geoip_all_1.json rename to tests/testdata/unit/geoip_enricher/rules/geoip_all.json diff --git a/tests/testdata/unit/geoip_enricher/rules/geoip_all_2.json b/tests/testdata/unit/geoip_enricher/rules/geoip_all_2.json deleted file mode 100644 index 60dd51013..000000000 --- a/tests/testdata/unit/geoip_enricher/rules/geoip_all_2.json +++ /dev/null @@ -1,21 +0,0 @@ -[ - { - "filter": "specific.client.ip AND NOT winlog.computer_name", - "geoip_enricher": { - "source_fields": [ - "client.ip" - ] - }, - "description": "" - }, - { - "filter": "specific.source.ip", - "geoip_enricher": { - "source_fields": [ - "source.ip" - ], - "target_field": "source.geo.ip" - }, - "description": "" - } -] \ No newline at end of file diff --git a/tests/unit/processor/geoip_enricher/test_geoip_enricher.py b/tests/unit/processor/geoip_enricher/test_geoip_enricher.py index 9ab1cea26..c96817bc1 100644 --- a/tests/unit/processor/geoip_enricher/test_geoip_enricher.py +++ b/tests/unit/processor/geoip_enricher/test_geoip_enricher.py @@ -119,7 +119,6 @@ def test_no_geoip_data_added_if_source_field_is_none(self): def test_source_field_is_none_emits_missing_fields_warning(self): document = {"client": {"ip": None}} expected = {"client": {"ip": None}, "tags": ["_geoip_enricher_missing_field_warning"]} - self._load_rule(self.object.rules[0]) self.object.process(document) assert document == expected assert len(self.object.result.warnings) == 1 From 08c2a59aae02fed9f98349abef70939ecdcec081 Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Wed, 18 Dec 2024 10:40:32 +0100 Subject: [PATCH 09/13] Remove redundant "_tree_rules" from processors and use "rules" instead --- .../processor_examples/field_manager.ipynb | 2 +- .../processor_examples/string_splitter.ipynb | 2 +- logprep/abc/processor.py | 17 ++--------------- logprep/processor/labeler/processor.py | 2 +- logprep/processor/list_comparison/processor.py | 2 +- logprep/processor/pseudonymizer/processor.py | 2 +- .../util/auto_rule_tester/auto_rule_tester.py | 4 +--- tests/unit/processor/base.py | 2 +- .../list_comparison/test_list_comparison.py | 2 +- .../pseudonymizer/test_pseudonymizer.py | 2 +- tests/unit/test_factory.py | 10 +++++----- 11 files changed, 16 insertions(+), 31 deletions(-) diff --git a/doc/source/development/notebooks/processor_examples/field_manager.ipynb b/doc/source/development/notebooks/processor_examples/field_manager.ipynb index 97ba0f2de..b6106c361 100644 --- a/doc/source/development/notebooks/processor_examples/field_manager.ipynb +++ b/doc/source/development/notebooks/processor_examples/field_manager.ipynb @@ -177,7 +177,7 @@ "for rule in rules:\n", " processor._rule_tree.add_rule(rule)\n", " \n", - "processor._tree_rules" + "processor._rules" ] }, { diff --git a/doc/source/development/notebooks/processor_examples/string_splitter.ipynb b/doc/source/development/notebooks/processor_examples/string_splitter.ipynb index b89fd52ba..9b4d8eb39 100644 --- a/doc/source/development/notebooks/processor_examples/string_splitter.ipynb +++ b/doc/source/development/notebooks/processor_examples/string_splitter.ipynb @@ -168,7 +168,7 @@ "for rule in rules:\n", " processor._rule_tree.add_rule(rule)\n", " \n", - "processor._tree_rules" + "processor.rules" ] }, { diff --git a/logprep/abc/processor.py b/logprep/abc/processor.py index 5ebde3a0e..391b366ac 100644 --- a/logprep/abc/processor.py +++ b/logprep/abc/processor.py @@ -112,7 +112,6 @@ class Config(Component.Config): "_rule_tree", "result", "_bypass_rule_tree", - "_rules", ] rule_class: "Rule" @@ -120,7 +119,6 @@ class Config(Component.Config): _rule_tree: RuleTree _strategy = None _bypass_rule_tree: bool - _rules: tuple["Rule"] result: ProcessorResult def __init__(self, name: str, configuration: "Processor.Config"): @@ -131,19 +129,8 @@ def __init__(self, name: str, configuration: "Processor.Config"): self._bypass_rule_tree = False if os.environ.get("LOGPREP_BYPASS_RULE_TREE"): self._bypass_rule_tree = True - self._rules = self.rules logger.debug("Bypassing rule tree for processor %s", self.name) - @property - def _tree_rules(self): - """Returns all rules - - Returns - ------- - rules: list[Rule] - """ - return self._rule_tree.rules - @property def rules(self): """Returns all rules @@ -152,7 +139,7 @@ def rules(self): ------- rules: list[Rule] """ - return [*self._tree_rules] + return self._rule_tree.rules @property def metric_labels(self) -> dict: @@ -195,7 +182,7 @@ def _process_rule(rule, event): rule.metrics.number_of_processed_events += 1 return event - for rule in self._rules: + for rule in self.rules: if rule.matches(event): _process_rule(rule, event) diff --git a/logprep/processor/labeler/processor.py b/logprep/processor/labeler/processor.py index d825e8ad7..278aff0dd 100644 --- a/logprep/processor/labeler/processor.py +++ b/logprep/processor/labeler/processor.py @@ -64,7 +64,7 @@ def __init__(self, name: str, configuration: Processor.Config): def setup(self): super().setup() - for rule in self._tree_rules: + for rule in self.rules: if self._config.include_parent_labels: rule.add_parent_labels_from_schema(self._schema) rule.conforms_to_schema(self._schema) diff --git a/logprep/processor/list_comparison/processor.py b/logprep/processor/list_comparison/processor.py index 2886aa910..4bf85ad4d 100644 --- a/logprep/processor/list_comparison/processor.py +++ b/logprep/processor/list_comparison/processor.py @@ -51,7 +51,7 @@ class Config(Processor.Config): def setup(self): super().setup() - for rule in [*self._tree_rules]: + for rule in self.rules: rule.init_list_comparison(self._config.list_search_base_path) def _apply_rules(self, event, rule): diff --git a/logprep/processor/pseudonymizer/processor.py b/logprep/processor/pseudonymizer/processor.py index 1bd31db84..dfeb9d1d0 100644 --- a/logprep/processor/pseudonymizer/processor.py +++ b/logprep/processor/pseudonymizer/processor.py @@ -216,7 +216,7 @@ def setup(self): self._replace_regex_keywords_by_regex_expression() def _replace_regex_keywords_by_regex_expression(self): - for rule in self._tree_rules: + for rule in self.rules: for dotted_field, regex_keyword in rule.pseudonyms.items(): if regex_keyword in self._regex_mapping: rule.pseudonyms[dotted_field] = re.compile(self._regex_mapping[regex_keyword]) diff --git a/logprep/util/auto_rule_tester/auto_rule_tester.py b/logprep/util/auto_rule_tester/auto_rule_tester.py index ee00811d1..5dccb0e83 100644 --- a/logprep/util/auto_rule_tester/auto_rule_tester.py +++ b/logprep/util/auto_rule_tester/auto_rule_tester.py @@ -392,8 +392,6 @@ def _load_rules(self, processor: "Processor"): type """ processor.load_rules(self._empty_rules_dirs) - if processor._bypass_rule_tree: - processor._rules = processor.rules processor.setup() def _prepare_test_eval( @@ -496,7 +494,7 @@ def _reset(processor: "Processor"): processor : Processor processor to reset tree on """ - if hasattr(processor, "_rules"): + if hasattr(processor, "rules"): processor.rules.clear() if hasattr(processor, "_rule_tree"): processor._rule_tree = RuleTree() diff --git a/tests/unit/processor/base.py b/tests/unit/processor/base.py index adc15507a..25a3b43b2 100644 --- a/tests/unit/processor/base.py +++ b/tests/unit/processor/base.py @@ -174,7 +174,7 @@ def test_no_redundant_rules_are_added_to_rule_tree(self): def test_rules_returns_all_rules(self): rules = self.rules - object_rules = self.object._tree_rules + object_rules = self.object.rules assert len(rules) == len(object_rules) @mock.patch("logging.Logger.debug") diff --git a/tests/unit/processor/list_comparison/test_list_comparison.py b/tests/unit/processor/list_comparison/test_list_comparison.py index 4bfabeb5c..a5b023227 100644 --- a/tests/unit/processor/list_comparison/test_list_comparison.py +++ b/tests/unit/processor/list_comparison/test_list_comparison.py @@ -275,6 +275,6 @@ def test_list_comparison_loads_rule_with_http_template_in_list_search_base_path( rule = processor.rule_class._create_from_dict(rule_dict) processor._rule_tree.add_rule(rule) processor.setup() - assert processor._tree_rules[0].compare_sets == { + assert processor.rules[0].compare_sets == { "bad_users.list": {"Franz", "Heinz", "Hans"} } diff --git a/tests/unit/processor/pseudonymizer/test_pseudonymizer.py b/tests/unit/processor/pseudonymizer/test_pseudonymizer.py index 39c3e4382..3f071effc 100644 --- a/tests/unit/processor/pseudonymizer/test_pseudonymizer.py +++ b/tests/unit/processor/pseudonymizer/test_pseudonymizer.py @@ -1065,7 +1065,7 @@ def test_setup_raises_invalid_configuration_on_missing_regex_mapping(self): }, } self._load_rule(rule_dict) - self.object._tree_rules[0].mapping["winlog.event_data.param2"] = "RE_DOES_NOT_EXIST" + self.object.rules[0].mapping["winlog.event_data.param2"] = "RE_DOES_NOT_EXIST" error_message = ( r"Regex keyword 'RE_DOES_NOT_EXIST' not found in regex_mapping '.*\/regex_mapping.yml'" ) diff --git a/tests/unit/test_factory.py b/tests/unit/test_factory.py index 56ad306ce..5f2d44c8d 100644 --- a/tests/unit/test_factory.py +++ b/tests/unit/test_factory.py @@ -139,7 +139,7 @@ def test_creates_calculator_with_inline_rules(): } } ) - assert len(processor._tree_rules) == 1 + assert len(processor.rules) == 1 def test_creates_calculator_with_inline_rules_and_files(): @@ -157,9 +157,9 @@ def test_creates_calculator_with_inline_rules_and_files(): } } ) - assert len(processor._tree_rules) == 2 - assert processor._tree_rules[0].filter_str == "message1: *" - assert processor._tree_rules[1].filter_str == "(field1: * AND field2: *)" + assert len(processor.rules) == 2 + assert processor.rules[0].filter_str == "message1: *" + assert processor.rules[1].filter_str == "(field1: * AND field2: *)" def test_creates_calculator_with_inline_rules_and_file_and_directory(): @@ -177,7 +177,7 @@ def test_creates_calculator_with_inline_rules_and_file_and_directory(): } } ) - assert len(processor._tree_rules) == 3 + assert len(processor.rules) == 3 def test_dummy_input_creates_dummy_input_connector(): From 63044c1380f5c5c28a1aee6b8a4c05888c9b12ad Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Wed, 18 Dec 2024 10:55:19 +0100 Subject: [PATCH 10/13] Fix black formatting --- tests/unit/processor/list_comparison/test_list_comparison.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/processor/list_comparison/test_list_comparison.py b/tests/unit/processor/list_comparison/test_list_comparison.py index a5b023227..aead77eac 100644 --- a/tests/unit/processor/list_comparison/test_list_comparison.py +++ b/tests/unit/processor/list_comparison/test_list_comparison.py @@ -275,6 +275,4 @@ def test_list_comparison_loads_rule_with_http_template_in_list_search_base_path( rule = processor.rule_class._create_from_dict(rule_dict) processor._rule_tree.add_rule(rule) processor.setup() - assert processor.rules[0].compare_sets == { - "bad_users.list": {"Franz", "Heinz", "Hans"} - } + assert processor.rules[0].compare_sets == {"bad_users.list": {"Franz", "Heinz", "Hans"}} From b1b6ee414193487f368eb08b613a115a1cfaec7d Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Wed, 18 Dec 2024 11:56:56 +0100 Subject: [PATCH 11/13] Update diagrams for single rule tree --- .../architecture/diagramms/pipeline.drawio | 598 +++++++++--------- .../diagramms/pipeline.drawio.html | 7 +- .../diagramms/process-Combined.drawio | 94 ++- .../diagramms/process-Combined.drawio.html | 7 +- 4 files changed, 343 insertions(+), 363 deletions(-) diff --git a/doc/source/development/architecture/diagramms/pipeline.drawio b/doc/source/development/architecture/diagramms/pipeline.drawio index 576852ac0..5fda47137 100644 --- a/doc/source/development/architecture/diagramms/pipeline.drawio +++ b/doc/source/development/architecture/diagramms/pipeline.drawio @@ -1,299 +1,299 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/development/architecture/diagramms/pipeline.drawio.html b/doc/source/development/architecture/diagramms/pipeline.drawio.html index e29358005..00467e61d 100644 --- a/doc/source/development/architecture/diagramms/pipeline.drawio.html +++ b/doc/source/development/architecture/diagramms/pipeline.drawio.html @@ -5,7 +5,8 @@ pipeline -
- + +
+ - \ No newline at end of file + diff --git a/doc/source/development/architecture/diagramms/process-Combined.drawio b/doc/source/development/architecture/diagramms/process-Combined.drawio index affbba022..4750edeef 100644 --- a/doc/source/development/architecture/diagramms/process-Combined.drawio +++ b/doc/source/development/architecture/diagramms/process-Combined.drawio @@ -1,31 +1,23 @@ - + - + - + - - - + + + - - - - - - - - - + @@ -83,7 +75,7 @@ - + @@ -91,20 +83,6 @@ - - - - - - - - - - - - - - @@ -133,7 +111,7 @@ - + @@ -264,7 +242,7 @@ - + @@ -343,7 +321,7 @@ - + @@ -511,7 +489,7 @@ - + @@ -595,7 +573,7 @@ - + @@ -660,7 +638,7 @@ - + @@ -685,7 +663,7 @@ - + @@ -748,7 +726,7 @@ - + @@ -878,7 +856,7 @@ - + @@ -1024,7 +1002,7 @@ - + @@ -1090,7 +1068,7 @@ - + @@ -1262,7 +1240,7 @@ - + @@ -1366,7 +1344,7 @@ - + @@ -1462,7 +1440,7 @@ - + @@ -1566,7 +1544,7 @@ - + @@ -1660,7 +1638,7 @@ - + @@ -1737,7 +1715,7 @@ - + @@ -1868,7 +1846,7 @@ - + @@ -1944,7 +1922,7 @@ - + @@ -1982,7 +1960,7 @@ - + @@ -2062,7 +2040,7 @@ - + @@ -2158,7 +2136,7 @@ - + @@ -2250,7 +2228,7 @@ - + @@ -2399,7 +2377,7 @@ - + @@ -2487,7 +2465,7 @@ - + @@ -2625,7 +2603,7 @@ - + @@ -2696,7 +2674,7 @@ - + @@ -2827,7 +2805,7 @@ - + diff --git a/doc/source/development/architecture/diagramms/process-Combined.drawio.html b/doc/source/development/architecture/diagramms/process-Combined.drawio.html index 372496ae6..c83f705e9 100644 --- a/doc/source/development/architecture/diagramms/process-Combined.drawio.html +++ b/doc/source/development/architecture/diagramms/process-Combined.drawio.html @@ -5,7 +5,8 @@ process-Combined -
- + +
+ - \ No newline at end of file + From d0906a4b0897035e6f508d99c1a01f73046cad01 Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Thu, 19 Dec 2024 08:01:38 +0100 Subject: [PATCH 12/13] Remove some remaining references to specific and generic rule tree --- logprep/processor/base/rule.py | 5 +---- tests/acceptance/test_full_configuration.py | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/logprep/processor/base/rule.py b/logprep/processor/base/rule.py index 08861f844..acbdb9137 100644 --- a/logprep/processor/base/rule.py +++ b/logprep/processor/base/rule.py @@ -15,10 +15,7 @@ Each file contains multiple YAML documents or a JSON array of JSON objects. The YAML format is preferred, since it is a superset of JSON and has better readability. -Depending on the filter, a rule can trigger for different types of messages or just for specific log -messages. -In general, specific rules are being applied first. -It depends on the directory where the rule is located if it is considered specific or generic. +Depending on the filter, a rule can trigger for different types of messages. Further details can be found in the section for processors. diff --git a/tests/acceptance/test_full_configuration.py b/tests/acceptance/test_full_configuration.py index 1f98a96bd..35c807d7b 100644 --- a/tests/acceptance/test_full_configuration.py +++ b/tests/acceptance/test_full_configuration.py @@ -234,12 +234,12 @@ def test_logprep_exposes_prometheus_metrics_and_healthchecks(tmp_path): assert re.search(first_calculator, metrics), "First calculator not found" assert ( len(re.findall(first_calculator, metrics)) == 2 - ), "More or less than two rules (specific, generic) were found for first calculator" + ), "More or less than two rules were found for first calculator" second_calculator = r"logprep_number_of_processed_events_total\{component=\"rule\",description=\"id:.+\",name=\"calculator2\",type\=\"calculator\"}" assert re.search(second_calculator, metrics), "Second calculator not found" assert ( len(re.findall(second_calculator, metrics)) == 2 - ), "More or less than two rules (specific, generic) were found for second calculator" + ), "More or less than two rules were found for second calculator" both_calculators = r"logprep_number_of_processed_events_total\{component=\"rule\",description=\"id:.+\",name=\".+\",type\=\"calculator\"}" assert ( len(re.findall(both_calculators, metrics)) == 4 From 12cdae7360ce2a92c62b2421b96bd35a32c2e506 Mon Sep 17 00:00:00 2001 From: Piotr Pauksztelo Date: Thu, 19 Dec 2024 08:02:21 +0100 Subject: [PATCH 13/13] Remove redundant property from processor unit tests --- tests/unit/processor/concatenator/test_concatenator.py | 4 ---- .../processor/datetime_extractor/test_datetime_extractor.py | 4 ---- .../domain_label_extractor/test_domain_label_extractor.py | 4 ---- tests/unit/processor/dropper/test_dropper.py | 4 ---- tests/unit/processor/generic_adder/test_generic_adder.py | 4 ---- .../unit/processor/generic_resolver/test_generic_resolver.py | 5 ----- tests/unit/processor/geoip_enricher/test_geoip_enricher.py | 4 ---- tests/unit/processor/labeler/test_labeler.py | 5 ----- .../processor/template_replacer/test_template_replacer.py | 4 ---- 9 files changed, 38 deletions(-) diff --git a/tests/unit/processor/concatenator/test_concatenator.py b/tests/unit/processor/concatenator/test_concatenator.py index 1f7c69873..649091b7d 100644 --- a/tests/unit/processor/concatenator/test_concatenator.py +++ b/tests/unit/processor/concatenator/test_concatenator.py @@ -14,10 +14,6 @@ class TestConcatenator(BaseProcessorTestCase): "tree_config": "tests/testdata/unit/shared_data/tree_config.json", } - @property - def rules_dirs(self): - return self.CONFIG["rules"] - @pytest.mark.parametrize( ["test_case", "rule", "document", "expected_output"], [ diff --git a/tests/unit/processor/datetime_extractor/test_datetime_extractor.py b/tests/unit/processor/datetime_extractor/test_datetime_extractor.py index d64329681..a1698b35b 100644 --- a/tests/unit/processor/datetime_extractor/test_datetime_extractor.py +++ b/tests/unit/processor/datetime_extractor/test_datetime_extractor.py @@ -16,10 +16,6 @@ class TestDatetimeExtractor(BaseProcessorTestCase): "rules": ["tests/testdata/unit/datetime_extractor/rules"], } - @property - def rules_dirs(self): - return self.CONFIG.get("rules") - def test_an_event_extracted_datetime_utc(self): timestamp = "2019-07-30T14:37:42.861Z" document = {"@timestamp": timestamp, "winlog": {"event_id": 123}} diff --git a/tests/unit/processor/domain_label_extractor/test_domain_label_extractor.py b/tests/unit/processor/domain_label_extractor/test_domain_label_extractor.py index fd8358e32..6a8dde0b2 100644 --- a/tests/unit/processor/domain_label_extractor/test_domain_label_extractor.py +++ b/tests/unit/processor/domain_label_extractor/test_domain_label_extractor.py @@ -14,10 +14,6 @@ class TestDomainLabelExtractor(BaseProcessorTestCase): "tree_config": "tests/testdata/unit/shared_data/tree_config.json", } - @property - def rules_dirs(self): - return self.CONFIG.get("rules") - def test_domain_extraction_from_full_url(self): document = {"url": {"domain": "https://url.full.domain.de/path/file?param=1"}} expected_output = { diff --git a/tests/unit/processor/dropper/test_dropper.py b/tests/unit/processor/dropper/test_dropper.py index 7fed2fa1b..890c91273 100644 --- a/tests/unit/processor/dropper/test_dropper.py +++ b/tests/unit/processor/dropper/test_dropper.py @@ -13,10 +13,6 @@ class TestDropper(BaseProcessorTestCase): "tree_config": "tests/testdata/unit/shared_data/tree_config.json", } - @property - def rules_dirs(self): - return self.CONFIG["rules"] - def test_dropper_instantiates(self): rule = {"filter": "drop_me", "dropper": {"drop": ["drop_me"]}} self._load_rule(rule) diff --git a/tests/unit/processor/generic_adder/test_generic_adder.py b/tests/unit/processor/generic_adder/test_generic_adder.py index 6625bece8..6fa7944d8 100644 --- a/tests/unit/processor/generic_adder/test_generic_adder.py +++ b/tests/unit/processor/generic_adder/test_generic_adder.py @@ -313,10 +313,6 @@ class TestGenericAdder(BaseProcessorTestCase): "rules": ["tests/testdata/unit/generic_adder/rules"], } - @property - def rules_dirs(self): - return self.CONFIG.get("rules") - @pytest.mark.parametrize("testcase, rule, event, expected", test_cases) def test_generic_adder_testcases( self, testcase, rule, event, expected diff --git a/tests/unit/processor/generic_resolver/test_generic_resolver.py b/tests/unit/processor/generic_resolver/test_generic_resolver.py index ca6324902..1b909d809 100644 --- a/tests/unit/processor/generic_resolver/test_generic_resolver.py +++ b/tests/unit/processor/generic_resolver/test_generic_resolver.py @@ -25,11 +25,6 @@ class TestGenericResolver(BaseProcessorTestCase): "logprep_generic_resolver_cache_load", ] - @property - def rules_dirs(self): - """Return the paths of the rules""" - return self.CONFIG["rules"] - def test_resolve_generic_instantiates(self): rule = {"filter": "anything", "generic_resolver": {"field_mapping": {}}} self._load_rule(rule) diff --git a/tests/unit/processor/geoip_enricher/test_geoip_enricher.py b/tests/unit/processor/geoip_enricher/test_geoip_enricher.py index c96817bc1..d7c2c5a8d 100644 --- a/tests/unit/processor/geoip_enricher/test_geoip_enricher.py +++ b/tests/unit/processor/geoip_enricher/test_geoip_enricher.py @@ -93,10 +93,6 @@ class TestGeoipEnricher(BaseProcessorTestCase): "tree_config": "tests/testdata/unit/shared_data/tree_config.json", } - @property - def rules_dirs(self): - return self.CONFIG["rules"] - def test_geoip_data_added(self): document = {"client": {"ip": "1.2.3.4"}} diff --git a/tests/unit/processor/labeler/test_labeler.py b/tests/unit/processor/labeler/test_labeler.py index a1e9f1681..2d6daff45 100644 --- a/tests/unit/processor/labeler/test_labeler.py +++ b/tests/unit/processor/labeler/test_labeler.py @@ -64,11 +64,6 @@ class TestLabeler(BaseProcessorTestCase): "rules": ["tests/testdata/unit/labeler/rules"], } - @property - def rules_dirs(self): - """Return path to rule directories""" - return self.CONFIG["rules"] - def _load_rule(self, rule, schema=None): # pylint: disable=arguments-differ rule = LabelerRule._create_from_dict(rule) if schema: diff --git a/tests/unit/processor/template_replacer/test_template_replacer.py b/tests/unit/processor/template_replacer/test_template_replacer.py index 8b8ed1a6c..7dd209f5e 100644 --- a/tests/unit/processor/template_replacer/test_template_replacer.py +++ b/tests/unit/processor/template_replacer/test_template_replacer.py @@ -23,10 +23,6 @@ class TestTemplateReplacer(BaseProcessorTestCase): "tree_config": "tests/testdata/unit/shared_data/tree_config.json", } - @property - def rules_dirs(self): - return self.CONFIG.get("rules") - def setup_method(self): super().setup_method() self.object.setup()