From 5c1609ba5ee85fdd45788a34b75e8afdd0aca0f7 Mon Sep 17 00:00:00 2001 From: Jason Thomas Date: Wed, 13 Sep 2023 09:23:44 -0600 Subject: [PATCH 1/2] DISABLE_REDUCER option for targets --- .../plugins/packages/openc3-cosmos-demo/plugin.txt | 14 ++++++++++++-- .../openc3-cosmos-demo/targets/EXAMPLE/target.txt | 4 +--- .../openc3-cosmos-demo/targets/INST/target.txt | 2 -- openc3/data/config/target.yaml | 2 ++ openc3/lib/openc3/models/target_model.rb | 13 ++++++++++--- 5 files changed, 25 insertions(+), 10 deletions(-) diff --git a/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/plugin.txt b/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/plugin.txt index e424f43bd..9692e45d5 100644 --- a/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/plugin.txt +++ b/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/plugin.txt @@ -39,6 +39,8 @@ VARIABLE reduced_log_retain_time 2592000 LOG_RETAIN_TIME <%= log_retain_time %> REDUCED_LOG_RETAIN_TIME <%= reduced_log_retain_time %> TLM_LOG_CYCLE_TIME 300 + # Allow the reducer microservice to take 50% of the cpu (default 30%) + REDUCER_MAX_CPU_UTILIZATION 50 <% end %> <% if include_inst2 %> @@ -46,18 +48,26 @@ VARIABLE reduced_log_retain_time 2592000 LOG_RETAIN_TIME <%= log_retain_time %> REDUCED_LOG_RETAIN_TIME <%= reduced_log_retain_time %> TLM_LOG_CYCLE_TIME 600 + # Allow the reducer microservice to take 10% of the cpu (default 30%) + REDUCER_MAX_CPU_UTILIZATION 10 <% end %> <% if include_example %> TARGET EXAMPLE <%= example_target_name %> LOG_RETAIN_TIME <%= log_retain_time %> - REDUCED_LOG_RETAIN_TIME <%= reduced_log_retain_time %> + # Disable data reduction (min, hour, day) on this target data + # No 'ruby reducer_microservice.rb DEFAULT__REDUCER__EXAMPLE' + # will appear in the the process list + DISABLE_REDUCER <% end %> <% if include_templated %> TARGET TEMPLATED <%= templated_target_name %> LOG_RETAIN_TIME <%= log_retain_time %> - REDUCED_LOG_RETAIN_TIME <%= reduced_log_retain_time %> + # Disable data reduction (min, hour, day) on this target data + # No 'ruby reducer_microservice.rb DEFAULT__REDUCER__TEMPLATED' + # will appear in the the process list + DISABLE_REDUCER <% end %> <% if include_system %> diff --git a/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/EXAMPLE/target.txt b/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/EXAMPLE/target.txt index d3714064e..89dd0f6b1 100644 --- a/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/EXAMPLE/target.txt +++ b/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/EXAMPLE/target.txt @@ -1,9 +1,7 @@ REQUIRE example_interface.rb -# Ignored Parameters -# IGNORE_PARAMETER parameter_name +# Ignored Parameters and Items IGNORE_PARAMETER PACKET_ID - IGNORE_ITEM RECEIVED_COUNT IGNORE_ITEM RECEIVED_TIMESECONDS IGNORE_ITEM RECEIVED_TIMEFORMATTED diff --git a/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST/target.txt b/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST/target.txt index 3fe3fddfc..7f550d0ff 100644 --- a/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST/target.txt +++ b/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST/target.txt @@ -3,7 +3,6 @@ REQUIRE example_limits_response.rb REQUIRE sim_inst.rb # Ignored Parameters -# IGNORE_PARAMETER parameter_name IGNORE_PARAMETER CCSDSVER IGNORE_PARAMETER CCSDSTYPE IGNORE_PARAMETER CCSDSSHF @@ -14,7 +13,6 @@ IGNORE_PARAMETER CCSDSLENGTH IGNORE_PARAMETER PKTID # Ignored Items -# IGNORE_ITEM item_name IGNORE_ITEM CCSDSVER IGNORE_ITEM CCSDSTYPE IGNORE_ITEM CCSDSSHF diff --git a/openc3/data/config/target.yaml b/openc3/data/config/target.yaml index 9fb3e59cf..086860f26 100644 --- a/openc3/data/config/target.yaml +++ b/openc3/data/config/target.yaml @@ -156,6 +156,8 @@ TARGET: required: true description: Number of seconds between runs of the cleanup process (default = 900 = 15 minutes) values: \d+ + DISABLE_REDUCER: + summary: Disables the data reduction microservice for the target REDUCER_MAX_CPU_UTILIZATION: summary: Maximum amount of CPU utilization to apply to data reduction parameters: diff --git a/openc3/lib/openc3/models/target_model.rb b/openc3/lib/openc3/models/target_model.rb index 6df0fe5ce..888f29395 100644 --- a/openc3/lib/openc3/models/target_model.rb +++ b/openc3/lib/openc3/models/target_model.rb @@ -346,6 +346,7 @@ def initialize( cleanup_poll_time: 900, needs_dependencies: false, target_microservices: {'REDUCER' => [[]]}, + disable_reducer: false, reducer_max_cpu_utilization: 30.0, scope: ) @@ -361,7 +362,7 @@ def initialize( reduced_minute_log_retain_time: reduced_minute_log_retain_time, reduced_hour_log_retain_time: reduced_hour_log_retain_time, reduced_day_log_retain_time: reduced_day_log_retain_time, cleanup_poll_time: cleanup_poll_time, needs_dependencies: needs_dependencies, target_microservices: target_microservices, - reducer_max_cpu_utilization: reducer_max_cpu_utilization, + disable_reducer: disable_reducer, reducer_max_cpu_utilization: reducer_max_cpu_utilization, scope: scope) @folder_name = folder_name @requires = requires @@ -392,6 +393,7 @@ def initialize( @cleanup_poll_time = cleanup_poll_time @needs_dependencies = needs_dependencies @target_microservices = target_microservices + @disable_reducer = disable_reducer @reducer_max_cpu_utilization = reducer_max_cpu_utilization @bucket = Bucket.getClient() @children = [] @@ -431,6 +433,7 @@ def as_json(*a) 'cleanup_poll_time' => @cleanup_poll_time, 'needs_dependencies' => @needs_dependencies, 'target_microservices' => @target_microservices.as_json(:allow_nan => true), + 'disable_reducer' => @disable_reducer, 'reducer_max_cpu_utilization' => @reducer_max_cpu_utilization } end @@ -513,6 +516,8 @@ def handle_config(parser, keyword, parameters) @reduced_hour_log_retain_time = reduced_log_retain_time.to_i @reduced_day_log_retain_time = reduced_log_retain_time.to_i end + when 'DISABLE_REDUCER', 'DISABLE_REDUCED' # Handle typos + @disable_reducer = true when 'REDUCER_MAX_CPU_UTILIZATION', 'REDUCED_MAX_CPU_UTILIZATION' # Handle typos parser.verify_num_parameters(1, 1, "#{keyword} ") @reducer_max_cpu_utilization = Float(parameters[0]) @@ -1063,8 +1068,10 @@ def deploy_microservices(gem_path, variables, system) end # Reducer Microservice - deploy_target_microservices('REDUCER', decom_topic_list, "#{@scope}__DECOM__{#{@name}}") do |topics, instance, parent| - deploy_reducer_microservice(gem_path, variables, topics, instance, parent) + unless @disable_reducer + deploy_target_microservices('REDUCER', decom_topic_list, "#{@scope}__DECOM__{#{@name}}") do |topics, instance, parent| + deploy_reducer_microservice(gem_path, variables, topics, instance, parent) + end end end From a01a562b79beb4eb6d6596a07cf7043cc0587b32 Mon Sep 17 00:00:00 2001 From: Jason Thomas Date: Wed, 13 Sep 2023 14:16:56 -0600 Subject: [PATCH 2/2] REDUCER_DISABLE instead of DISABLE_REDUCER --- .../plugins/packages/openc3-cosmos-demo/plugin.txt | 6 ++---- openc3/data/config/target.yaml | 2 +- openc3/lib/openc3/models/target_model.rb | 14 +++++++------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/plugin.txt b/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/plugin.txt index 9692e45d5..ac6f568c9 100644 --- a/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/plugin.txt +++ b/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/plugin.txt @@ -48,8 +48,6 @@ VARIABLE reduced_log_retain_time 2592000 LOG_RETAIN_TIME <%= log_retain_time %> REDUCED_LOG_RETAIN_TIME <%= reduced_log_retain_time %> TLM_LOG_CYCLE_TIME 600 - # Allow the reducer microservice to take 10% of the cpu (default 30%) - REDUCER_MAX_CPU_UTILIZATION 10 <% end %> <% if include_example %> @@ -58,7 +56,7 @@ VARIABLE reduced_log_retain_time 2592000 # Disable data reduction (min, hour, day) on this target data # No 'ruby reducer_microservice.rb DEFAULT__REDUCER__EXAMPLE' # will appear in the the process list - DISABLE_REDUCER + REDUCER_DISABLE <% end %> <% if include_templated %> @@ -67,7 +65,7 @@ VARIABLE reduced_log_retain_time 2592000 # Disable data reduction (min, hour, day) on this target data # No 'ruby reducer_microservice.rb DEFAULT__REDUCER__TEMPLATED' # will appear in the the process list - DISABLE_REDUCER + REDUCER_DISABLE <% end %> <% if include_system %> diff --git a/openc3/data/config/target.yaml b/openc3/data/config/target.yaml index 086860f26..ec2f57d40 100644 --- a/openc3/data/config/target.yaml +++ b/openc3/data/config/target.yaml @@ -156,7 +156,7 @@ TARGET: required: true description: Number of seconds between runs of the cleanup process (default = 900 = 15 minutes) values: \d+ - DISABLE_REDUCER: + REDUCER_DISABLE: summary: Disables the data reduction microservice for the target REDUCER_MAX_CPU_UTILIZATION: summary: Maximum amount of CPU utilization to apply to data reduction diff --git a/openc3/lib/openc3/models/target_model.rb b/openc3/lib/openc3/models/target_model.rb index 888f29395..0629b86a9 100644 --- a/openc3/lib/openc3/models/target_model.rb +++ b/openc3/lib/openc3/models/target_model.rb @@ -346,7 +346,7 @@ def initialize( cleanup_poll_time: 900, needs_dependencies: false, target_microservices: {'REDUCER' => [[]]}, - disable_reducer: false, + reducer_disable: false, reducer_max_cpu_utilization: 30.0, scope: ) @@ -362,7 +362,7 @@ def initialize( reduced_minute_log_retain_time: reduced_minute_log_retain_time, reduced_hour_log_retain_time: reduced_hour_log_retain_time, reduced_day_log_retain_time: reduced_day_log_retain_time, cleanup_poll_time: cleanup_poll_time, needs_dependencies: needs_dependencies, target_microservices: target_microservices, - disable_reducer: disable_reducer, reducer_max_cpu_utilization: reducer_max_cpu_utilization, + reducer_disable: reducer_disable, reducer_max_cpu_utilization: reducer_max_cpu_utilization, scope: scope) @folder_name = folder_name @requires = requires @@ -393,7 +393,7 @@ def initialize( @cleanup_poll_time = cleanup_poll_time @needs_dependencies = needs_dependencies @target_microservices = target_microservices - @disable_reducer = disable_reducer + @reducer_disable = reducer_disable @reducer_max_cpu_utilization = reducer_max_cpu_utilization @bucket = Bucket.getClient() @children = [] @@ -433,7 +433,7 @@ def as_json(*a) 'cleanup_poll_time' => @cleanup_poll_time, 'needs_dependencies' => @needs_dependencies, 'target_microservices' => @target_microservices.as_json(:allow_nan => true), - 'disable_reducer' => @disable_reducer, + 'reducer_disable' => @reducer_disable, 'reducer_max_cpu_utilization' => @reducer_max_cpu_utilization } end @@ -516,8 +516,8 @@ def handle_config(parser, keyword, parameters) @reduced_hour_log_retain_time = reduced_log_retain_time.to_i @reduced_day_log_retain_time = reduced_log_retain_time.to_i end - when 'DISABLE_REDUCER', 'DISABLE_REDUCED' # Handle typos - @disable_reducer = true + when 'REDUCER_DISABLE', 'REDUCER_DISABLED' # Handle typos + @reducer_disable = true when 'REDUCER_MAX_CPU_UTILIZATION', 'REDUCED_MAX_CPU_UTILIZATION' # Handle typos parser.verify_num_parameters(1, 1, "#{keyword} ") @reducer_max_cpu_utilization = Float(parameters[0]) @@ -1068,7 +1068,7 @@ def deploy_microservices(gem_path, variables, system) end # Reducer Microservice - unless @disable_reducer + unless @reducer_disable deploy_target_microservices('REDUCER', decom_topic_list, "#{@scope}__DECOM__{#{@name}}") do |topics, instance, parent| deploy_reducer_microservice(gem_path, variables, topics, instance, parent) end