From d7a0121a5349e8e313470ec29dfc99f3ed93ccf2 Mon Sep 17 00:00:00 2001 From: Jason Thomas Date: Mon, 11 Sep 2023 18:15:38 -0600 Subject: [PATCH 1/3] Comment out ERB --- openc3/lib/openc3/config/config_parser.rb | 4 +- .../lib/openc3/config/meta_config_parser.rb | 2 +- openc3/lib/openc3/core_ext/string.rb | 15 +++++- .../lib/openc3/models/microservice_model.rb | 2 +- openc3/lib/openc3/models/target_model.rb | 6 ++- openc3/lib/openc3/models/tool_model.rb | 2 +- openc3/lib/openc3/models/widget_model.rb | 2 +- openc3/lib/openc3/utilities/cli_generator.rb | 2 +- openc3/spec/config/config_parser_spec.rb | 17 +++++++ openc3/spec/core_ext/string_spec.rb | 46 ++++++++++++++++++- 10 files changed, 87 insertions(+), 11 deletions(-) diff --git a/openc3/lib/openc3/config/config_parser.rb b/openc3/lib/openc3/config/config_parser.rb index f34c4b571..11fba98f3 100644 --- a/openc3/lib/openc3/config/config_parser.rb +++ b/openc3/lib/openc3/config/config_parser.rb @@ -166,7 +166,7 @@ def render(template_name, options = {}) options[:locals].each { |key, value| b.local_variable_set(key, value) } end - return ERB.new(read_file(template_name), trim_mode: "-").result(b) + return ERB.new(read_file(template_name).comment_erb(), trim_mode: "-").result(b) end # Can be called during parsing to read a referenced file @@ -385,7 +385,7 @@ def create_parsed_output_file(filename, run_erb, variables) output = nil if run_erb OpenC3.set_working_dir(File.dirname(filename)) do - output = ERB.new(File.read(filename), trim_mode: "-").result(binding.set_variables(variables)) + output = ERB.new(File.read(filename).comment_erb(), trim_mode: "-").result(binding.set_variables(variables)) end else output = File.read(filename) diff --git a/openc3/lib/openc3/config/meta_config_parser.rb b/openc3/lib/openc3/config/meta_config_parser.rb index 101094f3a..4c7bf2e28 100644 --- a/openc3/lib/openc3/config/meta_config_parser.rb +++ b/openc3/lib/openc3/config/meta_config_parser.rb @@ -17,7 +17,7 @@ # All changes Copyright 2022, OpenC3, Inc. # All Rights Reserved # -# This file may also be used under the terms of a commercial license +# This file may also be used under the terms of a commercial license # if purchased from OpenC3, Inc. require 'erb' diff --git a/openc3/lib/openc3/core_ext/string.rb b/openc3/lib/openc3/core_ext/string.rb index 3f3a90c08..d2029c257 100644 --- a/openc3/lib/openc3/core_ext/string.rb +++ b/openc3/lib/openc3/core_ext/string.rb @@ -389,4 +389,17 @@ def to_utf8! end end -end # class String + def comment_erb + # Split with -1 to avoid stripping empty strings + output = self.split("\n", -1).collect! do |line| + # If we have a commented out line (starts with #) + # which contains an ERB statement (<% ...) + # then comment out the ERB statement (<%# ...) + if line.strip() =~ /^#.*<%/ + line.gsub!('<%', '<%#') + end + line.chomp + end + return output.join("\n") + end +end diff --git a/openc3/lib/openc3/models/microservice_model.rb b/openc3/lib/openc3/models/microservice_model.rb index ac6bd529b..340096769 100644 --- a/openc3/lib/openc3/models/microservice_model.rb +++ b/openc3/lib/openc3/models/microservice_model.rb @@ -221,7 +221,7 @@ def deploy(gem_path, variables, validate_only: false) # Load microservice files data = File.read(filename, mode: "rb") OpenC3.set_working_dir(File.dirname(filename)) do - data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable? and File.basename(filename)[0] != '_' + data = ERB.new(data.comment_erb(), trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable? and File.basename(filename)[0] != '_' end unless validate_only @bucket.put_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key, body: data) diff --git a/openc3/lib/openc3/models/target_model.rb b/openc3/lib/openc3/models/target_model.rb index 6df0fe5ce..47a96cc17 100644 --- a/openc3/lib/openc3/models/target_model.rb +++ b/openc3/lib/openc3/models/target_model.rb @@ -563,7 +563,9 @@ def deploy(gem_path, variables, validate_only: false) data = File.read(filename, mode: "rb") begin OpenC3.set_working_dir(File.dirname(filename)) do - data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable? and File.basename(filename)[0] != '_' + if data.is_printable? and File.basename(filename)[0] != '_' + data = ERB.new(data.comment_erb(), trim_mode: "-").result(binding.set_variables(variables)) + end end rescue => error # ERB error parsing a screen is just a logger error because life can go on @@ -673,7 +675,7 @@ def render(template_name, options = {}) begin OpenC3.set_working_dir(File.dirname(path)) do - return ERB.new(File.read(path), trim_mode: "-").result(b) + return ERB.new(File.read(path.comment_erb()), trim_mode: "-").result(b) end rescue => error raise "ERB error parsing: #{path}: #{error.formatted}" diff --git a/openc3/lib/openc3/models/tool_model.rb b/openc3/lib/openc3/models/tool_model.rb index f5c4905fb..90e649e94 100644 --- a/openc3/lib/openc3/models/tool_model.rb +++ b/openc3/lib/openc3/models/tool_model.rb @@ -240,7 +240,7 @@ def deploy(gem_path, variables, validate_only: false) # Load tool files data = File.read(filename, mode: "rb") - data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable? + data = ERB.new(data.comment_erb(), trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable? unless validate_only client = Bucket.getClient() cache_control = BucketUtilities.get_cache_control(filename) diff --git a/openc3/lib/openc3/models/widget_model.rb b/openc3/lib/openc3/models/widget_model.rb index 9c7af34c0..9e1a57ff5 100644 --- a/openc3/lib/openc3/models/widget_model.rb +++ b/openc3/lib/openc3/models/widget_model.rb @@ -124,7 +124,7 @@ def deploy(gem_path, variables, validate_only: false) # Load widget file data = File.read(filename, mode: "rb") OpenC3.set_working_dir(File.dirname(filename)) do - data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable? + data = ERB.new(data.comment_erb(), trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable? end unless validate_only cache_control = BucketUtilities.get_cache_control(@filename) diff --git a/openc3/lib/openc3/utilities/cli_generator.rb b/openc3/lib/openc3/utilities/cli_generator.rb index a100b736f..5aa50e0db 100644 --- a/openc3/lib/openc3/utilities/cli_generator.rb +++ b/openc3/lib/openc3/utilities/cli_generator.rb @@ -51,7 +51,7 @@ def self.process_template(template_dir, the_binding) FileUtils.mkdir(base_name) unless File.exist?(base_name) next end - output = ERB.new(File.read(file), trim_mode: "-").result(the_binding) + output = ERB.new(File.read(file).comment_erb(), trim_mode: "-").result(the_binding) File.open(base_name, 'w') do |file| file.write output end diff --git a/openc3/spec/config/config_parser_spec.rb b/openc3/spec/config/config_parser_spec.rb index 9d770d3ea..fc176a57d 100644 --- a/openc3/spec/config/config_parser_spec.rb +++ b/openc3/spec/config/config_parser_spec.rb @@ -79,6 +79,23 @@ module OpenC3 tf.unlink end + it "ignores commented out ERB syntax" do + tf = Tempfile.new('unittest') + tf.puts "# KEYWORD <%= raise 'boom' %>" + tf.puts "# <%= render '_ccsds_cmd.txt', locals: {id: 4} %>" + tf.puts "# KEYWORD <% if true %>" + tf.puts "# KEYWORD <% raise 'dead' %>" + tf.puts "# KEYWORD <% end %>" + tf.puts "OTHER stuff" + tf.close + + @cp.parse_file(tf.path) do |keyword, params| + expect(keyword).to eql "OTHER" + expect(params[0]).to eql "stuff" + end + tf.unlink + end + it "requires ERB partials begin with an underscore" do tf = Tempfile.new('unittest') tf.puts "<%= render 'partial.txt' %>" diff --git a/openc3/spec/core_ext/string_spec.rb b/openc3/spec/core_ext/string_spec.rb index a5b14c5da..03643f7e9 100644 --- a/openc3/spec/core_ext/string_spec.rb +++ b/openc3/spec/core_ext/string_spec.rb @@ -17,7 +17,7 @@ # All changes Copyright 2022, OpenC3, Inc. # All Rights Reserved # -# This file may also be used under the terms of a commercial license +# This file may also be used under the terms of a commercial license # if purchased from OpenC3, Inc. require 'spec_helper' @@ -296,6 +296,50 @@ expect(output.length).to eql 3 expect(output.force_encoding('ASCII-8BIT')).to eql "\xC2\xB0\xE2\x84\xA2\xE2\x80\xA6" end + end + + describe "comment_erb" do + it "comments out ERB syntax" do + input = "\n<%= ERB1 %>\n\n" + input += "#<% ERB2 %>\n" + input += "\n # <% ERB3 %>\n" + input += "test # <% ERB4 %>\n" + input += "puts \#{test} <%= ERB5 %>\n\n" + + output = "\n<%= ERB1 %>\n\n" # normal no comment + output += "#<%# ERB2 %>\n" # comment + output += "\n # <%# ERB3 %>\n" # comment + # We're not trying to handle trailing comments with ERB + output += "test # <% ERB4 %>\n" + # Normal string interpolation #{} code doesn't mean comment + output += "puts \#{test} <%= ERB5 %>\n\n" + expect(input.comment_erb()).to eql output + end end + + # require 'benchmark' + # input = "HI" + # 10000.times do |x| + # input += "this is a very long line that will be part of the code that we look at\n" + # if x % 10 == 0 + # input += "# <%= 'ERB input' %>\n" + # end + # end + # File.open('test.txt', 'w') do |file| + # file.write(input.comment_erb()) + # end + # Benchmark.bm do |x| + # x.report do + # input.comment_erb() + # end + # x.report do + # input + # end + # end + # Macbook Air M2 results: + # bundle exec rspec spec/core_ext/string_spec.rb + # user system total real + # 0.004457 0.000286 0.004743 ( 0.004743) + # 0.000002 0.000001 0.000003 ( 0.000002) end From 93337b645afca493219dcf3c5d96f8a2be1b34fc Mon Sep 17 00:00:00 2001 From: Jason Thomas Date: Tue, 12 Sep 2023 23:11:27 -0600 Subject: [PATCH 2/3] Escape ERB commenting --- openc3/lib/openc3/core_ext/string.rb | 9 ++++++--- openc3/spec/core_ext/string_spec.rb | 23 ++++++++++++++--------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/openc3/lib/openc3/core_ext/string.rb b/openc3/lib/openc3/core_ext/string.rb index d2029c257..85431a95d 100644 --- a/openc3/lib/openc3/core_ext/string.rb +++ b/openc3/lib/openc3/core_ext/string.rb @@ -392,10 +392,13 @@ def to_utf8! def comment_erb # Split with -1 to avoid stripping empty strings output = self.split("\n", -1).collect! do |line| - # If we have a commented out line (starts with #) + # If we have a commented out line that starts with # + # but not followed by % (allows for disabling ERB comments), # which contains an ERB statement (<% ...) - # then comment out the ERB statement (<%# ...) - if line.strip() =~ /^#.*<%/ + # then comment out the ERB statement (<%# ...). + # We explicitly don't comment out trailing ERB statements + # as that is not typical and is difficult to regex + if line.strip() =~ /^#[^%]*<%/ line.gsub!('<%', '<%#') end line.chomp diff --git a/openc3/spec/core_ext/string_spec.rb b/openc3/spec/core_ext/string_spec.rb index 03643f7e9..c6cc6ad20 100644 --- a/openc3/spec/core_ext/string_spec.rb +++ b/openc3/spec/core_ext/string_spec.rb @@ -304,15 +304,20 @@ input += "#<% ERB2 %>\n" input += "\n # <% ERB3 %>\n" input += "test # <% ERB4 %>\n" - input += "puts \#{test} <%= ERB5 %>\n\n" - - output = "\n<%= ERB1 %>\n\n" # normal no comment - output += "#<%# ERB2 %>\n" # comment - output += "\n # <%# ERB3 %>\n" # comment - # We're not trying to handle trailing comments with ERB - output += "test # <% ERB4 %>\n" - # Normal string interpolation #{} code doesn't mean comment - output += "puts \#{test} <%= ERB5 %>\n\n" + input += "puts \"\#{test} <%= ERB5 %>\"\n\n" + input += " #puts \"\#{test} <%= ERB6 %>\"\n" + input += "#\{ <%= ERB7 %>\n\n" + input += "#% <% ERB8 %>\n" + + output = "\n<%= ERB1 %>\n\n" # normal no ERB comment + output += "#<%# ERB2 %>\n" # ERB comment + output += "\n # <%# ERB3 %>\n" # ERB comment + output += "test # <% ERB4 %>\n" # trailing, no ERB comment + # string interpolation #{} code, no ERB comment + output += "puts \"\#{test} <%= ERB5 %>\"\n\n" + output += " #puts \"\#{test} <%#= ERB6 %>\"\n" # ERB comment + output += "#\{ <%#= ERB7 %>\n\n" # ERB comment + output += "#% <% ERB8 %>\n" # Special #% comment, no ERB comment expect(input.comment_erb()).to eql output end From 9588f1865f073466439f7fbded2043e786020f01 Mon Sep 17 00:00:00 2001 From: Jason Thomas Date: Wed, 13 Sep 2023 10:51:13 -0600 Subject: [PATCH 3/3] Improve performance --- openc3/lib/openc3/core_ext/string.rb | 9 ++++----- openc3/spec/core_ext/string_spec.rb | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/openc3/lib/openc3/core_ext/string.rb b/openc3/lib/openc3/core_ext/string.rb index 85431a95d..4345a8646 100644 --- a/openc3/lib/openc3/core_ext/string.rb +++ b/openc3/lib/openc3/core_ext/string.rb @@ -390,19 +390,18 @@ def to_utf8! end def comment_erb - # Split with -1 to avoid stripping empty strings - output = self.split("\n", -1).collect! do |line| + output = self.lines.collect! do |line| # If we have a commented out line that starts with # # but not followed by % (allows for disabling ERB comments), # which contains an ERB statement (<% ...) # then comment out the ERB statement (<%# ...). # We explicitly don't comment out trailing ERB statements # as that is not typical and is difficult to regex - if line.strip() =~ /^#[^%]*<%/ + if line =~ /^\s*#[^%]*<%/ line.gsub!('<%', '<%#') end - line.chomp + line end - return output.join("\n") + return output.join("") end end diff --git a/openc3/spec/core_ext/string_spec.rb b/openc3/spec/core_ext/string_spec.rb index c6cc6ad20..dd7091918 100644 --- a/openc3/spec/core_ext/string_spec.rb +++ b/openc3/spec/core_ext/string_spec.rb @@ -345,6 +345,6 @@ # Macbook Air M2 results: # bundle exec rspec spec/core_ext/string_spec.rb # user system total real - # 0.004457 0.000286 0.004743 ( 0.004743) + # 0.003438 0.000112 0.003550 ( 0.003550) # 0.000002 0.000001 0.000003 ( 0.000002) end