From 8a14e137087b3cc581d7753fa13bff68a10a625a Mon Sep 17 00:00:00 2001 From: Andrew Szczepanski Date: Fri, 24 Feb 2023 16:23:55 -0500 Subject: [PATCH] Fix stack overflow for large projects When large projects have a lot of missing objects in the first pass of parsing Ruby files we would run into stack overflows. This happens because, when there was a missing object in a parsed Ruby file, we would recursively call `#parse_remaining_files`. When this happens a lot the stack would get huge. We fix this by instead keeping a list of files that we want to retry and re-parse them in another pass. When we can no longer resolve any more files we break the loop. --- lib/yard/handlers/base.rb | 11 +++-------- lib/yard/handlers/c/base.rb | 17 +++++++--------- lib/yard/handlers/processor.rb | 13 ------------ lib/yard/parser/source_parser.rb | 34 +++++++++++++++++++++----------- 4 files changed, 33 insertions(+), 42 deletions(-) diff --git a/lib/yard/handlers/base.rb b/lib/yard/handlers/base.rb index 989eb1008..7863a621b 100644 --- a/lib/yard/handlers/base.rb +++ b/lib/yard/handlers/base.rb @@ -562,14 +562,9 @@ def ensure_loaded!(object, max_retries = 1) return if object.root? return object unless object.is_a?(Proxy) - retries = 0 - while object.is_a?(Proxy) - raise NamespaceMissingError, object if retries > max_retries - log.debug "Missing object #{object} in file `#{parser.file}', moving it to the back of the line." - parser.parse_remaining_files - retries += 1 - end - object + log.debug "Missing object #{object} in file `#{parser.file}', moving it to the back of the line." + globals.ordered_parser.files_to_retry << parser.file if globals.ordered_parser + raise NamespaceMissingError, object end # @group Macro Support diff --git a/lib/yard/handlers/c/base.rb b/lib/yard/handlers/c/base.rb index 78420d132..5936dc928 100644 --- a/lib/yard/handlers/c/base.rb +++ b/lib/yard/handlers/c/base.rb @@ -75,20 +75,17 @@ def namespace_for_variable(var) end def ensure_variable_defined!(var, max_retries = 1) - retries = 0 - object = nil + object = namespace_for_variable(var) + return object unless object.is_a?(Proxy) - loop do - object = namespace_for_variable(var) - break unless object.is_a?(Proxy) + log.debug "Missing object #{object} in file `#{parser.file}', moving it to the back of the line." - raise NamespaceMissingError, object if retries > max_retries - log.debug "Missing namespace variable #{var} in file `#{parser.file}', moving it to the back of the line." - parser.parse_remaining_files - retries += 1 + if globals.ordered_parser + retryable_file = parser.file == "(stdin)" ? StringIO.new("void Init_Foo() { #{statement.source} }") : parser.file + globals.ordered_parser.files_to_retry << retryable_file end - object + raise NamespaceMissingError, object end def namespaces diff --git a/lib/yard/handlers/processor.rb b/lib/yard/handlers/processor.rb index d6ea675ad..31a4a1b58 100644 --- a/lib/yard/handlers/processor.rb +++ b/lib/yard/handlers/processor.rb @@ -131,19 +131,6 @@ def process(statements) end end - # Continue parsing the remainder of the files in the +globals.ordered_parser+ - # object. After the remainder of files are parsed, processing will continue - # on the current file. - # - # @return [void] - # @see Parser::OrderedParser - def parse_remaining_files - if globals.ordered_parser - globals.ordered_parser.parse - log.debug("Re-processing #{@file}...") - end - end - # Searches for all handlers in {Base.subclasses} that match the +statement+ # # @param statement the statement object to match. diff --git a/lib/yard/parser/source_parser.rb b/lib/yard/parser/source_parser.rb index 8d133e193..c46dce115 100644 --- a/lib/yard/parser/source_parser.rb +++ b/lib/yard/parser/source_parser.rb @@ -12,16 +12,14 @@ class UndocumentableError < RuntimeError; end # Raised when the parser sees a Ruby syntax error class ParserSyntaxError < UndocumentableError; end - # Responsible for parsing a list of files in order. The - # {#parse} method of this class can be called from the - # {SourceParser#globals} globals state list to re-enter - # parsing for the remainder of files in the list recursively. - # - # @see Processor#parse_remaining_files + # Responsible for parsing a list of files in order. class OrderedParser # @return [Array] the list of remaining files to parse attr_accessor :files + # @return [Array] files to parse again after our first pass + attr_accessor :files_to_retry + # Creates a new OrderedParser with the global state and a list # of files to parse. # @@ -33,19 +31,33 @@ class OrderedParser def initialize(global_state, files) @global_state = global_state @files = files.dup + @files_to_retry = [] @global_state.ordered_parser = self end - # Parses the remainder of the {#files} list. - # - # @see Processor#parse_remaining_files + # Parses the remainder of the {#files} list. Any files that had issues + # parsing will be done in multiple passes until the size of the files + # remaining does not change. def parse - until files.empty? - file = files.shift + files.each do |file| log.capture("Parsing #{file}") do SourceParser.new(SourceParser.parser_type, @global_state).parse(file) end end + + loop do + prev_length = @files_to_retry.length + files_to_parse_now = @files_to_retry.dup + @files_to_retry = [] + + files_to_parse_now.each do |file| + log.capture("Re-processing #{file}") do + SourceParser.new(SourceParser.parser_type, @global_state).parse(file) + end + end + + break if @files_to_retry.length >= prev_length + end end end