PROPOSAL: add a fallthrough! method for continuing to check later rules

lib/rouge/regex_lexer.rb

@@ -37,6 +37,12 @@ def to_s
       end
     end
 
+    # exception class for fallthrough - using an exception is slightly faster
+    # than catch { ... }, but it is not semantically an error that should be
+    # rescued from.
+    class Fallthrough < Exception # rubocop:disable Lint/InheritException
+    end
+
     # A rule is a tuple of a regular expression to test, and a callback
     # to perform if the test succeeds.
     #
@@ -365,8 +371,6 @@ def step(state, stream)
           if (size = stream.skip(rule.re))
             puts "    got: #{stream[0].inspect}" if @debug
 
-            instance_exec(stream, &rule.callback)
-
             if size.zero?
               @null_steps += 1
               if @null_steps > MAX_NULL_SCANS
@@ -377,6 +381,13 @@ def step(state, stream)
               @null_steps = 0
             end
 
+            begin
+              instance_exec(stream, &rule.callback)
+            rescue Fallthrough
+              stream.unscan
+              next
+            end
+
             return true
           end
         end
@@ -435,6 +446,10 @@ def recurse(text=nil)
       delegate(self.class, text)
     end
 
+    def fallthrough!
+      raise Fallthrough
+    end
+
     # Push a state onto the stack.  If no state name is given and you've
     # passed a block, a state will be dynamically created using the
     # {StateDSL}.

spec/lexer_spec.rb

@@ -243,4 +243,31 @@ def initialize(*)
     assert { php.instance_variable_get(:@start_inline) == :guess }
     assert { inline_php.instance_variable_get(:@start_inline) == true }
   end
+
+  it 'falls through' do
+    new_lexer = Class.new(Rouge::RegexLexer) do
+      state :root do
+        rule %r/\w+\b/ do |m|
+          fallthrough! unless %w(testy1 testy2).include?(m[0])
+
+          token 'builtin'
+        end
+
+        rule %r/\w+\b/ do |m|
+          token 'function'
+        end
+
+        rule %r/\s+/, 'text'
+      end
+    end
+
+    result = new_lexer.lex('testy1 foobar testy2').to_a
+
+    assert { result.size == 5 }
+    assert { result[0] == ['builtin', 'testy1'] }
+    assert { result[1] == ['text', ' '] }
+    assert { result[2] == ['function', 'foobar'] }
+    assert { result[3] == ['text', ' '] }
+    assert { result[4] == ['builtin', 'testy2'] }
+  end
 end