Realign TOML lexers with the Pygments counterpart

This also helps to address a number of TOML bugs around handling of the
value attributes.

Author: tancnle

lib/rouge/lexers/toml.rb

@@ -11,63 +11,56 @@ class TOML < RegexLexer
       filenames '*.toml', 'Pipfile', 'poetry.lock'
       mimetypes 'text/x-toml'
 
-      # bare keys and quoted keys
-      identifier = %r/(?:\S+|"[^"]+"|'[^']+')/
-
-      state :basic do
-        rule %r/\s+/, Text
-        rule %r/#.*?$/, Comment
-        rule %r/(true|false)/, Keyword::Constant
-
-        rule %r/(#{identifier})(\s*)(=)(\s*)(\{)/ do
-          groups Name::Property, Text, Operator, Text, Punctuation
-          push :inline
-        end
-      end
-
       state :root do
-        mixin :basic
+        mixin :whitespace
 
-        rule %r/(?<!=)\s*\[.*?\]+/, Name::Namespace
+        mixin :key
 
-        rule %r/(#{identifier})(\s*)(=)/ do
-          groups Name::Property, Text, Punctuation
+        rule %r/(=)(\s*)/ do
+          groups Operator, Text::Whitespace
           push :value
         end
-      end
 
-      state :value do
-        rule %r/\n/, Text, :pop!
-        mixin :content
+        rule %r/\[\[?/, Keyword, :table_key
       end
 
-      state :content do
-        mixin :basic
-
-        rule %r/(#{identifier})(\s*)(=)/ do
-          groups Name::Property, Text, Punctuation
-        end
+      state :key do
+        rule %r/[A-Za-z0-9_-]+/, Name
 
-        rule %r/\d{4}-\d{2}-\d{2}(?:[Tt ]\d{2}:\d{2}:\d{2}(?:[Zz]|[+-]\d{2}:\d{2})?)?/, Literal::Date
-        rule %r/\d{2}:\d{2}:\d{2}/, Literal::Date
-
-        rule %r/[+-]?\d+(?:_\d+)*\.\d+(?:_\d+)*(?:[eE][+-]?\d+(?:_\d+)*)?/, Num::Float
-        rule %r/[+-]?\d+(?:_\d+)*[eE][+-]?\d+(?:_\d+)*/, Num::Float
-        rule %r/[+-]?(?:nan|inf)/, Num::Float
+        rule %r/"/, Str, :dq
+        rule %r/'/, Str, :sq
+        rule %r/\./, Punctuation
+      end
 
-        rule %r/0x\h+(?:_\h+)*/, Num::Hex
-        rule %r/0o[0-7]+(?:_[0-7]+)*/, Num::Oct
-        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
-        rule %r/[+-]?\d+(?:_\d+)*/, Num::Integer
+      state :table_key do
+        rule %r/[A-Za-z0-9_-]+/, Name
 
-        rule %r/"""/, Str, :mdq
         rule %r/"/, Str, :dq
-        rule %r/'''/, Str, :msq
         rule %r/'/, Str, :sq
-        mixin :esc_str
-        rule %r/\,/, Punctuation
-        rule %r/\[/, Punctuation, :array
-        rule %r/\{/, Punctuation, :inline
+        rule %r/\./, Keyword
+        rule %r/\]\]?/, Keyword, :pop!
+        rule %r/[ \t]+/, Text::Whitespace
+      end
+
+      state :value do
+        rule %r/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z/, Literal::Date, :pop!
+        rule %r/\d\d:\d\d:\d\d(\.\d+)?/, Literal::Date, :pop!
+        rule %r/[+-]?\d+(?:_\d+)*\.\d+(?:_\d+)*(?:[eE][+-]?\d+(?:_\d+)*)?/, Num::Float, :pop!
+        rule %r/[+-]?\d+(?:_\d+)*[eE][+-]?\d+(?:_\d+)*/, Num::Float, :pop!
+        rule %r/[+-]?(?:nan|inf)/, Num::Float, :pop!
+        rule %r/0x\h+(?:_\h+)*/, Num::Hex, :pop!
+        rule %r/0o[0-7]+(?:_[0-7]+)*/, Num::Oct, :pop!
+        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin, :pop!
+        rule %r/[+-]?\d+(?:_\d+)*/, Num::Integer, :pop!
+
+        rule %r/"""/, Str, [:pop!, :mdq]
+        rule %r/"/, Str, [:pop!, :dq]
+        rule %r/'''/, Str, [:pop!, :msq]
+        rule %r/'/, Str, [:pop!, :sq]
+
+        rule %r/(true|false)/, Keyword::Constant, :pop!
+        rule %r/\[/, Punctuation, [:pop!, :array]
+        rule %r/\{/, Punctuation, [:pop!, :inline]
       end
 
       state :dq do
@@ -101,15 +94,31 @@ class TOML < RegexLexer
       end
 
       state :array do
-        mixin :content
+        mixin :whitespace
+        rule %r/,/, Punctuation
+
         rule %r/\]/, Punctuation, :pop!
+
+        rule %r//, Token, :value
       end
 
       state :inline do
-        mixin :content
+        rule %r/[ \t]+/, Text::Whitespace
 
+        mixin :key
+        rule %r/(=)(\s*)/ do
+          groups Punctuation, Text::Whitespace
+          push :value
+        end
+
+        rule %r/,/, Punctuation
         rule %r/\}/, Punctuation, :pop!
       end
+
+      state :whitespace do
+        rule %r/\s+/, Text
+        rule %r/#.*?$/, Comment
+      end
     end
   end
 end

spec/visual/samples/toml

@@ -8,10 +8,10 @@ organization = "GitHub"
 bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
 # First class dates? Why not?
 dob = [
-	1979-05-27T07:32:00Z,
-	1979-05-27 07:32:00
-	1979-05-27,
-	07:32:00,
+  1979-05-27T07:32:00Z,
+  1979-05-27 07:32:00
+  1979-05-27,
+  07:32:00,
 ]
 
 [database]
@@ -106,16 +106,14 @@ test_string = "You'll hate me after this - #"          # " Annoying, isn't it?
     harder_test_string = " And when \"'s are in the string, along with # \""   # "and comments are there too"
     # Things will get harder
 
-        [the.hard.bit#]
-        what? = "You don't think some user won't do that?"
+        [the.hard."bit#"]
+        "what?" = "You don't think some user won't do that?"
         multi_line_array = [
             "]",
             "Oi!\n",
             # ] Oh yes I did
             ]
 
-東京都 = 123
-
 # Inline table
 name = { first = "Tom", last = "Preston-Werner" }
 point = { x = 1, y = 2 }
@@ -127,6 +125,7 @@ point = { x = 1, y = 2 }
 "ʎǝʞ" = "value"
 'key2' = "value"
 'quoted "value"' = "value"
+"東京都" = 123
 
 # Array
 integers = [ 1, 2, 3 ]
@@ -187,8 +186,21 @@ inf-3 = {inf=inf, nan=nan}
 str-inf-1 = 'inf'
 str-inf-2 = "inf"
 str-inf-3 = """ inf nan
-	inf nan
+  inf nan
 """
 str-inf-4 = """ inf nan
-	inf nan
+  inf nan
 """
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.11"
+idna = "3.4"
+inflection = "0.5.1"
+influxdb = "5.3.1"
+requests = "2.28.1"
+pywinpty = {version = ">=1.1.0", markers = "os_name == \"nt\""}
+
+dependencies = [
+  "aiohttp>=3.11.16,<4.0.0",
+  "backoff>=2.2.1,<3.0.0",
+]