Update [[:digit:]] in UCP mode to match Perl

Author: PhilipHazel

ChangeLog

@@ -123,6 +123,14 @@ extension, so PCRE2 follows ECMAScript rather than Perl.
 pcre2_match() was not fully resetting all captures that had been set within a
 (possibly recursive) subroutine call such as (?3).
 
+32. Changed the meaning of \w (and its synonyms) in UCP mode to match Perl. It
+now matches characters whose general categories are L or N or whose particular
+categories are Mn (non-spacing mark) or Pc (combining puntuation). The latter
+includes underscore.
+
+33. Changed the meaning of [:digit:] in UCP mode to match Perl. It now also
+matches the "fullwidth" versions of the hex digits.
+
 
 Version 10.42 11-December-2022
 ------------------------------
@@ -146,11 +154,6 @@ maximum of 65535 is now silently applied.
 
 5. Merged @carenas patch #175 which fixes #86 - segfault on aarch64 (ARM),
 
-6. Changed the meaning of \w (and its synonyms) in UCP mode to match Perl. It 
-now matches characters whose general categories are L or N or whose particular 
-categories are Mn (non-spacing mark) or Pc (combining puntuation). The latter 
-includes underscore.
-
 
 Version 10.41 06-December-2022
 ------------------------------

doc/html/pcre2pattern.html

@@ -1552,7 +1552,7 @@ <h1>pcre2pattern man page</h1>
   [:upper:]  becomes  \p{Lu}
   [:word:]   becomes  \p{Xwd}
 </pre>
-Negated versions, such as [:^alpha:] use \P instead of \p. Three other POSIX
+Negated versions, such as [:^alpha:] use \P instead of \p. Four other POSIX
 classes are handled specially in UCP mode:
 </P>
 <P>
@@ -1579,6 +1579,12 @@ <h1>pcre2pattern man page</h1>
 property.
 </P>
 <P>
+[:xdigit:]
+In addition to the ASCII hexadecimal digits, this also matches the "fullwidth" 
+versions of those characters, whose Unicode code points start at U+FF10. This 
+is a change that was made in PCRE release 10.43 for Perl compatibility.
+</P>
+<P>
 The other POSIX classes are unchanged by PCRE2_UCP, and match only characters
 with code points less than 256. The effect of PCRE2_UCP on POSIX classes can be
 negated by setting the PCRE2_EXTRA_ASCII_POSIX option, either when calling

doc/pcre2.txt

@@ -1546,7 +1546,7 @@ follows:
   [:upper:]  becomes  \ep{Lu}
   [:word:]   becomes  \ep{Xwd}
 .sp
-Negated versions, such as [:^alpha:] use \eP instead of \ep. Three other POSIX
+Negated versions, such as [:^alpha:] use \eP instead of \ep. Four other POSIX
 classes are handled specially in UCP mode:
 .TP 10
 [:graph:]
@@ -1567,6 +1567,11 @@ not controls, that is, characters with the Zs property.
 This matches all characters that have the Unicode P (punctuation) property,
 plus those characters with code points less than 256 that have the S (Symbol)
 property.
+.TP 10
+[:xdigit:]
+In addition to the ASCII hexadecimal digits, this also matches the "fullwidth" 
+versions of those characters, whose Unicode code points start at U+FF10. This 
+is a change that was made in PCRE release 10.43 for Perl compatibility.
 .P
 The other POSIX classes are unchanged by PCRE2_UCP, and match only characters
 with code points less than 256. The effect of PCRE2_UCP on POSIX classes can be

doc/pcre2pattern.3

@@ -706,9 +706,10 @@ static const char posix_names[] =
 static const uint8_t posix_name_lengths[] = {
   5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 6, 0 };
 
-#define PC_GRAPH  8
-#define PC_PRINT  9
-#define PC_PUNCT 10
+#define PC_GRAPH   8
+#define PC_PRINT   9
+#define PC_PUNCT  10
+#define PC_XDIGIT 13
 
 /* Table of class bit maps for each POSIX class. Each class is formed from a
 base map, with an optional addition or removal of another map. Then, for some
@@ -756,7 +757,7 @@ static int posix_substitutes[] = {
   PT_PXPUNCT, 0,    /* punct */
   PT_PXSPACE, 0,    /* space */   /* Xps is POSIX space, but from 8.34 */
   PT_WORD, 0,       /* word  */   /* Perl and POSIX space are the same */
-  -1, 0             /* xdigit, treat as non-UCP */
+  PT_PXXDIGIT, 0    /* xdigit */  /* Perl has additional hex digits */
 };
 #define POSIX_SUBSIZE (sizeof(posix_substitutes) / (2*sizeof(uint32_t)))
 #endif  /* SUPPORT_UNICODE */
@@ -6027,7 +6028,8 @@ for (;; pptr++)
             *class_uchardata++ = local_negate? XCL_NOTPROP : XCL_PROP;
             *class_uchardata++ = (PCRE2_UCHAR)
               ((posix_class == PC_GRAPH)? PT_PXGRAPH :
-               (posix_class == PC_PRINT)? PT_PXPRINT : PT_PXPUNCT);
+               (posix_class == PC_PRINT)? PT_PXPRINT : 
+               (posix_class == PC_XDIGIT)? PT_PXXDIGIT : PT_PXPUNCT);
             *class_uchardata++ = 0;
             xclass_has_prop = TRUE;
             goto CONTINUE_CLASS;

src/pcre2_compile.c

@@ -1299,7 +1299,7 @@ match. */
 #define PT_ALNUM      6    /* Alphanumeric - the union of L and N */
 #define PT_SPACE      7    /* Perl space - general category Z plus 9,10,12,13 */
 #define PT_PXSPACE    8    /* POSIX space - Z plus 9,10,11,12,13 */
-#define PT_WORD       9    /* Word - L plus N plus underscore */
+#define PT_WORD       9    /* Word - L, N, Mn, or Pc */
 #define PT_CLIST     10    /* Pseudo-property: match character list */
 #define PT_UCNC      11    /* Universal Character nameable character */
 #define PT_BIDICL    12    /* Specified bidi class */
@@ -1315,6 +1315,7 @@ table. */
 #define PT_PXGRAPH   14    /* [:graph:] - characters that mark the paper */
 #define PT_PXPRINT   15    /* [:print:] - [:graph:] plus non-control spaces */
 #define PT_PXPUNCT   16    /* [:punct:] - punctuation characters */
+#define PT_PXXDIGIT  17    /* [:xdigit:] - hex digits */
 
 /* This value is used when parsing \p and \P escapes to indicate that neither
 \p{script:...} nor \p{scx:...} has been encountered. */

src/pcre2_internal.h

@@ -778,6 +778,10 @@ for(;;)
               fprintf(f, "[:%spunct:]", notch);
               break;
 
+              case PT_PXXDIGIT:
+              fprintf(f, "[:%sxdigit:]", notch);
+              break;
+
               default:
               s = get_ucpname(ptype, pvalue);
               fprintf(f, "\\%c{%c%s}", ((notch[0] == '^')? 'P':'p'),

src/pcre2_printint.c

@@ -276,6 +276,18 @@ while ((t = *data++) != XCL_END)
         return !negated;
       break;
 
+      /* Perl has two sets of hex digits */
+
+      case PT_PXXDIGIT:
+      if (((c >= CHAR_0 && c <= CHAR_9) ||
+           (c >= CHAR_A && c <= CHAR_F) ||
+           (c >= CHAR_a && c <= CHAR_f) ||
+           (c >= 0xff10 && c <= 0xff19) ||  /* Fullwidth digits */
+           (c >= 0xff21 && c <= 0xff26) ||  /* Fullwidth letters */
+           (c >= 0xff41 && c <= 0xff46)) == isprop)
+        return !negated;
+      break;
+
       /* This should never occur, but compilers may mutter if there is no
       default. */
 

src/pcre2_xclass.c

@@ -1497,7 +1497,7 @@
     Az_\x{aa}\x{c0}\x{1c5}\x{2b0}\x{3b6}\x{1d7c9}\x{2fa1d}1\x{660}\x{bef}\x{16ee}
 
 /^[[:xdigit:]]*/utf,ucp
-    1a\x{660}\x{bef}\x{16ee}
+    1a\x{660}\x{bef}\x{16ee}\=no_jit
 
 /^\d+/utf,ucp
     1\x{660}\x{bef}\x{16ee}
@@ -2853,4 +2853,7 @@
 /[[:word:]]+/utf,ucp
     --cafe\x{300}_au\x{203f}lait!\=no_jit
 
+/[[:xdigit:]]+/utf,ucp
+    --123ef\x{ff10}\x{ff19}\x{ff21}\x{ff26}\x{ff1a}\=no_jit
+
 # End of testinput4

testdata/testinput4

@@ -2450,7 +2450,7 @@ No match
  0: Az_\x{aa}\x{c0}\x{1c5}\x{2b0}\x{3b6}\x{1d7c9}\x{2fa1d}1\x{660}\x{bef}\x{16ee}
 
 /^[[:xdigit:]]*/utf,ucp
-    1a\x{660}\x{bef}\x{16ee}
+    1a\x{660}\x{bef}\x{16ee}\=no_jit
  0: 1a
 
 /^\d+/utf,ucp
@@ -4563,4 +4563,8 @@ No match
     --cafe\x{300}_au\x{203f}lait!\=no_jit
  0: cafe\x{300}_au\x{203f}lait
 
+/[[:xdigit:]]+/utf,ucp
+    --123ef\x{ff10}\x{ff19}\x{ff21}\x{ff26}\x{ff1a}\=no_jit
+ 0: 123ef\x{ff10}\x{ff19}\x{ff21}\x{ff26}
+
 # End of testinput4