gh-95555: Support Unicode property escapes \p{...} in regular expressions

Add support for \p{property} and \P{property} in Unicode (str) regular
expressions, for the properties the engine can resolve without the
unicodedata database.  They are matched either as CATEGORY opcodes
(character predicates and combinations of them, see sre.c) or as fixed
sets of character ranges.

Supported properties:

* many General_Category values -- the groups L, N, Z, C and the values Lu,
  Lt, Lm, Nd, Nl, No, Zs, Zl, Zp, Cc, Cf, Cs, Co and Cn;
* the binary properties Alphabetic, Lowercase, Uppercase, Numeric,
  Printable, XID_Start, XID_Continue, Cased and Case_Ignorable;
* the POSIX compatibility classes alpha, alnum, blank, cntrl, digit, graph,
  lower, print, space, upper, word and xdigit;
* the code-point classes ASCII, Any, Assigned, Noncharacter_Code_Point,
  Join_Control and the immutable Pattern_Syntax and Pattern_White_Space.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: serhiy-storchaka

Doc/library/re.rst

@@ -591,7 +591,7 @@ character ``'$'``.
 
       Matches ``[0-9]`` if the :py:const:`~re.ASCII` flag is used.
 
-      __ https://www.unicode.org/versions/Unicode15.0.0/ch04.pdf#G134153
+      __ https://www.unicode.org/versions/Unicode17.0.0/core-spec/chapter-4/#G124142
 
    For 8-bit (bytes) patterns:
       Matches any decimal digit in the ASCII character set;
@@ -658,6 +658,50 @@ character ``'$'``.
    matches characters which are neither alphanumeric in the current locale
    nor the underscore.
 
+.. index:: single: \p; in regular expressions
+           single: \P; in regular expressions
+
+``\p{property=value}``, ``\p{value}``
+   Matches any character with the given Unicode property
+   (see `Unicode Technical Standard #18
+   <https://unicode.org/reports/tr18/>`_, requirement RL1.2 "Properties").
+   Property and value names are matched loosely:
+   case, whitespace, ``'-'`` and ``'_'`` are ignored.
+   The following properties are supported:
+
+   * The ``General_Category`` property (short name ``gc``),
+     spelled ``\p{Lu}``, ``\p{gc=Lu}`` or, for a one-letter group, ``\p{L}``.
+     The supported values are the groups ``L``, ``N``, ``Z`` and ``C`` and the
+     values ``Lu``, ``Lt``, ``Lm``, ``Nd``, ``Nl``, ``No``, ``Zs``, ``Zl``,
+     ``Zp``, ``Cc``, ``Cf``, ``Cs``, ``Co`` and ``Cn``.
+   * The binary properties ``XID_Start``, ``XID_Continue``, ``Alphabetic``,
+     ``Lowercase``, ``Uppercase``, ``Numeric``, ``Printable``, ``Cased`` and
+     ``Case_Ignorable``.  A binary property may also be spelled
+     ``\p{name=yes}`` or ``\p{name=no}``.
+   * The POSIX compatibility classes ``alpha``, ``alnum``, ``blank``,
+     ``cntrl``, ``digit``, ``graph``, ``lower``, ``print``, ``space``,
+     ``upper``, ``word`` and ``xdigit``.
+   * The properties ``ASCII``, ``Any``, ``Assigned``,
+     ``Noncharacter_Code_Point``, ``Join_Control``, ``Pattern_Syntax`` and
+     ``Pattern_White_Space``.
+
+   Where a supported property corresponds to a :mod:`unicodedata` accessor or
+   :class:`str` method, the set of characters it matches is exactly the one
+   they report.  For consistency with these, ``space`` follows
+   :py:meth:`str.isspace` (like ``\s``) and ``xdigit`` matches only the ASCII
+   hexadecimal digits.
+
+   This is only recognized in Unicode (str) patterns.
+   In bytes patterns it is an error.
+
+   .. versionadded:: next
+
+``\P{...}``
+   Matches any character which does *not* have the given Unicode property.
+   This is the opposite of ``\p``.
+
+   .. versionadded:: next
+
 .. index:: single: \z; in regular expressions
            single: \Z; in regular expressions
 

Doc/whatsnew/3.16.rst

@@ -142,6 +142,17 @@ os
   (Contributed by Maurycy Pawłowski-Wieroński in :gh:`149464`.)
 
 
+re
+--
+
+* Regular expressions now support Unicode property escapes ``\p{...}`` and
+  ``\P{...}``, which match a character by a Unicode property -- for example
+  ``\p{Lu}`` (an uppercase letter), ``\p{Cased}`` or ``\p{ASCII}``.  See
+  :ref:`the regular expression syntax <re-syntax>` for the supported
+  properties.
+  (Contributed by Serhiy Storchaka in :gh:`95555`.)
+
+
 shlex
 -----
 

Lib/re/_constants.py

@@ -13,7 +13,7 @@
 
 # update when constants are added or removed
 
-MAGIC = 20230612
+MAGIC = 20260628
 
 from _sre import MAXREPEAT, MAXGROUPS  # noqa: F401
 
@@ -150,6 +150,35 @@ def _makecodes(*names):
     'CATEGORY_UNI_SPACE', 'CATEGORY_UNI_NOT_SPACE',
     'CATEGORY_UNI_WORD', 'CATEGORY_UNI_NOT_WORD',
     'CATEGORY_UNI_LINEBREAK', 'CATEGORY_UNI_NOT_LINEBREAK',
+
+    # Unicode property categories.  These are not affected by the ASCII,
+    # LOCALE or UNICODE flags.
+    'CATEGORY_ALPHA', 'CATEGORY_NOT_ALPHA',
+    'CATEGORY_LOWER', 'CATEGORY_NOT_LOWER',
+    'CATEGORY_UPPER', 'CATEGORY_NOT_UPPER',
+    'CATEGORY_NUMERIC', 'CATEGORY_NOT_NUMERIC',
+    'CATEGORY_PRINTABLE', 'CATEGORY_NOT_PRINTABLE',
+    'CATEGORY_ALNUM', 'CATEGORY_NOT_ALNUM',
+    'CATEGORY_XID_START', 'CATEGORY_NOT_XID_START',
+    'CATEGORY_XID_CONTINUE', 'CATEGORY_NOT_XID_CONTINUE',
+    'CATEGORY_TITLE', 'CATEGORY_NOT_TITLE',
+    'CATEGORY_CASED', 'CATEGORY_NOT_CASED',
+    'CATEGORY_CASE_IGNORABLE', 'CATEGORY_NOT_CASE_IGNORABLE',
+    # Compound categories: Lu = uppercase letter, N = number.
+    'CATEGORY_LU', 'CATEGORY_NOT_LU',
+    'CATEGORY_N', 'CATEGORY_NOT_N',
+    'CATEGORY_LM', 'CATEGORY_NOT_LM',
+    'CATEGORY_NL', 'CATEGORY_NOT_NL',
+    'CATEGORY_NO', 'CATEGORY_NOT_NO',
+    'CATEGORY_CF', 'CATEGORY_NOT_CF',
+    'CATEGORY_Z', 'CATEGORY_NOT_Z',
+    'CATEGORY_ZS', 'CATEGORY_NOT_ZS',
+    'CATEGORY_C', 'CATEGORY_NOT_C',
+    'CATEGORY_CN', 'CATEGORY_NOT_CN',
+    'CATEGORY_ASSIGNED', 'CATEGORY_NOT_ASSIGNED',
+    'CATEGORY_BLANK', 'CATEGORY_NOT_BLANK',
+    'CATEGORY_GRAPH', 'CATEGORY_NOT_GRAPH',
+    'CATEGORY_PRINT', 'CATEGORY_NOT_PRINT',
 )
 
 
@@ -206,6 +235,39 @@ def _makecodes(*names):
     CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK
 }
 
+# The Unicode property categories are the same regardless of the flags.
+CH_PROPERTY = (
+    CATEGORY_ALPHA, CATEGORY_NOT_ALPHA,
+    CATEGORY_LOWER, CATEGORY_NOT_LOWER,
+    CATEGORY_UPPER, CATEGORY_NOT_UPPER,
+    CATEGORY_NUMERIC, CATEGORY_NOT_NUMERIC,
+    CATEGORY_PRINTABLE, CATEGORY_NOT_PRINTABLE,
+    CATEGORY_ALNUM, CATEGORY_NOT_ALNUM,
+    CATEGORY_XID_START, CATEGORY_NOT_XID_START,
+    CATEGORY_XID_CONTINUE, CATEGORY_NOT_XID_CONTINUE,
+    CATEGORY_TITLE, CATEGORY_NOT_TITLE,
+    CATEGORY_CASED, CATEGORY_NOT_CASED,
+    CATEGORY_CASE_IGNORABLE, CATEGORY_NOT_CASE_IGNORABLE,
+    CATEGORY_LU, CATEGORY_NOT_LU,
+    CATEGORY_N, CATEGORY_NOT_N,
+    CATEGORY_LM, CATEGORY_NOT_LM,
+    CATEGORY_NL, CATEGORY_NOT_NL,
+    CATEGORY_NO, CATEGORY_NOT_NO,
+    CATEGORY_CF, CATEGORY_NOT_CF,
+    CATEGORY_Z, CATEGORY_NOT_Z,
+    CATEGORY_ZS, CATEGORY_NOT_ZS,
+    CATEGORY_C, CATEGORY_NOT_C,
+    CATEGORY_CN, CATEGORY_NOT_CN,
+    CATEGORY_ASSIGNED, CATEGORY_NOT_ASSIGNED,
+    CATEGORY_BLANK, CATEGORY_NOT_BLANK,
+    CATEGORY_GRAPH, CATEGORY_NOT_GRAPH,
+    CATEGORY_PRINT, CATEGORY_NOT_PRINT,
+)
+for _cat in CH_PROPERTY:
+    CH_LOCALE[_cat] = _cat
+    CH_UNICODE[_cat] = _cat
+del _cat
+
 CH_NEGATE = dict(zip(CHCODES[::2] + CHCODES[1::2], CHCODES[1::2] + CHCODES[::2]))
 
 # flags

Lib/re/_parser.py

@@ -309,6 +309,22 @@ def checkgroupname(self, name, offset):
             msg = "bad character in group name %r" % name
             raise self.error(msg, len(name) + offset)
 
+def _property_escape(source, escape, in_set=False):
+    # handle \p{...} and \P{...} (UTS #18 1.2.4, "Property Syntax")
+    from . import _properties
+    if not source.match('{'):
+        raise source.error("missing {, expected property name")
+    name = source.getuntil('}', 'property name')
+    code = _properties.parse_property(name, escape[1] == 'P')
+    if code is None:
+        raise source.error("unknown property name %r" % name,
+                           len(name) + len(r'\p{}'))
+    if in_set and code[1][0] == (NEGATE, None):
+        # A negated multi-range property cannot be a member of a set.
+        raise source.error("bad escape %s in character class" % escape,
+                           len(name) + len(r'\p{}'))
+    return code
+
 def _class_escape(source, escape):
     # handle escape code inside character class
     code = ESCAPES.get(escape)
@@ -351,6 +367,8 @@ def _class_escape(source, escape):
                 raise source.error("undefined character name %r" % charname,
                                    len(charname) + len(r'\N{}')) from None
             return LITERAL, c
+        elif c in "pP" and source.istext:
+            return _property_escape(source, escape, in_set=True)
         elif c in OCTDIGITS:
             # octal escape (up to three digits)
             escape += source.getwhile(2, OCTDIGITS)
@@ -411,6 +429,8 @@ def _escape(source, escape, state):
                 raise source.error("undefined character name %r" % charname,
                                    len(charname) + len(r'\N{}')) from None
             return LITERAL, c
+        elif c in "pP" and source.istext:
+            return _property_escape(source, escape)
         elif c == "0":
             # octal escape
             escape += source.getwhile(2, OCTDIGITS)
@@ -591,8 +611,9 @@ def _parse(source, state, verbose, nested, first=False):
                                            source.tell() - here)
                     if that == "]":
                         if code1[0] is IN:
-                            code1 = code1[1][0]
-                        setappend(code1)
+                            set.extend(code1[1])
+                        else:
+                            setappend(code1)
                         setappend((LITERAL, _ord("-")))
                         break
                     if that[0] == "\\":
@@ -617,8 +638,9 @@ def _parse(source, state, verbose, nested, first=False):
                     setappend((RANGE, (lo, hi)))
                 else:
                     if code1[0] is IN:
-                        code1 = code1[1][0]
-                    setappend(code1)
+                        set.extend(code1[1])
+                    else:
+                        setappend(code1)
 
             set = _uniq(set)
             # XXX: <fl> should move set optimization to compiler!