From 815988716c817553b47c539b305028a7bebfe941 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 23 Jun 2026 22:23:06 +0300 Subject: [PATCH] gh-152026: Support redefinition of named capture groups The same name can now be used for more than one capturing group. All such groups share a single group number; the name and number refer to whichever of the groups matched (the last one, if more than one matched). The group's width range spans all the definitions. This is chiefly useful for giving the same name to corresponding groups in alternative spellings of a pattern: (?P\d+)/(?P\d+)/(?P\d+)|(?P\d+)-(?P\d+)-(?P\d+) Previously each group name had to be unique within a regular expression. Sharing a group number also requires a fix in the matching engine. Outside of repeats the matcher restores capture-group marks lazily, only rewinding the lastmark high-water index, which assumes each group number is written at a single place in the bytecode. A reused group number could otherwise leak a mark from a branch that matched and was then backtracked away, or raise SystemError. Reused group numbers are detected during code validation, and such patterns save and restore the full mark array on backtracking, exactly as is already done inside repeats; other patterns are unaffected. The same flag also lets the possessive quantifier handler request mark saving directly, replacing the placeholder repeat context that was previously installed for that purpose (the gh-101955 workaround), so a possessive repeat no longer allocates a repeat context. Co-Authored-By: Claude Opus 4.8 --- Doc/library/re.rst | 16 ++++- Doc/whatsnew/3.16.rst | 7 ++ Lib/re/_parser.py | 16 +++-- Lib/test/test_re.py | 70 ++++++++++++++++++- ...-06-23-19-00-00.gh-issue-152026.Re1NaM.rst | 5 ++ Modules/_sre/sre.c | 51 ++++++++++---- Modules/_sre/sre.h | 5 +- 7 files changed, 144 insertions(+), 26 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-06-23-19-00-00.gh-issue-152026.Re1NaM.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 1aefb91e0ceb40..4c1488db9cdb1b 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -436,9 +436,15 @@ The special characters are: Similar to regular parentheses, but the substring matched by the group is accessible via the symbolic group name *name*. Group names must be valid Python identifiers, and in :class:`bytes` patterns they can only contain - bytes in the ASCII range. Each group name must be defined only once within - a regular expression. A symbolic group is also a numbered group, just as if - the group were not named. + bytes in the ASCII range. A symbolic group is also a numbered group, just as + if the group were not named. + + A group name may be used for more than one group. All such groups share a + single group number, and the name (and that number) refer to whichever of + them matched; if more than one matched, they refer to the last. This is + chiefly useful for giving the same name to corresponding groups in + alternative spellings of a pattern, for example + ``(?P\d{4})-(?P\d\d)|(?P\d\d)/(?P\d{4})``. Named groups can be referenced in three contexts. If the pattern is ``(?P['"]).*?(?P=quote)`` (i.e. matching a string quoted with either @@ -462,6 +468,10 @@ The special characters are: In :class:`bytes` patterns, group *name* can only contain bytes in the ASCII range (``b'\x00'``-``b'\x7f'``). + .. versionchanged:: next + A group name can be used for more than one group. Previously each name + could be defined only once in a regular expression. + .. index:: single: (?P=; in regular expressions ``(?P=name)`` diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index e8c75d2f571061..aec0d46d639b6c 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -305,6 +305,13 @@ re properties. (Contributed by Serhiy Storchaka in :gh:`95555`.) +* A capturing group name can now be used for more than one group in a regular + expression. Such groups share a single group number, and the name refers to + whichever of them matched. This is useful for giving the same name to + corresponding groups in alternative spellings of a pattern, such as + ``(?P\d{4})-(?P\d\d)|(?P\d\d)/(?P\d{4})``. + (Contributed by Serhiy Storchaka in :gh:`152026`.) + shlex ----- diff --git a/Lib/re/_parser.py b/Lib/re/_parser.py index aab9b59168015c..c8457745a71758 100644 --- a/Lib/re/_parser.py +++ b/Lib/re/_parser.py @@ -86,18 +86,26 @@ def groups(self): return len(self.groupwidths) def opengroup(self, name=None): gid = self.groups - self.groupwidths.append(None) if self.groups > MAXGROUPS: raise error("too many groups") if name is not None: ogid = self.groupdict.get(name, None) if ogid is not None: - raise error("redefinition of group name %r as group %d; " - "was group %d" % (name, gid, ogid)) + # The same name may be used for more than one group. All such + # groups share a single group number, and the name refers to + # whichever of them matched. + return ogid self.groupdict[name] = gid + self.groupwidths.append(None) return gid def closegroup(self, gid, p): - self.groupwidths[gid] = p.getwidth() + # A reused group number may be closed more than once; its width spans + # the union of all the definitions. + w = p.getwidth() + wold = self.groupwidths[gid] + if wold is not None: + w = (min(wold[0], w[0]), max(wold[1], w[1])) + self.groupwidths[gid] = w def checkgroup(self, gid): return gid < self.groups and self.groupwidths[gid] is not None diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index af6e4612dcfaef..444ef3401bd1f5 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -297,9 +297,6 @@ def test_symbolic_groups(self): self.assertEqual(re.match(pat, 'xc8yz').span(), (0, 5)) def test_symbolic_groups_errors(self): - self.checkPatternError(r'(?P)(?P)', - "redefinition of group name 'a' as group 2; " - "was group 1") self.checkPatternError(r'(?P(?P=a))', "cannot refer to an open group", 10) self.checkPatternError(r'(?Pxy)', 'unknown extension ?Px') @@ -380,6 +377,73 @@ def test_symbolic_refs_errors(self): self.checkTemplateError('(?Px)', r'\g<१>', 'xx', "bad character in group name '१'", 3) + def test_redefined_named_groups(self): + # The same name may be used for more than one group. + p = re.compile(r'(?P\d+)/(?P\d+)/(?P\d+)|' + r'(?P\d+)-(?P\d+)-(?P\d+)') + self.assertEqual(p.fullmatch('07/09/2023').groupdict(), + {'m': '07', 'd': '09', 'y': '2023'}) + self.assertEqual(p.fullmatch('2023-07-09').groupdict(), + {'m': '07', 'd': '09', 'y': '2023'}) + # Reused groups share a single group number. + self.assertEqual(p.groups, 3) + self.assertEqual(p.groupindex, {'m': 1, 'd': 2, 'y': 3}) + # If more than one of the groups matches, the name refers to the last. + p = re.compile(r'(?P\w)(?P\w)') + self.assertEqual(p.match('xy').group('a'), 'y') + # A reused group may be left unset by the branch that does not match. + p = re.compile(r'(?P\d)(?:-(?P\d))?') + self.assertEqual(p.match('1').group('a'), '1') + self.assertEqual(p.match('1-2').group('a'), '2') + # A definition that is backtracked away does not leak into the group. + p = re.compile(r'(?Px)(?:(?Py)|z)') + self.assertEqual(p.match('xz').group('g'), 'x') + self.assertEqual(p.match('xy').group('g'), 'y') + + def test_redefined_named_groups_backref(self): + # A backreference to a redefined name refers to whichever definition + # participated in the match. + p = re.compile(r'(?:(?Pa)|(?Pb))(?P=g)') + self.assertEqual(p.match('aa').group(), 'aa') + self.assertEqual(p.match('bb').group(), 'bb') + self.assertIsNone(p.match('ab')) + # A numeric backreference to the shared group number works too. + p = re.compile(r'(?:(?Pa)|(?Pb))\1') + self.assertEqual(p.match('aa').group(), 'aa') + self.assertIsNone(p.match('ab')) + + def test_redefined_named_groups_conditional(self): + p = re.compile(r'(?:(?Pa)|b)(?(g)X|Y)') + self.assertEqual(p.match('aX').group(), 'aX') + self.assertEqual(p.match('bY').group(), 'bY') + self.assertIsNone(p.match('aY')) + self.assertIsNone(p.match('bX')) + + def test_redefined_named_groups_width(self): + # closegroup() widens the shared group to the union of the definitions. + p = re.compile(r'(?:(?Pa)|(?Pbb))!') + self.assertEqual(p.match('a!').group('g'), 'a') + self.assertEqual(p.match('bb!').group('g'), 'bb') + # A fixed-width union may be used in a look-behind... + p = re.compile(r'(?:(?Paa)|(?Pbb))(?<=(?P=g))') + self.assertEqual(p.match('aa').group(), 'aa') + # ...but a variable-width union may not. + self.checkPatternError(r'x(?<=(?:(?Pa)|(?Pbb)))', + 'look-behind requires fixed-width pattern') + + def test_redefined_named_groups_backtracking(self): + # A definition that matches and is then backtracked away (here the + # other branch of the inner alternation is taken) must not leak a stale + # capture into the shared group number. + p = re.compile(r'(?:(?Pa)|(?P(?Pb)(?:b|a)|(?:bbb)*))') + self.assertIsNone(p.match('b').group('g')) + self.assertEqual(p.match('a').group('g'), 'a') + self.assertEqual(p.match('bb').group('g'), 'b') + # A redefinition in the tail of a lazy repeat matches and then + # fails; it must not clobber the capture from the body. + m = re.search(r'(?:(?Pa)|x)+?(?:(?Px)q|y)', 'axy') + self.assertEqual(m.span('g'), (0, 1)) + def test_re_subn(self): self.assertEqual(re.subn("(?i)b+", "x", "bbbb BBBB"), ('x x', 2)) self.assertEqual(re.subn("b+", "x", "bbbb BBBB"), ('x BBBB', 1)) diff --git a/Misc/NEWS.d/next/Library/2026-06-23-19-00-00.gh-issue-152026.Re1NaM.rst b/Misc/NEWS.d/next/Library/2026-06-23-19-00-00.gh-issue-152026.Re1NaM.rst new file mode 100644 index 00000000000000..84f2e88fa33679 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-23-19-00-00.gh-issue-152026.Re1NaM.rst @@ -0,0 +1,5 @@ +A group name can now be used for more than one group in a regular expression. +All such groups share a single group number, and the name refers to whichever +of them matched (the last one, if more than one matched). This is chiefly +useful for giving the same name to corresponding groups in alternative +spellings of a pattern, e.g. ``(?P\d{4})-(?P\d\d)|(?P\d\d)/(?P\d{4})``. diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index e742a25e4891fc..e3978abcee274e 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -738,7 +738,7 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, state->charsize = charsize; state->match_all = 0; state->must_advance = 0; - state->save_marks = 0; + state->save_marks = pattern->reused_groups; state->debug = ((pattern->flags & SRE_FLAG_DEBUG) != 0); state->beginning = ptr; @@ -1792,6 +1792,7 @@ _sre_compile_impl(PyObject *module, PyObject *pattern, int flags, self->pattern = NULL; self->groupindex = NULL; self->indexgroup = NULL; + self->reused_groups = 0; #ifdef Py_DEBUG self->fail_after_count = -1; self->fail_after_exc = NULL; @@ -2133,7 +2134,8 @@ _validate_charset(SRE_CODE *code, SRE_CODE *end) /* Returns 0 on success, -1 on failure, and 1 if the last op is JUMP. */ static int -_validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) +_validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups, + char *seen, int *reused) { /* Some variables are manipulated by the macros above */ SRE_CODE op; @@ -2158,6 +2160,14 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) VTRACE(("arg=%d, groups=%d\n", (int)arg, (int)groups)); FAIL; } + /* A mark index appears exactly once in well-formed code, unless + the same group number is opened in more than one place (a + redefined named group). Such groups need full mark save/restore + on backtracking. */ + if (seen[arg]) + *reused = 1; + else + seen[arg] = 1; break; case SRE_OP_LITERAL: @@ -2291,7 +2301,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) if (skip == 0) break; /* Stop 2 before the end; we check the JUMP below */ - if (_validate_inner(code, code+skip-3, groups)) + if (_validate_inner(code, code+skip-3, groups, seen, reused)) FAIL; code += skip-3; /* Check that it ends with a JUMP, and that each JUMP @@ -2322,7 +2332,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) FAIL; if (max > SRE_MAXREPEAT) FAIL; - if (_validate_inner(code, code+skip-4, groups)) + if (_validate_inner(code, code+skip-4, groups, seen, reused)) FAIL; code += skip-4; GET_OP; @@ -2342,7 +2352,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) FAIL; if (max > SRE_MAXREPEAT) FAIL; - if (_validate_inner(code, code+skip-3, groups)) + if (_validate_inner(code, code+skip-3, groups, seen, reused)) FAIL; code += skip-3; GET_OP; @@ -2360,7 +2370,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) case SRE_OP_ATOMIC_GROUP: { GET_SKIP; - if (_validate_inner(code, code+skip-2, groups)) + if (_validate_inner(code, code+skip-2, groups, seen, reused)) FAIL; code += skip-2; GET_OP; @@ -2413,12 +2423,12 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) for a JUMP opcode preceding our skip target. */ VTRACE(("then part:\n")); - int rc = _validate_inner(code+1, code+skip-1, groups); + int rc = _validate_inner(code+1, code+skip-1, groups, seen, reused); if (rc == 1) { VTRACE(("else part:\n")); code += skip-2; /* Position after JUMP, at */ GET_SKIP; - rc = _validate_inner(code, code+skip-1, groups); + rc = _validate_inner(code, code+skip-1, groups, seen, reused); } if (rc) FAIL; @@ -2431,7 +2441,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) GET_ARG; /* 0 for lookahead, width for lookbehind */ code--; /* Back up over arg to simplify math below */ /* Stop 1 before the end; we check the SUCCESS below */ - if (_validate_inner(code+1, code+skip-2, groups)) + if (_validate_inner(code+1, code+skip-2, groups, seen, reused)) FAIL; code += skip-2; GET_OP; @@ -2456,24 +2466,35 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) } static int -_validate_outer(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) +_validate_outer(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups, + char *seen, int *reused) { if (groups < 0 || (size_t)groups > SRE_MAXGROUPS || code >= end || end[-1] != SRE_OP_SUCCESS) FAIL; - return _validate_inner(code, end-1, groups); + return _validate_inner(code, end-1, groups, seen, reused); } static int _validate(PatternObject *self) { - if (_validate_outer(self->code, self->code+self->codesize, self->groups)) - { + /* seen[i] tracks whether mark index i has already been emitted, so that a + reused group number (the same mark seen twice) can be detected. */ + int reused = 0; + char *seen = PyMem_Calloc(2 * (size_t)self->groups + 1, 1); + if (seen == NULL) { + PyErr_NoMemory(); + return 0; + } + int invalid = _validate_outer(self->code, self->code+self->codesize, + self->groups, seen, &reused); + PyMem_Free(seen); + if (invalid) { PyErr_SetString(PyExc_RuntimeError, "invalid SRE code"); return 0; } - else - VTRACE(("Success!\n")); + self->reused_groups = reused; + VTRACE(("Success!\n")); return 1; } diff --git a/Modules/_sre/sre.h b/Modules/_sre/sre.h index bec4ae21d1e10f..790fd08b4eb2ae 100644 --- a/Modules/_sre/sre.h +++ b/Modules/_sre/sre.h @@ -34,6 +34,8 @@ typedef struct { int flags; /* flags used when compiling pattern source */ PyObject *weakreflist; /* List of weak references */ int isbytes; /* pattern type (1 - bytes, 0 - string, -1 - None) */ + int reused_groups; /* a group number is opened in more than one place + (a redefined named group) */ #ifdef Py_DEBUG /* for simulation of user interruption */ int fail_after_count; @@ -100,7 +102,8 @@ typedef struct { int save_marks; /* if nonzero, save and restore mark values on backtracking instead of only rewinding the lastmark index; counts enclosing repeat contexts and - possessive bodies */ + possessive bodies, over a base of 1 if the pattern + reuses group numbers */ /* dynamically allocated stuff */ char* data_stack; size_t data_stack_size;