From 4b4612803dfd61675e34758eac4f464dba949cb8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 25 Jun 2026 23:52:39 +0300 Subject: [PATCH] gh-152100: Fuse set intersection into a single charset via INVERT Extend the set-operation charset fusion to intersection (&&), not just difference (--). A new charset-body opcode INVERT is the dual of NEGATE: NEGATE flips the running verdict, INVERT flips each following membership test, so [A&&B] -- and chained or nested set operations -- compile to a single charset instead of a positive lookbehind assertion. A --/&& chain is a pure conjunction, so _fuse_difference generalizes to _fuse_setops by appending one more kind of fail item. An intersection operand must reduce to a single member and is not fused under IGNORECASE (case folding could split it); the optimizer walk threads the flag context through groups, so a scoped (?i:...) is skipped too. Intersection-bearing set operations match 4-6x faster; pure difference compiles bit-for-bit as before. Co-Authored-By: Claude Opus 4.8 --- Lib/re/_compiler.py | 4 +- Lib/re/_constants.py | 4 +- Lib/re/_optimizer.py | 115 ++++++++++++++++++++++++----------- Lib/test/test_re.py | 7 +++ Modules/_sre/sre.c | 1 + Modules/_sre/sre_constants.h | 3 +- Modules/_sre/sre_lib.h | 39 ++++++++---- Modules/_sre/sre_targets.h | 3 +- 8 files changed, 123 insertions(+), 53 deletions(-) diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index 58a24964c3b374f..e9d4965330545d9 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -221,7 +221,7 @@ def _code(p, flags): flags = p.state.flags | flags # run the optimizer passes over the parsed pattern - optimize(p) + optimize(p, flags) code = [] @@ -267,7 +267,7 @@ def print_2(*args): i += 1 op = OPCODES[op] if op in (SUCCESS, FAILURE, ANY, ANY_ALL, - MAX_UNTIL, MIN_UNTIL, NEGATE): + MAX_UNTIL, MIN_UNTIL, NEGATE, INVERT): print_(op) elif op in (LITERAL, NOT_LITERAL, LITERAL_IGNORE, NOT_LITERAL_IGNORE, diff --git a/Lib/re/_constants.py b/Lib/re/_constants.py index 6e99dae53501512..244973f5233202e 100644 --- a/Lib/re/_constants.py +++ b/Lib/re/_constants.py @@ -13,7 +13,7 @@ # update when constants are added or removed -MAGIC = 20260622 +MAGIC = 20260625 from _sre import MAXREPEAT, MAXGROUPS # noqa: F401 @@ -120,6 +120,8 @@ def _makecodes(*names): 'NOT_LITERAL_UNI_IGNORE', 'RANGE_UNI_IGNORE', + 'INVERT', # charset-body item: dual of NEGATE, inverts member tests + # The following opcodes are only occurred in the parser output, # but not in the compiled code. 'MIN_REPEAT', 'MAX_REPEAT', diff --git a/Lib/re/_optimizer.py b/Lib/re/_optimizer.py index 6a0bb5a2973eaec..0961c500a860f8a 100644 --- a/Lib/re/_optimizer.py +++ b/Lib/re/_optimizer.py @@ -33,7 +33,7 @@ def _compile_charset(charset, flags, code): emit = code.append for op, av in charset: emit(op) - if op is NEGATE: + if op in (NEGATE, INVERT): pass elif op is LITERAL: emit(av) @@ -58,31 +58,40 @@ def _compile_charset(charset, flags, code): def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): # internal: optimize character set. # - # The engine's charset() walk toggles polarity on every NEGATE (see - # Modules/_sre/sre_lib.h), so NEGATE markers split the set into - # alternating-polarity segments: a leading NEGATE is a complemented class - # [^...], an interior one is set difference (RL1.3). Each segment is a - # plain union, optimized on its own with the NEGATE boundaries kept in place. - negates = [i for i, (op, _av) in enumerate(charset) if op is NEGATE] - if not negates or negates == [0]: + # The engine's charset() walk toggles its return polarity on every NEGATE + # and its membership-test direction on every INVERT (see sre_lib.h) -- + # together they express complement, difference and intersection (RL1.3). + # Each toggle-delimited segment is optimized on its own. + bounds = [i for i, (op, _av) in enumerate(charset) + if op in (NEGATE, INVERT)] + if not bounds or (bounds == [0] and charset[0][0] is NEGATE): # Fast path: a plain union, optionally complemented as a whole -- every # charset the parser produces today, optimized as before. return _optimize_charset_segment(charset, iscased, fixup, fixes) - # Optimize each NEGATE-delimited run on its own. _allow_anyall is off: the + # Optimize each toggle-delimited run on its own. _allow_anyall is off: the # [\s\S] -> ANY_ALL / [^\s\S] -> empty shortcuts rewrite a whole set and - # would inject or drop a NEGATE mid-segment. + # would inject or drop a toggle mid-segment. out = [] hascased = False start = 0 - for i in negates + [len(charset)]: + inv = False + for i in bounds + [len(charset)]: if i > start: # skip an empty run (e.g. a leading NEGATE) - opt, cased = _optimize_charset_segment( - charset[start:i], iscased, fixup, fixes, _allow_anyall=False) - out.extend(opt) - hascased |= cased + if inv: + # An INVERT run intersects its members: re-emit them as + # they are (already optimized, see _single_member). + out.extend(charset[start:i]) + else: + opt, cased = _optimize_charset_segment( + charset[start:i], iscased, fixup, fixes, _allow_anyall=False) + out.extend(opt) + hascased |= cased if i < len(charset): - out.append((NEGATE, None)) + tok = charset[i] + out.append(tok) # re-emit the toggle in place + if tok[0] is INVERT: + inv = not inv start = i + 1 return out, hascased @@ -467,43 +476,75 @@ def _fuse_branch(av): items += cs return items if tail is None else items + tail -def _fuse_difference(data): - # Replace (? (? [NEGATE] [INVERT] A. The chain + # is a pure conjunction, so each lookbehind appends one fail item: + # "ch in B" for a difference, "ch not in C" (under INVERT) for an + # intersection. Not fused under IGNORECASE, where case folding could + # split a single-member intersection operand (see _single_member). out = [] - head = None # _flat_items(A) for the fused difference now at out[-1] - subtrahend = None # its accumulated B items, or None when not fusing + head = None # _flat_items(A) for the fused set operation now at out[-1] + fails = None # its accumulated fail items, or None when not fusing + inv = 0 # the INVERT state at the end of fails for op, av in data: - if op is ASSERT_NOT and av[0] < 0: # a negative lookbehind - b = _parser._flat_items(av[1].data) + if op in (ASSERT, ASSERT_NOT) and av[0] < 0: # a lookbehind + if op is ASSERT_NOT: # -- difference + b = _parser._flat_items(av[1].data) + tinv = 0 + elif not flags & SRE_FLAG_IGNORECASE: # && intersection + b = _single_member(av[1].data) + tinv = 1 + else: + b = None if b is not None: - if subtrahend is None and out: + if fails is None and out: # the first lookbehind of a run: only now is it worth # checking whether the preceding item A is a flat charset. head = _parser._flat_items([out[-1]]) if head is not None: - subtrahend = [] - if subtrahend is not None: - subtrahend += b - out[-1] = (IN, [(NEGATE, None)] + subtrahend - + [(NEGATE, None)] + head) + fails = [] + inv = 0 + if fails is not None: + if inv != tinv: + fails.append((INVERT, None)) + inv ^= 1 + fails += b + tail = [(NEGATE, None)] + if inv: + tail.append((INVERT, None)) + out[-1] = (IN, [(NEGATE, None)] + fails + tail + head) continue - head = subtrahend = None + head = fails = None out.append((op, av)) data[:] = out -def _walk(seq): +def _walk(seq, flags): for i, (op, av) in enumerate(seq): - for sub in _subpatterns(op, av): - _walk(sub.data) + if op is SUBPATTERN: + # A group can change the flag context, e.g. (?i:...). + _walk(av[3].data, _combine_flags(flags, av[1], av[2])) + else: + for sub in _subpatterns(op, av): + _walk(sub.data, flags) if op is BRANCH: items = _fuse_branch(av) if items is not None: seq[i] = (IN, items) - _fuse_difference(seq) + _fuse_setops(seq, flags) -def optimize(pattern): +def optimize(pattern, flags): """Rewrite a parsed pattern in place and return it.""" - _walk(pattern.data) + _walk(pattern.data, flags) return pattern diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index af6e4612dcfaef5..ff18fdb690b3919 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1496,6 +1496,13 @@ def test_set_operations(self): self.assertEqual(re.findall(r'[\d&&1]', s), list('1')) # A leading '&' is a literal. self.assertEqual(re.findall(r'[&&1]', s), list('&1')) + # Intersection is not fused under IGNORECASE (case folding can + # split an operand), including when the flag comes from a scoped group. + for pat in (r'(?i)[\w&&[s]]', r'(?i:[\w&&[s]])'): + for c in 'sSſ': + self.assertTrue(re.fullmatch(pat, c), (pat, c)) + self.assertTrue(re.fullmatch(r'(?-i:[\w&&[s]])', 's', re.I)) + self.assertIsNone(re.fullmatch(r'(?-i:[\w&&[s]])', 'S', re.I)) # Nested sets and lookbehind-mapped operands. self.assertEqual(re.findall(r'[a-z--[aeiou]]', s), diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 7d03b909226f244..dee7cc74e8a39f7 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -2078,6 +2078,7 @@ _validate_charset(SRE_CODE *code, SRE_CODE *end) switch (op) { case SRE_OP_NEGATE: + case SRE_OP_INVERT: break; case SRE_OP_LITERAL: diff --git a/Modules/_sre/sre_constants.h b/Modules/_sre/sre_constants.h index d5ecefb37647555..bc52b2fc8ab8769 100644 --- a/Modules/_sre/sre_constants.h +++ b/Modules/_sre/sre_constants.h @@ -11,7 +11,7 @@ * See the sre.c file for information on usage and redistribution. */ -#define SRE_MAGIC 20260622 +#define SRE_MAGIC 20260625 #define SRE_OP_FAILURE 0 #define SRE_OP_SUCCESS 1 #define SRE_OP_ANY 2 @@ -55,6 +55,7 @@ #define SRE_OP_LITERAL_UNI_IGNORE 40 #define SRE_OP_NOT_LITERAL_UNI_IGNORE 41 #define SRE_OP_RANGE_UNI_IGNORE 42 +#define SRE_OP_INVERT 43 #define SRE_AT_BEGINNING 0 #define SRE_AT_BEGINNING_LINE 1 #define SRE_AT_BEGINNING_STRING 2 diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index 6e6ae46f05a50f0..78881835736b3cd 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -94,6 +94,10 @@ SRE(charset)(SRE_STATE* state, const SRE_CODE* set, SRE_CODE ch) /* check if character is a member of the given set */ int ok = 1; + /* INVERT toggles inv, which inverts each following membership test + (set difference/intersection within a single charset). Member tests + are normalized to 0/1 before the ^ inv. */ + int inv = 0; for (;;) { switch (*set++) { @@ -103,29 +107,32 @@ SRE(charset)(SRE_STATE* state, const SRE_CODE* set, SRE_CODE ch) case SRE_OP_LITERAL: /* */ - if (ch == set[0]) + if ((ch == set[0]) ^ inv) return ok; set++; break; case SRE_OP_CATEGORY: /* */ - if (sre_category(set[0], (int) ch)) + if (!!sre_category(set[0], (int) ch) ^ inv) return ok; set++; break; case SRE_OP_CHARSET: /* */ - if (ch < 256 && - (set[ch/SRE_CODE_BITS] & (1u << (ch & (SRE_CODE_BITS-1))))) + { + int m = ch < 256 && + (set[ch/SRE_CODE_BITS] & (1u << (ch & (SRE_CODE_BITS-1)))); + if (m ^ inv) return ok; set += 256/SRE_CODE_BITS; break; + } case SRE_OP_RANGE: /* */ - if (set[0] <= ch && ch <= set[1]) + if ((set[0] <= ch && ch <= set[1]) ^ inv) return ok; set += 2; break; @@ -134,11 +141,14 @@ SRE(charset)(SRE_STATE* state, const SRE_CODE* set, SRE_CODE ch) /* */ { SRE_CODE uch; + int m; /* ch is already lower cased */ - if (set[0] <= ch && ch <= set[1]) - return ok; - uch = sre_upper_unicode(ch); - if (set[0] <= uch && uch <= set[1]) + m = set[0] <= ch && ch <= set[1]; + if (!m) { + uch = sre_upper_unicode(ch); + m = set[0] <= uch && uch <= set[1]; + } + if (m ^ inv) return ok; set += 2; break; @@ -148,10 +158,15 @@ SRE(charset)(SRE_STATE* state, const SRE_CODE* set, SRE_CODE ch) ok = !ok; break; + case SRE_OP_INVERT: + inv = !inv; + break; + case SRE_OP_BIGCHARSET: /* <256 blockindices> */ { Py_ssize_t count, block; + int m; count = *(set++); if (ch < 0x10000u) @@ -159,9 +174,10 @@ SRE(charset)(SRE_STATE* state, const SRE_CODE* set, SRE_CODE ch) else block = -1; set += 256/sizeof(SRE_CODE); - if (block >=0 && + m = block >= 0 && (set[(block * 256 + (ch & 255))/SRE_CODE_BITS] & - (1u << (ch & (SRE_CODE_BITS-1))))) + (1u << (ch & (SRE_CODE_BITS-1)))); + if (m ^ inv) return ok; set += count * (256/SRE_CODE_BITS); break; @@ -1616,6 +1632,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) TARGET(SRE_OP_SUBPATTERN): TARGET(SRE_OP_RANGE): TARGET(SRE_OP_NEGATE): + TARGET(SRE_OP_INVERT): TARGET(SRE_OP_BIGCHARSET): TARGET(SRE_OP_CHARSET): TRACE(("|%p|%p|UNKNOWN %d\n", pattern, ptr, diff --git a/Modules/_sre/sre_targets.h b/Modules/_sre/sre_targets.h index 62761a0000d8368..3f0c05e70fa4dd7 100644 --- a/Modules/_sre/sre_targets.h +++ b/Modules/_sre/sre_targets.h @@ -11,7 +11,7 @@ * See the sre.c file for information on usage and redistribution. */ -static void *sre_targets[43] = { +static void *sre_targets[44] = { &&TARGET_SRE_OP_FAILURE, &&TARGET_SRE_OP_SUCCESS, &&TARGET_SRE_OP_ANY, @@ -55,4 +55,5 @@ static void *sre_targets[43] = { &&TARGET_SRE_OP_LITERAL_UNI_IGNORE, &&TARGET_SRE_OP_NOT_LITERAL_UNI_IGNORE, &&TARGET_SRE_OP_RANGE_UNI_IGNORE, + &&TARGET_SRE_OP_INVERT, };