From 169fcfc5d1adf49372013d2d30ffb18e1ff08e11 Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Sun, 5 Jul 2026 09:59:35 +0800 Subject: [PATCH 1/4] gh-153056: Fix a data race compiling the string.Template pattern in free-threading builds Template compiles its substitution pattern lazily and caches it on the class. On the free-threaded build two concurrent first uses could race: a thread that observed the pattern another thread had just compiled would try to recompile it, and re.compile() rejects flags on an already-compiled pattern, raising a spurious ValueError. Return the already-compiled pattern instead. As a side effect, a subclass that supplies an already-compiled pattern now works too; previously it raised the same ValueError at class definition. --- Lib/string/__init__.py | 3 ++ .../test_string_template_race.py | 38 +++++++++++++++++++ Lib/test/test_string/test_string.py | 16 ++++++++ ...-07-05-12-00-00.gh-issue-153056.tMpLat.rst | 2 + 4 files changed, 59 insertions(+) create mode 100644 Lib/test/test_free_threading/test_string_template_race.py create mode 100644 Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst diff --git a/Lib/string/__init__.py b/Lib/string/__init__.py index b788d7136f1ae3..ef71dea75a9ad3 100644 --- a/Lib/string/__init__.py +++ b/Lib/string/__init__.py @@ -84,6 +84,9 @@ def _compile_pattern(cls): import re # deferred import, for performance pattern = cls.__dict__.get('pattern', _TemplatePattern) + if isinstance(pattern, re.Pattern): + # Already compiled; reuse it (re.compile() rejects flags on a Pattern). + return pattern if pattern is _TemplatePattern: delim = re.escape(cls.delimiter) id = cls.idpattern diff --git a/Lib/test/test_free_threading/test_string_template_race.py b/Lib/test/test_free_threading/test_string_template_race.py new file mode 100644 index 00000000000000..57c7072f65a932 --- /dev/null +++ b/Lib/test/test_free_threading/test_string_template_race.py @@ -0,0 +1,38 @@ +import string +import unittest +from string import Template + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestTemplateCompileRace(unittest.TestCase): + def test_concurrent_first_use(self): + # Template compiles its pattern lazily on first use and caches it on the + # class. Race that lazy compile from many threads and confirm none hits + # a spurious ValueError from recompiling an already-compiled pattern. + # Use a throwaway subclass each round so the shared string.Template is + # never mutated; subclasses precompile in __init_subclass__, so restore + # the sentinel descriptor (string._TemplatePattern) to re-arm the lazy + # path before racing. + uncompiled = string._TemplatePattern + errors = [] + + def use_template(cls): + try: + cls("$x and ${y}").substitute(x=1, y=2) + except Exception as e: + errors.append(e) + + for _ in range(20): + class T(Template): + pass + T.pattern = uncompiled + T.flags = None + threading_helper.run_concurrently(use_template, nthreads=10, args=(T,)) + + self.assertEqual(errors, [], msg=f"unexpected errors: {errors}") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_string/test_string.py b/Lib/test/test_string/test_string.py index 5394fe4e12cd41..98881c8c232222 100644 --- a/Lib/test/test_string/test_string.py +++ b/Lib/test/test_string/test_string.py @@ -299,6 +299,22 @@ def test_SafeTemplate(self): eq(s.safe_substitute(dict(who='tim', what='ham', meal='dinner')), 'tim likes ham for dinner') + def test_precompiled_pattern(self): + # A subclass may supply an already-compiled pattern; it must be reused, + # not recompiled (re.compile() rejects flags on a compiled pattern). + # This is the non-threaded form of the free-threading lazy-compile race + # where a thread observes the pattern another thread just compiled. + import re + compiled = re.compile( + r'\$(?:(?P\$)|(?P[a-z]+)|' + r'\{(?P[a-z]+)\}|(?P))') + class MyTemplate(Template): + pattern = compiled + self.assertIs(MyTemplate.pattern, compiled) + self.assertEqual( + MyTemplate('$who likes $what').substitute(who='tim', what='ham'), + 'tim likes ham') + def test_invalid_placeholders(self): raises = self.assertRaises s = Template('$who likes $') diff --git a/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst b/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst new file mode 100644 index 00000000000000..5485927d03519b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst @@ -0,0 +1,2 @@ +Fix a race in :class:`string.Template` where concurrent first use of a +template on the free-threaded build could raise a spurious :exc:`ValueError`. From 21fd115a7bba92e53b3b04bdd47f7ea021defbe9 Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Sun, 5 Jul 2026 10:44:28 +0800 Subject: [PATCH 2/4] Trim test comments and NEWS wording --- .../test_free_threading/test_string_template_race.py | 11 ++++------- Lib/test/test_string/test_string.py | 2 -- .../2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst | 4 ++-- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_free_threading/test_string_template_race.py b/Lib/test/test_free_threading/test_string_template_race.py index 57c7072f65a932..dc4b4e9f18d286 100644 --- a/Lib/test/test_free_threading/test_string_template_race.py +++ b/Lib/test/test_free_threading/test_string_template_race.py @@ -8,13 +8,10 @@ @threading_helper.requires_working_threading() class TestTemplateCompileRace(unittest.TestCase): def test_concurrent_first_use(self): - # Template compiles its pattern lazily on first use and caches it on the - # class. Race that lazy compile from many threads and confirm none hits - # a spurious ValueError from recompiling an already-compiled pattern. - # Use a throwaway subclass each round so the shared string.Template is - # never mutated; subclasses precompile in __init_subclass__, so restore - # the sentinel descriptor (string._TemplatePattern) to re-arm the lazy - # path before racing. + # Racing the lazy pattern compile must not raise a spurious ValueError + # from recompiling an already-compiled pattern. A throwaway subclass, + # re-armed to the sentinel each round, keeps string.Template unmutated + # (subclasses precompile eagerly in __init_subclass__). uncompiled = string._TemplatePattern errors = [] diff --git a/Lib/test/test_string/test_string.py b/Lib/test/test_string/test_string.py index 98881c8c232222..350784a9f00817 100644 --- a/Lib/test/test_string/test_string.py +++ b/Lib/test/test_string/test_string.py @@ -302,8 +302,6 @@ def test_SafeTemplate(self): def test_precompiled_pattern(self): # A subclass may supply an already-compiled pattern; it must be reused, # not recompiled (re.compile() rejects flags on a compiled pattern). - # This is the non-threaded form of the free-threading lazy-compile race - # where a thread observes the pattern another thread just compiled. import re compiled = re.compile( r'\$(?:(?P\$)|(?P[a-z]+)|' diff --git a/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst b/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst index 5485927d03519b..63ff9c3df5e31a 100644 --- a/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst +++ b/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst @@ -1,2 +1,2 @@ -Fix a race in :class:`string.Template` where concurrent first use of a -template on the free-threaded build could raise a spurious :exc:`ValueError`. +Fix a race in :class:`string.Template` where concurrent first use on the +free-threaded build could raise a spurious :exc:`ValueError`. From 9518fcb702d2f8d9e0b080299b9e7c9714abaefd Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Tue, 7 Jul 2026 10:46:37 +0800 Subject: [PATCH 3/4] Document that the pattern attribute accepts a string or a compiled regex --- Doc/library/string.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index be968a3c53d843..77d5a34d99c41e 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -971,7 +971,8 @@ attributes: Alternatively, you can provide the entire regular expression pattern by overriding the class attribute *pattern*. If you do this, the value must be a -regular expression object with four named capturing groups. The capturing +string containing the regular expression, or a compiled regular expression +object, with four named capturing groups. The capturing groups correspond to the rules given above, along with the invalid placeholder rule: From 764fa3f02e9bc38abc6830aedf84f417e6053403 Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Tue, 7 Jul 2026 10:50:47 +0800 Subject: [PATCH 4/4] Comment the three states of pattern and note the documented-behavior fix in NEWS --- Lib/string/__init__.py | 6 +++++- .../Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Lib/string/__init__.py b/Lib/string/__init__.py index ef71dea75a9ad3..88f1ae1b9b023e 100644 --- a/Lib/string/__init__.py +++ b/Lib/string/__init__.py @@ -83,9 +83,13 @@ def __init_subclass__(cls): def _compile_pattern(cls): import re # deferred import, for performance + # `pattern` may be the `_TemplatePattern` sentinel (not yet compiled), an + # already-compiled regular expression object (as documented), or a string + # regular expression. An already-compiled object is returned as-is; the + # other two are compiled and cached back on the class. pattern = cls.__dict__.get('pattern', _TemplatePattern) if isinstance(pattern, re.Pattern): - # Already compiled; reuse it (re.compile() rejects flags on a Pattern). + # re.compile() rejects flags on an already-compiled pattern. return pattern if pattern is _TemplatePattern: delim = re.escape(cls.delimiter) diff --git a/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst b/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst index 63ff9c3df5e31a..a2bc89f25a53c1 100644 --- a/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst +++ b/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst @@ -1,2 +1,4 @@ -Fix a race in :class:`string.Template` where concurrent first use on the -free-threaded build could raise a spurious :exc:`ValueError`. +Fix :class:`string.Template` raising a spurious :exc:`ValueError` when the +*pattern* attribute is a compiled regular expression object, which the +documentation allows. On the free-threaded build this also occurred as a data +race on the first concurrent use.