diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index eaa0ba54af18e7..7b62b9208412e9 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2527,6 +2527,12 @@ types. .. versionadded:: 3.8 + .. deprecated-removed:: 3.15 3.20 + It is deprecated to call :func:`isinstance` and :func:`issubclass` checks on + protocol classes that were not explicitly decorated with :func:`!runtime_checkable` + but that inherit from a runtime-checkable protocol class. This will throw + a :exc:`TypeError` in Python 3.20. + .. decorator:: runtime_checkable Mark a protocol class as a runtime protocol. @@ -2548,6 +2554,18 @@ types. import threading assert isinstance(threading.Thread(name='Bob'), Named) + Runtime checkability of protocols is not inherited. A subclass of a runtime-checkable protocol + is only runtime-checkable if it is explicitly marked as such, regardless of class hierarchy:: + + @runtime_checkable + class Iterable(Protocol): + def __iter__(self): ... + + # Without @runtime_checkable, Reversible would no longer be runtime-checkable. + @runtime_checkable + class Reversible(Iterable, Protocol): + def __reversed__(self): ... + This decorator raises :exc:`TypeError` when applied to a non-protocol class. .. note:: @@ -2588,6 +2606,11 @@ types. protocol. See :ref:`What's new in Python 3.12 ` for more details. + .. deprecated-removed:: 3.15 3.20 + It is deprecated to call :func:`isinstance` and :func:`issubclass` checks on + protocol classes that were not explicitly decorated with :func:`!runtime_checkable` + but that inherit from a runtime-checkable protocol class. This will throw + a :exc:`TypeError` in Python 3.20. .. class:: TypedDict(dict) diff --git a/Include/datetime.h b/Include/datetime.h index ed36e6e48c87d2..ed7e55009d2208 100644 --- a/Include/datetime.h +++ b/Include/datetime.h @@ -196,8 +196,23 @@ typedef struct { /* Define global variable for the C API and a macro for setting it. */ static PyDateTime_CAPI *PyDateTimeAPI = NULL; -#define PyDateTime_IMPORT \ - PyDateTimeAPI = (PyDateTime_CAPI *)PyCapsule_Import(PyDateTime_CAPSULE_NAME, 0) +static inline PyDateTime_CAPI * +_PyDateTime_IMPORT(void) { + PyDateTime_CAPI *val = _Py_atomic_load_ptr(&PyDateTimeAPI); + if (val == NULL) { + PyDateTime_CAPI *capi = (PyDateTime_CAPI *)PyCapsule_Import( + PyDateTime_CAPSULE_NAME, 0); + if (capi != NULL) { + /* if the compare exchange fails then in that case + another thread would have initialized it */ + _Py_atomic_compare_exchange_ptr(&PyDateTimeAPI, &val, (void *)capi); + return capi; + } + } + return val; +} + +#define PyDateTime_IMPORT _PyDateTime_IMPORT() /* Macro for access to the UTC singleton */ #define PyDateTime_TimeZone_UTC PyDateTimeAPI->TimeZone_UTC diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 3ed7a360d64e3c..55e5f06c8071ea 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -9,7 +9,6 @@ import tempfile import textwrap import unittest -import warnings from test import support from test.support import os_helper from test.support import force_not_colorized @@ -943,21 +942,15 @@ def test_python_asyncio_debug(self): @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") def test_python_dump_refs(self): - code = 'import sys; sys._clear_type_cache()' - # TODO: Remove warnings context manager once sys._clear_type_cache is removed - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFS='1') + code = 'import sys; sys._clear_internal_caches()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFS='1') self.assertEqual(rc, 0) @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") def test_python_dump_refs_file(self): with tempfile.NamedTemporaryFile() as dump_file: - code = 'import sys; sys._clear_type_cache()' - # TODO: Remove warnings context manager once sys._clear_type_cache is removed - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFSFILE=dump_file.name) + code = 'import sys; sys._clear_internal_caches()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFSFILE=dump_file.name) self.assertEqual(rc, 0) with open(dump_file.name, 'r') as file: contents = file.read() diff --git a/Lib/test/test_cppext/extension.cpp b/Lib/test/test_cppext/extension.cpp index 038f67bbbe3f74..a8cd70aacbc805 100644 --- a/Lib/test/test_cppext/extension.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -16,10 +16,9 @@ // gh-135906: Check for compiler warnings in the internal C API # include "internal/pycore_frame.h" // mimalloc emits many compiler warnings when Python is built in debug - // mode (when MI_DEBUG is not zero) - // mimalloc emits compiler warnings when Python is built on Windows - // in free-threaded mode. -# if !defined(Py_DEBUG) && !(defined(MS_WINDOWS) && defined(Py_GIL_DISABLED)) + // mode (when MI_DEBUG is not zero). + // mimalloc emits compiler warnings when Python is built on Windows. +# if !defined(Py_DEBUG) && !defined(MS_WINDOWS) # include "internal/pycore_backoff.h" # include "internal/pycore_cell.h" # endif diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 59c6dc4587c93d..f66e2987d34850 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -45,6 +45,7 @@ Py_GIL_DISABLED, no_rerun, force_not_colorized_test_class, + catch_unraisable_exception ) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, ready_to_import, @@ -2517,6 +2518,32 @@ def test_disallowed_reimport(self): excsnap = _interpreters.run_string(interpid, script) self.assertIsNot(excsnap, None) + @requires_subinterpreters + def test_pyinit_function_raises_exception(self): + # gh-144601: PyInit functions that raised exceptions would cause a + # crash when imported from a subinterpreter. + import _testsinglephase + filename = _testsinglephase.__file__ + script = f"""if True: + from test.test_import import import_extension_from_file + + import_extension_from_file('_testsinglephase_raise_exception', {filename!r})""" + + interp = _interpreters.create() + try: + with catch_unraisable_exception() as cm: + exception = _interpreters.run_string(interp, script) + unraisable = cm.unraisable + finally: + _interpreters.destroy(interp) + + self.assertIsNotNone(exception) + self.assertIsNotNone(exception.type.__name__, "ImportError") + self.assertIsNotNone(exception.msg, "failed to import from subinterpreter due to exception") + self.assertIsNotNone(unraisable) + self.assertIs(unraisable.exc_type, RuntimeError) + self.assertEqual(str(unraisable.exc_value), "evil") + class TestSinglePhaseSnapshot(ModuleSnapshot): """A representation of a single-phase init module for testing. diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 61bea9dba07fec..dc64288085fa74 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -733,6 +733,27 @@ def keyfunc(obj): keyfunc.skip = 1 self.assertRaises(ExpectedError, gulp, [None, None], keyfunc) + def test_groupby_reentrant_eq_does_not_crash(self): + # regression test for gh-143543 + class Key: + def __init__(self, do_advance): + self.do_advance = do_advance + + def __eq__(self, other): + if self.do_advance: + self.do_advance = False + next(g) + return NotImplemented + return False + + def keys(): + yield Key(True) + yield Key(False) + + g = itertools.groupby([None, None], keys().send) + next(g) + next(g) # must pass with address sanitizer + def test_filter(self): self.assertEqual(list(filter(isEven, range(6))), [0,2,4]) self.assertEqual(list(filter(None, [0,1,0,2,0])), [1,2]) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index e896df518447c5..72ae7776ab9062 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -51,7 +51,7 @@ from test.support import ( captured_stderr, cpython_only, requires_docstrings, import_helper, run_code, - EqualToForwardRef, + subTests, EqualToForwardRef, ) from test.typinganndata import ( ann_module695, mod_generics_cache, _typed_dict_helper, @@ -3885,8 +3885,8 @@ def meth(self): pass self.assertIsNot(get_protocol_members(PR), P.__protocol_attrs__) acceptable_extra_attrs = { - '_is_protocol', '_is_runtime_protocol', '__parameters__', - '__init__', '__annotations__', '__subclasshook__', '__annotate__', + '_is_protocol', '_is_runtime_protocol', '__typing_is_deprecated_inherited_runtime_protocol__', + '__parameters__', '__init__', '__annotations__', '__subclasshook__', '__annotate__', '__annotations_cache__', '__annotate_func__', } self.assertLessEqual(vars(NonP).keys(), vars(C).keys() | acceptable_extra_attrs) @@ -4458,6 +4458,70 @@ class P(Protocol): with self.assertRaisesRegex(TypeError, "@runtime_checkable"): isinstance(1, P) + @subTests(['check_obj', 'check_func'], ([42, isinstance], [frozenset, issubclass])) + def test_inherited_runtime_protocol_deprecated(self, check_obj, check_func): + """See GH-132604.""" + + class BareProto(Protocol): + """I am not runtime-checkable.""" + + @runtime_checkable + class RCProto1(Protocol): + """I am runtime-checkable.""" + + class InheritedRCProto1(RCProto1, Protocol): + """I am accidentally runtime-checkable (by inheritance).""" + + @runtime_checkable + class RCProto2(InheritedRCProto1, Protocol): + """Explicit RC -> inherited RC -> explicit RC.""" + def spam(self): ... + + @runtime_checkable + class RCProto3(BareProto, Protocol): + """Not RC -> explicit RC.""" + + class InheritedRCProto2(RCProto3, Protocol): + """Not RC -> explicit RC -> inherited RC.""" + def eggs(self): ... + + class InheritedRCProto3(RCProto2, Protocol): + """Explicit RC -> inherited RC -> explicit RC -> inherited RC.""" + + class Concrete1(BareProto): + pass + + class Concrete2(InheritedRCProto2): + pass + + class Concrete3(InheritedRCProto3): + pass + + depr_message_re = ( + r" isn't explicitly decorated " + r"with @runtime_checkable but it is used in issubclass\(\) or " + r"isinstance\(\). Instance and class checks can only be used with " + r"@runtime_checkable protocols. This will raise a TypeError in Python 3.20." + ) + + for inherited_runtime_proto in InheritedRCProto1, InheritedRCProto2, InheritedRCProto3: + with self.assertWarnsRegex(DeprecationWarning, depr_message_re): + check_func(check_obj, inherited_runtime_proto) + + # Don't warn for explicitly checkable protocols and concrete implementations. + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + + for checkable in RCProto1, RCProto2, RCProto3, Concrete1, Concrete2, Concrete3: + check_func(check_obj, checkable) + + # Don't warn for uncheckable protocols. + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + + with self.assertRaises(TypeError): # Self-test. Protocol below can't be runtime-checkable. + check_func(check_obj, BareProto) + def test_super_call_init(self): class P(Protocol): x: int diff --git a/Lib/typing.py b/Lib/typing.py index 1a2ef8c086f772..71a08a5f1df811 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1826,6 +1826,7 @@ class _TypingEllipsis: _TYPING_INTERNALS = frozenset({ '__parameters__', '__orig_bases__', '__orig_class__', '_is_protocol', '_is_runtime_protocol', '__protocol_attrs__', + '__typing_is_deprecated_inherited_runtime_protocol__', '__non_callable_proto_members__', '__type_params__', }) @@ -2015,6 +2016,16 @@ def __subclasscheck__(cls, other): "Instance and class checks can only be used with " "@runtime_checkable protocols" ) + if getattr(cls, '__typing_is_deprecated_inherited_runtime_protocol__', False): + # See GH-132604. + import warnings + depr_message = ( + f"{cls!r} isn't explicitly decorated with @runtime_checkable but " + "it is used in issubclass() or isinstance(). Instance and class " + "checks can only be used with @runtime_checkable protocols. " + "This will raise a TypeError in Python 3.20." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=2) if ( # this attribute is set by @runtime_checkable: cls.__non_callable_proto_members__ @@ -2044,6 +2055,18 @@ def __instancecheck__(cls, instance): raise TypeError("Instance and class checks can only be used with" " @runtime_checkable protocols") + if getattr(cls, '__typing_is_deprecated_inherited_runtime_protocol__', False): + # See GH-132604. + import warnings + + depr_message = ( + f"{cls!r} isn't explicitly decorated with @runtime_checkable but " + "it is used in issubclass() or isinstance(). Instance and class " + "checks can only be used with @runtime_checkable protocols. " + "This will raise a TypeError in Python 3.20." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=2) + if _abc_instancecheck(cls, instance): return True @@ -2136,6 +2159,11 @@ def __init_subclass__(cls, *args, **kwargs): if not cls.__dict__.get('_is_protocol', False): cls._is_protocol = any(b is Protocol for b in cls.__bases__) + # Mark inherited runtime checkability (deprecated). See GH-132604. + if cls._is_protocol and getattr(cls, '_is_runtime_protocol', False): + # This flag is set to False by @runtime_checkable. + cls.__typing_is_deprecated_inherited_runtime_protocol__ = True + # Set (or override) the protocol subclass hook. if '__subclasshook__' not in cls.__dict__: cls.__subclasshook__ = _proto_hook @@ -2282,6 +2310,9 @@ def close(self): ... raise TypeError('@runtime_checkable can be only applied to protocol classes,' ' got %r' % cls) cls._is_runtime_protocol = True + # See GH-132604. + if hasattr(cls, '__typing_is_deprecated_inherited_runtime_protocol__'): + cls.__typing_is_deprecated_inherited_runtime_protocol__ = False # PEP 544 prohibits using issubclass() # with protocols that have non-method members. # See gh-113320 for why we compute this attribute here, diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-02-02-17-07-34.gh-issue-141563.GheXjr.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-02-02-17-07-34.gh-issue-141563.GheXjr.rst new file mode 100644 index 00000000000000..4059525f090c50 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-02-02-17-07-34.gh-issue-141563.GheXjr.rst @@ -0,0 +1 @@ +Fix thread safety of :c:macro:`! PyDateTime_IMPORT`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-02-08-12-47-27.gh-issue-144601.E4Yi9J.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-02-08-12-47-27.gh-issue-144601.E4Yi9J.rst new file mode 100644 index 00000000000000..1c7772e2f3ca26 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-02-08-12-47-27.gh-issue-144601.E4Yi9J.rst @@ -0,0 +1,2 @@ +Fix crash when importing a module whose ``PyInit`` function raises an +exception from a subinterpreter. diff --git a/Misc/NEWS.d/next/Library/2026-01-13-10-38-43.gh-issue-143543.DeQRCO.rst b/Misc/NEWS.d/next/Library/2026-01-13-10-38-43.gh-issue-143543.DeQRCO.rst new file mode 100644 index 00000000000000..14622a395ec22e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-01-13-10-38-43.gh-issue-143543.DeQRCO.rst @@ -0,0 +1,2 @@ +Fix a crash in itertools.groupby that could occur when a user-defined +:meth:`~object.__eq__` method re-enters the iterator during key comparison. diff --git a/Misc/NEWS.d/next/Library/2026-01-13-15-56-03.gh-issue-132604.lvjNTr.rst b/Misc/NEWS.d/next/Library/2026-01-13-15-56-03.gh-issue-132604.lvjNTr.rst new file mode 100644 index 00000000000000..92c4dbb536cdf6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-01-13-15-56-03.gh-issue-132604.lvjNTr.rst @@ -0,0 +1,4 @@ +Previously, :class:`~typing.Protocol` classes that were not decorated with :deco:`~typing.runtime_checkable`, +but that inherited from another ``Protocol`` class that did have this decorator, could be used in :func:`isinstance` +and :func:`issubclass` checks. This behavior is now deprecated and such checks will throw a :exc:`TypeError` +in Python 3.20. Patch by Bartosz Sławecki. diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index ee38d61b43a82a..7ea77c6312c59e 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -801,3 +801,11 @@ PyInit__testsinglephase_circular(void) } return Py_NewRef(static_module_circular); } + + +PyMODINIT_FUNC +PyInit__testsinglephase_raise_exception(void) +{ + PyErr_SetString(PyExc_RuntimeError, "evil"); + return NULL; +} diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 7e73f76bc20b58..ff0e2fd2b3569d 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -544,9 +544,19 @@ groupby_next(PyObject *op) else if (gbo->tgtkey == NULL) break; else { - int rcmp; + /* A user-defined __eq__ can re-enter groupby and advance the iterator, + mutating gbo->tgtkey / gbo->currkey while we are comparing them. + Take local snapshots and hold strong references so INCREF/DECREF + apply to the same objects even under re-entrancy. */ + PyObject *tgtkey = gbo->tgtkey; + PyObject *currkey = gbo->currkey; + + Py_INCREF(tgtkey); + Py_INCREF(currkey); + int rcmp = PyObject_RichCompareBool(tgtkey, currkey, Py_EQ); + Py_DECREF(tgtkey); + Py_DECREF(currkey); - rcmp = PyObject_RichCompareBool(gbo->tgtkey, gbo->currkey, Py_EQ); if (rcmp == -1) return NULL; else if (rcmp == 0) diff --git a/Python/import.c b/Python/import.c index 466c5868ab7ee8..e7f803d84f1367 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2176,13 +2176,29 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, } main_finally: + if (rc < 0) { + _Py_ext_module_loader_result_apply_error(&res, name_buf); + } + /* Switch back to the subinterpreter. */ if (switched) { + // gh-144601: The exception object can't be transferred across + // interpreters. Instead, we print out an unraisable exception, and + // then raise a different exception for the calling interpreter. + if (rc < 0) { + assert(PyErr_Occurred()); + PyErr_FormatUnraisable("Exception while importing from subinterpreter"); + } assert(main_tstate != tstate); switch_back_from_main_interpreter(tstate, main_tstate, mod); /* Any module we got from the init function will have to be * reloaded in the subinterpreter. */ mod = NULL; + if (rc < 0) { + PyErr_SetString(PyExc_ImportError, + "failed to import from subinterpreter due to exception"); + goto error; + } } /*****************************************************************/ @@ -2191,7 +2207,6 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, /* Finally we handle the error return from _PyImport_RunModInitFunc(). */ if (rc < 0) { - _Py_ext_module_loader_result_apply_error(&res, name_buf); goto error; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 381b2500158ef0..7bd6970e5fd2dc 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -83,7 +83,7 @@ dump_abstract_stack(_Py_UOpsAbstractFrame *frame, JitOptRef *stack_pointer) static void dump_uop(JitOptContext *ctx, const char *label, int index, - const _PyUOpInstruction *instr, JitOptRef *stack_pointer) + const _PyUOpInstruction *instr, JitOptRef *stack_pointer) { if (get_lltrace() >= 3) { printf("%4d %s: ", index, label); @@ -95,11 +95,24 @@ dump_uop(JitOptContext *ctx, const char *label, int index, } } +static void +dump_uops(JitOptContext *ctx, const char *label, + _PyUOpInstruction *start, JitOptRef *stack_pointer) +{ + int current_len = uop_buffer_length(&ctx->out_buffer); + int added_count = (int)(ctx->out_buffer.next - start); + for (int j = 0; j < added_count; j++) { + dump_uop(ctx, label, current_len - added_count + j, &start[j], stack_pointer); + } +} + #define DUMP_UOP dump_uop +#define DUMP_UOPS dump_uops #else #define DPRINTF(level, ...) #define DUMP_UOP(ctx, label, index, instr, stack_pointer) + #define DUMP_UOPS(ctx, label, start, stack_pointer) #endif static int @@ -508,7 +521,7 @@ optimize_uops( *(ctx->out_buffer.next++) = *this_instr; } assert(ctx->frame != NULL); - DUMP_UOP(ctx, "out", uop_buffer_length(&ctx->out_buffer) - 1, out_ptr, stack_pointer); + DUMP_UOPS(ctx, "out", out_ptr, stack_pointer); if (!CURRENT_FRAME_IS_INIT_SHIM() && !ctx->done) { DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); ctx->frame->stack_pointer = stack_pointer; diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 08aba5ef5423fe..00fd0edd8537bf 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -232,9 +232,12 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: if file.name == "reusable-windows-msi.yml": run_windows_msi = True if file.name == "reusable-macos.yml": + run_tests = True platforms_changed.add("macos") if file.name == "reusable-wasi.yml": + run_tests = True platforms_changed.add("wasi") + continue if not doc_file and file not in RUN_TESTS_IGNORE: run_tests = True