[3.13] gh-150436: Skip subprocess test on STATUS_DLL_INIT_FAILED (#150704) (#150721)

gh-150436: Skip subprocess test on STATUS_DLL_INIT_FAILED (#150704)

If a subprocess spawned with CREATE_NEW_CONSOLE creation flag fails
with STATUS_DLL_INIT_FAILED return code, skip the test. It's likely a
memory allocation failure in the desktop heap memory which caused the
DLL init failure.

(cherry picked from commit e8034dd)

Author: vstinner

Lib/test/support/__init__.py

@@ -2863,3 +2863,18 @@ def control_characters_c0() -> list[str]:
     C0 control characters defined as the byte range 0x00-0x1F, and 0x7F.
     """
     return [chr(c) for c in range(0x00, 0x20)] + ["\x7F"]
+
+
+STATUS_DLL_INIT_FAILED = 0xC0000142
+def skip_on_low_desktop_heap_memory_subprocess(returncode):
+    if sys.platform not in ('win32', 'cygwin'):
+        return
+    # On Windows, STATUS_DLL_INIT_FAILED is a generic error code that could
+    # come from any of the DLLs being loaded when a new Python process is
+    # created. In practice, it's likely a memory allocation failure in the
+    # desktop heap memory which caused the DLL init failure, especially on
+    # process created with CREATE_NEW_CONSOLE creation flag. See the article:
+    # https://learn.microsoft.com/en-us/troubleshoot/windows-server/performance/desktop-heap-limitation-out-of-memory
+    if returncode == STATUS_DLL_INIT_FAILED:
+        raise unittest.SkipTest('gh-150436: DLL init failed, likely because '
+                                'of low desktop heap memory')

Lib/test/test_msvcrt.py

@@ -4,6 +4,7 @@
 import unittest
 from textwrap import dedent
 
+from test import support
 from test.support import os_helper, requires_resource
 from test.support.os_helper import TESTFN, TESTFN_ASCII
 
@@ -67,8 +68,12 @@ def run_in_separated_process(self, code):
         # Run test in a separated process to avoid stdin conflicts.
         # See: gh-110147
         cmd = [sys.executable, '-c', code]
-        subprocess.run(cmd, check=True, capture_output=True,
-                       creationflags=subprocess.CREATE_NEW_CONSOLE)
+        try:
+            subprocess.run(cmd, check=True, capture_output=True,
+                           creationflags=subprocess.CREATE_NEW_CONSOLE)
+        except subprocess.CalledProcessError as exc:
+            support.skip_on_low_desktop_heap_memory_subprocess(exc.returncode)
+            raise
 
     def test_kbhit(self):
         code = dedent('''

Lib/test/test_subprocess.py

@@ -3761,13 +3761,17 @@ def test_startupinfo_copy(self):
             self.assertEqual(startupinfo.wShowWindow, subprocess.SW_HIDE)
             self.assertEqual(startupinfo.lpAttributeList, {"handle_list": []})
 
+    # CREATE_NEW_CONSOLE creates a "popup" window.
+    @support.requires_resource('gui')
     def test_creationflags(self):
         # creationflags argument
         CREATE_NEW_CONSOLE = 16
         sys.stderr.write("    a DOS box should flash briefly ...\n")
-        subprocess.call(sys.executable +
-                        ' -c "import time; time.sleep(0.25)"',
-                        creationflags=CREATE_NEW_CONSOLE)
+        rc = subprocess.call(sys.executable +
+                             ' -c "import time; time.sleep(0.25)"',
+                             creationflags=CREATE_NEW_CONSOLE)
+        support.skip_on_low_desktop_heap_memory_subprocess(rc)
+        self.assertEqual(rc, 0)
 
     def test_invalid_args(self):
         # invalid arguments should raise ValueError