Add subinterpreter support

Dockerfile

@@ -66,6 +66,9 @@ RUN apt-get update \
     python3.13-dev \
     python3.13-dbg \
     python3.13-venv \
+    python3.14-dev \
+    python3.14-dbg \
+    python3.14-venv \
     make \
     cmake \
     gdb \

news/279.bugfix.rst

@@ -0,0 +1,5 @@
+Fix an issue where the PID offset could not be determined when multiple
+subinterpreters were present. Previously, pystack only checked the first
+interpreter in the linked list, which was not guaranteed to be the main
+interpreter. The fix now iterates over all interpreters and correctly locates
+the TID.

news/279.feature.rst

@@ -0,0 +1,3 @@
+Add support for subinterpreters when reporting pure Python stacks. Threads
+running in subinterpreters are now identified and grouped by interpreter ID.
+Native stack reporting for subinterpreters is not yet supported.

setup.py

@@ -77,6 +77,7 @@
         "src/pystack/_pystack.pyx",
         "src/pystack/_pystack/corefile.cpp",
         "src/pystack/_pystack/elf_common.cpp",
+        "src/pystack/_pystack/interpreter.cpp",
         "src/pystack/_pystack/logging.cpp",
         "src/pystack/_pystack/mem.cpp",
         "src/pystack/_pystack/process.cpp",

src/pystack/__init__.py

@@ -1,7 +1,5 @@
 from ._version import __version__
-from .traceback_formatter import print_thread
 
 __all__ = [
     "__version__",
-    "print_thread",
 ]

src/pystack/__main__.py

@@ -19,13 +19,13 @@
 from pystack.process import is_gzip
 
 from . import errors
-from . import print_thread
 from .colors import colored
 from .engine import CoreFileAnalyzer
 from .engine import NativeReportingMode
 from .engine import StackMethod
 from .engine import get_process_threads
 from .engine import get_process_threads_for_core
+from .traceback_formatter import TracebackPrinter
 
 PERMISSION_ERROR_MSG = "Operation not permitted"
 NO_SUCH_PROCESS_ERROR_MSG = "No such process"
@@ -285,14 +285,24 @@ def process_remote(parser: argparse.ArgumentParser, args: argparse.Namespace) ->
     if not args.block and args.native_mode != NativeReportingMode.OFF:
         parser.error("Native traces are only available in blocking mode")
 
-    for thread in get_process_threads(
+    threads = get_process_threads(
         args.pid,
         stop_process=args.block,
         native_mode=args.native_mode,
         locals=args.locals,
         method=StackMethod.ALL if args.exhaustive else StackMethod.AUTO,
-    ):
-        print_thread(thread, args.native_mode)
+    )
+
+    has_multiple_subinterpreters = (
+        len(set(thread.interpreter_id for thread in threads)) > 1
+    )
+
+    printer = TracebackPrinter(
+        native_mode=args.native_mode,
+        include_subinterpreters=has_multiple_subinterpreters,
+    )
+    for thread in threads:
+        printer.print_thread(thread)
 
 
 def format_psinfo_information(psinfo: Dict[str, Any]) -> str:
@@ -414,15 +424,25 @@ def process_core(parser: argparse.ArgumentParser, args: argparse.Namespace) -> N
                 elf_id if elf_id else "<MISSING>",
             )
 
-    for thread in get_process_threads_for_core(
+    threads = get_process_threads_for_core(
         corefile,
         executable,
         library_search_path=lib_search_path,
         native_mode=args.native_mode,
         locals=args.locals,
         method=StackMethod.ALL if args.exhaustive else StackMethod.AUTO,
-    ):
-        print_thread(thread, args.native_mode)
+    )
+
+    has_multiple_subinterpreters = (
+        len(set(thread.interpreter_id for thread in threads)) > 1
+    )
+
+    printer = TracebackPrinter(
+        args.native_mode, include_subinterpreters=has_multiple_subinterpreters
+    )
+
+    for thread in threads:
+        printer.print_thread(thread)
 
 
 if __name__ == "__main__":  # pragma: no cover

src/pystack/_pystack.pyx

@@ -22,6 +22,7 @@ from _pystack.elf_common cimport CoreFileAnalyzer as NativeCoreFileAnalyzer
 from _pystack.elf_common cimport ProcessAnalyzer as NativeProcessAnalyzer
 from _pystack.elf_common cimport SectionInfo
 from _pystack.elf_common cimport getSectionInfo
+from _pystack.interpreter cimport InterpreterUtils
 from _pystack.logging cimport initializePythonLoggerInterface
 from _pystack.mem cimport AbstractRemoteMemoryManager
 from _pystack.mem cimport MemoryMapInformation as CppMemoryMapInformation
@@ -65,6 +66,7 @@ from .types import NativeFrame
 from .types import PyCodeObject
 from .types import PyFrame
 from .types import PyThread
+from .types import frame_type
 
 LOGGER = logging.getLogger(__file__)
 
@@ -462,6 +464,7 @@ cdef object _construct_threads_from_interpreter_state(
     bint add_native_traces,
     bint resolve_locals,
 ):
+    interpreter_id = InterpreterUtils.getInterpreterId(manager, head)
     LOGGER.info("Fetching Python threads")
     threads = []
 
@@ -486,7 +489,9 @@ cdef object _construct_threads_from_interpreter_state(
                 current_thread.isGilHolder(),
                 current_thread.isGCCollecting(),
                 python_version,
+                interpreter_id,
                 name=get_thread_name(pid, current_thread.Tid()),
+                stack_anchor=current_thread.StackAnchor(),
             )
         )
         current_thread = (
@@ -495,6 +500,78 @@ cdef object _construct_threads_from_interpreter_state(
 
     return threads
 
+
+def _entry_frame_count(thread: PyThread) -> int:
+    return sum(1 for frame in thread.all_frames if frame.is_entry)
+
+
+def _slice_native_stacks_for_same_tid_threads(threads) -> None:
+    canonical = next((thread for thread in threads if thread.native_frames), None)
+    if canonical is None:
+        return
+
+    canonical_frames = list(canonical.native_frames)
+    eval_positions = [
+        index
+        for index, native_frame in enumerate(canonical_frames)
+        if frame_type(native_frame, canonical.python_version) == NativeFrame.FrameType.EVAL
+    ]
+    if not eval_positions:
+        return
+
+    entry_counts = [_entry_frame_count(thread) for thread in threads]
+    if sum(entry_counts) != len(eval_positions):
+        LOGGER.debug(
+            "Skipping same-tid native slicing for tid %s due to mismatched counts: "
+            "entry=%s eval=%s",
+            threads[0].tid,
+            sum(entry_counts),
+            len(eval_positions),
+        )
+        return
+
+    ordered_threads = sorted(
+        enumerate(threads),
+        key=lambda item: (
+            item[1].stack_anchor is None,  # Everything with a stack anchor before everything without one
+            -(item[1].stack_anchor or 0),  # The stack anchor in descending order
+            item[0],  # The index of the item in the list in ascending order
+        ),
+    )
+
+    cursor = 0
+    for _, thread in ordered_threads:
+        required_eval_frames = _entry_frame_count(thread)
+        if required_eval_frames == 0:
+            thread.native_frames = []
+            continue
+
+        group_start = cursor
+        group_end = cursor + required_eval_frames
+        prev_eval = eval_positions[group_start - 1] if group_start > 0 else -1
+        next_eval = (
+            eval_positions[group_end]
+            if group_end < len(eval_positions)
+            else len(canonical_frames)
+        )
+        thread.native_frames = canonical_frames[prev_eval + 1 : next_eval]
+        cursor = group_end
+
+
+def _normalize_python_threads(threads, native_mode: NativeReportingMode):
+    if native_mode == NativeReportingMode.OFF:
+        return threads
+
+    threads_by_tid = {}
+    for thread in threads:
+        threads_by_tid.setdefault(thread.tid, []).append(thread)
+
+    for group in threads_by_tid.values():
+        if len(group) <= 1:
+            continue
+        _slice_native_stacks_for_same_tid_threads(group)
+    return threads
+
 cdef object _construct_os_thread(
     shared_ptr[AbstractProcessManager] manager, int pid, int tid
 ):
@@ -622,7 +699,8 @@ def _get_process_threads(
         )
 
     all_tids = list(manager.get().Tids())
-    if head:
+    threads = []
+    while head:
         add_native_traces = native_mode != NativeReportingMode.OFF
         for thread in _construct_threads_from_interpreter_state(
             manager,
@@ -634,7 +712,11 @@ def _get_process_threads(
         ):
             if thread.tid in all_tids:
                 all_tids.remove(thread.tid)
-            yield thread
+            threads.append(thread)
+        head = InterpreterUtils.getNextInterpreter(manager, head)
+
+    for thread in _normalize_python_threads(threads, native_mode):
+        yield thread
 
     if native_mode == NativeReportingMode.ALL:
         yield from _construct_os_threads(manager, pid, all_tids)
@@ -768,15 +850,25 @@ def _get_process_threads_for_core(
         )
 
     all_tids = list(manager.get().Tids())
+    threads = []
 
-    if head:
-        native = native_mode in {NativeReportingMode.PYTHON, NativeReportingMode.ALL}
+    while head:
+        add_native_traces = native_mode != NativeReportingMode.OFF
         for thread in _construct_threads_from_interpreter_state(
-            manager, head, pymanager.pid, pymanager.python_version, native, locals
+            manager,
+            head,
+            pymanager.pid,
+            pymanager.python_version,
+            add_native_traces,
+            locals,
         ):
             if thread.tid in all_tids:
                 all_tids.remove(thread.tid)
-            yield thread
+            threads.append(thread)
+        head = InterpreterUtils.getNextInterpreter(manager, head)
+
+    for thread in _normalize_python_threads(threads, native_mode):
+        yield thread
 
     if native_mode == NativeReportingMode.ALL:
         yield from _construct_os_threads(manager, pymanager.pid, all_tids)

src/pystack/_pystack/CMakeLists.txt

@@ -21,6 +21,7 @@ add_library(_pystack STATIC
             pythread.cpp
             version.cpp
             elf_common.cpp
-            pytypes.cpp)
+            pytypes.cpp
+            interpreter.cpp)
 set_property(TARGET _pystack PROPERTY POSITION_INDEPENDENT_CODE ON)
 include_directories("." "cpython" ${PYTHON_INCLUDE_DIRS})

src/pystack/_pystack/cpython/interpreter.h

@@ -375,10 +375,10 @@ struct _gil_runtime_state
     int locked;
     unsigned long switch_number;
     pthread_cond_t cond;
-    pthread_cond_t mutex;
+    pthread_mutex_t mutex;
 #ifdef FORCE_SWITCHING
     pthread_cond_t switch_cond;
-    pthread_cond_t switch_mutex;
+    pthread_mutex_t switch_mutex;
 #endif
 };
 

src/pystack/_pystack/interpreter.cpp

@@ -0,0 +1,36 @@
+#include <memory>
+
+#include "interpreter.h"
+#include "logging.h"
+#include "process.h"
+#include "structure.h"
+#include "version.h"
+
+namespace pystack {
+
+remote_addr_t
+InterpreterUtils::getNextInterpreter(
+        const std::shared_ptr<const AbstractProcessManager>& manager,
+        remote_addr_t interpreter_addr)
+{
+    Structure<py_is_v> is(manager, interpreter_addr);
+    return is.getField(&py_is_v::o_next);
+}
+
+int64_t
+InterpreterUtils::getInterpreterId(
+        const std::shared_ptr<const AbstractProcessManager>& manager,
+        remote_addr_t interpreter_addr)
+{
+    if (!manager->versionIsAtLeast(3, 7)) {
+        // No support for subinterpreters so the only interpreter is ID 0.
+        return 0;
+    }
+
+    Structure<py_is_v> is(manager, interpreter_addr);
+    int64_t id_value = is.getField(&py_is_v::o_id);
+
+    return id_value;
+}
+
+}  // namespace pystack

src/pystack/_pystack/interpreter.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <cstdint>
+#include <memory>
+
+#include "mem.h"
+#include "process.h"
+
+namespace pystack {
+
+class InterpreterUtils
+{
+  public:
+    // Static Methods
+    static remote_addr_t getNextInterpreter(
+            const std::shared_ptr<const AbstractProcessManager>& manager,
+            remote_addr_t interpreter_addr);
+
+    static int64_t getInterpreterId(
+            const std::shared_ptr<const AbstractProcessManager>& manager,
+            remote_addr_t interpreter_addr);
+};
+
+}  // namespace pystack

src/pystack/_pystack/interpreter.pxd

@@ -0,0 +1,13 @@
+from _pystack.mem cimport remote_addr_t
+from _pystack.process cimport AbstractProcessManager
+from libc.stdint cimport int64_t
+from libcpp.memory cimport shared_ptr
+
+
+cdef extern from "interpreter.h" namespace "pystack":
+    cdef cppclass  InterpreterUtils:
+        @staticmethod
+        remote_addr_t getNextInterpreter(shared_ptr[AbstractProcessManager] manager, remote_addr_t interpreter_addr) except +
+
+        @staticmethod
+        int64_t getInterpreterId(shared_ptr[AbstractProcessManager] manager, remote_addr_t interpreter_addr) except +

src/pystack/_pystack/process.cpp

@@ -964,6 +964,7 @@ AbstractProcessManager::copyDebugOffsets(Structure<py_runtime_v>& py_runtime, py
     set_offset(py_is.o_sysdict, &py_runtime_v::o_dbg_off_interpreter_state_sysdict);
     set_offset(py_is.o_builtins, &py_runtime_v::o_dbg_off_interpreter_state_builtins);
     set_offset(py_is.o_gil_runtime_state, &py_runtime_v::o_dbg_off_interpreter_state_ceval_gil);
+    set_offset(py_is.o_id, &py_runtime_v::o_dbg_off_interpreter_state_id);
 
     set_size(py_thread, &py_runtime_v::o_dbg_off_thread_state_struct_size);
     set_offset(py_thread.o_prev, &py_runtime_v::o_dbg_off_thread_state_prev);