From 5239c2072999b37ce0f3b7c116613d248919e524 Mon Sep 17 00:00:00 2001 From: Saul Cooperman Date: Sun, 22 Feb 2026 17:28:12 +0000 Subject: [PATCH 1/6] Support --- setup.py | 1 + src/pystack/__init__.py | 2 - src/pystack/__main__.py | 10 +- src/pystack/_pystack.pyx | 7 +- src/pystack/_pystack/CMakeLists.txt | 3 +- src/pystack/_pystack/interpreter.cpp | 36 ++++ src/pystack/_pystack/interpreter.h | 24 +++ src/pystack/_pystack/interpreter.pxd | 12 ++ src/pystack/_pystack/process.cpp | 1 + src/pystack/_pystack/pythread.cpp | 133 ++++++------ src/pystack/_pystack/version.cpp | 47 ++++- src/pystack/_pystack/version.h | 3 +- src/pystack/traceback_formatter.py | 33 ++- src/pystack/types.py | 1 + tests/integration/test_subinterpreters.py | 91 +++++++++ tests/unit/test_main.py | 176 +++++++++------- tests/unit/test_traceback_formatter.py | 238 +++++++++++++++++++++- tests/utils.py | 7 + 18 files changed, 668 insertions(+), 157 deletions(-) create mode 100644 src/pystack/_pystack/interpreter.cpp create mode 100644 src/pystack/_pystack/interpreter.h create mode 100644 src/pystack/_pystack/interpreter.pxd create mode 100644 tests/integration/test_subinterpreters.py diff --git a/setup.py b/setup.py index d76c9375..58bc40d2 100644 --- a/setup.py +++ b/setup.py @@ -77,6 +77,7 @@ "src/pystack/_pystack.pyx", "src/pystack/_pystack/corefile.cpp", "src/pystack/_pystack/elf_common.cpp", + "src/pystack/_pystack/interpreter.cpp", "src/pystack/_pystack/logging.cpp", "src/pystack/_pystack/mem.cpp", "src/pystack/_pystack/process.cpp", diff --git a/src/pystack/__init__.py b/src/pystack/__init__.py index e973464d..a98c6c19 100644 --- a/src/pystack/__init__.py +++ b/src/pystack/__init__.py @@ -1,7 +1,5 @@ from ._version import __version__ -from .traceback_formatter import print_thread __all__ = [ "__version__", - "print_thread", ] diff --git a/src/pystack/__main__.py b/src/pystack/__main__.py index 55102c91..c206e08a 100644 --- a/src/pystack/__main__.py +++ b/src/pystack/__main__.py @@ -19,13 +19,13 @@ from pystack.process import is_gzip from . import errors -from . import print_thread from .colors import colored from .engine import CoreFileAnalyzer from .engine import NativeReportingMode from .engine import StackMethod from .engine import get_process_threads from .engine import get_process_threads_for_core +from .traceback_formatter import TracebackPrinter PERMISSION_ERROR_MSG = "Operation not permitted" NO_SUCH_PROCESS_ERROR_MSG = "No such process" @@ -285,6 +285,9 @@ def process_remote(parser: argparse.ArgumentParser, args: argparse.Namespace) -> if not args.block and args.native_mode != NativeReportingMode.OFF: parser.error("Native traces are only available in blocking mode") + printer = TracebackPrinter( + native_mode=args.native_mode, + ) for thread in get_process_threads( args.pid, stop_process=args.block, @@ -292,7 +295,7 @@ def process_remote(parser: argparse.ArgumentParser, args: argparse.Namespace) -> locals=args.locals, method=StackMethod.ALL if args.exhaustive else StackMethod.AUTO, ): - print_thread(thread, args.native_mode) + printer.print_thread(thread) def format_psinfo_information(psinfo: Dict[str, Any]) -> str: @@ -414,6 +417,7 @@ def process_core(parser: argparse.ArgumentParser, args: argparse.Namespace) -> N elf_id if elf_id else "", ) + printer = TracebackPrinter(args.native_mode) for thread in get_process_threads_for_core( corefile, executable, @@ -422,7 +426,7 @@ def process_core(parser: argparse.ArgumentParser, args: argparse.Namespace) -> N locals=args.locals, method=StackMethod.ALL if args.exhaustive else StackMethod.AUTO, ): - print_thread(thread, args.native_mode) + printer.print_thread(thread) if __name__ == "__main__": # pragma: no cover diff --git a/src/pystack/_pystack.pyx b/src/pystack/_pystack.pyx index de16701d..80f6bb46 100644 --- a/src/pystack/_pystack.pyx +++ b/src/pystack/_pystack.pyx @@ -22,6 +22,7 @@ from _pystack.elf_common cimport CoreFileAnalyzer as NativeCoreFileAnalyzer from _pystack.elf_common cimport ProcessAnalyzer as NativeProcessAnalyzer from _pystack.elf_common cimport SectionInfo from _pystack.elf_common cimport getSectionInfo +from _pystack.interpreter cimport InterpreterUtils from _pystack.logging cimport initializePythonLoggerInterface from _pystack.mem cimport AbstractRemoteMemoryManager from _pystack.mem cimport MemoryMapInformation as CppMemoryMapInformation @@ -462,6 +463,7 @@ cdef object _construct_threads_from_interpreter_state( bint add_native_traces, bint resolve_locals, ): + interp_id = InterpreterUtils.getInterpreterId(manager, head) LOGGER.info("Fetching Python threads") threads = [] @@ -486,6 +488,7 @@ cdef object _construct_threads_from_interpreter_state( current_thread.isGilHolder(), current_thread.isGCCollecting(), python_version, + interp_id, name=get_thread_name(pid, current_thread.Tid()), ) ) @@ -622,7 +625,7 @@ def _get_process_threads( ) all_tids = list(manager.get().Tids()) - if head: + while head: add_native_traces = native_mode != NativeReportingMode.OFF for thread in _construct_threads_from_interpreter_state( manager, @@ -635,6 +638,7 @@ def _get_process_threads( if thread.tid in all_tids: all_tids.remove(thread.tid) yield thread + head = InterpreterUtils.getNextInterpreter(manager, head); if native_mode == NativeReportingMode.ALL: yield from _construct_os_threads(manager, pid, all_tids) @@ -762,6 +766,7 @@ def _get_process_threads_for_core( manager.get(), method, core=True ) + if not head and native_mode != NativeReportingMode.ALL: raise NotEnoughInformation( "Could not gather enough information to extract the Python frame information" diff --git a/src/pystack/_pystack/CMakeLists.txt b/src/pystack/_pystack/CMakeLists.txt index 5a0fd8a8..74183d11 100644 --- a/src/pystack/_pystack/CMakeLists.txt +++ b/src/pystack/_pystack/CMakeLists.txt @@ -21,6 +21,7 @@ add_library(_pystack STATIC pythread.cpp version.cpp elf_common.cpp - pytypes.cpp) + pytypes.cpp + interpreter.cpp) set_property(TARGET _pystack PROPERTY POSITION_INDEPENDENT_CODE ON) include_directories("." "cpython" ${PYTHON_INCLUDE_DIRS}) diff --git a/src/pystack/_pystack/interpreter.cpp b/src/pystack/_pystack/interpreter.cpp new file mode 100644 index 00000000..850d9f44 --- /dev/null +++ b/src/pystack/_pystack/interpreter.cpp @@ -0,0 +1,36 @@ +#include + +#include "interpreter.h" +#include "logging.h" +#include "process.h" +#include "structure.h" +#include "version.h" + +namespace pystack { + +remote_addr_t +InterpreterUtils::getNextInterpreter( + const std::shared_ptr& manager, + remote_addr_t interpreter_addr) +{ + Structure is(manager, interpreter_addr); + return is.getField(&py_is_v::o_next); +} + +int +InterpreterUtils::getInterpreterId( + const std::shared_ptr& manager, + remote_addr_t interpreter_addr) +{ + if (!manager->versionIsAtLeast(3, 7)) { + // No support for subinterpreters so the only interpreter is ID 0. + return 0; + } + + Structure is(manager, interpreter_addr); + int64_t id_value = is.getField(&py_is_v::o_id); + + return id_value; +} + +} // namespace pystack diff --git a/src/pystack/_pystack/interpreter.h b/src/pystack/_pystack/interpreter.h new file mode 100644 index 00000000..c3be687a --- /dev/null +++ b/src/pystack/_pystack/interpreter.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +#include "mem.h" +#include "process.h" + +namespace pystack { + +class InterpreterUtils +{ + public: + // Static Methods + static remote_addr_t getNextInterpreter( + const std::shared_ptr& manager, + remote_addr_t interpreter_addr); + + static int getInterpreterId( + const std::shared_ptr& manager, + remote_addr_t interpreter_addr); +}; + +} // namespace pystack diff --git a/src/pystack/_pystack/interpreter.pxd b/src/pystack/_pystack/interpreter.pxd new file mode 100644 index 00000000..92d0eb46 --- /dev/null +++ b/src/pystack/_pystack/interpreter.pxd @@ -0,0 +1,12 @@ +from _pystack.mem cimport remote_addr_t +from _pystack.process cimport AbstractProcessManager +from libcpp.memory cimport shared_ptr + + +cdef extern from "interpreter.h" namespace "pystack": + cdef cppclass InterpreterUtils: + @staticmethod + remote_addr_t getNextInterpreter(shared_ptr[AbstractProcessManager] manager, remote_addr_t interpreter_addr) except+ + + @staticmethod + int getInterpreterId(shared_ptr[AbstractProcessManager] manager, remote_addr_t interpreter_addr) except+ diff --git a/src/pystack/_pystack/process.cpp b/src/pystack/_pystack/process.cpp index 56e32bae..761509a0 100644 --- a/src/pystack/_pystack/process.cpp +++ b/src/pystack/_pystack/process.cpp @@ -964,6 +964,7 @@ AbstractProcessManager::copyDebugOffsets(Structure& py_runtime, py set_offset(py_is.o_sysdict, &py_runtime_v::o_dbg_off_interpreter_state_sysdict); set_offset(py_is.o_builtins, &py_runtime_v::o_dbg_off_interpreter_state_builtins); set_offset(py_is.o_gil_runtime_state, &py_runtime_v::o_dbg_off_interpreter_state_ceval_gil); + set_offset(py_is.o_id, &py_runtime_v::o_dbg_off_interpreter_state_id); set_size(py_thread, &py_runtime_v::o_dbg_off_thread_state_struct_size); set_offset(py_thread.o_prev, &py_runtime_v::o_dbg_off_thread_state_prev); diff --git a/src/pystack/_pystack/pythread.cpp b/src/pystack/_pystack/pythread.cpp index d50e4126..5d51aaa7 100644 --- a/src/pystack/_pystack/pythread.cpp +++ b/src/pystack/_pystack/pythread.cpp @@ -2,6 +2,8 @@ #include #include +#include "cpython/pthread.h" +#include "interpreter.h" #include "logging.h" #include "mem.h" #include "native_frame.h" @@ -11,8 +13,6 @@ #include "structure.h" #include "version.h" -#include "cpython/pthread.h" - namespace pystack { Thread::Thread(pid_t pid, pid_t tid) @@ -47,79 +47,88 @@ findPthreadTidOffset( remote_addr_t interp_state_addr) { LOG(DEBUG) << "Attempting to locate tid offset in pthread structure"; - Structure is(manager, interp_state_addr); - auto current_thread_addr = is.getField(&py_is_v::o_tstate_head); + // If interp_state_addr does not point to the main interpreter we won't find the + // PID == TID in the interpreter threads (as it is in the main interpreter). Hence, + // we traverse the linked list of interpreters and pray. + + while (interp_state_addr != 0) { + Structure is(manager, interp_state_addr); + + auto current_thread_addr = is.getField(&py_is_v::o_tstate_head); - auto thread_head = current_thread_addr; + auto thread_head = current_thread_addr; - // Iterate over all Python threads until we find a thread that has a tid equal to - // the process pid. This works because in the main thread the tid is equal to the pid, - // so when this happens it has to happen on the main thread. Note that the main thread - // is not necessarily at the head of the Python thread linked list + // Iterate over all Python threads until we find a thread that has a tid equal to + // the process pid. This works because in the main thread the tid is equal to the pid, + // so when this happens it has to happen on the main thread. Note that the main thread + // is not necessarily at the head of the Python thread linked list #if defined(__GLIBC__) - // If we detect GLIBC, we can try the two main known structs for 'struct - // pthread' that we know about to avoid having to do guess-work by doing a - // linear scan over the struct. - while (current_thread_addr != (remote_addr_t) nullptr) { - Structure current_thread(manager, current_thread_addr); - auto pthread_id_addr = current_thread.getField(&py_thread_v::o_thread_id); - - pid_t the_tid; - std::vector glibc_pthread_offset_candidates = { - offsetof(_pthread_structure_with_simple_header, tid), - offsetof(_pthread_structure_with_tcbhead, tid)}; - for (off_t candidate : glibc_pthread_offset_candidates) { - manager->copyObjectFromProcess((remote_addr_t)(pthread_id_addr + candidate), &the_tid); - if (the_tid == manager->Pid()) { - LOG(DEBUG) << "Tid offset located using GLIBC offsets at offset " << std::showbase - << std::hex << candidate << " in pthread structure"; - return candidate; + // If we detect GLIBC, we can try the two main known structs for 'struct + // pthread' that we know about to avoid having to do guess-work by doing a + // linear scan over the struct. + while (current_thread_addr != (remote_addr_t) nullptr) { + Structure current_thread(manager, current_thread_addr); + auto pthread_id_addr = current_thread.getField(&py_thread_v::o_thread_id); + + pid_t the_tid; + std::vector glibc_pthread_offset_candidates = { + offsetof(_pthread_structure_with_simple_header, tid), + offsetof(_pthread_structure_with_tcbhead, tid)}; + for (off_t candidate : glibc_pthread_offset_candidates) { + manager->copyObjectFromProcess((remote_addr_t)(pthread_id_addr + candidate), &the_tid); + if (the_tid == manager->Pid()) { + LOG(DEBUG) << "Tid offset located using GLIBC offsets at offset " << std::showbase + << std::hex << candidate << " in pthread structure"; + return candidate; + } } + remote_addr_t next_thread_addr = current_thread.getField(&py_thread_v::o_next); + if (next_thread_addr == current_thread_addr) { + break; + } + current_thread_addr = next_thread_addr; } - remote_addr_t next_thread_addr = current_thread.getField(&py_thread_v::o_next); - if (next_thread_addr == current_thread_addr) { - break; - } - current_thread_addr = next_thread_addr; - } #endif - current_thread_addr = thread_head; - - while (current_thread_addr != (remote_addr_t) nullptr) { - Structure current_thread(manager, current_thread_addr); - auto pthread_id_addr = current_thread.getField(&py_thread_v::o_thread_id); - - // Attempt to locate a field in the pthread struct that's equal to the pid. - uintptr_t buffer[100]; - size_t buffer_size = sizeof(buffer); - while (buffer_size > 0) { - try { - LOG(DEBUG) << "Trying to copy a buffer of " << buffer_size << " bytes to get pthread ID"; - manager->copyMemoryFromProcess(pthread_id_addr, buffer_size, &buffer); - break; - } catch (const RemoteMemCopyError& ex) { - LOG(DEBUG) << "Failed to copy buffer to get pthread ID"; - buffer_size /= 2; + current_thread_addr = thread_head; + + while (current_thread_addr != (remote_addr_t) nullptr) { + Structure current_thread(manager, current_thread_addr); + auto pthread_id_addr = current_thread.getField(&py_thread_v::o_thread_id); + + // Attempt to locate a field in the pthread struct that's equal to the pid. + uintptr_t buffer[100]; + size_t buffer_size = sizeof(buffer); + while (buffer_size > 0) { + try { + LOG(DEBUG) << "Trying to copy a buffer of " << buffer_size + << " bytes to get pthread ID"; + manager->copyMemoryFromProcess(pthread_id_addr, buffer_size, &buffer); + break; + } catch (const RemoteMemCopyError& ex) { + LOG(DEBUG) << "Failed to copy buffer to get pthread ID"; + buffer_size /= 2; + } } - } - LOG(DEBUG) << "Copied a buffer of " << buffer_size << " bytes to get pthread ID"; - for (size_t i = 0; i < buffer_size / sizeof(uintptr_t); i++) { - if (static_cast(buffer[i]) == manager->Pid()) { - off_t offset = sizeof(uintptr_t) * i; - LOG(DEBUG) << "Tid offset located by scanning at offset " << std::showbase << std::hex - << offset << " in pthread structure"; - return offset; + LOG(DEBUG) << "Copied a buffer of " << buffer_size << " bytes to get pthread ID"; + for (size_t i = 0; i < buffer_size / sizeof(uintptr_t); i++) { + if (static_cast(buffer[i]) == manager->Pid()) { + off_t offset = sizeof(uintptr_t) * i; + LOG(DEBUG) << "Tid offset located by scanning at offset " << std::showbase + << std::hex << offset << " in pthread structure"; + return offset; + } } - } - remote_addr_t next_thread_addr = current_thread.getField(&py_thread_v::o_next); - if (next_thread_addr == current_thread_addr) { - break; + remote_addr_t next_thread_addr = current_thread.getField(&py_thread_v::o_next); + if (next_thread_addr == current_thread_addr) { + break; + } + current_thread_addr = next_thread_addr; } - current_thread_addr = next_thread_addr; + interp_state_addr = InterpreterUtils::getNextInterpreter(manager, interp_state_addr); } LOG(ERROR) << "Could not find tid offset in pthread structure"; return 0; diff --git a/src/pystack/_pystack/version.cpp b/src/pystack/_pystack/version.cpp index f58ff878..a31ad1fc 100644 --- a/src/pystack/_pystack/version.cpp +++ b/src/pystack/_pystack/version.cpp @@ -179,6 +179,23 @@ py_is() }; } +template +constexpr py_is_v +py_isv7() +{ + return { + sizeof(T), + {offsetof(T, next)}, + {offsetof(T, tstate_head)}, + {offsetof(T, gc)}, + {offsetof(T, modules)}, + {offsetof(T, sysdict)}, + {offsetof(T, builtins)}, + {0}, + {offsetof(T, id)}, + }; +} + template constexpr py_is_v py_isv311() @@ -191,6 +208,8 @@ py_isv311() {offsetof(T, modules)}, {offsetof(T, sysdict)}, {offsetof(T, builtins)}, + {0}, + {offsetof(T, id)}, }; } @@ -207,6 +226,24 @@ py_isv312() {offsetof(T, sysdict)}, {offsetof(T, builtins)}, {offsetof(T, ceval.gil)}, + {offsetof(T, id)}, + }; +} + +template +constexpr py_is_v +py_isv314() +{ + return { + sizeof(T), + {offsetof(T, next)}, + {offsetof(T, threads.head)}, + {offsetof(T, gc)}, + {offsetof(T, imports.modules)}, + {offsetof(T, sysdict)}, + {offsetof(T, builtins)}, + {offsetof(T, _gil)}, + {offsetof(T, id)}, }; } @@ -578,7 +615,7 @@ python_v python_v3_7 = { py_code(), py_frame(), py_thread(), - py_is(), + py_isv7(), py_runtime(), py_gc(), }; @@ -600,7 +637,7 @@ python_v python_v3_8 = { py_code(), py_frame(), py_thread(), - py_is(), + py_isv7(), py_runtime(), py_gc(), }; @@ -622,7 +659,7 @@ python_v python_v3_9 = { py_code(), py_frame(), py_thread(), - py_is(), + py_isv7(), py_runtime(), py_gc(), }; @@ -644,7 +681,7 @@ python_v python_v3_10 = { py_code(), py_frame(), py_thread(), - py_is(), + py_isv7(), py_runtime(), py_gc(), }; @@ -737,7 +774,7 @@ python_v python_v3_14 = { py_codev311(), py_framev314(), py_threadv313(), - py_isv312(), + py_isv314(), py_runtimev313(), py_gc(), py_cframe(), diff --git a/src/pystack/_pystack/version.h b/src/pystack/_pystack/version.h index c56851ac..4f7d4992 100644 --- a/src/pystack/_pystack/version.h +++ b/src/pystack/_pystack/version.h @@ -236,11 +236,12 @@ struct py_is_v FieldOffset o_next; FieldOffset o_tstate_head; FieldOffset o_gc; // Using char because we can only use the offset, - // as the size and members change between versions + // as the size and members change between versions FieldOffset o_modules; FieldOffset o_sysdict; FieldOffset o_builtins; FieldOffset o_gil_runtime_state; + FieldOffset o_id; }; struct py_gc_v diff --git a/src/pystack/traceback_formatter.py b/src/pystack/traceback_formatter.py index 35637b97..6318366d 100644 --- a/src/pystack/traceback_formatter.py +++ b/src/pystack/traceback_formatter.py @@ -12,9 +12,36 @@ from .types import frame_type -def print_thread(thread: PyThread, native_mode: NativeReportingMode) -> None: - for line in format_thread(thread, native_mode): - print(line, file=sys.stdout, flush=True) +class TracebackPrinter: + def __init__( + self, native_mode: NativeReportingMode, include_subinterpreters: bool = False + ): + self.native_mode = native_mode + self.include_subinterpreters = include_subinterpreters + self._current_interp_id = -1 + + def print_thread(self, thread: PyThread) -> None: + # Print interpreter header if we've switched interpreters + if self.include_subinterpreters: + if thread.interp_id != self._current_interp_id: + self._print_interpreter_header(thread.interp_id) + self._current_interp_id = thread.interp_id + + # Print the thread with indentation + for line in format_thread(thread, self.native_mode): + if self.include_subinterpreters: + print(" " * 2, end="") + print(line, file=sys.stdout, flush=True) + + def _print_interpreter_header(self, interp_id: Optional[int]) -> None: + header = "Interpreter-" + if interp_id is not None: + header += str(interp_id) + else: + header += "Unknown" + if interp_id == 0: + header += " (main)" + print(header, file=sys.stdout, flush=True) def format_frame(frame: PyFrame) -> Iterable[str]: diff --git a/src/pystack/types.py b/src/pystack/types.py index b58106dd..bb8381be 100644 --- a/src/pystack/types.py +++ b/src/pystack/types.py @@ -108,6 +108,7 @@ class PyThread: holds_the_gil: int is_gc_collecting: int python_version: Optional[Tuple[int, int]] + interp_id: Optional[int] = None name: Optional[str] = None @property diff --git a/tests/integration/test_subinterpreters.py b/tests/integration/test_subinterpreters.py new file mode 100644 index 00000000..b93348f2 --- /dev/null +++ b/tests/integration/test_subinterpreters.py @@ -0,0 +1,91 @@ +import io +from contextlib import redirect_stdout +from pathlib import Path + +from pystack.engine import NativeReportingMode +from pystack.engine import get_process_threads +from pystack.traceback_formatter import TracebackPrinter +from tests.utils import ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +from tests.utils import spawn_child_process + +NUM_INTERPRETERS = 3 + +PROGRAM = f"""\ +import sys +import threading +import time + +from concurrent import interpreters + +NUM_INTERPRETERS = {NUM_INTERPRETERS} + + +def start_interpreter_async(interp, code): + t = threading.Thread(target=interp.exec, args=(code,)) + t.daemon = True + t.start() + return t + + +CODE = '''\\ +import time +while True: + time.sleep(1) +''' + +threads = [] +for i in range(NUM_INTERPRETERS): + interp = interpreters.create() + t = start_interpreter_async(interp, CODE) + threads.append(t) + +# Give sub-interpreters time to start executing +time.sleep(1) + +fifo = sys.argv[1] +with open(fifo, "w") as f: + f.write("ready") + +while True: + time.sleep(1) +""" + + +@ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +def test_subinterpreters(python, tmpdir): + """Test that pystack can detect and report multiple sub-interpreters.""" + + # GIVEN + _, python_executable = python + test_file = Path(str(tmpdir)) / "subinterpreters_program.py" + test_file.write_text(PROGRAM) + + # WHEN + with spawn_child_process(python_executable, test_file, tmpdir) as child_process: + threads = list(get_process_threads(child_process.pid, stop_process=True)) + + # Collect all interpreter IDs from the threads + interp_ids = {thread.interp_id for thread in threads} + + # THEN + + # We expect the main interpreter (0) plus NUM_INTERPRETERS sub-interpreters + assert 0 in interp_ids + assert len(interp_ids) == NUM_INTERPRETERS + 1 + + # Verify the TracebackPrinter output contains the interpreter headers + printer = TracebackPrinter( + native_mode=NativeReportingMode.OFF, + include_subinterpreters=True, + ) + output = io.StringIO() + with redirect_stdout(output): + for thread in threads: + printer.print_thread(thread) + + result = output.getvalue() + assert "Interpreter-0 (main)" in result + for interp_id in interp_ids: + if interp_id == 0: + continue + assert f"Interpreter-{interp_id}" in result diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 2401d554..2aef5a21 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -191,8 +191,8 @@ def test_process_remote_default(): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ): get_process_threads_mock.return_value = threads @@ -207,8 +207,9 @@ def test_process_remote_default(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with(native_mode=NativeReportingMode.OFF) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -224,8 +225,8 @@ def test_process_remote_no_block(): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ): get_process_threads_mock.return_value = threads @@ -240,8 +241,9 @@ def test_process_remote_no_block(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with(native_mode=NativeReportingMode.OFF) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -265,8 +267,8 @@ def test_process_remote_native(argument, mode): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ): get_process_threads_mock.return_value = threads @@ -281,7 +283,10 @@ def test_process_remote_native(argument, mode): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, mode) for thread in threads] + TracebackPrinterMock.assert_called_once_with(native_mode=mode) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads + ] def test_process_remote_locals(): @@ -296,8 +301,8 @@ def test_process_remote_locals(): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ): get_process_threads_mock.return_value = threads @@ -312,8 +317,9 @@ def test_process_remote_locals(): locals=True, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with(native_mode=NativeReportingMode.OFF) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -329,8 +335,8 @@ def test_process_remote_native_no_block(capsys): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ): get_process_threads_mock.return_value = threads @@ -340,7 +346,8 @@ def test_process_remote_native_no_block(capsys): main() get_process_threads_mock.assert_not_called() - print_thread_mock.assert_not_called() + TracebackPrinterMock.assert_not_called() + TracebackPrinterMock.return_value.print_thread.assert_not_called() def test_process_remote_exhaustive(): @@ -355,8 +362,8 @@ def test_process_remote_exhaustive(): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ): get_process_threads_mock.return_value = threads @@ -371,8 +378,9 @@ def test_process_remote_exhaustive(): locals=False, method=StackMethod.ALL, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with(native_mode=NativeReportingMode.OFF) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -389,8 +397,8 @@ def test_process_remote_error(exception, exval, capsys): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -403,7 +411,8 @@ def test_process_remote_error(exception, exval, capsys): # THEN get_process_threads_mock.assert_called_once() - print_thread_mock.assert_not_called() + TracebackPrinterMock.assert_called_once_with(native_mode=NativeReportingMode.OFF) + TracebackPrinterMock.return_value.print_thread.assert_not_called() capture = capsys.readouterr() assert "Oh no!" in capture.err @@ -420,8 +429,8 @@ def test_process_core_default_without_executable(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -448,8 +457,9 @@ def test_process_core_default_without_executable(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -471,8 +481,8 @@ def test_process_core_default_gzip_without_executable(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -504,8 +514,9 @@ def test_process_core_default_gzip_without_executable(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] gzip_open_mock.assert_called_with(Path("corefile.gz"), "rb") @@ -575,8 +586,8 @@ def test_process_core_default_with_executable(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -600,8 +611,9 @@ def test_process_core_default_with_executable(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -625,8 +637,8 @@ def test_process_core_native(argument, mode): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -650,7 +662,10 @@ def test_process_core_native(argument, mode): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, mode) for thread in threads] + TracebackPrinterMock.assert_called_once_with(mode) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads + ] def test_process_core_locals(): @@ -665,8 +680,8 @@ def test_process_core_locals(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -690,8 +705,9 @@ def test_process_core_locals(): locals=True, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -714,8 +730,8 @@ def test_process_core_with_search_path(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -739,8 +755,9 @@ def test_process_core_with_search_path(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -756,8 +773,8 @@ def test_process_core_with_search_root(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -789,8 +806,9 @@ def test_process_core_with_search_root(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -802,7 +820,7 @@ def test_process_core_with_not_readable_search_root(): # WHEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("pathlib.Path.exists", return_value=True), patch( "pystack.__main__.CoreFileAnalyzer" ), patch( @@ -826,7 +844,7 @@ def test_process_core_with_invalid_search_root(): # WHEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("pathlib.Path.exists", return_value=True), patch( "pystack.__main__.CoreFileAnalyzer" ), patch( @@ -851,8 +869,8 @@ def path_exists(what): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch.object( Path, "exists", path_exists @@ -865,7 +883,8 @@ def path_exists(what): # THEN get_process_threads_mock.assert_not_called() - print_thread_mock.assert_not_called() + TracebackPrinterMock.assert_not_called() + TracebackPrinterMock.return_value.print_thread.assert_not_called() def test_process_core_executable_does_not_exit(): @@ -883,8 +902,8 @@ def does_exit(what): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "pystack.__main__.is_gzip", return_value=False ), patch( "sys.argv", argv @@ -898,7 +917,8 @@ def does_exit(what): # THEN get_process_threads_mock.assert_not_called() - print_thread_mock.assert_not_called() + TracebackPrinterMock.assert_not_called() + TracebackPrinterMock.return_value.print_thread.assert_not_called() @pytest.mark.parametrize( @@ -914,8 +934,8 @@ def test_process_core_error(exception, exval, capsys): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -935,7 +955,8 @@ def test_process_core_error(exception, exval, capsys): # THEN get_process_threads_mock.assert_called_once() - print_thread_mock.assert_not_called() + TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + TracebackPrinterMock.return_value.print_thread.assert_not_called() capture = capsys.readouterr() assert "Oh no!" in capture.err @@ -951,8 +972,8 @@ def test_process_core_exhaustive(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -976,8 +997,9 @@ def test_process_core_exhaustive(): locals=False, method=StackMethod.ALL, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -990,7 +1012,7 @@ def test_default_colored_output(): # WHEN with patch("pystack.__main__.get_process_threads"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("os.environ", environ): main() @@ -1008,7 +1030,7 @@ def test_nocolor_output(): # WHEN with patch("pystack.__main__.get_process_threads"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("os.environ", environ): main() @@ -1026,7 +1048,7 @@ def test_nocolor_output_at_the_front_for_process(): # WHEN with patch("pystack.__main__.get_process_threads"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("os.environ", environ): main() @@ -1043,7 +1065,7 @@ def test_nocolor_output_at_the_front_for_core(): # WHEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("os.environ", environ), patch( "pathlib.Path.exists", return_value=True ), patch( @@ -1069,7 +1091,7 @@ def test_global_options_can_be_placed_at_any_point(option): # WHEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("os.environ", environ), patch( "pathlib.Path.exists", return_value=True ), patch( @@ -1092,7 +1114,7 @@ def test_verbose_as_global_options_sets_correctly_the_logger(): # WHEN with patch("pystack.__main__.get_process_threads"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("os.environ", environ), patch( "pathlib.Path.exists", return_value=True ), patch( @@ -1241,7 +1263,7 @@ def test_process_core_does_not_crash_if_core_analyzer_fails(method): # WHEN / THEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("pystack.__main__.is_elf", return_value=True), patch( "pystack.__main__.is_gzip", return_value=False ), patch( @@ -1268,7 +1290,7 @@ def test_core_file_missing_modules_are_logged(caplog, native): # WHEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("pystack.__main__.is_elf", return_value=True), patch( "pystack.__main__.is_gzip", return_value=False ), patch( @@ -1301,7 +1323,7 @@ def test_core_file_missing_build_ids_are_logged(caplog, native): # WHEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("pystack.__main__.is_elf", return_value=True), patch( "pystack.__main__.is_gzip", return_value=False ), patch( @@ -1342,7 +1364,7 @@ def test_executable_is_not_elf_uses_the_first_map(): with patch( "pystack.__main__.get_process_threads_for_core" - ) as get_process_threads_mock, patch("pystack.__main__.print_thread"), patch( + ) as get_process_threads_mock, patch("pystack.__main__.TracebackPrinter"), patch( "pystack.__main__.is_elf", lambda x: x == real_executable ), patch( "pystack.__main__.is_gzip", return_value=False diff --git a/tests/unit/test_traceback_formatter.py b/tests/unit/test_traceback_formatter.py index 636cfc6b..3e304808 100644 --- a/tests/unit/test_traceback_formatter.py +++ b/tests/unit/test_traceback_formatter.py @@ -4,8 +4,8 @@ import pytest from pystack.engine import NativeReportingMode +from pystack.traceback_formatter import TracebackPrinter from pystack.traceback_formatter import format_thread -from pystack.traceback_formatter import print_thread from pystack.types import SYMBOL_IGNORELIST from pystack.types import LocationInfo from pystack.types import NativeFrame @@ -1205,6 +1205,7 @@ def test_traceback_formatter_native_last(): def test_print_thread(capsys): + printer = TracebackPrinter(NativeReportingMode.OFF) # GIVEN thread = PyThread( tid=1, @@ -1220,7 +1221,9 @@ def test_print_thread(capsys): "pystack.traceback_formatter.format_thread", return_value=("1", "2", "3"), ): - print_thread(thread, NativeReportingMode.OFF) + printer.print_thread( + thread, + ) # THEN @@ -1629,3 +1632,234 @@ def test_native_traceback_with_shim_frames(): colored_mock.assert_any_call("x =", color="blue") colored_mock.assert_any_call('"This is the line 2" ', color="blue") colored_mock.assert_any_call("(1+1)", color="blue") + + +@pytest.mark.parametrize( + "native_mode", + [ + NativeReportingMode.OFF, + NativeReportingMode.ALL, + NativeReportingMode.PYTHON, + NativeReportingMode.LAST, + ], +) +def test_traceback_printer_created_with_native_level(native_mode): + # GIVEN / WHEN + printer = TracebackPrinter(native_mode) + + # THEN + assert printer.native_mode is native_mode + assert printer.include_subinterpreters is False + assert printer._current_interp_id == -1 + + +def test_traceback_printer_created_with_subinterpreters(): + # GIVEN / WHEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + + # THEN + assert printer.native_mode is NativeReportingMode.OFF + assert printer.include_subinterpreters is True + + +def test_print_thread_passes_native_mode_to_format_thread(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.ALL) + thread = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1", "line2"), + ) as format_mock: + printer.print_thread(thread) + + # THEN + format_mock.assert_called_once_with(thread, NativeReportingMode.ALL) + captured = capsys.readouterr() + assert captured.out == "line1\nline2\n" + + +def test_print_thread_with_subinterpreters(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + thread = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interp_id=0, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1", "line2"), + ): + printer.print_thread(thread) + + # THEN + captured = capsys.readouterr() + assert "Interpreter-Unknown (main)" in captured.out + # Lines should be indented with 2 spaces + assert " line1\n" in captured.out + assert " line2\n" in captured.out + + +def test_print_thread_with_subinterpreters_nonzero_interp(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + thread = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interp_id=2, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1",), + ): + printer.print_thread(thread) + + # THEN + captured = capsys.readouterr() + assert "Interpreter-2\n" in captured.out + assert " line1\n" in captured.out + + +def test_print_thread_with_subinterpreters_none_interp(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + thread = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interp_id=None, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1",), + ): + printer.print_thread(thread) + + # THEN + captured = capsys.readouterr() + assert "Interpreter-Unknown\n" in captured.out + + +def test_print_thread_with_subinterpreters_same_interp_no_repeat_header(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + thread1 = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interp_id=1, + ) + thread2 = PyThread( + tid=2, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interp_id=1, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1",), + ): + printer.print_thread(thread1) + printer.print_thread(thread2) + + # THEN + captured = capsys.readouterr() + # Header should appear only once + assert captured.out.count("Interpreter-1") == 1 + + +def test_print_thread_with_subinterpreters_different_interps_prints_headers(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + thread1 = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interp_id=1, + ) + thread2 = PyThread( + tid=2, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interp_id=2, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1",), + ): + printer.print_thread(thread1) + printer.print_thread(thread2) + + # THEN + captured = capsys.readouterr() + assert "Interpreter-1\n" in captured.out + assert "Interpreter-2\n" in captured.out + + +def test_print_thread_without_subinterpreters_no_indentation(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=False) + thread = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interp_id=1, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1", "line2"), + ): + printer.print_thread(thread) + + # THEN + captured = capsys.readouterr() + # No interpreter header and no indentation + assert "Interpreter" not in captured.out + assert captured.out == "line1\nline2\n" diff --git a/tests/utils.py b/tests/utils.py index dacb18fa..dcf1f639 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -307,6 +307,13 @@ def all_pystack_combinations(corefile=False, native=False): ) +ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS = pytest.mark.parametrize( + "python", + [python[:2] for python in AVAILABLE_PYTHONS if python.version >= (3, 14)], + ids=[python[1].name for python in AVAILABLE_PYTHONS if python.version >= (3, 14)], +) + + ALL_PYTHONS_THAT_DO_NOT_SUPPORT_ELF_DATA = pytest.mark.parametrize( "python", [python[:2] for python in AVAILABLE_PYTHONS if python.version < (3, 10)], From 2d5e5829c4303c14a4485c65c85a0864147ec8d1 Mon Sep 17 00:00:00 2001 From: Saul Cooperman Date: Sun, 22 Feb 2026 19:44:53 +0000 Subject: [PATCH 2/6] Tidy --- src/pystack/__main__.py | 4 ++-- src/pystack/_pystack.pyx | 5 ++--- src/pystack/_pystack/pythread.cpp | 6 +++--- src/pystack/_pystack/version.h | 2 +- src/pystack/traceback_formatter.py | 20 +++++++++----------- src/pystack/types.py | 2 +- tests/integration/test_subinterpreters.py | 12 ++++++------ tests/unit/test_traceback_formatter.py | 18 +++++++++--------- 8 files changed, 33 insertions(+), 36 deletions(-) diff --git a/src/pystack/__main__.py b/src/pystack/__main__.py index c206e08a..5744d7f7 100644 --- a/src/pystack/__main__.py +++ b/src/pystack/__main__.py @@ -286,7 +286,7 @@ def process_remote(parser: argparse.ArgumentParser, args: argparse.Namespace) -> parser.error("Native traces are only available in blocking mode") printer = TracebackPrinter( - native_mode=args.native_mode, + native_mode=args.native_mode, include_subinterpreters=True ) for thread in get_process_threads( args.pid, @@ -417,7 +417,7 @@ def process_core(parser: argparse.ArgumentParser, args: argparse.Namespace) -> N elf_id if elf_id else "", ) - printer = TracebackPrinter(args.native_mode) + printer = TracebackPrinter(args.native_mode, include_subinterpreters=True) for thread in get_process_threads_for_core( corefile, executable, diff --git a/src/pystack/_pystack.pyx b/src/pystack/_pystack.pyx index 80f6bb46..91b9516e 100644 --- a/src/pystack/_pystack.pyx +++ b/src/pystack/_pystack.pyx @@ -463,7 +463,7 @@ cdef object _construct_threads_from_interpreter_state( bint add_native_traces, bint resolve_locals, ): - interp_id = InterpreterUtils.getInterpreterId(manager, head) + interpreter_id = InterpreterUtils.getInterpreterId(manager, head) LOGGER.info("Fetching Python threads") threads = [] @@ -488,7 +488,7 @@ cdef object _construct_threads_from_interpreter_state( current_thread.isGilHolder(), current_thread.isGCCollecting(), python_version, - interp_id, + interpreter_id, name=get_thread_name(pid, current_thread.Tid()), ) ) @@ -766,7 +766,6 @@ def _get_process_threads_for_core( manager.get(), method, core=True ) - if not head and native_mode != NativeReportingMode.ALL: raise NotEnoughInformation( "Could not gather enough information to extract the Python frame information" diff --git a/src/pystack/_pystack/pythread.cpp b/src/pystack/_pystack/pythread.cpp index 5d51aaa7..b6b90a24 100644 --- a/src/pystack/_pystack/pythread.cpp +++ b/src/pystack/_pystack/pythread.cpp @@ -48,9 +48,9 @@ findPthreadTidOffset( { LOG(DEBUG) << "Attempting to locate tid offset in pthread structure"; - // If interp_state_addr does not point to the main interpreter we won't find the - // PID == TID in the interpreter threads (as it is in the main interpreter). Hence, - // we traverse the linked list of interpreters and pray. + // If interp_state_addr does not point to the main interpreter (id 0) we won't find the + // PID == TID in the interpreter threads. Hence, we traverse the linked list of interpreters. The + // main interpreter is not necessarily the head of the linked lists of interpreters. while (interp_state_addr != 0) { Structure is(manager, interp_state_addr); diff --git a/src/pystack/_pystack/version.h b/src/pystack/_pystack/version.h index 4f7d4992..d9b2b2de 100644 --- a/src/pystack/_pystack/version.h +++ b/src/pystack/_pystack/version.h @@ -236,7 +236,7 @@ struct py_is_v FieldOffset o_next; FieldOffset o_tstate_head; FieldOffset o_gc; // Using char because we can only use the offset, - // as the size and members change between versions + // as the size and members change between versions FieldOffset o_modules; FieldOffset o_sysdict; FieldOffset o_builtins; diff --git a/src/pystack/traceback_formatter.py b/src/pystack/traceback_formatter.py index 6318366d..56f7921f 100644 --- a/src/pystack/traceback_formatter.py +++ b/src/pystack/traceback_formatter.py @@ -18,14 +18,14 @@ def __init__( ): self.native_mode = native_mode self.include_subinterpreters = include_subinterpreters - self._current_interp_id = -1 + self._current_interpreter_id = -1 def print_thread(self, thread: PyThread) -> None: # Print interpreter header if we've switched interpreters if self.include_subinterpreters: - if thread.interp_id != self._current_interp_id: - self._print_interpreter_header(thread.interp_id) - self._current_interp_id = thread.interp_id + if thread.interpreter_id != self._current_interpreter_id: + self._print_interpreter_header(thread.interpreter_id) + self._current_interpreter_id = thread.interpreter_id # Print the thread with indentation for line in format_thread(thread, self.native_mode): @@ -33,13 +33,11 @@ def print_thread(self, thread: PyThread) -> None: print(" " * 2, end="") print(line, file=sys.stdout, flush=True) - def _print_interpreter_header(self, interp_id: Optional[int]) -> None: - header = "Interpreter-" - if interp_id is not None: - header += str(interp_id) - else: - header += "Unknown" - if interp_id == 0: + def _print_interpreter_header(self, interpreter_id: Optional[int]) -> None: + header = ( + f"Interpreter-{interpreter_id if interpreter_id is not None else 'Unknown'}" + ) + if interpreter_id == 0: header += " (main)" print(header, file=sys.stdout, flush=True) diff --git a/src/pystack/types.py b/src/pystack/types.py index bb8381be..b2b445cd 100644 --- a/src/pystack/types.py +++ b/src/pystack/types.py @@ -108,7 +108,7 @@ class PyThread: holds_the_gil: int is_gc_collecting: int python_version: Optional[Tuple[int, int]] - interp_id: Optional[int] = None + interpreter_id: Optional[int] = None name: Optional[str] = None @property diff --git a/tests/integration/test_subinterpreters.py b/tests/integration/test_subinterpreters.py index b93348f2..56ee074c 100644 --- a/tests/integration/test_subinterpreters.py +++ b/tests/integration/test_subinterpreters.py @@ -65,13 +65,13 @@ def test_subinterpreters(python, tmpdir): threads = list(get_process_threads(child_process.pid, stop_process=True)) # Collect all interpreter IDs from the threads - interp_ids = {thread.interp_id for thread in threads} + interpreter_ids = {thread.interpreter_id for thread in threads} # THEN # We expect the main interpreter (0) plus NUM_INTERPRETERS sub-interpreters - assert 0 in interp_ids - assert len(interp_ids) == NUM_INTERPRETERS + 1 + assert 0 in interpreter_ids + assert len(interpreter_ids) == NUM_INTERPRETERS + 1 # Verify the TracebackPrinter output contains the interpreter headers printer = TracebackPrinter( @@ -85,7 +85,7 @@ def test_subinterpreters(python, tmpdir): result = output.getvalue() assert "Interpreter-0 (main)" in result - for interp_id in interp_ids: - if interp_id == 0: + for interpreter_id in interpreter_ids: + if interpreter_id == 0: continue - assert f"Interpreter-{interp_id}" in result + assert f"Interpreter-{interpreter_id}" in result diff --git a/tests/unit/test_traceback_formatter.py b/tests/unit/test_traceback_formatter.py index 3e304808..2f592b05 100644 --- a/tests/unit/test_traceback_formatter.py +++ b/tests/unit/test_traceback_formatter.py @@ -1650,7 +1650,7 @@ def test_traceback_printer_created_with_native_level(native_mode): # THEN assert printer.native_mode is native_mode assert printer.include_subinterpreters is False - assert printer._current_interp_id == -1 + assert printer._current_interpreter_id == -1 def test_traceback_printer_created_with_subinterpreters(): @@ -1697,7 +1697,7 @@ def test_print_thread_with_subinterpreters(capsys): holds_the_gil=False, is_gc_collecting=False, python_version=(3, 8), - interp_id=0, + interpreter_id=0, ) # WHEN @@ -1725,7 +1725,7 @@ def test_print_thread_with_subinterpreters_nonzero_interp(capsys): holds_the_gil=False, is_gc_collecting=False, python_version=(3, 8), - interp_id=2, + interpreter_id=2, ) # WHEN @@ -1751,7 +1751,7 @@ def test_print_thread_with_subinterpreters_none_interp(capsys): holds_the_gil=False, is_gc_collecting=False, python_version=(3, 8), - interp_id=None, + interpreter_id=None, ) # WHEN @@ -1776,7 +1776,7 @@ def test_print_thread_with_subinterpreters_same_interp_no_repeat_header(capsys): holds_the_gil=False, is_gc_collecting=False, python_version=(3, 8), - interp_id=1, + interpreter_id=1, ) thread2 = PyThread( tid=2, @@ -1785,7 +1785,7 @@ def test_print_thread_with_subinterpreters_same_interp_no_repeat_header(capsys): holds_the_gil=False, is_gc_collecting=False, python_version=(3, 8), - interp_id=1, + interpreter_id=1, ) # WHEN @@ -1812,7 +1812,7 @@ def test_print_thread_with_subinterpreters_different_interps_prints_headers(caps holds_the_gil=False, is_gc_collecting=False, python_version=(3, 8), - interp_id=1, + interpreter_id=1, ) thread2 = PyThread( tid=2, @@ -1821,7 +1821,7 @@ def test_print_thread_with_subinterpreters_different_interps_prints_headers(caps holds_the_gil=False, is_gc_collecting=False, python_version=(3, 8), - interp_id=2, + interpreter_id=2, ) # WHEN @@ -1848,7 +1848,7 @@ def test_print_thread_without_subinterpreters_no_indentation(capsys): holds_the_gil=False, is_gc_collecting=False, python_version=(3, 8), - interp_id=1, + interpreter_id=1, ) # WHEN From 4434816e5bded386121222696f66ba9540865337 Mon Sep 17 00:00:00 2001 From: Saul Cooperman Date: Sun, 22 Feb 2026 19:57:17 +0000 Subject: [PATCH 3/6] Tidy --- src/pystack/_pystack/interpreter.cpp | 2 +- src/pystack/_pystack/interpreter.h | 2 +- src/pystack/_pystack/interpreter.pxd | 5 +++-- tests/unit/test_traceback_formatter.py | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/pystack/_pystack/interpreter.cpp b/src/pystack/_pystack/interpreter.cpp index 850d9f44..4f52e043 100644 --- a/src/pystack/_pystack/interpreter.cpp +++ b/src/pystack/_pystack/interpreter.cpp @@ -17,7 +17,7 @@ InterpreterUtils::getNextInterpreter( return is.getField(&py_is_v::o_next); } -int +int64_t InterpreterUtils::getInterpreterId( const std::shared_ptr& manager, remote_addr_t interpreter_addr) diff --git a/src/pystack/_pystack/interpreter.h b/src/pystack/_pystack/interpreter.h index c3be687a..0138ff84 100644 --- a/src/pystack/_pystack/interpreter.h +++ b/src/pystack/_pystack/interpreter.h @@ -16,7 +16,7 @@ class InterpreterUtils const std::shared_ptr& manager, remote_addr_t interpreter_addr); - static int getInterpreterId( + static int64_t getInterpreterId( const std::shared_ptr& manager, remote_addr_t interpreter_addr); }; diff --git a/src/pystack/_pystack/interpreter.pxd b/src/pystack/_pystack/interpreter.pxd index 92d0eb46..0248f468 100644 --- a/src/pystack/_pystack/interpreter.pxd +++ b/src/pystack/_pystack/interpreter.pxd @@ -1,12 +1,13 @@ from _pystack.mem cimport remote_addr_t from _pystack.process cimport AbstractProcessManager +from libc.stdint cimport int64_t from libcpp.memory cimport shared_ptr cdef extern from "interpreter.h" namespace "pystack": cdef cppclass InterpreterUtils: @staticmethod - remote_addr_t getNextInterpreter(shared_ptr[AbstractProcessManager] manager, remote_addr_t interpreter_addr) except+ + remote_addr_t getNextInterpreter(shared_ptr[AbstractProcessManager] manager, remote_addr_t interpreter_addr) except + @staticmethod - int getInterpreterId(shared_ptr[AbstractProcessManager] manager, remote_addr_t interpreter_addr) except+ + int64_t getInterpreterId(shared_ptr[AbstractProcessManager] manager, remote_addr_t interpreter_addr) except + diff --git a/tests/unit/test_traceback_formatter.py b/tests/unit/test_traceback_formatter.py index 2f592b05..c2a230b6 100644 --- a/tests/unit/test_traceback_formatter.py +++ b/tests/unit/test_traceback_formatter.py @@ -1709,7 +1709,7 @@ def test_print_thread_with_subinterpreters(capsys): # THEN captured = capsys.readouterr() - assert "Interpreter-Unknown (main)" in captured.out + assert "Interpreter-0 (main)" in captured.out # Lines should be indented with 2 spaces assert " line1\n" in captured.out assert " line2\n" in captured.out From 7460c907744712f85bcab5cf5b4e5e77757cf453 Mon Sep 17 00:00:00 2001 From: Saul Cooperman Date: Tue, 24 Feb 2026 21:50:14 +0000 Subject: [PATCH 4/6] Fix assert_called_with --- tests/unit/test_main.py | 58 ++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 2aef5a21..6f591dd9 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -207,7 +207,9 @@ def test_process_remote_default(): locals=False, method=StackMethod.AUTO, ) - TracebackPrinterMock.assert_called_once_with(native_mode=NativeReportingMode.OFF) + TracebackPrinterMock.assert_called_once_with( + native_mode=NativeReportingMode.OFF, include_subinterpreters=True + ) assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ call(thread) for thread in threads ] @@ -241,7 +243,9 @@ def test_process_remote_no_block(): locals=False, method=StackMethod.AUTO, ) - TracebackPrinterMock.assert_called_once_with(native_mode=NativeReportingMode.OFF) + TracebackPrinterMock.assert_called_once_with( + native_mode=NativeReportingMode.OFF, include_subinterpreters=True + ) assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ call(thread) for thread in threads ] @@ -283,7 +287,9 @@ def test_process_remote_native(argument, mode): locals=False, method=StackMethod.AUTO, ) - TracebackPrinterMock.assert_called_once_with(native_mode=mode) + TracebackPrinterMock.assert_called_once_with( + native_mode=mode, include_subinterpreters=True + ) assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ call(thread) for thread in threads ] @@ -317,7 +323,9 @@ def test_process_remote_locals(): locals=True, method=StackMethod.AUTO, ) - TracebackPrinterMock.assert_called_once_with(native_mode=NativeReportingMode.OFF) + TracebackPrinterMock.assert_called_once_with( + native_mode=NativeReportingMode.OFF, include_subinterpreters=True + ) assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ call(thread) for thread in threads ] @@ -378,7 +386,9 @@ def test_process_remote_exhaustive(): locals=False, method=StackMethod.ALL, ) - TracebackPrinterMock.assert_called_once_with(native_mode=NativeReportingMode.OFF) + TracebackPrinterMock.assert_called_once_with( + native_mode=NativeReportingMode.OFF, include_subinterpreters=True + ) assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ call(thread) for thread in threads ] @@ -411,7 +421,9 @@ def test_process_remote_error(exception, exval, capsys): # THEN get_process_threads_mock.assert_called_once() - TracebackPrinterMock.assert_called_once_with(native_mode=NativeReportingMode.OFF) + TracebackPrinterMock.assert_called_once_with( + native_mode=NativeReportingMode.OFF, include_subinterpreters=True + ) TracebackPrinterMock.return_value.print_thread.assert_not_called() capture = capsys.readouterr() assert "Oh no!" in capture.err @@ -457,7 +469,9 @@ def test_process_core_default_without_executable(): locals=False, method=StackMethod.AUTO, ) - TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ call(thread) for thread in threads ] @@ -514,7 +528,9 @@ def test_process_core_default_gzip_without_executable(): locals=False, method=StackMethod.AUTO, ) - TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ call(thread) for thread in threads ] @@ -611,7 +627,9 @@ def test_process_core_default_with_executable(): locals=False, method=StackMethod.AUTO, ) - TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ call(thread) for thread in threads ] @@ -662,7 +680,7 @@ def test_process_core_native(argument, mode): locals=False, method=StackMethod.AUTO, ) - TracebackPrinterMock.assert_called_once_with(mode) + TracebackPrinterMock.assert_called_once_with(mode, include_subinterpreters=True) assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ call(thread) for thread in threads ] @@ -705,7 +723,9 @@ def test_process_core_locals(): locals=True, method=StackMethod.AUTO, ) - TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ call(thread) for thread in threads ] @@ -755,7 +775,9 @@ def test_process_core_with_search_path(): locals=False, method=StackMethod.AUTO, ) - TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ call(thread) for thread in threads ] @@ -806,7 +828,9 @@ def test_process_core_with_search_root(): locals=False, method=StackMethod.AUTO, ) - TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ call(thread) for thread in threads ] @@ -955,7 +979,9 @@ def test_process_core_error(exception, exval, capsys): # THEN get_process_threads_mock.assert_called_once() - TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) TracebackPrinterMock.return_value.print_thread.assert_not_called() capture = capsys.readouterr() assert "Oh no!" in capture.err @@ -997,7 +1023,9 @@ def test_process_core_exhaustive(): locals=False, method=StackMethod.ALL, ) - TracebackPrinterMock.assert_called_once_with(NativeReportingMode.OFF) + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ call(thread) for thread in threads ] From 6f9df58f937e136f65376dd5bb9f6ac8335ff219 Mon Sep 17 00:00:00 2001 From: Saul Cooperman Date: Tue, 24 Feb 2026 22:12:04 +0000 Subject: [PATCH 5/6] Lint files --- src/pystack/traceback_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pystack/traceback_formatter.py b/src/pystack/traceback_formatter.py index 56f7921f..3dc44add 100644 --- a/src/pystack/traceback_formatter.py +++ b/src/pystack/traceback_formatter.py @@ -25,7 +25,7 @@ def print_thread(self, thread: PyThread) -> None: if self.include_subinterpreters: if thread.interpreter_id != self._current_interpreter_id: self._print_interpreter_header(thread.interpreter_id) - self._current_interpreter_id = thread.interpreter_id + self._current_interpreter_id = thread.interpreter_id or -1 # Print the thread with indentation for line in format_thread(thread, self.native_mode): From c959f75d5db3ea045ee4b3f3104b8292a2aebeea Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 28 Feb 2026 02:31:36 +0000 Subject: [PATCH 6/6] Fix native/subinterpreter handling and strengthen integration coverage --- src/pystack/_pystack.pyx | 14 +- src/pystack/_pystack/cpython/interpreter.h | 4 +- src/pystack/traceback_formatter.py | 6 +- tests/integration/test_subinterpreters.py | 291 +++++++++++++++++++-- tests/unit/test_traceback_formatter.py | 36 +++ 5 files changed, 326 insertions(+), 25 deletions(-) diff --git a/src/pystack/_pystack.pyx b/src/pystack/_pystack.pyx index 91b9516e..d9a0488b 100644 --- a/src/pystack/_pystack.pyx +++ b/src/pystack/_pystack.pyx @@ -638,7 +638,7 @@ def _get_process_threads( if thread.tid in all_tids: all_tids.remove(thread.tid) yield thread - head = InterpreterUtils.getNextInterpreter(manager, head); + head = InterpreterUtils.getNextInterpreter(manager, head) if native_mode == NativeReportingMode.ALL: yield from _construct_os_threads(manager, pid, all_tids) @@ -773,14 +773,20 @@ def _get_process_threads_for_core( all_tids = list(manager.get().Tids()) - if head: - native = native_mode in {NativeReportingMode.PYTHON, NativeReportingMode.ALL} + while head: + add_native_traces = native_mode != NativeReportingMode.OFF for thread in _construct_threads_from_interpreter_state( - manager, head, pymanager.pid, pymanager.python_version, native, locals + manager, + head, + pymanager.pid, + pymanager.python_version, + add_native_traces, + locals, ): if thread.tid in all_tids: all_tids.remove(thread.tid) yield thread + head = InterpreterUtils.getNextInterpreter(manager, head) if native_mode == NativeReportingMode.ALL: yield from _construct_os_threads(manager, pymanager.pid, all_tids) diff --git a/src/pystack/_pystack/cpython/interpreter.h b/src/pystack/_pystack/cpython/interpreter.h index b35b26ba..d75558fe 100644 --- a/src/pystack/_pystack/cpython/interpreter.h +++ b/src/pystack/_pystack/cpython/interpreter.h @@ -375,10 +375,10 @@ struct _gil_runtime_state int locked; unsigned long switch_number; pthread_cond_t cond; - pthread_cond_t mutex; + pthread_mutex_t mutex; #ifdef FORCE_SWITCHING pthread_cond_t switch_cond; - pthread_cond_t switch_mutex; + pthread_mutex_t switch_mutex; #endif }; diff --git a/src/pystack/traceback_formatter.py b/src/pystack/traceback_formatter.py index 3dc44add..ea2ef539 100644 --- a/src/pystack/traceback_formatter.py +++ b/src/pystack/traceback_formatter.py @@ -25,7 +25,11 @@ def print_thread(self, thread: PyThread) -> None: if self.include_subinterpreters: if thread.interpreter_id != self._current_interpreter_id: self._print_interpreter_header(thread.interpreter_id) - self._current_interpreter_id = thread.interpreter_id or -1 + self._current_interpreter_id = ( + thread.interpreter_id + if thread.interpreter_id is not None + else -1 + ) # Print the thread with indentation for line in format_thread(thread, self.native_mode): diff --git a/tests/integration/test_subinterpreters.py b/tests/integration/test_subinterpreters.py index 56ee074c..693ce58c 100644 --- a/tests/integration/test_subinterpreters.py +++ b/tests/integration/test_subinterpreters.py @@ -1,14 +1,24 @@ import io +from collections import Counter from contextlib import redirect_stdout from pathlib import Path +import pytest + from pystack.engine import NativeReportingMode +from pystack.engine import StackMethod from pystack.engine import get_process_threads +from pystack.engine import get_process_threads_for_core from pystack.traceback_formatter import TracebackPrinter +from pystack.types import NativeFrame +from pystack.types import frame_type from tests.utils import ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +from tests.utils import generate_core_file from tests.utils import spawn_child_process NUM_INTERPRETERS = 3 +NUM_INTERPRETERS_WITH_THREADS = 2 +NUM_THREADS_PER_SUBINTERPRETER = 2 PROGRAM = f"""\ import sys @@ -34,7 +44,7 @@ def start_interpreter_async(interp, code): ''' threads = [] -for i in range(NUM_INTERPRETERS): +for _ in range(NUM_INTERPRETERS): interp = interpreters.create() t = start_interpreter_async(interp, CODE) threads.append(t) @@ -51,31 +61,87 @@ def start_interpreter_async(interp, code): """ -@ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS -def test_subinterpreters(python, tmpdir): - """Test that pystack can detect and report multiple sub-interpreters.""" +PROGRAM_WITH_THREADS = f"""\ +import sys +import threading +import time - # GIVEN - _, python_executable = python +from concurrent import interpreters + +NUM_INTERPRETERS = {NUM_INTERPRETERS_WITH_THREADS} + + +def start_interpreter_async(interp, code): + t = threading.Thread(target=interp.exec, args=(code,)) + t.daemon = True + t.start() + return t + + +CODE = '''\\ +import threading +import time + +NUM_THREADS = {NUM_THREADS_PER_SUBINTERPRETER} + +def worker(): + while True: + time.sleep(1) + +threads = [] +for _ in range(NUM_THREADS): + t = threading.Thread(target=worker) + # daemon threads are disabled in isolated subinterpreters + t.start() + threads.append(t) + +while True: + time.sleep(1) +''' + +threads = [] +for _ in range(NUM_INTERPRETERS): + interp = interpreters.create() + t = start_interpreter_async(interp, CODE) + threads.append(t) + +# Give sub-interpreters and their internal workers time to start. +time.sleep(2) + +fifo = sys.argv[1] +with open(fifo, "w") as f: + f.write("ready") + +while True: + time.sleep(1) +""" + + +def _collect_threads( + python_executable: Path, + tmpdir: Path, + native_mode: NativeReportingMode = NativeReportingMode.OFF, +): test_file = Path(str(tmpdir)) / "subinterpreters_program.py" test_file.write_text(PROGRAM) - # WHEN with spawn_child_process(python_executable, test_file, tmpdir) as child_process: - threads = list(get_process_threads(child_process.pid, stop_process=True)) + return list( + get_process_threads( + child_process.pid, + stop_process=True, + native_mode=native_mode, + ) + ) - # Collect all interpreter IDs from the threads - interpreter_ids = {thread.interpreter_id for thread in threads} - # THEN - - # We expect the main interpreter (0) plus NUM_INTERPRETERS sub-interpreters - assert 0 in interpreter_ids - assert len(interpreter_ids) == NUM_INTERPRETERS + 1 - - # Verify the TracebackPrinter output contains the interpreter headers +def _assert_interpreter_headers( + threads, + native_mode: NativeReportingMode, + interpreter_ids, +) -> str: printer = TracebackPrinter( - native_mode=NativeReportingMode.OFF, + native_mode=native_mode, include_subinterpreters=True, ) output = io.StringIO() @@ -89,3 +155,192 @@ def test_subinterpreters(python, tmpdir): if interpreter_id == 0: continue assert f"Interpreter-{interpreter_id}" in result + return result + + +def _count_threads_by_interpreter(threads): + return dict( + Counter( + thread.interpreter_id + for thread in threads + if thread.interpreter_id is not None + ) + ) + + +def _interpreter_ids(threads) -> set[int]: + return { + thread.interpreter_id for thread in threads if thread.interpreter_id is not None + } + + +def _assert_subinterpreter_coverage(threads) -> set[int]: + interpreter_ids = _interpreter_ids(threads) + assert 0 in interpreter_ids + assert len(interpreter_ids) == NUM_INTERPRETERS + 1 + return interpreter_ids + + +def _assert_native_eval_symbols(threads) -> None: + eval_frames = [ + frame + for thread in threads + for frame in thread.native_frames + if frame_type(frame, thread.python_version) == NativeFrame.FrameType.EVAL + ] + assert eval_frames + assert all("?" not in frame.symbol for frame in eval_frames) + if any(frame.linenumber == 0 for frame in eval_frames): # pragma: no cover + assert all(frame.linenumber == 0 for frame in eval_frames) + assert all(frame.path == "???" for frame in eval_frames) + else: # pragma: no cover + assert all(frame.linenumber != 0 for frame in eval_frames) + assert any(frame.path and "?" not in frame.path for frame in eval_frames) + + +@ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +def test_subinterpreters(python, tmpdir): + _, python_executable = python + + threads = _collect_threads( + python_executable=python_executable, + tmpdir=tmpdir, + native_mode=NativeReportingMode.OFF, + ) + + interpreter_ids = _assert_subinterpreter_coverage(threads) + assert all(not thread.native_frames for thread in threads) + _assert_interpreter_headers( + threads=threads, + native_mode=NativeReportingMode.OFF, + interpreter_ids=interpreter_ids, + ) + + +@ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +@pytest.mark.parametrize( + "native_mode", + [ + NativeReportingMode.PYTHON, + NativeReportingMode.LAST, + NativeReportingMode.ALL, + ], + ids=["python", "last", "all"], +) +def test_subinterpreters_with_native(python, tmpdir, native_mode): + _, python_executable = python + + threads = _collect_threads( + python_executable=python_executable, + tmpdir=tmpdir, + native_mode=native_mode, + ) + + interpreter_ids = _assert_subinterpreter_coverage(threads) + assert any(thread.native_frames for thread in threads) + _assert_native_eval_symbols(threads) + + output = _assert_interpreter_headers( + threads=threads, + native_mode=native_mode, + interpreter_ids=interpreter_ids, + ) + assert "(C)" in output or "Unable to merge native stack" in output + + +@ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +def test_subinterpreters_many_threads_with_native(python, tmpdir): + _, python_executable = python + + test_file = Path(str(tmpdir)) / "subinterpreters_with_threads_program.py" + test_file.write_text(PROGRAM_WITH_THREADS) + + with spawn_child_process(python_executable, test_file, tmpdir) as child_process: + threads = list( + get_process_threads( + child_process.pid, + stop_process=True, + native_mode=NativeReportingMode.PYTHON, + method=StackMethod.DEBUG_OFFSETS, + ) + ) + + interpreter_ids = _interpreter_ids(threads) + assert 0 in interpreter_ids + assert len(interpreter_ids) == NUM_INTERPRETERS_WITH_THREADS + 1 + + counts_by_interpreter = _count_threads_by_interpreter(threads) + assert all( + counts_by_interpreter.get(interpreter_id, 0) >= 1 + for interpreter_id in interpreter_ids + ) + # At least one sub-interpreter should expose multiple Python threads. + assert any( + count > 1 + for interpreter_id, count in counts_by_interpreter.items() + if interpreter_id != 0 + ) + + assert any(thread.native_frames for thread in threads) + _assert_native_eval_symbols(threads) + + +@ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +def test_subinterpreters_for_core(python, tmpdir): + _, python_executable = python + + test_file = Path(str(tmpdir)) / "subinterpreters_program.py" + test_file.write_text(PROGRAM) + + with generate_core_file(python_executable, test_file, tmpdir) as core_file: + threads = list( + get_process_threads_for_core( + core_file, + python_executable, + native_mode=NativeReportingMode.OFF, + ) + ) + + interpreter_ids = _assert_subinterpreter_coverage(threads) + assert all(not thread.native_frames for thread in threads) + _assert_interpreter_headers( + threads=threads, + native_mode=NativeReportingMode.OFF, + interpreter_ids=interpreter_ids, + ) + + +@ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +@pytest.mark.parametrize( + "native_mode", + [ + NativeReportingMode.PYTHON, + NativeReportingMode.LAST, + NativeReportingMode.ALL, + ], + ids=["python", "last", "all"], +) +def test_subinterpreters_for_core_with_native(python, tmpdir, native_mode): + _, python_executable = python + + test_file = Path(str(tmpdir)) / "subinterpreters_program.py" + test_file.write_text(PROGRAM) + + with generate_core_file(python_executable, test_file, tmpdir) as core_file: + threads = list( + get_process_threads_for_core( + core_file, + python_executable, + native_mode=native_mode, + ) + ) + + interpreter_ids = _assert_subinterpreter_coverage(threads) + assert any(thread.native_frames for thread in threads) + _assert_native_eval_symbols(threads) + output = _assert_interpreter_headers( + threads=threads, + native_mode=native_mode, + interpreter_ids=interpreter_ids, + ) + assert "(C)" in output or "Unable to merge native stack" in output diff --git a/tests/unit/test_traceback_formatter.py b/tests/unit/test_traceback_formatter.py index c2a230b6..86c9c3ad 100644 --- a/tests/unit/test_traceback_formatter.py +++ b/tests/unit/test_traceback_formatter.py @@ -1802,6 +1802,42 @@ def test_print_thread_with_subinterpreters_same_interp_no_repeat_header(capsys): assert captured.out.count("Interpreter-1") == 1 +def test_print_thread_with_subinterpreters_main_interp_no_repeat_header(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + thread1 = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interpreter_id=0, + ) + thread2 = PyThread( + tid=2, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interpreter_id=0, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1",), + ): + printer.print_thread(thread1) + printer.print_thread(thread2) + + # THEN + captured = capsys.readouterr() + # Header should appear only once + assert captured.out.count("Interpreter-0 (main)") == 1 + + def test_print_thread_with_subinterpreters_different_interps_prints_headers(capsys): # GIVEN printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True)