diff --git a/setup.py b/setup.py index d76c9375..58bc40d2 100644 --- a/setup.py +++ b/setup.py @@ -77,6 +77,7 @@ "src/pystack/_pystack.pyx", "src/pystack/_pystack/corefile.cpp", "src/pystack/_pystack/elf_common.cpp", + "src/pystack/_pystack/interpreter.cpp", "src/pystack/_pystack/logging.cpp", "src/pystack/_pystack/mem.cpp", "src/pystack/_pystack/process.cpp", diff --git a/src/pystack/__init__.py b/src/pystack/__init__.py index e973464d..a98c6c19 100644 --- a/src/pystack/__init__.py +++ b/src/pystack/__init__.py @@ -1,7 +1,5 @@ from ._version import __version__ -from .traceback_formatter import print_thread __all__ = [ "__version__", - "print_thread", ] diff --git a/src/pystack/__main__.py b/src/pystack/__main__.py index 55102c91..5744d7f7 100644 --- a/src/pystack/__main__.py +++ b/src/pystack/__main__.py @@ -19,13 +19,13 @@ from pystack.process import is_gzip from . import errors -from . import print_thread from .colors import colored from .engine import CoreFileAnalyzer from .engine import NativeReportingMode from .engine import StackMethod from .engine import get_process_threads from .engine import get_process_threads_for_core +from .traceback_formatter import TracebackPrinter PERMISSION_ERROR_MSG = "Operation not permitted" NO_SUCH_PROCESS_ERROR_MSG = "No such process" @@ -285,6 +285,9 @@ def process_remote(parser: argparse.ArgumentParser, args: argparse.Namespace) -> if not args.block and args.native_mode != NativeReportingMode.OFF: parser.error("Native traces are only available in blocking mode") + printer = TracebackPrinter( + native_mode=args.native_mode, include_subinterpreters=True + ) for thread in get_process_threads( args.pid, stop_process=args.block, @@ -292,7 +295,7 @@ def process_remote(parser: argparse.ArgumentParser, args: argparse.Namespace) -> locals=args.locals, method=StackMethod.ALL if args.exhaustive else StackMethod.AUTO, ): - print_thread(thread, args.native_mode) + printer.print_thread(thread) def format_psinfo_information(psinfo: Dict[str, Any]) -> str: @@ -414,6 +417,7 @@ def process_core(parser: argparse.ArgumentParser, args: argparse.Namespace) -> N elf_id if elf_id else "", ) + printer = TracebackPrinter(args.native_mode, include_subinterpreters=True) for thread in get_process_threads_for_core( corefile, executable, @@ -422,7 +426,7 @@ def process_core(parser: argparse.ArgumentParser, args: argparse.Namespace) -> N locals=args.locals, method=StackMethod.ALL if args.exhaustive else StackMethod.AUTO, ): - print_thread(thread, args.native_mode) + printer.print_thread(thread) if __name__ == "__main__": # pragma: no cover diff --git a/src/pystack/_pystack.pyx b/src/pystack/_pystack.pyx index de16701d..d9a0488b 100644 --- a/src/pystack/_pystack.pyx +++ b/src/pystack/_pystack.pyx @@ -22,6 +22,7 @@ from _pystack.elf_common cimport CoreFileAnalyzer as NativeCoreFileAnalyzer from _pystack.elf_common cimport ProcessAnalyzer as NativeProcessAnalyzer from _pystack.elf_common cimport SectionInfo from _pystack.elf_common cimport getSectionInfo +from _pystack.interpreter cimport InterpreterUtils from _pystack.logging cimport initializePythonLoggerInterface from _pystack.mem cimport AbstractRemoteMemoryManager from _pystack.mem cimport MemoryMapInformation as CppMemoryMapInformation @@ -462,6 +463,7 @@ cdef object _construct_threads_from_interpreter_state( bint add_native_traces, bint resolve_locals, ): + interpreter_id = InterpreterUtils.getInterpreterId(manager, head) LOGGER.info("Fetching Python threads") threads = [] @@ -486,6 +488,7 @@ cdef object _construct_threads_from_interpreter_state( current_thread.isGilHolder(), current_thread.isGCCollecting(), python_version, + interpreter_id, name=get_thread_name(pid, current_thread.Tid()), ) ) @@ -622,7 +625,7 @@ def _get_process_threads( ) all_tids = list(manager.get().Tids()) - if head: + while head: add_native_traces = native_mode != NativeReportingMode.OFF for thread in _construct_threads_from_interpreter_state( manager, @@ -635,6 +638,7 @@ def _get_process_threads( if thread.tid in all_tids: all_tids.remove(thread.tid) yield thread + head = InterpreterUtils.getNextInterpreter(manager, head) if native_mode == NativeReportingMode.ALL: yield from _construct_os_threads(manager, pid, all_tids) @@ -769,14 +773,20 @@ def _get_process_threads_for_core( all_tids = list(manager.get().Tids()) - if head: - native = native_mode in {NativeReportingMode.PYTHON, NativeReportingMode.ALL} + while head: + add_native_traces = native_mode != NativeReportingMode.OFF for thread in _construct_threads_from_interpreter_state( - manager, head, pymanager.pid, pymanager.python_version, native, locals + manager, + head, + pymanager.pid, + pymanager.python_version, + add_native_traces, + locals, ): if thread.tid in all_tids: all_tids.remove(thread.tid) yield thread + head = InterpreterUtils.getNextInterpreter(manager, head) if native_mode == NativeReportingMode.ALL: yield from _construct_os_threads(manager, pymanager.pid, all_tids) diff --git a/src/pystack/_pystack/CMakeLists.txt b/src/pystack/_pystack/CMakeLists.txt index 5a0fd8a8..74183d11 100644 --- a/src/pystack/_pystack/CMakeLists.txt +++ b/src/pystack/_pystack/CMakeLists.txt @@ -21,6 +21,7 @@ add_library(_pystack STATIC pythread.cpp version.cpp elf_common.cpp - pytypes.cpp) + pytypes.cpp + interpreter.cpp) set_property(TARGET _pystack PROPERTY POSITION_INDEPENDENT_CODE ON) include_directories("." "cpython" ${PYTHON_INCLUDE_DIRS}) diff --git a/src/pystack/_pystack/cpython/interpreter.h b/src/pystack/_pystack/cpython/interpreter.h index b35b26ba..d75558fe 100644 --- a/src/pystack/_pystack/cpython/interpreter.h +++ b/src/pystack/_pystack/cpython/interpreter.h @@ -375,10 +375,10 @@ struct _gil_runtime_state int locked; unsigned long switch_number; pthread_cond_t cond; - pthread_cond_t mutex; + pthread_mutex_t mutex; #ifdef FORCE_SWITCHING pthread_cond_t switch_cond; - pthread_cond_t switch_mutex; + pthread_mutex_t switch_mutex; #endif }; diff --git a/src/pystack/_pystack/interpreter.cpp b/src/pystack/_pystack/interpreter.cpp new file mode 100644 index 00000000..4f52e043 --- /dev/null +++ b/src/pystack/_pystack/interpreter.cpp @@ -0,0 +1,36 @@ +#include + +#include "interpreter.h" +#include "logging.h" +#include "process.h" +#include "structure.h" +#include "version.h" + +namespace pystack { + +remote_addr_t +InterpreterUtils::getNextInterpreter( + const std::shared_ptr& manager, + remote_addr_t interpreter_addr) +{ + Structure is(manager, interpreter_addr); + return is.getField(&py_is_v::o_next); +} + +int64_t +InterpreterUtils::getInterpreterId( + const std::shared_ptr& manager, + remote_addr_t interpreter_addr) +{ + if (!manager->versionIsAtLeast(3, 7)) { + // No support for subinterpreters so the only interpreter is ID 0. + return 0; + } + + Structure is(manager, interpreter_addr); + int64_t id_value = is.getField(&py_is_v::o_id); + + return id_value; +} + +} // namespace pystack diff --git a/src/pystack/_pystack/interpreter.h b/src/pystack/_pystack/interpreter.h new file mode 100644 index 00000000..0138ff84 --- /dev/null +++ b/src/pystack/_pystack/interpreter.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +#include "mem.h" +#include "process.h" + +namespace pystack { + +class InterpreterUtils +{ + public: + // Static Methods + static remote_addr_t getNextInterpreter( + const std::shared_ptr& manager, + remote_addr_t interpreter_addr); + + static int64_t getInterpreterId( + const std::shared_ptr& manager, + remote_addr_t interpreter_addr); +}; + +} // namespace pystack diff --git a/src/pystack/_pystack/interpreter.pxd b/src/pystack/_pystack/interpreter.pxd new file mode 100644 index 00000000..0248f468 --- /dev/null +++ b/src/pystack/_pystack/interpreter.pxd @@ -0,0 +1,13 @@ +from _pystack.mem cimport remote_addr_t +from _pystack.process cimport AbstractProcessManager +from libc.stdint cimport int64_t +from libcpp.memory cimport shared_ptr + + +cdef extern from "interpreter.h" namespace "pystack": + cdef cppclass InterpreterUtils: + @staticmethod + remote_addr_t getNextInterpreter(shared_ptr[AbstractProcessManager] manager, remote_addr_t interpreter_addr) except + + + @staticmethod + int64_t getInterpreterId(shared_ptr[AbstractProcessManager] manager, remote_addr_t interpreter_addr) except + diff --git a/src/pystack/_pystack/process.cpp b/src/pystack/_pystack/process.cpp index 56e32bae..761509a0 100644 --- a/src/pystack/_pystack/process.cpp +++ b/src/pystack/_pystack/process.cpp @@ -964,6 +964,7 @@ AbstractProcessManager::copyDebugOffsets(Structure& py_runtime, py set_offset(py_is.o_sysdict, &py_runtime_v::o_dbg_off_interpreter_state_sysdict); set_offset(py_is.o_builtins, &py_runtime_v::o_dbg_off_interpreter_state_builtins); set_offset(py_is.o_gil_runtime_state, &py_runtime_v::o_dbg_off_interpreter_state_ceval_gil); + set_offset(py_is.o_id, &py_runtime_v::o_dbg_off_interpreter_state_id); set_size(py_thread, &py_runtime_v::o_dbg_off_thread_state_struct_size); set_offset(py_thread.o_prev, &py_runtime_v::o_dbg_off_thread_state_prev); diff --git a/src/pystack/_pystack/pythread.cpp b/src/pystack/_pystack/pythread.cpp index d50e4126..b6b90a24 100644 --- a/src/pystack/_pystack/pythread.cpp +++ b/src/pystack/_pystack/pythread.cpp @@ -2,6 +2,8 @@ #include #include +#include "cpython/pthread.h" +#include "interpreter.h" #include "logging.h" #include "mem.h" #include "native_frame.h" @@ -11,8 +13,6 @@ #include "structure.h" #include "version.h" -#include "cpython/pthread.h" - namespace pystack { Thread::Thread(pid_t pid, pid_t tid) @@ -47,79 +47,88 @@ findPthreadTidOffset( remote_addr_t interp_state_addr) { LOG(DEBUG) << "Attempting to locate tid offset in pthread structure"; - Structure is(manager, interp_state_addr); - auto current_thread_addr = is.getField(&py_is_v::o_tstate_head); + // If interp_state_addr does not point to the main interpreter (id 0) we won't find the + // PID == TID in the interpreter threads. Hence, we traverse the linked list of interpreters. The + // main interpreter is not necessarily the head of the linked lists of interpreters. + + while (interp_state_addr != 0) { + Structure is(manager, interp_state_addr); + + auto current_thread_addr = is.getField(&py_is_v::o_tstate_head); - auto thread_head = current_thread_addr; + auto thread_head = current_thread_addr; - // Iterate over all Python threads until we find a thread that has a tid equal to - // the process pid. This works because in the main thread the tid is equal to the pid, - // so when this happens it has to happen on the main thread. Note that the main thread - // is not necessarily at the head of the Python thread linked list + // Iterate over all Python threads until we find a thread that has a tid equal to + // the process pid. This works because in the main thread the tid is equal to the pid, + // so when this happens it has to happen on the main thread. Note that the main thread + // is not necessarily at the head of the Python thread linked list #if defined(__GLIBC__) - // If we detect GLIBC, we can try the two main known structs for 'struct - // pthread' that we know about to avoid having to do guess-work by doing a - // linear scan over the struct. - while (current_thread_addr != (remote_addr_t) nullptr) { - Structure current_thread(manager, current_thread_addr); - auto pthread_id_addr = current_thread.getField(&py_thread_v::o_thread_id); - - pid_t the_tid; - std::vector glibc_pthread_offset_candidates = { - offsetof(_pthread_structure_with_simple_header, tid), - offsetof(_pthread_structure_with_tcbhead, tid)}; - for (off_t candidate : glibc_pthread_offset_candidates) { - manager->copyObjectFromProcess((remote_addr_t)(pthread_id_addr + candidate), &the_tid); - if (the_tid == manager->Pid()) { - LOG(DEBUG) << "Tid offset located using GLIBC offsets at offset " << std::showbase - << std::hex << candidate << " in pthread structure"; - return candidate; + // If we detect GLIBC, we can try the two main known structs for 'struct + // pthread' that we know about to avoid having to do guess-work by doing a + // linear scan over the struct. + while (current_thread_addr != (remote_addr_t) nullptr) { + Structure current_thread(manager, current_thread_addr); + auto pthread_id_addr = current_thread.getField(&py_thread_v::o_thread_id); + + pid_t the_tid; + std::vector glibc_pthread_offset_candidates = { + offsetof(_pthread_structure_with_simple_header, tid), + offsetof(_pthread_structure_with_tcbhead, tid)}; + for (off_t candidate : glibc_pthread_offset_candidates) { + manager->copyObjectFromProcess((remote_addr_t)(pthread_id_addr + candidate), &the_tid); + if (the_tid == manager->Pid()) { + LOG(DEBUG) << "Tid offset located using GLIBC offsets at offset " << std::showbase + << std::hex << candidate << " in pthread structure"; + return candidate; + } } + remote_addr_t next_thread_addr = current_thread.getField(&py_thread_v::o_next); + if (next_thread_addr == current_thread_addr) { + break; + } + current_thread_addr = next_thread_addr; } - remote_addr_t next_thread_addr = current_thread.getField(&py_thread_v::o_next); - if (next_thread_addr == current_thread_addr) { - break; - } - current_thread_addr = next_thread_addr; - } #endif - current_thread_addr = thread_head; - - while (current_thread_addr != (remote_addr_t) nullptr) { - Structure current_thread(manager, current_thread_addr); - auto pthread_id_addr = current_thread.getField(&py_thread_v::o_thread_id); - - // Attempt to locate a field in the pthread struct that's equal to the pid. - uintptr_t buffer[100]; - size_t buffer_size = sizeof(buffer); - while (buffer_size > 0) { - try { - LOG(DEBUG) << "Trying to copy a buffer of " << buffer_size << " bytes to get pthread ID"; - manager->copyMemoryFromProcess(pthread_id_addr, buffer_size, &buffer); - break; - } catch (const RemoteMemCopyError& ex) { - LOG(DEBUG) << "Failed to copy buffer to get pthread ID"; - buffer_size /= 2; + current_thread_addr = thread_head; + + while (current_thread_addr != (remote_addr_t) nullptr) { + Structure current_thread(manager, current_thread_addr); + auto pthread_id_addr = current_thread.getField(&py_thread_v::o_thread_id); + + // Attempt to locate a field in the pthread struct that's equal to the pid. + uintptr_t buffer[100]; + size_t buffer_size = sizeof(buffer); + while (buffer_size > 0) { + try { + LOG(DEBUG) << "Trying to copy a buffer of " << buffer_size + << " bytes to get pthread ID"; + manager->copyMemoryFromProcess(pthread_id_addr, buffer_size, &buffer); + break; + } catch (const RemoteMemCopyError& ex) { + LOG(DEBUG) << "Failed to copy buffer to get pthread ID"; + buffer_size /= 2; + } } - } - LOG(DEBUG) << "Copied a buffer of " << buffer_size << " bytes to get pthread ID"; - for (size_t i = 0; i < buffer_size / sizeof(uintptr_t); i++) { - if (static_cast(buffer[i]) == manager->Pid()) { - off_t offset = sizeof(uintptr_t) * i; - LOG(DEBUG) << "Tid offset located by scanning at offset " << std::showbase << std::hex - << offset << " in pthread structure"; - return offset; + LOG(DEBUG) << "Copied a buffer of " << buffer_size << " bytes to get pthread ID"; + for (size_t i = 0; i < buffer_size / sizeof(uintptr_t); i++) { + if (static_cast(buffer[i]) == manager->Pid()) { + off_t offset = sizeof(uintptr_t) * i; + LOG(DEBUG) << "Tid offset located by scanning at offset " << std::showbase + << std::hex << offset << " in pthread structure"; + return offset; + } } - } - remote_addr_t next_thread_addr = current_thread.getField(&py_thread_v::o_next); - if (next_thread_addr == current_thread_addr) { - break; + remote_addr_t next_thread_addr = current_thread.getField(&py_thread_v::o_next); + if (next_thread_addr == current_thread_addr) { + break; + } + current_thread_addr = next_thread_addr; } - current_thread_addr = next_thread_addr; + interp_state_addr = InterpreterUtils::getNextInterpreter(manager, interp_state_addr); } LOG(ERROR) << "Could not find tid offset in pthread structure"; return 0; diff --git a/src/pystack/_pystack/version.cpp b/src/pystack/_pystack/version.cpp index f58ff878..a31ad1fc 100644 --- a/src/pystack/_pystack/version.cpp +++ b/src/pystack/_pystack/version.cpp @@ -179,6 +179,23 @@ py_is() }; } +template +constexpr py_is_v +py_isv7() +{ + return { + sizeof(T), + {offsetof(T, next)}, + {offsetof(T, tstate_head)}, + {offsetof(T, gc)}, + {offsetof(T, modules)}, + {offsetof(T, sysdict)}, + {offsetof(T, builtins)}, + {0}, + {offsetof(T, id)}, + }; +} + template constexpr py_is_v py_isv311() @@ -191,6 +208,8 @@ py_isv311() {offsetof(T, modules)}, {offsetof(T, sysdict)}, {offsetof(T, builtins)}, + {0}, + {offsetof(T, id)}, }; } @@ -207,6 +226,24 @@ py_isv312() {offsetof(T, sysdict)}, {offsetof(T, builtins)}, {offsetof(T, ceval.gil)}, + {offsetof(T, id)}, + }; +} + +template +constexpr py_is_v +py_isv314() +{ + return { + sizeof(T), + {offsetof(T, next)}, + {offsetof(T, threads.head)}, + {offsetof(T, gc)}, + {offsetof(T, imports.modules)}, + {offsetof(T, sysdict)}, + {offsetof(T, builtins)}, + {offsetof(T, _gil)}, + {offsetof(T, id)}, }; } @@ -578,7 +615,7 @@ python_v python_v3_7 = { py_code(), py_frame(), py_thread(), - py_is(), + py_isv7(), py_runtime(), py_gc(), }; @@ -600,7 +637,7 @@ python_v python_v3_8 = { py_code(), py_frame(), py_thread(), - py_is(), + py_isv7(), py_runtime(), py_gc(), }; @@ -622,7 +659,7 @@ python_v python_v3_9 = { py_code(), py_frame(), py_thread(), - py_is(), + py_isv7(), py_runtime(), py_gc(), }; @@ -644,7 +681,7 @@ python_v python_v3_10 = { py_code(), py_frame(), py_thread(), - py_is(), + py_isv7(), py_runtime(), py_gc(), }; @@ -737,7 +774,7 @@ python_v python_v3_14 = { py_codev311(), py_framev314(), py_threadv313(), - py_isv312(), + py_isv314(), py_runtimev313(), py_gc(), py_cframe(), diff --git a/src/pystack/_pystack/version.h b/src/pystack/_pystack/version.h index c56851ac..d9b2b2de 100644 --- a/src/pystack/_pystack/version.h +++ b/src/pystack/_pystack/version.h @@ -241,6 +241,7 @@ struct py_is_v FieldOffset o_sysdict; FieldOffset o_builtins; FieldOffset o_gil_runtime_state; + FieldOffset o_id; }; struct py_gc_v diff --git a/src/pystack/traceback_formatter.py b/src/pystack/traceback_formatter.py index 35637b97..ea2ef539 100644 --- a/src/pystack/traceback_formatter.py +++ b/src/pystack/traceback_formatter.py @@ -12,9 +12,38 @@ from .types import frame_type -def print_thread(thread: PyThread, native_mode: NativeReportingMode) -> None: - for line in format_thread(thread, native_mode): - print(line, file=sys.stdout, flush=True) +class TracebackPrinter: + def __init__( + self, native_mode: NativeReportingMode, include_subinterpreters: bool = False + ): + self.native_mode = native_mode + self.include_subinterpreters = include_subinterpreters + self._current_interpreter_id = -1 + + def print_thread(self, thread: PyThread) -> None: + # Print interpreter header if we've switched interpreters + if self.include_subinterpreters: + if thread.interpreter_id != self._current_interpreter_id: + self._print_interpreter_header(thread.interpreter_id) + self._current_interpreter_id = ( + thread.interpreter_id + if thread.interpreter_id is not None + else -1 + ) + + # Print the thread with indentation + for line in format_thread(thread, self.native_mode): + if self.include_subinterpreters: + print(" " * 2, end="") + print(line, file=sys.stdout, flush=True) + + def _print_interpreter_header(self, interpreter_id: Optional[int]) -> None: + header = ( + f"Interpreter-{interpreter_id if interpreter_id is not None else 'Unknown'}" + ) + if interpreter_id == 0: + header += " (main)" + print(header, file=sys.stdout, flush=True) def format_frame(frame: PyFrame) -> Iterable[str]: diff --git a/src/pystack/types.py b/src/pystack/types.py index fbd1eb13..5eb77ced 100644 --- a/src/pystack/types.py +++ b/src/pystack/types.py @@ -115,6 +115,7 @@ class PyThread: holds_the_gil: int is_gc_collecting: int python_version: Optional[Tuple[int, int]] + interpreter_id: Optional[int] = None name: Optional[str] = None @property diff --git a/tests/integration/test_subinterpreters.py b/tests/integration/test_subinterpreters.py new file mode 100644 index 00000000..693ce58c --- /dev/null +++ b/tests/integration/test_subinterpreters.py @@ -0,0 +1,346 @@ +import io +from collections import Counter +from contextlib import redirect_stdout +from pathlib import Path + +import pytest + +from pystack.engine import NativeReportingMode +from pystack.engine import StackMethod +from pystack.engine import get_process_threads +from pystack.engine import get_process_threads_for_core +from pystack.traceback_formatter import TracebackPrinter +from pystack.types import NativeFrame +from pystack.types import frame_type +from tests.utils import ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +from tests.utils import generate_core_file +from tests.utils import spawn_child_process + +NUM_INTERPRETERS = 3 +NUM_INTERPRETERS_WITH_THREADS = 2 +NUM_THREADS_PER_SUBINTERPRETER = 2 + +PROGRAM = f"""\ +import sys +import threading +import time + +from concurrent import interpreters + +NUM_INTERPRETERS = {NUM_INTERPRETERS} + + +def start_interpreter_async(interp, code): + t = threading.Thread(target=interp.exec, args=(code,)) + t.daemon = True + t.start() + return t + + +CODE = '''\\ +import time +while True: + time.sleep(1) +''' + +threads = [] +for _ in range(NUM_INTERPRETERS): + interp = interpreters.create() + t = start_interpreter_async(interp, CODE) + threads.append(t) + +# Give sub-interpreters time to start executing +time.sleep(1) + +fifo = sys.argv[1] +with open(fifo, "w") as f: + f.write("ready") + +while True: + time.sleep(1) +""" + + +PROGRAM_WITH_THREADS = f"""\ +import sys +import threading +import time + +from concurrent import interpreters + +NUM_INTERPRETERS = {NUM_INTERPRETERS_WITH_THREADS} + + +def start_interpreter_async(interp, code): + t = threading.Thread(target=interp.exec, args=(code,)) + t.daemon = True + t.start() + return t + + +CODE = '''\\ +import threading +import time + +NUM_THREADS = {NUM_THREADS_PER_SUBINTERPRETER} + +def worker(): + while True: + time.sleep(1) + +threads = [] +for _ in range(NUM_THREADS): + t = threading.Thread(target=worker) + # daemon threads are disabled in isolated subinterpreters + t.start() + threads.append(t) + +while True: + time.sleep(1) +''' + +threads = [] +for _ in range(NUM_INTERPRETERS): + interp = interpreters.create() + t = start_interpreter_async(interp, CODE) + threads.append(t) + +# Give sub-interpreters and their internal workers time to start. +time.sleep(2) + +fifo = sys.argv[1] +with open(fifo, "w") as f: + f.write("ready") + +while True: + time.sleep(1) +""" + + +def _collect_threads( + python_executable: Path, + tmpdir: Path, + native_mode: NativeReportingMode = NativeReportingMode.OFF, +): + test_file = Path(str(tmpdir)) / "subinterpreters_program.py" + test_file.write_text(PROGRAM) + + with spawn_child_process(python_executable, test_file, tmpdir) as child_process: + return list( + get_process_threads( + child_process.pid, + stop_process=True, + native_mode=native_mode, + ) + ) + + +def _assert_interpreter_headers( + threads, + native_mode: NativeReportingMode, + interpreter_ids, +) -> str: + printer = TracebackPrinter( + native_mode=native_mode, + include_subinterpreters=True, + ) + output = io.StringIO() + with redirect_stdout(output): + for thread in threads: + printer.print_thread(thread) + + result = output.getvalue() + assert "Interpreter-0 (main)" in result + for interpreter_id in interpreter_ids: + if interpreter_id == 0: + continue + assert f"Interpreter-{interpreter_id}" in result + return result + + +def _count_threads_by_interpreter(threads): + return dict( + Counter( + thread.interpreter_id + for thread in threads + if thread.interpreter_id is not None + ) + ) + + +def _interpreter_ids(threads) -> set[int]: + return { + thread.interpreter_id for thread in threads if thread.interpreter_id is not None + } + + +def _assert_subinterpreter_coverage(threads) -> set[int]: + interpreter_ids = _interpreter_ids(threads) + assert 0 in interpreter_ids + assert len(interpreter_ids) == NUM_INTERPRETERS + 1 + return interpreter_ids + + +def _assert_native_eval_symbols(threads) -> None: + eval_frames = [ + frame + for thread in threads + for frame in thread.native_frames + if frame_type(frame, thread.python_version) == NativeFrame.FrameType.EVAL + ] + assert eval_frames + assert all("?" not in frame.symbol for frame in eval_frames) + if any(frame.linenumber == 0 for frame in eval_frames): # pragma: no cover + assert all(frame.linenumber == 0 for frame in eval_frames) + assert all(frame.path == "???" for frame in eval_frames) + else: # pragma: no cover + assert all(frame.linenumber != 0 for frame in eval_frames) + assert any(frame.path and "?" not in frame.path for frame in eval_frames) + + +@ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +def test_subinterpreters(python, tmpdir): + _, python_executable = python + + threads = _collect_threads( + python_executable=python_executable, + tmpdir=tmpdir, + native_mode=NativeReportingMode.OFF, + ) + + interpreter_ids = _assert_subinterpreter_coverage(threads) + assert all(not thread.native_frames for thread in threads) + _assert_interpreter_headers( + threads=threads, + native_mode=NativeReportingMode.OFF, + interpreter_ids=interpreter_ids, + ) + + +@ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +@pytest.mark.parametrize( + "native_mode", + [ + NativeReportingMode.PYTHON, + NativeReportingMode.LAST, + NativeReportingMode.ALL, + ], + ids=["python", "last", "all"], +) +def test_subinterpreters_with_native(python, tmpdir, native_mode): + _, python_executable = python + + threads = _collect_threads( + python_executable=python_executable, + tmpdir=tmpdir, + native_mode=native_mode, + ) + + interpreter_ids = _assert_subinterpreter_coverage(threads) + assert any(thread.native_frames for thread in threads) + _assert_native_eval_symbols(threads) + + output = _assert_interpreter_headers( + threads=threads, + native_mode=native_mode, + interpreter_ids=interpreter_ids, + ) + assert "(C)" in output or "Unable to merge native stack" in output + + +@ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +def test_subinterpreters_many_threads_with_native(python, tmpdir): + _, python_executable = python + + test_file = Path(str(tmpdir)) / "subinterpreters_with_threads_program.py" + test_file.write_text(PROGRAM_WITH_THREADS) + + with spawn_child_process(python_executable, test_file, tmpdir) as child_process: + threads = list( + get_process_threads( + child_process.pid, + stop_process=True, + native_mode=NativeReportingMode.PYTHON, + method=StackMethod.DEBUG_OFFSETS, + ) + ) + + interpreter_ids = _interpreter_ids(threads) + assert 0 in interpreter_ids + assert len(interpreter_ids) == NUM_INTERPRETERS_WITH_THREADS + 1 + + counts_by_interpreter = _count_threads_by_interpreter(threads) + assert all( + counts_by_interpreter.get(interpreter_id, 0) >= 1 + for interpreter_id in interpreter_ids + ) + # At least one sub-interpreter should expose multiple Python threads. + assert any( + count > 1 + for interpreter_id, count in counts_by_interpreter.items() + if interpreter_id != 0 + ) + + assert any(thread.native_frames for thread in threads) + _assert_native_eval_symbols(threads) + + +@ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +def test_subinterpreters_for_core(python, tmpdir): + _, python_executable = python + + test_file = Path(str(tmpdir)) / "subinterpreters_program.py" + test_file.write_text(PROGRAM) + + with generate_core_file(python_executable, test_file, tmpdir) as core_file: + threads = list( + get_process_threads_for_core( + core_file, + python_executable, + native_mode=NativeReportingMode.OFF, + ) + ) + + interpreter_ids = _assert_subinterpreter_coverage(threads) + assert all(not thread.native_frames for thread in threads) + _assert_interpreter_headers( + threads=threads, + native_mode=NativeReportingMode.OFF, + interpreter_ids=interpreter_ids, + ) + + +@ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS +@pytest.mark.parametrize( + "native_mode", + [ + NativeReportingMode.PYTHON, + NativeReportingMode.LAST, + NativeReportingMode.ALL, + ], + ids=["python", "last", "all"], +) +def test_subinterpreters_for_core_with_native(python, tmpdir, native_mode): + _, python_executable = python + + test_file = Path(str(tmpdir)) / "subinterpreters_program.py" + test_file.write_text(PROGRAM) + + with generate_core_file(python_executable, test_file, tmpdir) as core_file: + threads = list( + get_process_threads_for_core( + core_file, + python_executable, + native_mode=native_mode, + ) + ) + + interpreter_ids = _assert_subinterpreter_coverage(threads) + assert any(thread.native_frames for thread in threads) + _assert_native_eval_symbols(threads) + output = _assert_interpreter_headers( + threads=threads, + native_mode=native_mode, + interpreter_ids=interpreter_ids, + ) + assert "(C)" in output or "Unable to merge native stack" in output diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 2401d554..6f591dd9 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -191,8 +191,8 @@ def test_process_remote_default(): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ): get_process_threads_mock.return_value = threads @@ -207,8 +207,11 @@ def test_process_remote_default(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with( + native_mode=NativeReportingMode.OFF, include_subinterpreters=True + ) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -224,8 +227,8 @@ def test_process_remote_no_block(): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ): get_process_threads_mock.return_value = threads @@ -240,8 +243,11 @@ def test_process_remote_no_block(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with( + native_mode=NativeReportingMode.OFF, include_subinterpreters=True + ) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -265,8 +271,8 @@ def test_process_remote_native(argument, mode): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ): get_process_threads_mock.return_value = threads @@ -281,7 +287,12 @@ def test_process_remote_native(argument, mode): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, mode) for thread in threads] + TracebackPrinterMock.assert_called_once_with( + native_mode=mode, include_subinterpreters=True + ) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads + ] def test_process_remote_locals(): @@ -296,8 +307,8 @@ def test_process_remote_locals(): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ): get_process_threads_mock.return_value = threads @@ -312,8 +323,11 @@ def test_process_remote_locals(): locals=True, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with( + native_mode=NativeReportingMode.OFF, include_subinterpreters=True + ) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -329,8 +343,8 @@ def test_process_remote_native_no_block(capsys): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ): get_process_threads_mock.return_value = threads @@ -340,7 +354,8 @@ def test_process_remote_native_no_block(capsys): main() get_process_threads_mock.assert_not_called() - print_thread_mock.assert_not_called() + TracebackPrinterMock.assert_not_called() + TracebackPrinterMock.return_value.print_thread.assert_not_called() def test_process_remote_exhaustive(): @@ -355,8 +370,8 @@ def test_process_remote_exhaustive(): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ): get_process_threads_mock.return_value = threads @@ -371,8 +386,11 @@ def test_process_remote_exhaustive(): locals=False, method=StackMethod.ALL, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with( + native_mode=NativeReportingMode.OFF, include_subinterpreters=True + ) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -389,8 +407,8 @@ def test_process_remote_error(exception, exval, capsys): with patch( "pystack.__main__.get_process_threads" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -403,7 +421,10 @@ def test_process_remote_error(exception, exval, capsys): # THEN get_process_threads_mock.assert_called_once() - print_thread_mock.assert_not_called() + TracebackPrinterMock.assert_called_once_with( + native_mode=NativeReportingMode.OFF, include_subinterpreters=True + ) + TracebackPrinterMock.return_value.print_thread.assert_not_called() capture = capsys.readouterr() assert "Oh no!" in capture.err @@ -420,8 +441,8 @@ def test_process_core_default_without_executable(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -448,8 +469,11 @@ def test_process_core_default_without_executable(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -471,8 +495,8 @@ def test_process_core_default_gzip_without_executable(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -504,8 +528,11 @@ def test_process_core_default_gzip_without_executable(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] gzip_open_mock.assert_called_with(Path("corefile.gz"), "rb") @@ -575,8 +602,8 @@ def test_process_core_default_with_executable(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -600,8 +627,11 @@ def test_process_core_default_with_executable(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -625,8 +655,8 @@ def test_process_core_native(argument, mode): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -650,7 +680,10 @@ def test_process_core_native(argument, mode): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, mode) for thread in threads] + TracebackPrinterMock.assert_called_once_with(mode, include_subinterpreters=True) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads + ] def test_process_core_locals(): @@ -665,8 +698,8 @@ def test_process_core_locals(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -690,8 +723,11 @@ def test_process_core_locals(): locals=True, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -714,8 +750,8 @@ def test_process_core_with_search_path(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -739,8 +775,11 @@ def test_process_core_with_search_path(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -756,8 +795,8 @@ def test_process_core_with_search_root(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -789,8 +828,11 @@ def test_process_core_with_search_root(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -802,7 +844,7 @@ def test_process_core_with_not_readable_search_root(): # WHEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("pathlib.Path.exists", return_value=True), patch( "pystack.__main__.CoreFileAnalyzer" ), patch( @@ -826,7 +868,7 @@ def test_process_core_with_invalid_search_root(): # WHEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("pathlib.Path.exists", return_value=True), patch( "pystack.__main__.CoreFileAnalyzer" ), patch( @@ -851,8 +893,8 @@ def path_exists(what): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch.object( Path, "exists", path_exists @@ -865,7 +907,8 @@ def path_exists(what): # THEN get_process_threads_mock.assert_not_called() - print_thread_mock.assert_not_called() + TracebackPrinterMock.assert_not_called() + TracebackPrinterMock.return_value.print_thread.assert_not_called() def test_process_core_executable_does_not_exit(): @@ -883,8 +926,8 @@ def does_exit(what): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "pystack.__main__.is_gzip", return_value=False ), patch( "sys.argv", argv @@ -898,7 +941,8 @@ def does_exit(what): # THEN get_process_threads_mock.assert_not_called() - print_thread_mock.assert_not_called() + TracebackPrinterMock.assert_not_called() + TracebackPrinterMock.return_value.print_thread.assert_not_called() @pytest.mark.parametrize( @@ -914,8 +958,8 @@ def test_process_core_error(exception, exval, capsys): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -935,7 +979,10 @@ def test_process_core_error(exception, exval, capsys): # THEN get_process_threads_mock.assert_called_once() - print_thread_mock.assert_not_called() + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) + TracebackPrinterMock.return_value.print_thread.assert_not_called() capture = capsys.readouterr() assert "Oh no!" in capture.err @@ -951,8 +998,8 @@ def test_process_core_exhaustive(): with patch( "pystack.__main__.get_process_threads_for_core" ) as get_process_threads_mock, patch( - "pystack.__main__.print_thread" - ) as print_thread_mock, patch( + "pystack.__main__.TracebackPrinter" + ) as TracebackPrinterMock, patch( "sys.argv", argv ), patch( "pathlib.Path.exists", return_value=True @@ -976,8 +1023,11 @@ def test_process_core_exhaustive(): locals=False, method=StackMethod.ALL, ) - assert print_thread_mock.mock_calls == [ - call(thread, NativeReportingMode.OFF) for thread in threads + TracebackPrinterMock.assert_called_once_with( + NativeReportingMode.OFF, include_subinterpreters=True + ) + assert TracebackPrinterMock.return_value.print_thread.mock_calls == [ + call(thread) for thread in threads ] @@ -990,7 +1040,7 @@ def test_default_colored_output(): # WHEN with patch("pystack.__main__.get_process_threads"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("os.environ", environ): main() @@ -1008,7 +1058,7 @@ def test_nocolor_output(): # WHEN with patch("pystack.__main__.get_process_threads"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("os.environ", environ): main() @@ -1026,7 +1076,7 @@ def test_nocolor_output_at_the_front_for_process(): # WHEN with patch("pystack.__main__.get_process_threads"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("os.environ", environ): main() @@ -1043,7 +1093,7 @@ def test_nocolor_output_at_the_front_for_core(): # WHEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("os.environ", environ), patch( "pathlib.Path.exists", return_value=True ), patch( @@ -1069,7 +1119,7 @@ def test_global_options_can_be_placed_at_any_point(option): # WHEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("os.environ", environ), patch( "pathlib.Path.exists", return_value=True ), patch( @@ -1092,7 +1142,7 @@ def test_verbose_as_global_options_sets_correctly_the_logger(): # WHEN with patch("pystack.__main__.get_process_threads"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("sys.argv", argv), patch("os.environ", environ), patch( "pathlib.Path.exists", return_value=True ), patch( @@ -1241,7 +1291,7 @@ def test_process_core_does_not_crash_if_core_analyzer_fails(method): # WHEN / THEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("pystack.__main__.is_elf", return_value=True), patch( "pystack.__main__.is_gzip", return_value=False ), patch( @@ -1268,7 +1318,7 @@ def test_core_file_missing_modules_are_logged(caplog, native): # WHEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("pystack.__main__.is_elf", return_value=True), patch( "pystack.__main__.is_gzip", return_value=False ), patch( @@ -1301,7 +1351,7 @@ def test_core_file_missing_build_ids_are_logged(caplog, native): # WHEN with patch("pystack.__main__.get_process_threads_for_core"), patch( - "pystack.__main__.print_thread" + "pystack.__main__.TracebackPrinter" ), patch("pystack.__main__.is_elf", return_value=True), patch( "pystack.__main__.is_gzip", return_value=False ), patch( @@ -1342,7 +1392,7 @@ def test_executable_is_not_elf_uses_the_first_map(): with patch( "pystack.__main__.get_process_threads_for_core" - ) as get_process_threads_mock, patch("pystack.__main__.print_thread"), patch( + ) as get_process_threads_mock, patch("pystack.__main__.TracebackPrinter"), patch( "pystack.__main__.is_elf", lambda x: x == real_executable ), patch( "pystack.__main__.is_gzip", return_value=False diff --git a/tests/unit/test_traceback_formatter.py b/tests/unit/test_traceback_formatter.py index 636cfc6b..86c9c3ad 100644 --- a/tests/unit/test_traceback_formatter.py +++ b/tests/unit/test_traceback_formatter.py @@ -4,8 +4,8 @@ import pytest from pystack.engine import NativeReportingMode +from pystack.traceback_formatter import TracebackPrinter from pystack.traceback_formatter import format_thread -from pystack.traceback_formatter import print_thread from pystack.types import SYMBOL_IGNORELIST from pystack.types import LocationInfo from pystack.types import NativeFrame @@ -1205,6 +1205,7 @@ def test_traceback_formatter_native_last(): def test_print_thread(capsys): + printer = TracebackPrinter(NativeReportingMode.OFF) # GIVEN thread = PyThread( tid=1, @@ -1220,7 +1221,9 @@ def test_print_thread(capsys): "pystack.traceback_formatter.format_thread", return_value=("1", "2", "3"), ): - print_thread(thread, NativeReportingMode.OFF) + printer.print_thread( + thread, + ) # THEN @@ -1629,3 +1632,270 @@ def test_native_traceback_with_shim_frames(): colored_mock.assert_any_call("x =", color="blue") colored_mock.assert_any_call('"This is the line 2" ', color="blue") colored_mock.assert_any_call("(1+1)", color="blue") + + +@pytest.mark.parametrize( + "native_mode", + [ + NativeReportingMode.OFF, + NativeReportingMode.ALL, + NativeReportingMode.PYTHON, + NativeReportingMode.LAST, + ], +) +def test_traceback_printer_created_with_native_level(native_mode): + # GIVEN / WHEN + printer = TracebackPrinter(native_mode) + + # THEN + assert printer.native_mode is native_mode + assert printer.include_subinterpreters is False + assert printer._current_interpreter_id == -1 + + +def test_traceback_printer_created_with_subinterpreters(): + # GIVEN / WHEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + + # THEN + assert printer.native_mode is NativeReportingMode.OFF + assert printer.include_subinterpreters is True + + +def test_print_thread_passes_native_mode_to_format_thread(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.ALL) + thread = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1", "line2"), + ) as format_mock: + printer.print_thread(thread) + + # THEN + format_mock.assert_called_once_with(thread, NativeReportingMode.ALL) + captured = capsys.readouterr() + assert captured.out == "line1\nline2\n" + + +def test_print_thread_with_subinterpreters(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + thread = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interpreter_id=0, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1", "line2"), + ): + printer.print_thread(thread) + + # THEN + captured = capsys.readouterr() + assert "Interpreter-0 (main)" in captured.out + # Lines should be indented with 2 spaces + assert " line1\n" in captured.out + assert " line2\n" in captured.out + + +def test_print_thread_with_subinterpreters_nonzero_interp(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + thread = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interpreter_id=2, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1",), + ): + printer.print_thread(thread) + + # THEN + captured = capsys.readouterr() + assert "Interpreter-2\n" in captured.out + assert " line1\n" in captured.out + + +def test_print_thread_with_subinterpreters_none_interp(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + thread = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interpreter_id=None, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1",), + ): + printer.print_thread(thread) + + # THEN + captured = capsys.readouterr() + assert "Interpreter-Unknown\n" in captured.out + + +def test_print_thread_with_subinterpreters_same_interp_no_repeat_header(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + thread1 = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interpreter_id=1, + ) + thread2 = PyThread( + tid=2, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interpreter_id=1, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1",), + ): + printer.print_thread(thread1) + printer.print_thread(thread2) + + # THEN + captured = capsys.readouterr() + # Header should appear only once + assert captured.out.count("Interpreter-1") == 1 + + +def test_print_thread_with_subinterpreters_main_interp_no_repeat_header(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + thread1 = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interpreter_id=0, + ) + thread2 = PyThread( + tid=2, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interpreter_id=0, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1",), + ): + printer.print_thread(thread1) + printer.print_thread(thread2) + + # THEN + captured = capsys.readouterr() + # Header should appear only once + assert captured.out.count("Interpreter-0 (main)") == 1 + + +def test_print_thread_with_subinterpreters_different_interps_prints_headers(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=True) + thread1 = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interpreter_id=1, + ) + thread2 = PyThread( + tid=2, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interpreter_id=2, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1",), + ): + printer.print_thread(thread1) + printer.print_thread(thread2) + + # THEN + captured = capsys.readouterr() + assert "Interpreter-1\n" in captured.out + assert "Interpreter-2\n" in captured.out + + +def test_print_thread_without_subinterpreters_no_indentation(capsys): + # GIVEN + printer = TracebackPrinter(NativeReportingMode.OFF, include_subinterpreters=False) + thread = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + interpreter_id=1, + ) + + # WHEN + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("line1", "line2"), + ): + printer.print_thread(thread) + + # THEN + captured = capsys.readouterr() + # No interpreter header and no indentation + assert "Interpreter" not in captured.out + assert captured.out == "line1\nline2\n" diff --git a/tests/utils.py b/tests/utils.py index dacb18fa..dcf1f639 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -307,6 +307,13 @@ def all_pystack_combinations(corefile=False, native=False): ) +ALL_PYTHONS_THAT_SUPPORT_SUBINTERPRETERS = pytest.mark.parametrize( + "python", + [python[:2] for python in AVAILABLE_PYTHONS if python.version >= (3, 14)], + ids=[python[1].name for python in AVAILABLE_PYTHONS if python.version >= (3, 14)], +) + + ALL_PYTHONS_THAT_DO_NOT_SUPPORT_ELF_DATA = pytest.mark.parametrize( "python", [python[:2] for python in AVAILABLE_PYTHONS if python.version < (3, 10)],