macOS dylib build is missing $(MODLIBS) #144942
Description
Bug report
Bug description:
I maintain a cross-platform Python distribution, notably with all of lib-dynload statically compiled into libpython (so there's no lib-dynload folder at all).
When upgrading it from 3.13 to 3.14, I ran into an issue where some of the HACL objects were never linked into the dylib, notably resulting in a null pointer crash when using the blake2 module.
I think this is two problems:
- the dylib build in
Makefile.pre.inis missing$(MODLIBS) - the dylib build has the flag
-undefined dynamic_lookup, which means the missing symbols will just be null pointers at runtime with no build-time errors.
In practice, the python exe and testembed both get the HACL objects linked into them directly, which I can't imagine is intentional. It seems like those should all be included in libpython.dylib or libpython.a and the embedded version shouldn't be linking any objects directly.
I believe this diff will fix the two main issues. I've tested it both in my build, and with a fairly normal 3.14.3 build. (I'm not sure what to do about the hacl objects being redundantly linked into python.exe / testembed, but that seems harmless in my test)
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 38a355a23f2..6b41310e70c 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1026,7 +1026,7 @@ libpython3.so: libpython$(LDVERSION).so
$(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^
libpython$(LDVERSION).dylib: $(LIBRARY_OBJS)
- $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \
+ $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \
libpython$(VERSION).sl: $(LIBRARY_OBJS)Here are the link commands observed in my build. Note specifically HACL objects like Modules/_hacl/Hacl_Hash_Blake2s.o are linked into python.exe but not libpython3.14t.dylib.
# libpython3.14t.dylib
gcc -dynamiclib -arch arm64 -arch x86_64 -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.2.sdk -I/var/folders/8c/340skkm557l4n5c1d7rfpyh80000gn/T/talon-vendor/include -L/var/folders/8c/340skkm557l4n5c1d7rfpyh80000gn/T/talon-vendor/lib Modules/_decimal/libmpdec/libmpdec.a Modules/expat/libexpat.a -lbz2
-llzma -lz -lzstd -lssl -lcrypto -lffi -lreadline -ltermcap -undefined dynamic_lookup -Wl,-install_name,/var/folders/8c/340skkm557l4n5c1d7rfpyh80000gn/T/talon-vendor/python/lib/libpython3.14t.dylib -Wl,-compatibility_version,3.14 -Wl,-current_version,3.14 -o libpython3.14t.dylib Modules/getbuildinfo.o Parser/token.o Parser/pegen.o Parser/pegen_errors.o
Parser/action_helpers.o Parser/parser.o Parser/string_parser.o Parser/peg_api.o Parser/lexer/buffer.o Parser/lexer/lexer.o Parser/lexer/state.o Parser/tokenizer/file_tokenizer.o Parser/tokenizer/readline_tokenizer.o Parser/tokenizer/string_tokenizer.o Parser/tokenizer/utf8_tokenizer.o Parser/tokenizer/helpers.o Parser/myreadline.o Objects/abstract.o
Objects/boolobject.o Objects/bytes_methods.o Objects/bytearrayobject.o Objects/bytesobject.o Objects/call.o Objects/capsule.o Objects/cellobject.o Objects/classobject.o Objects/codeobject.o Objects/complexobject.o Objects/descrobject.o Objects/enumobject.o Objects/exceptions.o Objects/genericaliasobject.o Objects/genobject.o Objects/fileobject.o Objects/
floatobject.o Objects/frameobject.o Objects/funcobject.o Objects/interpolationobject.o Objects/iterobject.o Objects/listobject.o Objects/longobject.o Objects/dictobject.o Objects/odictobject.o Objects/memoryobject.o Objects/methodobject.o Objects/moduleobject.o Objects/namespaceobject.o Objects/object.o Objects/obmalloc.o Objects/picklebufobject.o Objects/
rangeobject.o Objects/setobject.o Objects/sliceobject.o Objects/structseq.o Objects/templateobject.o Objects/tupleobject.o Objects/typeobject.o Objects/typevarobject.o Objects/unicodeobject.o Objects/unicodectype.o Objects/unionobject.o Objects/weakrefobject.o Python/_contextvars.o Python/_warnings.o Python/Python-ast.o Python/Python-tokenize.o Python/
asdl.o Python/assemble.o Python/ast.o Python/ast_preprocess.o Python/ast_unparse.o Python/bltinmodule.o Python/brc.o Python/ceval.o Python/codecs.o Python/codegen.o Python/compile.o Python/context.o Python/critical_section.o Python/crossinterp.o Python/dynamic_annotations.o Python/errors.o Python/flowgraph.o Python/frame.o Python/frozenmain.o Python/
future.o Python/gc.o Python/gc_free_threading.o Python/gc_gil.o Python/getargs.o Python/getcompiler.o Python/getcopyright.o Python/getplatform.o Python/getversion.o Python/ceval_gil.o Python/hamt.o Python/hashtable.o Python/import.o Python/importdl.o Python/index_pool.o Python/initconfig.o Python/interpconfig.o Python/instrumentation.o Python/
instruction_sequence.o Python/intrinsics.o Python/jit.o Python/legacy_tracing.o Python/lock.o Python/marshal.o Python/modsupport.o Python/mysnprintf.o Python/mystrtoul.o Python/object_stack.o Python/optimizer.o Python/optimizer_analysis.o Python/optimizer_symbols.o Python/parking_lot.o Python/pathconfig.o Python/preconfig.o Python/pyarena.o Python/pyctype.o
Python/pyfpe.o Python/pyhash.o Python/pylifecycle.o Python/pymath.o Python/pystate.o Python/pythonrun.o Python/pytime.o Python/qsbr.o Python/bootstrap_hash.o Python/specialize.o Python/stackrefs.o Python/structmember.o Python/symtable.o Python/sysmodule.o Python/thread.o Python/traceback.o Python/tracemalloc.o Python/uniqueid.o Python/getopt.o Python/
pystrcmp.o Python/pystrtod.o Python/pystrhex.o Python/dtoa.o Python/formatter_unicode.o Python/fileutils.o Python/suggestions.o Python/perf_trampoline.o Python/perf_jit_trampoline.o Python/remote_debugging.o Python/dynload_shlib.o Modules/config.o Modules/main.o Modules/gcmodule.o Modules/_asynciomodule.o Modules/_bisectmodule.o Modules/_csv.o
Modules/_datetimemodule.o Modules/_decimal/_decimal.o Modules/_heapqmodule.o Modules/_interpchannelsmodule.o Modules/_interpqueuesmodule.o Modules/_interpretersmodule.o Modules/_json.o Modules/_lsprof.o Modules/rotatingtree.o Modules/_multiprocessing/multiprocessing.o Modules/_multiprocessing/semaphore.o Modules/_opcode.o Modules/_pickle.o
Modules/_queuemodule.o Modules/_randommodule.o Modules/socketmodule.o Modules/_statisticsmodule.o Modules/_struct.o Modules/_typesmodule.o Modules/_typingmodule.o Modules/_zoneinfo.o Modules/arraymodule.o Modules/binascii.o Modules/cmathmodule.o Modules/mathmodule.o Modules/mmapmodule.o Modules/selectmodule.o Modules/_sysconfig.o Modules/
_elementtree.o Modules/pyexpat.o Modules/blake2module.o Modules/md5module.o Modules/sha1module.o Modules/sha2module.o Modules/sha3module.o Modules/cjkcodecs/_codecs_cn.o Modules/cjkcodecs/_codecs_hk.o Modules/cjkcodecs/_codecs_iso2022.o Modules/cjkcodecs/_codecs_jp.o Modules/cjkcodecs/_codecs_kr.o Modules/cjkcodecs/_codecs_tw.o Modules/
cjkcodecs/multibytecodec.o Modules/unicodedata.o Modules/_posixsubprocess.o Modules/fcntlmodule.o Modules/grpmodule.o Modules/resource.o Modules/syslogmodule.o Modules/termios.o Modules/_multiprocessing/posixshmem.o Modules/_bz2module.o Modules/_lzmamodule.o Modules/_uuidmodule.o Modules/_zstd/_zstdmodule.o Modules/_zstd/compressor.o Modules/
_zstd/decompressor.o Modules/_zstd/zstddict.o Modules/zlibmodule.o Modules/_sqlite/blob.o Modules/_sqlite/connection.o Modules/_sqlite/cursor.o Modules/_sqlite/microprotocols.o Modules/_sqlite/module.o Modules/_sqlite/prepare_protocol.o Modules/_sqlite/row.o Modules/_sqlite/statement.o Modules/_sqlite/util.o Modules/_ssl.o Modules/_hashopenssl.o
Modules/_scproxy.o Modules/_ctypes/_ctypes.o Modules/_ctypes/callbacks.o Modules/_ctypes/callproc.o Modules/_ctypes/stgdict.o Modules/_ctypes/cfield.o Modules/_ctypes/malloc_closure.o Modules/readline.o Modules/_cursesmodule.o Modules/_curses_panel.o Modules/atexitmodule.o Modules/faulthandler.o Modules/posixmodule.o Modules/signalmodule.o Modules/
_tracemalloc.o Modules/_suggestions.o Modules/_codecsmodule.o Modules/_collectionsmodule.o Modules/errnomodule.o Modules/_io/_iomodule.o Modules/_io/iobase.o Modules/_io/fileio.o Modules/_io/bytesio.o Modules/_io/bufferedio.o Modules/_io/textio.o Modules/_io/stringio.o Modules/itertoolsmodule.o Modules/_sre/sre.o Modules/_threadmodule.o Modules/
timemodule.o Modules/_weakref.o Modules/_abc.o Modules/_functoolsmodule.o Modules/_localemodule.o Modules/_operator.o Modules/_stat.o Modules/symtablemodule.o Modules/pwdmodule.o Modules/getpath.o Python/frozen.o -ldl -framework CoreFoundation ; \
# python.exe
gcc -arch arm64 -arch x86_64 -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.2.sdk -I/var/folders/8c/340skkm557l4n5c1d7rfpyh80000gn/T/talon-vendor/include -L/var/folders/8c/340skkm557l4n5c1d7rfpyh80000gn/T/talon-vendor/lib Modules/_decimal/libmpdec/libmpdec.a Modules/expat/libexpat.a -lbz2 -llzma -lz
-lzstd -lssl -lcrypto -lffi -lreadline -ltermcap -Wl,-stack_size,1000000 -framework CoreFoundation -o python.exe Programs/python.o -L. -lpython3.14t -ldl -framework CoreFoundation -lm Modules/_decimal/libmpdec/libmpdec.a -lz -lm Modules/expat/libexpat.a Modules/_hacl/Hacl_Hash_Blake2s.o Modules/_hacl/Hacl_Hash_Blake2b.o
Modules/_hacl/Lib_Memzero0.o Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o Modules/_hacl/Hacl_Hash_MD5.o Modules/_hacl/Hacl_Hash_SHA1.o Modules/_hacl/Hacl_Hash_SHA2.o Modules/_hacl/Hacl_Hash_SHA3.o -lbz2 -L/var/folders/8c/340skkm557l4n5c1d7rfpyh80000gn/T/talon-vendor/lib -llzma -lz
-lsqlite3 -L/var/folders/8c/340skkm557l4n5c1d7rfpyh80000gn/T/talon-vendor/lib -lssl -lcrypto -L/var/folders/8c/340skkm557l4n5c1d7rfpyh80000gn/T/talon-vendor/lib -lcrypto -framework SystemConfiguration -framework CoreFoundation -lffi -ldl -lreadline -lncurses -lpanel -lncurses
ld: warning: ignoring duplicate libraries: '-lbz2', '-lcrypto', '-ldl', '-lffi', '-llzma', '-lm', '-lncurses', '-lreadline', '-lssl', '-lz', 'Modules/_decimal/libmpdec/libmpdec.a', 'Modules/expat/libexpat.a'
ld: warning: ignoring duplicate libraries: '-lbz2', '-lcrypto', '-ldl', '-lffi', '-llzma', '-lm', '-lncurses', '-lreadline', '-lssl', '-lz', 'Modules/_decimal/libmpdec/libmpdec.a', 'Modules/expat/libexpat.a'
./python.exe -E -S -m sysconfig --generate-posix-vars ;\
if test $? -ne 0 ; then \
echo "generate-posix-vars failed" ; \
rm -f ./pybuilddir.txt ; \
exit 1 ; \
fi
# Written build/lib.macosx-10.15-universal2-3.14/_sysconfigdata_t_darwin_darwin.py
# Written build/lib.macosx-10.15-universal2-3.14/_sysconfig_vars_t_darwin_darwin.json
./python.exe -E -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform
# _testembed.o
gcc -arch arm64 -arch x86_64 -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.2.sdk -I/var/folders/8c/340skkm557l4n5c1d7rfpyh80000gn/T/talon-vendor/include -L/var/folders/8c/340skkm557l4n5c1d7rfpyh80000gn/T/talon-vendor/lib Modules/_decimal/libmpdec/libmpdec.a Modules/expat/libexpat.a -lbz2 -llzma -lz
-lzstd -lssl -lcrypto -lffi -lreadline -ltermcap -Wl,-stack_size,1000000 -framework CoreFoundation -o Programs/_testembed Programs/_testembed.o -L. -lpython3.14t -ldl -framework CoreFoundation -lm Modules/_decimal/libmpdec/libmpdec.a -lz -lm Modules/expat/libexpat.a Modules/_hacl/Hacl_Hash_Blake2s.o Modules/_hacl/
Hacl_Hash_Blake2b.o Modules/_hacl/Lib_Memzero0.o Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o Modules/_hacl/Hacl_Hash_MD5.o Modules/_hacl/Hacl_Hash_SHA1.o Modules/_hacl/Hacl_Hash_SHA2.o Modules/_hacl/Hacl_Hash_SHA3.o -lbz2 -L/var/folders/8c/340skkm557l4n5c1d7rfpyh80000gn/T/talon-
vendor/lib -llzma -lz -lsqlite3 -L/var/folders/8c/340skkm557l4n5c1d7rfpyh80000gn/T/talon-vendor/lib -lssl -lcrypto -L/var/folders/8c/340skkm557l4n5c1d7rfpyh80000gn/T/talon-vendor/lib -lcrypto -framework SystemConfiguration -framework CoreFoundation -lffi -ldl -lreadline -lncurses -lpanel -lncursesCPython versions tested on:
3.14
Operating systems tested on:
macOS