Add -headerpad_max_install_names flag for macOS dylib builds

This fixes the install_name_tool issue on macOS where the tool fails
with "larger updated load commands do not fit". The linker flag
-Wl,-headerpad_max_install_names ensures sufficient header padding
in the Mach-O binary for install_name_tool to modify install names.

Changes made:
- CMake: Added flag to libz3, z3java, z3jl shared libraries on Darwin
- Python build: Changed flag from ML-only to all macOS builds
- OCaml CMake: Added flag to OCaml stublib build on APPLE

Fixes #7623 (regression in 4.15.5)

Co-authored-by: NikolajBjorner <3085284+NikolajBjorner@users.noreply.github.com>

scripts/mk_util.py

@@ -2749,7 +2749,9 @@ def mk_config():
             CXXFLAGS = '%s -arch arm64' % CXXFLAGS
             LDFLAGS = '%s -arch arm64' % LDFLAGS
             SLIBEXTRAFLAGS = '%s -arch arm64' % SLIBEXTRAFLAGS
-        if IS_OSX and is_ml_enabled():
+        # Add header padding for macOS to allow install_name_tool to modify the dylib
+        # This fixes issues where install_name_tool fails with "larger updated load commands do not fit"
+        if IS_OSX:
             SLIBFLAGS += ' -Wl,-headerpad_max_install_names'
 
         config.write('PREFIX=%s\n' % PREFIX)

src/CMakeLists.txt

@@ -134,6 +134,10 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
     # Enable RPATH support
     MACOSX_RPATH TRUE
   )
+  # Add header padding to allow install_name_tool to modify the dylib
+  # This fixes issues where install_name_tool fails with "larger updated load commands do not fit"
+  # See: https://github.com/Z3Prover/z3/issues/7623
+  target_link_options(libz3 PRIVATE "-Wl,-headerpad_max_install_names")
 endif()
 
 if (NOT MSVC)

src/api/java/CMakeLists.txt

@@ -48,6 +48,10 @@ target_include_directories(z3java PRIVATE
   "${PROJECT_BINARY_DIR}/src/api"
   ${JNI_INCLUDE_DIRS}
 )
+# Add header padding for macOS to allow install_name_tool to modify the dylib
+if (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+  target_link_options(z3java PRIVATE "-Wl,-headerpad_max_install_names")
+endif()
 # FIXME: Should this library have SONAME and VERSION set?
 
 # This prevents CMake from automatically defining ``z3java_EXPORTS``

src/api/julia/CMakeLists.txt

@@ -34,6 +34,10 @@ target_include_directories(z3jl PRIVATE
     "${PROJECT_BINARY_DIR}/src/api"
     "${PROJECT_SOURCE_DIR}/src/api/c++"
 )
+# Add header padding for macOS to allow install_name_tool to modify the dylib
+if (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+  target_link_options(z3jl PRIVATE "-Wl,-headerpad_max_install_names")
+endif()
 
 option(Z3_INSTALL_JULIA_BINDINGS "Install Julia bindings when invoking install target" ON)
 if(Z3_INSTALL_JULIA_BINDINGS)

src/api/ml/CMakeLists.txt

@@ -159,6 +159,9 @@ endif()
 
 if( APPLE )
   # set(ocaml_rpath "@executable_path/../libz3${so_ext}")
+  # Add header padding to allow install_name_tool to modify the dylib
+  # This fixes issues where install_name_tool fails with "larger updated load commands do not fit"
+  list(APPEND c_lib_deps "-ldopt" "-Wl,-headerpad_max_install_names")
 elseif( UNIX )
   set(ocaml_rpath "\\$ORIGIN/../libz3${so_ext}")
   list(APPEND c_lib_deps "-dllpath" ${ocaml_rpath})