From c714d47f760e1e79cd8e7c6eef5167139e50ab11 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Fri, 2 Oct 2020 00:11:15 +0200 Subject: [PATCH 30/30] gh21564 --- Lib/_osx_support.py | 2 + Lib/ctypes/macholib/dyld.py | 5 + Lib/distutils/tests/test_build_ext.py | 2 +- Lib/test/test_bytes.py | 1 + Lib/test/test_platform.py | 2 +- Lib/test/test_unicode.py | 2 + Mac/BuildScript/build-installer.py | 47 +++++++- Mac/Tools/pythonw.c | 6 +- Modules/_ctypes/callbacks.c | 39 +++++-- Modules/_ctypes/callproc.c | 149 ++++++++++++++++++-------- Modules/_ctypes/ctypes.h | 8 ++ Modules/_ctypes/malloc_closure.c | 15 ++- configure | 16 ++- configure.ac | 16 ++- setup.py | 131 +++++++++++++--------- 15 files changed, 320 insertions(+), 121 deletions(-) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index e9efce7d7e..8ed1eeac85 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -481,6 +481,8 @@ def get_platform_osx(_config_vars, osname, release, machine): if len(archs) == 1: machine = archs[0] + elif archs == ('arm64', 'x86_64'): + machine = 'universal2' elif archs == ('i386', 'ppc'): machine = 'fat' elif archs == ('i386', 'x86_64'): diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index bf0e8e8dee..d230b23da9 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -6,6 +6,11 @@ import os from ctypes.macholib.framework import framework_info from ctypes.macholib.dylib import dylib_info from itertools import * +try: + from _ctypes import _dyld_shared_cache_contains_path +except ImportError: + def _dyld_shared_cache_contains_path(*args): + raise NotImplementedError try: from _ctypes import _dyld_shared_cache_contains_path diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index 5e47e0773a..5a32e03980 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -492,7 +492,7 @@ class BuildExtTestCase(TempdirManager, # format the target value as defined in the Apple # Availability Macros. We can't use the macro names since # at least one value we test with will not exist yet. - if target[1] < 10: + if target[:2] < (10, 10): # for 10.1 through 10.9.x -> "10n0" target = '%02d%01d0' % target else: diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 770e2c5592..05568f2c67 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1034,6 +1034,7 @@ class BytesTest(BaseBytesTest, unittest.TestCase): c_char_p) PyBytes_FromFormat = pythonapi.PyBytes_FromFormat + PyBytes_FromFormat.argtypes = (c_char_p,) PyBytes_FromFormat.restype = py_object # basic tests diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index a5c35dff79..bd953b5d67 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -245,7 +245,7 @@ class PlatformTest(unittest.TestCase): self.assertEqual(res[1], ('', '', '')) if sys.byteorder == 'little': - self.assertIn(res[2], ('i386', 'x86_64')) + self.assertIn(res[2], ('i386', 'x86_64', 'arm64')) else: self.assertEqual(res[2], 'PowerPC') diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 2ee4e64d63..23508c56e7 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2523,11 +2523,13 @@ class CAPITest(unittest.TestCase): def test_from_format(self): support.import_module('ctypes') from ctypes import ( + c_char_p, pythonapi, py_object, sizeof, c_int, c_long, c_longlong, c_ssize_t, c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) name = "PyUnicode_FromFormat" _PyUnicode_FromFormat = getattr(pythonapi, name) + _PyUnicode_FromFormat.argtypes = (c_char_p,) _PyUnicode_FromFormat.restype = py_object def PyUnicode_FromFormat(format, *args): diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index a58b922ce3..56fc20390c 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -116,7 +116,8 @@ WORKDIR = "/tmp/_py" DEPSRC = os.path.join(WORKDIR, 'third-party') DEPSRC = os.path.expanduser('~/Universal/other-sources') -universal_opts_map = { '32-bit': ('i386', 'ppc',), +universal_opts_map = { 'universal2': ('arm64', 'x86_64'), + '32-bit': ('i386', 'ppc',), '64-bit': ('x86_64', 'ppc64',), 'intel': ('i386', 'x86_64'), 'intel-32': ('i386',), @@ -124,6 +125,7 @@ universal_opts_map = { '32-bit': ('i386', 'ppc',), '3-way': ('ppc', 'i386', 'x86_64'), 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } default_target_map = { + 'universal2': '10.9', '64-bit': '10.5', '3-way': '10.5', 'intel': '10.5', @@ -148,6 +150,10 @@ SRCDIR = os.path.dirname( # $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level DEPTARGET = '10.5' +# If true only builds the 3th-party dependencies +# in $WORKDIR +DEPS_ONLY=False + def getDeptargetTuple(): return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) @@ -190,6 +196,27 @@ EXPECTED_SHARED_LIBS = {} def internalTk(): return getDeptargetTuple() >= (10, 6) + +def tweak_tcl_build(basedir, archList): + with open("Makefile", "r") as fp: + contents = fp.readlines() + + # For reasons I don't understand the tcl configure script + # decides that some stdlib symbols aren't present, before + # deciding that strtod is broken. + new_contents = [] + for line in contents: + if line.startswith("COMPAT_OBJS"): + # note: the space before strtod.o is intentional, + # the detection of a broken strtod results in + # "fixstrod.o" on this line. + for nm in ("strstr.o", "strtoul.o", " strtod.o"): + line = line.replace(nm, "") + new_contents.append(line) + + with open("Makefile", "w") as fp: + fp.writelines(new_contents) + # List of names of third party software built with this installer. # The names will be inserted into the rtf version of the License. THIRD_PARTY_LIBS = [] @@ -215,6 +242,9 @@ def library_recipes(): buildrecipe=build_universal_openssl, configure=None, install=None, + patches=[ + "openssl-mac-arm64.patch", + ], ), ]) @@ -231,6 +261,7 @@ def library_recipes(): '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), ], useLDFlags=False, + buildrecipe=tweak_tcl_build, install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), @@ -596,7 +627,7 @@ def checkEnvironment(): ev, os.environ[ev])) del os.environ[ev] - base_path = '/bin:/sbin:/usr/bin:/usr/sbin' + base_path = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin' if 'SDK_TOOLS_BIN' in os.environ: base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin; @@ -618,6 +649,7 @@ def parseOptions(args=None): global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX global FW_VERSION_PREFIX global FW_SSL_DIRECTORY + global DEPS_ONLY if args is None: args = sys.argv[1:] @@ -625,7 +657,7 @@ def parseOptions(args=None): try: options, args = getopt.getopt(args, '?hb', [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=', - 'dep-target=', 'universal-archs=', 'help' ]) + 'dep-target=', 'universal-archs=', 'deps-only', 'help' ]) except getopt.GetoptError: print(sys.exc_info()[1]) sys.exit(1) @@ -646,6 +678,9 @@ def parseOptions(args=None): elif k in ('--third-party',): DEPSRC=v + elif k in ('--deps-only',): + DEPS_ONLY=True + elif k in ('--sdk-path',): print(" WARNING: --sdk-path is no longer supported") @@ -691,6 +726,8 @@ def parseOptions(args=None): print(" -- Building a Python %s framework at patch level %s" % (getVersion(), getFullVersion())) print("") + if DEPS_ONLY: + print("Stopping after building third-party libraries") def extractArchive(builddir, archiveName): """ @@ -801,6 +838,7 @@ def build_universal_openssl(basedir, archList): arch_opts = { "i386": ["darwin-i386-cc"], "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"], + "arm64": ["darwin64-arm64-cc" ], "ppc": ["darwin-ppc-cc"], "ppc64": ["darwin64-ppc-cc"], } @@ -1656,6 +1694,9 @@ def main(): # Then build third-party libraries such as sleepycat DB4. buildLibraries() + if DEPS_ONLY: + sys.exit(1) + # Now build python itself buildPython() diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c index ae4c262129..78813e818e 100644 --- a/Mac/Tools/pythonw.c +++ b/Mac/Tools/pythonw.c @@ -95,9 +95,6 @@ setup_spawnattr(posix_spawnattr_t* spawnattr) size_t count; cpu_type_t cpu_types[1]; short flags = 0; -#ifdef __LP64__ - int ch; -#endif if ((errno = posix_spawnattr_init(spawnattr)) != 0) { err(2, "posix_spawnattr_int"); @@ -226,7 +223,8 @@ main(int argc, char **argv) { /* We're weak-linking to posix-spawnv to ensure that * an executable build on 10.5 can work on 10.4. */ - if (posix_spawn != NULL) { + + if (&posix_spawn != NULL) { posix_spawnattr_t spawnattr = NULL; setup_spawnattr(&spawnattr); diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 2abfa67cdc..77d4855277 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -1,6 +1,8 @@ #include "Python.h" #include "frameobject.h" +#include + #include #ifdef MS_WIN32 #include @@ -18,7 +20,7 @@ CThunkObject_dealloc(PyObject *myself) Py_XDECREF(self->callable); Py_XDECREF(self->restype); if (self->pcl_write) - ffi_closure_free(self->pcl_write); + Py_ffi_closure_free(self->pcl_write); PyObject_GC_Del(self); } @@ -361,8 +363,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, assert(CThunk_CheckExact((PyObject *)p)); - p->pcl_write = ffi_closure_alloc(sizeof(ffi_closure), - &p->pcl_exec); + p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec); if (p->pcl_write == NULL) { PyErr_NoMemory(); goto error; @@ -408,13 +409,35 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, "ffi_prep_cif failed with %d", result); goto error; } -#if defined(X86_DARWIN) || defined(POWERPC_DARWIN) - result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); +#if HAVE_FFI_PREP_CLOSURE_LOC +# if USING_APPLE_OS_LIBFFI +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# else +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME true +# endif + if (HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME) { + result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, + p, + p->pcl_exec); + } else +#endif + { +#if USING_APPLE_OS_LIBFFI && defined(__arm64__) + PyErr_Format(PyExc_NotImplementedError, "ffi_prep_closure_loc() is missing"); + goto error; #else - result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, - p, - p->pcl_exec); +#ifdef MACOSX + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif + result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); + +#ifdef MACOSX + #pragma clang diagnostic pop +#endif + +#endif + } if (result != FFI_OK) { PyErr_Format(PyExc_RuntimeError, "ffi_prep_closure failed with %d", result); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index f8007505ed..badd85c8fd 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -57,6 +57,8 @@ #include "Python.h" #include "structmember.h" // PyMemberDef +#include + #ifdef MS_WIN32 #include #include @@ -64,8 +66,16 @@ #include "ctypes_dlfcn.h" #endif -#if __APPLE__ && HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH -#include +#ifdef __APPLE__ +/* + * The API to query if a shared library is in the shared cache is + * private for now, this should change in beta 4. + * + * TODO: + * - Switch to that API + * - Add feature macro and runtime guards (as with the ffi.*loc API's) + */ +extern bool _dyld_shared_cache_contains_path(const char* path) __attribute__((weak_import)); #endif #ifdef MS_WIN32 @@ -816,7 +826,8 @@ static int _call_function_pointer(int flags, ffi_type **atypes, ffi_type *restype, void *resmem, - int argcount) + int argcount, + int argtypecount) { PyThreadState *_save = NULL; /* For Py_BLOCK_THREADS and Py_UNBLOCK_THREADS */ PyObject *error_object = NULL; @@ -839,14 +850,60 @@ static int _call_function_pointer(int flags, if ((flags & FUNCFLAG_CDECL) == 0) cc = FFI_STDCALL; #endif - if (FFI_OK != ffi_prep_cif(&cif, - cc, - argcount, - restype, - atypes)) { - PyErr_SetString(PyExc_RuntimeError, - "ffi_prep_cif failed"); - return -1; + +# if USING_APPLE_OS_LIBFFI +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# elif HAVE_FFI_PREP_CIF_VAR +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME true +# else +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME false +# endif + + /* Even on Apple-arm64 the calling convention for variadic functions conincides + * with the standard calling convention in the case that the function called + * only with its fixed arguments. Thus, we do not need a special flag to be + * set on variadic functions. We treat a function as variadic if it is called + * with a nonzero number of variadic arguments */ + bool is_variadic = (argtypecount != 0 && argcount > argtypecount); + (void) is_variadic; + +#if defined(__APPLE__) && defined(__arm64__) + if (is_variadic) { + if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { + } else { + PyErr_SetString(PyExc_NotImplementedError, "ffi_prep_cif_var() is missing"); + return -1; + } + } +#endif + +#if HAVE_FFI_PREP_CIF_VAR + if (is_variadic) { + if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { + if (FFI_OK != ffi_prep_cif_var(&cif, + cc, + argtypecount, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif_var failed"); + return -1; + } + } + } else +#endif + + { + if (FFI_OK != ffi_prep_cif(&cif, + cc, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif failed"); + return -1; + } } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { @@ -1216,9 +1273,8 @@ PyObject *_ctypes_callproc(PPROC pProc, if (-1 == _call_function_pointer(flags, pProc, avalues, atypes, rtype, resbuf, - Py_SAFE_DOWNCAST(argcount, - Py_ssize_t, - int))) + Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int), + Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int))) goto cleanup; #ifdef WORDS_BIGENDIAN @@ -1402,35 +1458,36 @@ copy_com_pointer(PyObject *self, PyObject *args) } #else -#if __APPLE__ && HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH -static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) -{ - if (__builtin_available(macos 11, ios 14, watchos 7, tvos 14, *)) { - PyObject *name, *name2; - char *name_str; - if (!PyArg_ParseTuple(args, "O", &name)) - return NULL; - - if (name == Py_None) - Py_RETURN_FALSE; - - if (PyUnicode_FSConverter(name, &name2) == 0) - return NULL; - if (PyBytes_Check(name2)) - name_str = PyBytes_AS_STRING(name2); - else - name_str = PyByteArray_AS_STRING(name2); - - if(_dyld_shared_cache_contains_path(name_str)) - Py_RETURN_TRUE; - else - Py_RETURN_FALSE; - } else { - PyErr_SetString(PyExc_NotImplementedError, "_dyld_shared_cache_contains_path symbol is missing"); - return NULL; - } -} -#endif +#ifdef __APPLE__ + static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) + { + PyObject *name, *name2; + char *name_str; + + if (_dyld_shared_cache_contains_path == NULL) { + PyErr_SetString(PyExc_NotImplementedError, "_dyld_shared_cache_contains_path symbol is missing"); + return NULL; + } + + if (!PyArg_ParseTuple(args, "O", &name)) + return NULL; + + if (name == Py_None) + Py_RETURN_FALSE; + + if (PyUnicode_FSConverter(name, &name2) == 0) + return NULL; + if (PyBytes_Check(name2)) + name_str = PyBytes_AS_STRING(name2); + else + name_str = PyByteArray_AS_STRING(name2); + + if(_dyld_shared_cache_contains_path(name_str)) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; + } + #endif static PyObject *py_dl_open(PyObject *self, PyObject *args) { @@ -1921,6 +1978,8 @@ buffer_info(PyObject *self, PyObject *arg) return Py_BuildValue("siN", dict->format, dict->ndim, shape); } + + PyMethodDef _ctypes_module_methods[] = { {"get_errno", get_errno, METH_NOARGS}, {"set_errno", set_errno, METH_VARARGS}, @@ -1943,8 +2002,8 @@ PyMethodDef _ctypes_module_methods[] = { {"dlclose", py_dl_close, METH_VARARGS, "dlclose a library"}, {"dlsym", py_dl_sym, METH_VARARGS, "find symbol in shared library"}, #endif -#if __APPLE__ && HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH - {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, +#ifdef __APPLE__ + {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, #endif {"alignment", align_func, METH_O, alignment_doc}, {"sizeof", sizeof_func, METH_O, sizeof_doc}, diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 1effccf9cc..3f20031d67 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -366,6 +366,14 @@ PyObject *_ctypes_get_errobj(int **pspace); extern PyObject *ComError; #endif +#if USING_MALLOC_CLOSURE_DOT_C +void Py_ffi_closure_free(void *p); +void *Py_ffi_closure_alloc(size_t size, void** codeloc); +#else +#define Py_ffi_closure_free ffi_closure_free +#define Py_ffi_closure_alloc ffi_closure_alloc +#endif + /* Local Variables: compile-command: "python setup.py -q build install --home ~" diff --git a/Modules/_ctypes/malloc_closure.c b/Modules/_ctypes/malloc_closure.c index f9cdb33695..4f220e42ff 100644 --- a/Modules/_ctypes/malloc_closure.c +++ b/Modules/_ctypes/malloc_closure.c @@ -89,16 +89,27 @@ static void more_core(void) /******************************************************************/ /* put the item back into the free list */ -void ffi_closure_free(void *p) +void Py_ffi_closure_free(void *p) { +#if USING_APPLE_OS_LIBFFI && HAVE_FFI_CLOSURE_ALLOC + if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { + ffi_closure_free(p); + return; + } +#endif ITEM *item = (ITEM *)p; item->next = free_list; free_list = item; } /* return one item from the free list, allocating more if needed */ -void *ffi_closure_alloc(size_t ignored, void** codeloc) +void *Py_ffi_closure_alloc(size_t size, void** codeloc) { +#if USING_APPLE_OS_LIBFFI && HAVE_FFI_CLOSURE_ALLOC + if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { + return ffi_closure_alloc(size, codeloc); + } +#endif ITEM *item; if (!free_list) more_core(); diff --git a/configure b/configure index 8f94ca7715..6691ea79bc 100755 --- a/configure +++ b/configure @@ -6959,7 +6959,7 @@ fi -# The -arch flags for universal builds on OSX +# The -arch flags for universal builds on macOS UNIVERSAL_ARCH_FLAGS= @@ -7486,6 +7486,11 @@ $as_echo "$CC" >&6; } LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; + universal2) + UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" @@ -7507,7 +7512,7 @@ $as_echo "$CC" >&6; } ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; *) - as_fn_error $? "proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way" "$LINENO" 5 + as_fn_error $? "proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way" "$LINENO" 5 ;; esac @@ -9339,7 +9344,7 @@ fi MACOSX_DEFAULT_ARCH="ppc" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error $? "Unexpected output of 'arch' on macOS" "$LINENO" 5 ;; esac else @@ -9349,9 +9354,12 @@ fi ;; ppc) MACOSX_DEFAULT_ARCH="ppc64" + ;; + arm64) + MACOSX_DEFAULT_ARCH="arm64" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error $? "Unexpected output of 'arch' on macOS" "$LINENO" 5 ;; esac diff --git a/configure.ac b/configure.ac index 40be5384d8..63923a9e2a 100644 --- a/configure.ac +++ b/configure.ac @@ -1589,7 +1589,7 @@ AC_SUBST(BASECFLAGS) AC_SUBST(CFLAGS_NODIST) AC_SUBST(LDFLAGS_NODIST) -# The -arch flags for universal builds on OSX +# The -arch flags for universal builds on macOS UNIVERSAL_ARCH_FLAGS= AC_SUBST(UNIVERSAL_ARCH_FLAGS) @@ -1890,6 +1890,11 @@ yes) LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; + universal2) + UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" @@ -1911,7 +1916,7 @@ yes) ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; *) - AC_MSG_ERROR([proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way]) + AC_MSG_ERROR([proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way]) ;; esac @@ -2481,7 +2486,7 @@ case $ac_sys_system/$ac_sys_release in MACOSX_DEFAULT_ARCH="ppc" ;; *) - AC_MSG_ERROR([Unexpected output of 'arch' on OSX]) + AC_MSG_ERROR([Unexpected output of 'arch' on macOS]) ;; esac else @@ -2491,9 +2496,12 @@ case $ac_sys_system/$ac_sys_release in ;; ppc) MACOSX_DEFAULT_ARCH="ppc64" + ;; + arm64) + MACOSX_DEFAULT_ARCH="arm64" ;; *) - AC_MSG_ERROR([Unexpected output of 'arch' on OSX]) + AC_MSG_ERROR([Unexpected output of 'arch' on macOS]) ;; esac diff --git a/setup.py b/setup.py index a95162b43a..1add5541ce 100644 --- a/setup.py +++ b/setup.py @@ -230,6 +230,35 @@ def macosx_sdk_specified(): return MACOS_SDK_SPECIFIED +def macosx_sdk_specified(): + """Returns true if an SDK was explicitly configured. + + True if an SDK was selected at configure time, either by specifying + --enable-universalsdk=(something other than no or /) or by adding a + -isysroot option to CFLAGS. In some cases, like when making + decisions about macOS Tk framework paths, we need to be able to + know whether the user explicitly asked to build with an SDK versus + the implicit use of an SDK when header files are no longer + installed on a running system by the Command Line Tools. + """ + global MACOS_SDK_SPECIFIED + + # If already called, return cached result. + if MACOS_SDK_SPECIFIED: + return MACOS_SDK_SPECIFIED + + # Find the sdk root and set MACOS_SDK_SPECIFIED + macosx_sdk_root() + return MACOS_SDK_SPECIFIED + +def is_macosx_at_least(vers): + if MACOS: + dep_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + if dep_target: + return tuple(map(int, dep_target.split('.'))) >= vers + return False + + def is_macosx_sdk_path(path): """ Returns True if 'path' can be located in an OSX SDK @@ -239,6 +268,13 @@ def is_macosx_sdk_path(path): or path.startswith('/Library/') ) +def grep_headers_for(function, headers): + for header in headers: + with open(header, 'r') as f: + if function in f.read(): + return True + return False + def find_file(filename, std_dirs, paths): """Searches for the directory where a given file is located, and returns a possibly-empty list of additional directories, or None @@ -278,6 +314,13 @@ def find_file(filename, std_dirs, paths): # Not found anywhere return None +def is_macosx_at_least(vers): + if MACOS: + dep_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + if dep_target: + return tuple(map(int, dep_target.split('.'))) >= vers + return False + def find_library_file(compiler, libname, std_dirs, paths): result = compiler.find_library_file(std_dirs + paths, libname) @@ -2108,43 +2151,18 @@ class PyBuildExt(build_ext): library_dirs=added_lib_dirs)) return True - def configure_ctypes_darwin(self, ext): - # Darwin (OS X) uses preconfigured files, in - # the Modules/_ctypes/libffi_osx directory. - ffi_srcdir = os.path.abspath(os.path.join(self.srcdir, 'Modules', - '_ctypes', 'libffi_osx')) - sources = [os.path.join(ffi_srcdir, p) - for p in ['ffi.c', - 'x86/darwin64.S', - 'x86/x86-darwin.S', - 'x86/x86-ffi_darwin.c', - 'x86/x86-ffi64.c', - 'powerpc/ppc-darwin.S', - 'powerpc/ppc-darwin_closure.S', - 'powerpc/ppc-ffi_darwin.c', - 'powerpc/ppc64-darwin_closure.S', - ]] - - # Add .S (preprocessed assembly) to C compiler source extensions. - self.compiler.src_extensions.append('.S') - - include_dirs = [os.path.join(ffi_srcdir, 'include'), - os.path.join(ffi_srcdir, 'powerpc')] - ext.include_dirs.extend(include_dirs) - ext.sources.extend(sources) - return True - def configure_ctypes(self, ext): - if not self.use_system_libffi: - if MACOS: - return self.configure_ctypes_darwin(ext) - print('INFO: Could not locate ffi libs and/or headers') - return False return True def detect_ctypes(self): # Thomas Heller's _ctypes module - self.use_system_libffi = False + + if (not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and MACOS and + (is_macosx_at_least((10,15)) or '-arch arm64' in sysconfig.get_config_var("CFLAGS"))): + self.use_system_libffi = True + else: + self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS") + include_dirs = [] extra_compile_args = ['-DPy_BUILD_CORE_MODULE'] extra_link_args = [] @@ -2157,11 +2175,10 @@ class PyBuildExt(build_ext): if MACOS: sources.append('_ctypes/malloc_closure.c') - sources.append('_ctypes/darwin/dlfcn_simple.c') + extra_compile_args.append('-DUSING_MALLOC_CLOSURE_DOT_C=1') + #sources.append('_ctypes/darwin/dlfcn_simple.c') extra_compile_args.append('-DMACOSX') include_dirs.append('_ctypes/darwin') - # XXX Is this still needed? - # extra_link_args.extend(['-read_only_relocs', 'warning']) elif HOST_PLATFORM == 'sunos5': # XXX This shouldn't be necessary; it appears that some @@ -2191,31 +2208,47 @@ class PyBuildExt(build_ext): sources=['_ctypes/_ctypes_test.c'], libraries=['m'])) + ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR") + ffi_lib = None + ffi_inc_dirs = self.inc_dirs.copy() if MACOS: - if '--with-system-ffi' not in sysconfig.get_config_var("CONFIG_ARGS"): - return - # OS X 10.5 comes with libffi.dylib; the include files are - # in /usr/include/ffi - ffi_inc_dirs.append('/usr/include/ffi') - - ffi_inc = [sysconfig.get_config_var("LIBFFI_INCLUDEDIR")] - if not ffi_inc or ffi_inc[0] == '': - ffi_inc = find_file('ffi.h', [], ffi_inc_dirs) - if ffi_inc is not None: - ffi_h = ffi_inc[0] + '/ffi.h' + # XXX: The define should only be added when actually using the system + # version (and not a locally compiled one) + ext.extra_compile_args.append("-DUSING_APPLE_OS_LIBFFI=1") + ffi_in_sdk = os.path.join(macosx_sdk_root(), "usr/include/ffi") + if os.path.exists(ffi_in_sdk): + ffi_inc = ffi_in_sdk + ffi_lib = 'ffi' + else: + # OS X 10.5 comes with libffi.dylib; the include files are + # in /usr/include/ffi + ffi_inc_dirs.append('/usr/include/ffi') + + if not ffi_inc: + found = find_file('ffi.h', [], ffi_inc_dirs) + if found: + ffi_inc = found[0] + if ffi_inc: + ffi_h = ffi_inc + '/ffi.h' if not os.path.exists(ffi_h): ffi_inc = None print('Header file {} does not exist'.format(ffi_h)) - ffi_lib = None - if ffi_inc is not None: + if ffi_lib is None and ffi_inc: for lib_name in ('ffi', 'ffi_pic'): if (self.compiler.find_library_file(self.lib_dirs, lib_name)): ffi_lib = lib_name break if ffi_inc and ffi_lib: - ext.include_dirs.extend(ffi_inc) + ffi_headers = glob(os.path.join(ffi_inc, '*.h')) + if grep_headers_for('ffi_prep_cif_var', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_PREP_CIF_VAR=1") + if grep_headers_for('ffi_prep_closure_loc', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_PREP_CLOSURE_LOC=1") + if grep_headers_for('ffi_closure_alloc', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_CLOSURE_ALLOC=1") + ext.include_dirs.append(ffi_inc) ext.libraries.append(ffi_lib) self.use_system_libffi = True -- 2.23.0