From 8179665c51ee194846b925747bcf9abbb8433d41 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Tue, 24 Dec 2019 18:37:17 +0100 Subject: [PATCH 13/24] Add CondaEcosystemModifyDllSearchPath() There are 2 modes depending on CONDA_DLL_SEARCH_MODIFICATION env variable - unset CONDA_DLL_SEARCH_MODIFICATION (Default) In this mode, the python interpreter works as if the python interpreter was called with the following conda directories. os.add_dll_directory(join(sys.prefix, 'bin')) os.add_dll_directory(join(sys.prefix, 'Scripts')) os.add_dll_directory(join(sys.prefix, 'Library', 'bin')) os.add_dll_directory(join(sys.prefix, 'Library', 'usr', 'bin')) os.add_dll_directory(join(sys.prefix, 'Library', 'mingw-w64', 'bin')) Search order - The directory that contains the DLL (if looking for a dependency) - Application (python.exe) directory - Directories added with os.add_dll_directory - The 5 conda directories - C:\Windows\System32 Note that the default behaviour changed in conda python 3.10 to make os.add_dll_directory work in user code. - CONDA_DLL_SEARCH_MODIFICATION=1 Search order is roughly, - The directory that contains the DLL (if looking for a dependency) - Application (python.exe) directory - C:\Windows - Current working directory - The 5 conda directories - PATH - Directories added with os.add_dll_directory - Old PATH entries (Deficiency in current patch) - Old working directories (Deficiency in current patch) - C:\Windows\System32 This changes the DLL search order so that C:\Windows\System32 does not get searched in before entries in PATH. Reviewed by Kai Tietz 7.2.2019 Updated a bit to include other directories. Made fwprintfs breakpointable From Shaun Walbridge: Fix CondaEcosystemModifyDllSearchPath for users of the Python DLL Co-authored-by: Isuru Fernando --- Modules/main.c | 370 +++++++++++++++++++++++++++++++++++++++++++ Python/dynload_win.c | 4 + Python/pylifecycle.c | 5 +- 3 files changed, 378 insertions(+), 1 deletion(-) diff --git a/Modules/main.c b/Modules/main.c index ea6250e28c..9cfcdc5e39 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -20,6 +20,10 @@ #endif #ifdef MS_WINDOWS # include /* STATUS_CONTROL_C_EXIT */ +# include +# include +# include +# include #endif /* End of includes for exit_sigint() */ @@ -703,10 +707,376 @@ Py_RunMain(void) return exitcode; } +#ifdef MS_WINDOWS +/* Please do not remove this function. It is needed for testing + CondaEcosystemModifyDllSearchPath(). */ + +/* +void LoadAndUnloadTestDLL(wchar_t* test_dll) +{ + wchar_t test_path[MAX_PATH + 1]; + HMODULE hDLL = LoadLibraryExW(&test_dll[0], NULL, 0); + if (hDLL == NULL) + { + wchar_t err_msg[256]; + DWORD err_code = GetLastError(); + FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + err_msg, (sizeof(err_msg) / sizeof(wchar_t)), NULL); + fwprintf(stderr, L"LoadAndUnloadTestDLL() :: ERROR :: Failed to load %ls, error is: %ls\n", &test_dll[0], &err_msg[0]); + } + GetModuleFileNameW(hDLL, &test_path[0], MAX_PATH); + fwprintf(stderr, L"LoadAndUnloadTestDLL() :: %ls loaded from %ls\n", &test_dll[0], &test_path[0]); + if (FreeLibrary(hDLL) == 0) + { + wchar_t err_msg[256]; + DWORD err_code = GetLastError(); + FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + err_msg, (sizeof(err_msg) / sizeof(wchar_t)), NULL); + fwprintf(stderr, L"LoadAndUnloadTestDLL() :: ERROR :: Failed to free %ls, error is: %ls\n", &test_dll[0], &err_msg[0]); + } +} +*/ + +/* + Provided CONDA_DLL_SEARCH_MODIFICATION_ENABLE is set (to anything at all!) + this function will modify the DLL search path so that C:\Windows\System32 + does not appear before entries in PATH. If it does appear in PATH then it + gets added at the position it was in in PATH. + + This is achieved via a call to SetDefaultDllDirectories() then calls to + AddDllDirectory() for each entry in PATH. We also take the opportunity to + clean-up these PATH entries such that any '/' are replaced with '\', no + double quotes occour and no PATH entry ends with '\'. + + Caution: Microsoft's documentation says that the search order of entries + passed to AddDllDirectory is not respected and arbitrary. I do not think + this will be the case but it is worth bearing in mind. +*/ + +#if !defined(LOAD_LIBRARY_SEARCH_DEFAULT_DIRS) +#define LOAD_LIBRARY_SEARCH_DEFAULT_DIRS 0x00001000 +#endif + +/* Caching of prior processed PATH environment */ +static wchar_t *sv_path_env = NULL; +typedef void (WINAPI *SDDD)(DWORD DirectoryFlags); +typedef void (WINAPI *SDD)(PCWSTR SetDir); +typedef void (WINAPI *ADD)(PCWSTR NewDirectory); +static SDDD pSetDefaultDllDirectories = NULL; +static SDD pSetDllDirectory = NULL; +static ADD pAddDllDirectory = NULL; +static int sv_failed_to_find_dll_fns = 0; +/* Have hidden this behind a define because it is clearly not code that + could be considered for upstreaming so clearly delimiting it makes it + easier to remove. */ +#define HARDCODE_CONDA_PATHS +#if defined(HARDCODE_CONDA_PATHS) +typedef struct +{ + wchar_t *p_relative; + wchar_t *p_name; +} CONDA_PATH; + +#define NUM_CONDA_PATHS 5 + +static CONDA_PATH condaPaths[NUM_CONDA_PATHS] = +{ + {L"Library\\mingw-w64\\bin", NULL}, + {L"Library\\usr\\bin", NULL}, + {L"Library\\bin", NULL}, + {L"Scripts", NULL}, + {L"bin", NULL} +}; +#endif /* HARDCODE_CONDA_PATHS */ +static wchar_t sv_dll_dirname[1024]; +static wchar_t sv_windows_directory[1024]; +static wchar_t *sv_added_windows_directory = NULL; +static wchar_t *sv_added_cwd = NULL; + +int CondaEcosystemModifyDllSearchPath_Init() +{ + int debug_it = _wgetenv(L"CONDA_DLL_SEARCH_MODIFICATION_DEBUG") ? 1 : 0; + wchar_t* enable = _wgetenv(L"CONDA_DLL_SEARCH_MODIFICATION_ENABLE"); + int res = 0; +#if defined(HARDCODE_CONDA_PATHS) + long long j; + CONDA_PATH *p_conda_path; +#endif /* defined(HARDCODE_CONDA_PATHS) */ + HMODULE dll_handle = NULL; + + if (pSetDefaultDllDirectories == NULL) + { + wchar_t *conda_prefix = _wgetenv(L"CONDA_PREFIX"); + wchar_t *build_prefix = _wgetenv(L"BUILD_PREFIX"); + wchar_t *prefix = _wgetenv(L"PREFIX"); + pSetDefaultDllDirectories = (SDDD)GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + pSetDllDirectory = (SDD)GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDllDirectoryW"); + pAddDllDirectory = (ADD)GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "AddDllDirectory"); + + /* Determine sv_dll_dirname */ + if (GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCSTR) &CondaEcosystemModifyDllSearchPath_Init, &dll_handle) == 0) + { + // Getting the pythonxx.dll path failed. Fall back to relative path of python.exe + // assuming that the executable that is running this code is python.exe + dll_handle = NULL; + } + GetModuleFileNameW(dll_handle, &sv_dll_dirname[0], sizeof(sv_dll_dirname)/sizeof(sv_dll_dirname[0])-1); + sv_dll_dirname[sizeof(sv_dll_dirname)/sizeof(sv_dll_dirname[0])-1] = L'\0'; + if (wcsrchr(sv_dll_dirname, L'\\')) + *wcsrchr(sv_dll_dirname, L'\\') = L'\0'; + +#if defined(HARDCODE_CONDA_PATHS) + for (p_conda_path = &condaPaths[0]; p_conda_path < &condaPaths[NUM_CONDA_PATHS]; ++p_conda_path) + { + size_t n_chars_dll_dirname = wcslen(sv_dll_dirname); + size_t n_chars_p_relative = wcslen(p_conda_path->p_relative); + p_conda_path->p_name = malloc(sizeof(wchar_t) * (n_chars_dll_dirname + n_chars_p_relative + 2)); + wcsncpy(p_conda_path->p_name, sv_dll_dirname, n_chars_dll_dirname+1); + wcsncat(p_conda_path->p_name, L"\\", 2); + wcsncat(p_conda_path->p_name, p_conda_path->p_relative, n_chars_p_relative+1); + } +#endif /* defined(HARDCODE_CONDA_PATHS) */ + + /* Determine sv_windows_directory */ + { + char tmp_ascii[1024]; + size_t convertedChars = 0; + GetWindowsDirectory(&tmp_ascii[0], sizeof(tmp_ascii) / sizeof(tmp_ascii[0]) - 1); + tmp_ascii[sizeof(tmp_ascii) / sizeof(tmp_ascii[0]) - 1] = L'\0'; + mbstowcs_s(&convertedChars, sv_windows_directory, strlen(tmp_ascii)+1, tmp_ascii, _TRUNCATE); + sv_windows_directory[sizeof(sv_windows_directory) / sizeof(sv_windows_directory[0]) - 1] = L'\0'; + } + } + + if (pSetDefaultDllDirectories == NULL || pSetDllDirectory == NULL || pAddDllDirectory == NULL) + { + if (debug_it) + fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: WARNING :: Please install KB2533623 from http://go.microsoft.com/fwlink/p/?linkid=217865\n"\ + L"CondaEcosystemModifyDllSearchPath() :: WARNING :: to improve conda ecosystem DLL isolation"); + sv_failed_to_find_dll_fns = 1; + res = 2; + } +#if defined(HARDCODE_CONDA_PATHS) + else if (enable == NULL || !wcscmp(enable, L"0")) { + for (j = NUM_CONDA_PATHS-1, p_conda_path = &condaPaths[NUM_CONDA_PATHS-1]; j > -1; --j, --p_conda_path) + { + if (debug_it) + fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: AddDllDirectory(%ls - ExePrefix)\n", p_conda_path->p_name); + pAddDllDirectory(p_conda_path->p_name); + } + } +#endif /* defined(HARDCODE_CONDA_PATHS) */ + return res; +} + +int CondaEcosystemModifyDllSearchPath(int add_windows_directory, int add_cwd) { + int debug_it = _wgetenv(L"CONDA_DLL_SEARCH_MODIFICATION_DEBUG") ? 1 : 0; + const wchar_t *path_env = _wgetenv(L"PATH"); + wchar_t current_working_directory[1024]; + const wchar_t *p_cwd = NULL; + long long entry_num = 0; + long long i; + wchar_t **path_entries; + wchar_t *path_end; + long long num_entries = 1; +#if defined(HARDCODE_CONDA_PATHS) + long long j; + CONDA_PATH *p_conda_path; + int foundCondaPath[NUM_CONDA_PATHS] = {0, 0, 0, 0, 0}; +#endif /* defined(HARDCODE_CONDA_PATHS) */ + wchar_t *enable; + + int SetDllDirectoryValue = LOAD_LIBRARY_SEARCH_DEFAULT_DIRS; + if (sv_failed_to_find_dll_fns) + return 1; + + /* Fix for embedding the Python DLL. Courtesy of Shaun Walbridge + * if the CondaEcosystemModifyDllSearchPath_Init(argc, argv) code hasn't been run + * or failed to bind to the required functions in kernel32.dll, fail early to avoid + * an access violation. */ + if (pSetDefaultDllDirectories == NULL || pSetDllDirectory == NULL || pAddDllDirectory == NULL) + return 1; + + enable = _wgetenv(L"CONDA_DLL_SEARCH_MODIFICATION_ENABLE"); + if (enable == NULL || !wcscmp(enable, L"0")) + return 0; + if (_wgetenv(L"CONDA_DLL_SEARCH_MODIFICATION_NEVER_ADD_WINDOWS_DIRECTORY")) + add_windows_directory = 0; + if (_wgetenv(L"CONDA_DLL_SEARCH_MODIFICATION_NEVER_ADD_CWD")) + add_cwd = 0; + + if (add_cwd) + { + _wgetcwd(¤t_working_directory[0], (sizeof(current_working_directory)/sizeof(current_working_directory[0])) - 1); + current_working_directory[sizeof(current_working_directory)/sizeof(current_working_directory[0]) - 1] = L'\0'; + p_cwd = ¤t_working_directory[0]; + } + + /* cache path to avoid multiple adds */ + if (sv_path_env != NULL && path_env != NULL && !wcscmp(path_env, sv_path_env)) + { + if ((add_windows_directory && sv_added_windows_directory != NULL) || + (!add_windows_directory && sv_added_windows_directory == NULL) ) + { + if ((p_cwd == NULL && sv_added_cwd == NULL) || + p_cwd != NULL && sv_added_cwd != NULL && !wcscmp(p_cwd, sv_added_cwd)) + { + if (_wgetenv(L"CONDA_DLL_SEARCH_MODIFICATION_NEVER_CACHE") == NULL) + { + if (debug_it) + fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: INFO :: Values unchanged\n"); + return 0; + } + } + } + } + /* Something has changed. + Reset to default search order */ + pSetDllDirectory(NULL); + + if (sv_path_env != NULL) + { + free(sv_path_env); + } + sv_path_env = (path_env == NULL) ? NULL : _wcsdup(path_env); + + if (path_env != NULL) + { + size_t len = wcslen(path_env); + wchar_t *path = (wchar_t *)alloca((len + 1) * sizeof(wchar_t)); + if (debug_it) + fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: PATH=%ls\n\b", path_env); + memcpy(path, path_env, (len + 1) * sizeof(wchar_t)); + /* Convert any / to \ */ + /* Replace slash with backslash */ + while ((path_end = wcschr(path, L'/'))) + *path_end = L'\\'; + /* Remove all double quotes */ + while ((path_end = wcschr(path, L'"'))) + memmove(path_end, path_end + 1, sizeof(wchar_t) * (len-- - (path_end - path))); + /* Remove all leading and double ';' */ + while (*path == L';') + memmove(path, path + 1, sizeof(wchar_t) * len--); + while ((path_end = wcsstr(path, L";;"))) + memmove(path_end, path_end + 1, sizeof(wchar_t) * (len-- - (path_end - path))); + /* Remove trailing ;'s */ + while(path[len-1] == L';') + path[len-- - 1] = L'\0'; + + if (len == 0) + return 2; + + /* Count the number of path entries */ + path_end = path; + while ((path_end = wcschr(path_end, L';'))) + { + ++num_entries; + ++path_end; + } + + path_entries = (wchar_t **)alloca((num_entries) * sizeof(wchar_t *)); + path_end = wcschr(path, L';'); + + if (getenv("CONDA_DLL_SET_DLL_DIRECTORY_VALUE") != NULL) + SetDllDirectoryValue = atoi(getenv("CONDA_DLL_SET_DLL_DIRECTORY_VALUE")); + pSetDefaultDllDirectories(SetDllDirectoryValue); + while (path != NULL) + { + if (path_end != NULL) + { + *path_end = L'\0'; + /* Hygiene, no \ at the end */ + while (path_end > path && path_end[-1] == L'\\') + { + --path_end; + *path_end = L'\0'; + } + } + if (wcslen(path) != 0) + path_entries[entry_num++] = path; + path = path_end; + if (path != NULL) + { + while (*path == L'\0') + ++path; + path_end = wcschr(path, L';'); + } + } + for (i = num_entries - 1; i > -1; --i) + { +#if defined(HARDCODE_CONDA_PATHS) + for (j = 0, p_conda_path = &condaPaths[0]; p_conda_path < &condaPaths[NUM_CONDA_PATHS]; ++j, ++p_conda_path) + { + if (!foundCondaPath[j] && !wcscmp(path_entries[i], p_conda_path->p_name)) + { + foundCondaPath[j] = 1; + break; + } + } +#endif /* defined(HARDCODE_CONDA_PATHS) */ + if (debug_it) + fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: AddDllDirectory(%ls)\n", path_entries[i]); + pAddDllDirectory(path_entries[i]); + } + } + +#if defined(HARDCODE_CONDA_PATHS) + if (_wgetenv(L"CONDA_DLL_SEARCH_MODIFICATION_DO_NOT_ADD_EXEPREFIX") == NULL) + { + for (j = NUM_CONDA_PATHS-1, p_conda_path = &condaPaths[NUM_CONDA_PATHS-1]; j > -1; --j, --p_conda_path) + { + if (debug_it) + fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: p_conda_path->p_name = %ls, foundCondaPath[%zd] = %d\n", p_conda_path->p_name, j, foundCondaPath[j]); + if (!foundCondaPath[j]) + { + if (debug_it) + fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: AddDllDirectory(%ls - ExePrefix)\n", p_conda_path->p_name); + pAddDllDirectory(p_conda_path->p_name); + } + } + } +#endif /* defined(HARDCODE_CONDA_PATHS) */ + + if (p_cwd) + { + if (sv_added_cwd != NULL && wcscmp(p_cwd, sv_added_cwd)) + { + free(sv_added_cwd); + } + sv_added_cwd = _wcsdup(p_cwd); + if (debug_it) + fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: AddDllDirectory(%ls - CWD)\n", sv_added_cwd); + pAddDllDirectory(sv_added_cwd); + } + + if (add_windows_directory) + { + sv_added_windows_directory = &sv_windows_directory[0]; + if (debug_it) + fwprintf(stderr, L"CondaEcosystemModifyDllSearchPath() :: AddDllDirectory(%ls - WinDir)\n", sv_windows_directory); + pAddDllDirectory(sv_windows_directory); + } + else + { + sv_added_windows_directory = NULL; + } + + return 0; +} +#endif + static int pymain_main(_PyArgv *args) { +#ifdef MS_WINDOWS + /* LoadAndUnloadTestDLL(L"libiomp5md.dll"); */ + CondaEcosystemModifyDllSearchPath_Init(args->argc, args->wchar_argv); + /* LoadAndUnloadTestDLL(L"libiomp5md.dll"); */ +#endif PyStatus status = pymain_init(args); if (_PyStatus_IS_EXIT(status)) { pymain_free(); diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 4896c6dc8c..ab90f62cfc 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -199,6 +199,10 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, to avoid DLL preloading attacks and enable use of the AddDllDirectory function. We add SEARCH_DLL_LOAD_DIR to ensure DLLs adjacent to the PYD are preferred. */ + /* This resyncs values in PATH to AddDllDirectory() */ + extern int CondaEcosystemModifyDllSearchPath(int, int); + CondaEcosystemModifyDllSearchPath(1, 1); + Py_BEGIN_ALLOW_THREADS hDLL = LoadLibraryExW(wpathname, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8732d814f8..10700e0a1b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -84,7 +84,10 @@ _PyRuntime_Initialize(void) return _PyStatus_OK(); } runtime_initialized = 1; - +#ifdef MS_WINDOWS + extern int CondaEcosystemModifyDllSearchPath_Init(); + CondaEcosystemModifyDllSearchPath_Init(); +#endif return _PyRuntimeState_Init(&_PyRuntime); }