diff options
| author | Alexandru Croitor <alexandru.croitor@qt.io> | 2021-09-29 19:01:51 +0200 |
|---|---|---|
| committer | Alexandru Croitor <alexandru.croitor@qt.io> | 2022-02-04 15:51:04 +0100 |
| commit | 57866a57586d401c784f809f9f7994b0e4623706 (patch) | |
| tree | 48b9d5a7d1a03d1edd15359858adcefd18430467 /build_scripts/setup_runner.py | |
| parent | 14e4527cc427ce8c5e7c1758a95a1bbce0498471 (diff) | |
setup.py: Add support for cross-building
setup.py can now be used to cross-compile PySide to a target Linux
distribution from a Linux host.
For example you could cross-compile PySide targeting an arm64
Raspberry Pi4 sysroot on an Ubuntu x86_64 host machine.
Cross-compiling PySide has a few requirements:
- a sysroot to cross-compile against, with a pre-installed Qt,
Python interpreter, library and development packages (which
provides C++ headers)
- a host Qt installation of the same version that is in the target
sysroot
- a host Python installation, preferably of the same version as the
target one (to run setup.py)
- a working cross-compiling toolchain (cross-compiler, linker, etc)
- a custom written CMake toolchain file
- CMake version 3.17+
- Qt version 6.3+
The CMake toolchain file is required to set up all the relevant
cross-compilation information: where the sysroot is, where the
toolchain is, the compiler name, compiler flags, etc.
Once are requirements are met, to cross-compile one has to specify a
few additional options when calling setup.py: the path to the cmake
toolchain file, the path to the host Qt installation
and the target python platform name.
An example setup.py invocation to build a wheel for an armv7 machine
might look like the following:
python setup.py bdist_wheel --parallel=8 --ignore-git --reuse-build
--cmake-toolchain-file=$PWD/rpi/toolchain_armv7.cmake
--qt-host-path=/opt/Qt/6.3.0/gcc_64
--plat-name=linux_armv7l
--limited-api=yes
--standalone
Sample platform names that can be used are: linux_armv7, linux_aarch64.
If the auto-detection code fails to find the target Python or Qt
installation, one can specify their location by providing the
--python-target-path=<path>
and
--qt-target-path=<path>
options to setup.py.
If the automatic build of the host shiboken code generator fails,
one can specify the path to a custom built host shiboken via the
--shiboken-host-path option.
Documentation about the build process and a sample CMake
toolchain file will be added in a separate change.
Implementation details.
Internally, setup.py will build a host shiboken executable using
the provided host Qt path, and then use it for the cross-build.
This is achieved via an extra setup.py sub-invocation with some
heuristics on which options should be passed to the sub-invocation.
The host shiboken is not included in the target wheels.
Introspection of where the host / target Qt and Python are located
is done via CMake compile tests, because we can't query information
from a qmake that is built for a different architecture / platform.
When limited API is enabled, we modify the wheel name to contain the
manylinux2014 tag, despite the wheel not fully qualifying for that
tag.
When copying the Qt libraries / plugins from the target sysroot in a
standalone build, we need to adjust all their rpaths to match the
destination directory layout of the wheel.
Fixes: PYSIDE-802
Task-number: PYSIDE-1033
Change-Id: I6e8c51ef5127d85949de650396d615ca95194db0
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>
Diffstat (limited to 'build_scripts/setup_runner.py')
| -rw-r--r-- | build_scripts/setup_runner.py | 139 |
1 files changed, 128 insertions, 11 deletions
diff --git a/build_scripts/setup_runner.py b/build_scripts/setup_runner.py index f77b9d1fe..362432d2a 100644 --- a/build_scripts/setup_runner.py +++ b/build_scripts/setup_runner.py @@ -39,6 +39,7 @@ import sys import os +import tempfile import textwrap from setuptools import setup # Import setuptools before distutils @@ -68,6 +69,16 @@ class SetupRunner(object): return any(arg for arg in list(args) if "--" + argument in arg) @staticmethod + def get_cmd_line_argument_in_args(argument, args): + """ Gets the value of a cmd line argument passed in args. """ + for arg in list(args): + if "--" + argument in arg: + prefix = f"--{argument}" + prefix_len = len(prefix) + 1 + return arg[prefix_len:] + return None + + @staticmethod def remove_cmd_line_argument_in_args(argument, args): """ Remove command line argument from args. """ return [arg for arg in list(args) if "--" + argument not in arg] @@ -83,20 +94,107 @@ class SetupRunner(object): def construct_internal_build_type_cmd_line_argument(internal_build_type): return SetupRunner.construct_cmd_line_argument("internal-build-type", internal_build_type) - def add_setup_internal_invocation(self, build_type, reuse_build=False): - """ Enqueues a script sub-invocation to be executed later. """ + def enqueue_setup_internal_invocation(self, setup_cmd): + self.invocations_list.append(setup_cmd) + + def add_setup_internal_invocation(self, build_type, reuse_build=False, extra_args=None): + setup_cmd = self.new_setup_internal_invocation(build_type, reuse_build, extra_args) + self.enqueue_setup_internal_invocation(setup_cmd) + + def new_setup_internal_invocation(self, build_type, + reuse_build=False, + extra_args=None, + replace_command_with=None): + """ Creates a script sub-invocation to be executed later. """ internal_build_type_arg = self.construct_internal_build_type_cmd_line_argument(build_type) - setup_cmd = [sys.executable] + self.sub_argv + [internal_build_type_arg] - command = self.sub_argv[0] + command_index = 0 + command = self.sub_argv[command_index] if command == 'setup.py' and len(self.sub_argv) > 1: - command = self.sub_argv[1] + command_index = 1 + command = self.sub_argv[command_index] + + # Make a copy + modified_argv = list(self.sub_argv) + + if replace_command_with: + modified_argv[command_index] = replace_command_with + + setup_cmd = [sys.executable] + modified_argv + [internal_build_type_arg] + + if extra_args: + for (name, value) in extra_args: + setup_cmd.append(self.construct_cmd_line_argument(name, value)) # Add --reuse-build option if requested and not already present. if (reuse_build and command in ('bdist_wheel', 'build', 'build_rst_docs', 'install') - and not self.cmd_line_argument_is_in_args("reuse-build", self.sub_argv)): + and not self.cmd_line_argument_is_in_args("reuse-build", modified_argv)): setup_cmd.append(self.construct_cmd_line_argument("reuse-build")) - self.invocations_list.append(setup_cmd) + return setup_cmd + + def add_host_tools_setup_internal_invocation(self, initialized_config): + extra_args = [] + extra_host_args = [] + + # When cross-compiling, build the host shiboken generator tool + # only if a path to an existing one was not provided. + if not self.cmd_line_argument_is_in_args("shiboken-host-path", self.sub_argv): + handle, initialized_config.shiboken_host_query_path = tempfile.mkstemp() + os.close(handle) + + # Tell the setup process to create a file with the location + # of the installed host shiboken as its contents. + extra_host_args.append( + ("internal-cmake-install-dir-query-file-path", + initialized_config.shiboken_host_query_path)) + + # Tell the other setup invocations to read that file and use + # the read path as the location of the host shiboken. + extra_args.append( + ("internal-shiboken-host-path-query-file", + initialized_config.shiboken_host_query_path) + ) + + # This is specifying shiboken_module_option_name + # instead of shiboken_generator_option_name, but it will + # actually build the generator. + host_cmd = self.new_setup_internal_invocation( + initialized_config.shiboken_module_option_name, + extra_args=extra_host_args, + replace_command_with="build") + + # To build the host tools, we reuse the initial target + # command line arguments, but we remove some options that + # don't make sense for the host build. + + # Drop the toolchain arg. + host_cmd = self.remove_cmd_line_argument_in_args("cmake-toolchain-file", + host_cmd) + + # Drop the target plat-name arg if there is one. + if self.cmd_line_argument_is_in_args("plat-name", host_cmd): + host_cmd = self.remove_cmd_line_argument_in_args("plat-name", host_cmd) + + # Drop the python-target-path arg if there is one. + if self.cmd_line_argument_is_in_args("python-target-path", host_cmd): + host_cmd = self.remove_cmd_line_argument_in_args("python-target-path", host_cmd) + + # Drop the target build-tests arg if there is one. + if self.cmd_line_argument_is_in_args("build-tests", host_cmd): + host_cmd = self.remove_cmd_line_argument_in_args("build-tests", host_cmd) + + # Make sure to pass the qt host path as the target path + # when doing the host build. And make sure to remove any + # existing qt target path. + if self.cmd_line_argument_is_in_args("qt-host-path", host_cmd): + qt_host_path = self.get_cmd_line_argument_in_args("qt-host-path", host_cmd) + host_cmd = self.remove_cmd_line_argument_in_args("qt-host-path", host_cmd) + host_cmd = self.remove_cmd_line_argument_in_args("qt-target-path", host_cmd) + host_cmd.append(self.construct_cmd_line_argument("qt-target-path", + qt_host_path)) + + self.enqueue_setup_internal_invocation(host_cmd) + return extra_args def run_setup(self): """ @@ -118,6 +216,7 @@ class SetupRunner(object): package_version=get_package_version(), ext_modules=get_setuptools_extension_modules(), setup_script_dir=self.setup_script_dir, + cmake_toolchain_file=OPTION["CMAKE_TOOLCHAIN_FILE"], quiet=OPTION["QUIET"]) # Enable logging for both the top-level invocation of setup.py @@ -149,18 +248,33 @@ class SetupRunner(object): # Build everything: shiboken6, shiboken6-generator and PySide6. help_requested = '--help' in self.sub_argv or '-h' in self.sub_argv + if help_requested: self.add_setup_internal_invocation(config.pyside_option_name) elif config.is_top_level_build_all(): - self.add_setup_internal_invocation(config.shiboken_module_option_name) + extra_args = [] + + # extra_args might contain the location of the built host + # shiboken, which needs to be passed to the other + # target invocations. + if config.is_cross_compile(): + extra_args = self.add_host_tools_setup_internal_invocation(config) + + self.add_setup_internal_invocation( + config.shiboken_module_option_name, + extra_args=extra_args) # Reuse the shiboken build for the generator package instead # of rebuilding it again. - self.add_setup_internal_invocation(config.shiboken_generator_option_name, - reuse_build=True) + # Don't build it in a cross-build though. + if not config.is_cross_compile(): + self.add_setup_internal_invocation( + config.shiboken_generator_option_name, + reuse_build=True) - self.add_setup_internal_invocation(config.pyside_option_name) + self.add_setup_internal_invocation(config.pyside_option_name, + extra_args=extra_args) elif config.is_top_level_build_shiboken_module(): self.add_setup_internal_invocation(config.shiboken_module_option_name) @@ -184,6 +298,9 @@ class SetupRunner(object): if help_requested: print(ADDITIONAL_OPTIONS) + # Cleanup temp query file. + if config.shiboken_host_query_path: + os.remove(config.shiboken_host_query_path) @staticmethod def run_setuptools_setup(): |