diff options
| author | Shyamnath Premnadh <Shyamnath.Premnadh@qt.io> | 2023-10-24 16:10:56 +0200 |
|---|---|---|
| committer | Shyamnath Premnadh <Shyamnath.Premnadh@qt.io> | 2024-02-29 10:32:54 +0100 |
| commit | ec6a0f8baef5c6d4e80e650e11a498756e6055e6 (patch) | |
| tree | d4f0c4ba59ae239a8841ac68683da58e5e0dd440 /sources/pyside-tools/deploy_lib/android/android_config.py | |
| parent | 489899819f4b417f8bc8923b1eb03728bed4bb5e (diff) | |
Deployment: Refactoring
- Functions in buildozer.py for finding the local_libs, plugin and Qt
module dependencies of the application are related to the overall
config of the application and not buildozer. Hence, these functions
are moved to android_config.py.
- `ALL_PYSIDE_MODULES` moved to a function under deploy_lib/__init__.py
and `platform_map` moved to deploy_lib/android/__init__.py.
- Enable the user to pass both arm64-v8a and aarch64 as the
architecture type. Same for all the other architecures that are
synonymous.
- `verify_and_set_recipe_dir()` is now called explicitly from
android_deploy.py due to `cleanup()` deleting the recipe directories
during config initialization.
- New property `dependency_files` for AndroidConfig class.
- Fix --dry-run for Android Deployment.
- Adapt tests.
Pick-to: 6.6
Task-number: PYSIDE-1612
Change-Id: Icdf14001ae2b07dc8614af3f458f9cad11eafdac
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
Diffstat (limited to 'sources/pyside-tools/deploy_lib/android/android_config.py')
| -rw-r--r-- | sources/pyside-tools/deploy_lib/android/android_config.py | 234 |
1 files changed, 223 insertions, 11 deletions
diff --git a/sources/pyside-tools/deploy_lib/android/android_config.py b/sources/pyside-tools/deploy_lib/android/android_config.py index 442672a23..1ea99411f 100644 --- a/sources/pyside-tools/deploy_lib/android/android_config.py +++ b/sources/pyside-tools/deploy_lib/android/android_config.py @@ -1,12 +1,19 @@ # Copyright (C) 2023 The Qt Company Ltd. # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +import re +import tempfile import logging +import zipfile +import xml.etree.ElementTree as ET from typing import List from pathlib import Path +from pkginfo import Wheel -from . import extract_and_copy_jar, get_wheel_android_arch -from .. import Config, find_pyside_modules +from . import (extract_and_copy_jar, get_wheel_android_arch, find_lib_dependencies, + get_llvm_readobj, find_qtlibs_in_wheel, platform_map, create_recipe) +from .. import (Config, find_pyside_modules, run_qmlimportscanner, get_all_pyside_modules, + MAJOR_VERSION) ANDROID_NDK_VERSION = "25c" ANDROID_DEPLOY_CACHE = Path.home() / ".pyside6_android_deploy" @@ -86,28 +93,52 @@ class AndroidConfig(Config): if jars_dir_temp and Path(jars_dir_temp).resolve().exists(): self.jars_dir = Path(jars_dir_temp).resolve() + self._arch = None + if self.get_value("buildozer", "arch"): + self.arch = self.get_value("buildozer", "arch") + else: + self._find_and_set_arch() + + # maps to correct platform name incase the instruction set was specified + self._arch = platform_map[self.arch] + + self._mode = self.get_value("buildozer", "mode") + + self.qt_libs_path: zipfile.Path = find_qtlibs_in_wheel(wheel_pyside=self.wheel_pyside) + logging.info(f"[DEPLOY] Qt libs path inside wheel: {str(self.qt_libs_path)}") + self._modules = [] if self.get_value("buildozer", "modules"): self.modules = self.get_value("buildozer", "modules").split(",") else: self._find_and_set_pysidemodules() self._find_and_set_qtquick_modules() + self.modules += self._find_dependent_qt_modules() + # remove duplicates + self.modules = list(set(self.modules)) - self._arch = None - if self.get_value("buildozer", "arch"): - self.arch = self.get_value("buildozer", "arch") - else: - self._find_and_set_arch() + # gets the xml dependency files from Qt installation path + self._dependency_files = [] + self._find_and_set_dependency_files() + + self._qt_plugins = [] + if self.get_value("android", "plugins"): + self._qt_plugins = self.get_value("android", "plugins").split(",") self._local_libs = [] if self.get_value("buildozer", "local_libs"): self.local_libs = self.get_value("buildozer", "local_libs").split(",") - self._qt_plugins = [] - if self.get_value("android", "plugins"): - self._qt_plugins = self.get_value("android", "plugins").split(",") + dependent_plugins = [] + # the local_libs can also store dependent plugins + local_libs, dependent_plugins = self._find_local_libs() + self._find_plugin_dependencies(dependent_plugins) + self.qt_plugins += dependent_plugins + self.local_libs += local_libs - self._mode = self.get_value("buildozer", "mode") + recipe_dir_temp = self.get_value("buildozer", "recipe_dir") + if recipe_dir_temp: + self.recipe_dir = Path(recipe_dir_temp) @property def qt_plugins(self): @@ -218,6 +249,14 @@ class AndroidConfig(Config): if self._wheel_shiboken: self.set_value("android", "wheel_shiboken", str(self._wheel_shiboken)) + @property + def dependency_files(self): + return self._dependency_files + + @dependency_files.setter + def dependency_files(self, dependency_files): + self._dependency_files = dependency_files + def _find_and_set_pysidemodules(self): self.modules = find_pyside_modules(project_dir=self.project_dir, extra_ignore_dirs=self.extra_ignore_dirs, @@ -246,6 +285,9 @@ class AndroidConfig(Config): """Identify if QtQuick is used in QML files and add them as dependency """ extra_modules = [] + if not self.qml_modules: + self.qml_modules = set(run_qmlimportscanner(qml_files=self.qml_files, + dry_run=self.dry_run)) if "QtQuick" in self.qml_modules: extra_modules.append("Quick") @@ -254,3 +296,173 @@ class AndroidConfig(Config): extra_modules.append("QuickControls2") self.modules += extra_modules + + def _find_dependent_qt_modules(self): + """ + Given pysidedeploy_config.modules, find all the other dependent Qt modules. This is + done by using llvm-readobj (readelf) to find the dependent libraries from the module + library. + """ + dependent_modules = set() + all_dependencies = set() + lib_pattern = re.compile(f"libQt6(?P<mod_name>.*)_{self.arch}") + + llvm_readobj = get_llvm_readobj(self.ndk_path) + if not llvm_readobj.exists(): + raise FileNotFoundError(f"[DEPLOY] {llvm_readobj} does not exist." + "Finding Qt dependencies failed") + + archive = zipfile.ZipFile(self.wheel_pyside) + lib_path_suffix = Path(str(self.qt_libs_path)).relative_to(self.wheel_pyside) + + with tempfile.TemporaryDirectory() as tmpdir: + archive.extractall(tmpdir) + qt_libs_tmpdir = Path(tmpdir) / lib_path_suffix + # find the lib folder where Qt libraries are stored + for module_name in sorted(self.modules): + qt_module_path = qt_libs_tmpdir / f"libQt6{module_name}_{self.arch}.so" + if not qt_module_path.exists(): + raise FileNotFoundError(f"[DEPLOY] libQt6{module_name}_{self.arch}.so not found" + " inside the wheel") + find_lib_dependencies(llvm_readobj=llvm_readobj, lib_path=qt_module_path, + dry_run=self.dry_run, + used_dependencies=all_dependencies) + + for dependency in all_dependencies: + match = lib_pattern.search(dependency) + if match: + module = match.group("mod_name") + if module not in self.modules: + dependent_modules.add(module) + + # check if the PySide6 binary for the Qt module actually exists + # eg: libQt6QmlModels.so exists and it includes QML types. Hence, it makes no + dependent_modules = [module for module in dependent_modules if module in + get_all_pyside_modules()] + dependent_modules_str = ",".join(dependent_modules) + logging.info("[DEPLOY] The following extra dependencies were found:" + f" {dependent_modules_str}") + + return dependent_modules + + def _find_and_set_dependency_files(self) -> List[zipfile.Path]: + """ + Based on `modules`, returns the Qt6{module}_{arch}-android-dependencies.xml file, which + contains the various dependencies of the module, like permissions, plugins etc + """ + needed_dependency_files = [(f"Qt{MAJOR_VERSION}{module}_{self.arch}" + "-android-dependencies.xml") for module in self.modules] + + for dependency_file_name in needed_dependency_files: + dependency_file = self.qt_libs_path / dependency_file_name + if dependency_file.exists(): + self._dependency_files.append(dependency_file) + + logging.info("[DEPLOY] The following dependency files were found: " + f"{*self._dependency_files,}") + + def _find_local_libs(self): + local_libs = set() + plugins = set() + lib_pattern = re.compile(f"lib(?P<lib_name>.*)_{self.arch}") + for dependency_file in self._dependency_files: + xml_content = dependency_file.read_text() + root = ET.fromstring(xml_content) + for local_lib in root.iter("lib"): + + if 'file' not in local_lib.attrib: + if 'name' not in local_lib.attrib: + logging.warning("[DEPLOY] Invalid android dependency file" + f" {str(dependency_file)}") + continue + + file = local_lib.attrib['file'] + if file.endswith(".so"): + # file_name starts with lib and ends with the platform name + # eg: lib<lib_name>_x86_64.so + file_name = Path(file).stem + + if file_name.startswith("libplugins_platforms_qtforandroid"): + # the platform library is a requisite and is already added from the + # configuration file + continue + + # we only need lib_name, because lib and arch gets re-added by + # python-for-android + match = lib_pattern.search(file_name) + if match: + lib_name = match.group("lib_name") + local_libs.add(lib_name) + if lib_name.startswith("plugins"): + plugin_name = lib_name.split('plugins_', 1)[1] + plugins.add(plugin_name) + + return list(local_libs), list(plugins) + + def _find_plugin_dependencies(self, dependent_plugins: List[str]): + # The `bundled` element in the dependency xml files points to the folder where + # additional dependencies for the application exists. Inspecting the depenency files + # in android, this always points to the specific Qt plugin dependency folder. + # eg: for application using Qt Multimedia, this looks like: + # <bundled file="./plugins/multimedia" /> + # The code recusively checks all these dependent folders and adds the necessary plugins + # as dependencies + lib_pattern = re.compile(f"libplugins_(?P<plugin_name>.*)_{self.arch}.so") + for dependency_file in self._dependency_files: + xml_content = dependency_file.read_text() + root = ET.fromstring(xml_content) + for bundled_element in root.iter("bundled"): + # the attribute 'file' can be misleading, but it always points to the plugin + # folder on inspecting the dependency files + if 'file' not in bundled_element.attrib: + logging.warning("[DEPLOY] Invalid Android dependency file" + f" {str(dependency_file)}") + continue + + # from "./plugins/multimedia" to absolute path in wheel + plugin_module_folder = bundled_element.attrib['file'] + # they all should start with `./plugins` + if plugin_module_folder.startswith("./plugins"): + plugin_module_folder = plugin_module_folder.partition("./plugins/")[2] + else: + continue + + absolute_plugin_module_folder = (self.qt_libs_path.parent / "plugins" + / plugin_module_folder) + + if not absolute_plugin_module_folder.is_dir(): + logging.warning(f"[DEPLOY] Qt plugin folder '{plugin_module_folder}' does not" + " exist or is not a directory for this Android platform") + continue + + for plugin in absolute_plugin_module_folder.iterdir(): + plugin_name = plugin.name + if plugin_name.endswith(".so") and plugin_name.startswith("libplugins"): + # we only need part of plugin_name, because `lib` prefix and `arch` suffix + # gets re-added by python-for-android + match = lib_pattern.search(plugin_name) + if match: + plugin_infix_name = match.group("plugin_name") + if plugin_infix_name not in dependent_plugins: + dependent_plugins.append(plugin_infix_name) + + def verify_and_set_recipe_dir(self): + # create recipes + # https://python-for-android.readthedocs.io/en/latest/recipes/ + # These recipes are manually added through buildozer.spec file to be used by + # python_for_android while building the distribution + + if not self.recipes_exist() and not self.dry_run: + logging.info("[DEPLOY] Creating p4a recipes for PySide6 and shiboken6") + version = Wheel(self.wheel_pyside).version + create_recipe(version=version, component=f"PySide{MAJOR_VERSION}", + wheel_path=self.wheel_pyside, + generated_files_path=self.generated_files_path, + qt_modules=self.modules, + local_libs=self.local_libs, + plugins=self.qt_plugins) + create_recipe(version=version, component=f"shiboken{MAJOR_VERSION}", + wheel_path=self.wheel_shiboken, + generated_files_path=self.generated_files_path) + self.recipe_dir = ((self.generated_files_path + / "recipes").resolve()) |