From 4c9191d4890418e7a59910d48f75ffab192c99dd Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Fri, 7 Jan 2022 10:16:44 -0600 Subject: [PATCH] Addon manager dependency resolver (#5339) Squashed: * Addon Manager: Refactor metadata.txt download * Addon Manager: Basic dependency walker * Addon Manager: Add basic support for dependencies * Addon Manager: Improve network detection messaging * Addon Manager: Black reformat * Addon Manager: Display dependency info in dialog * Addon Manager: Dependency dialog added * Addon Manager: Improve display of update all results * Addon Manager: Improve display of package list * Addon Manager: Fix codespell * Addon Manager: Clean up unused signal --- src/Mod/AddonManager/AddonManager.py | 278 ++++++++++++- src/Mod/AddonManager/AddonManagerRepo.py | 54 ++- src/Mod/AddonManager/CMakeLists.txt | 1 + src/Mod/AddonManager/addonmanager_metadata.py | 120 ++++-- .../AddonManager/addonmanager_utilities.py | 37 ++ src/Mod/AddonManager/addonmanager_workers.py | 387 +++++++----------- .../dependency_resolution_dialog.ui | 117 ++++++ src/Mod/AddonManager/package_list.py | 2 - 8 files changed, 722 insertions(+), 274 deletions(-) create mode 100644 src/Mod/AddonManager/dependency_resolution_dialog.ui diff --git a/src/Mod/AddonManager/AddonManager.py b/src/Mod/AddonManager/AddonManager.py index 2cb735e076..336c0f7ad1 100644 --- a/src/Mod/AddonManager/AddonManager.py +++ b/src/Mod/AddonManager/AddonManager.py @@ -74,6 +74,7 @@ class CommandAddonManager: """The main Addon Manager class and FreeCAD command""" workers = [ + "connection_checker", "update_worker", "check_worker", "show_worker", @@ -84,6 +85,7 @@ class CommandAddonManager: "load_macro_metadata_worker", "update_all_worker", "update_check_single_worker", + "dependency_installation_worker", ] lock = threading.Lock() @@ -171,7 +173,45 @@ class CommandAddonManager: pref.SetString("ProxyUrl", warning_dialog.lineEditProxy.text()) if readWarning: - self.launch() + # Check the connection in a new thread, so FreeCAD stays responsive + self.connection_checker = ConnectionChecker() + self.connection_checker.success.connect(self.launch) + self.connection_checker.failure.connect(self.network_connection_failed) + self.connection_checker.start() + + # If it takes longer than a half second to check the connection, show a message: + self.connection_message_timer = QtCore.QTimer.singleShot( + 500, self.show_connection_check_message + ) + + def show_connection_check_message(self): + if not self.connection_checker.isFinished(): + self.connection_check_message = QtWidgets.QMessageBox( + QtWidgets.QMessageBox.Information, + translate("AddonsInstaller", "Checking connection"), + translate("AddonsInstaller", "Checking for connection to GitHub..."), + QtWidgets.QMessageBox.Cancel, + ) + self.connection_check_message.buttonClicked.connect( + self.cancel_network_check + ) + self.connection_check_message.show() + + def cancel_network_check(self, button): + if not self.connection_checker.isFinished(): + self.connection_checker.success.disconnect(self.launch) + self.connection_checker.failure.disconnect(self.network_connection_failed) + self.connection_checker.requestInterruption() + self.connection_checker.wait(500) + self.connection_check_message.close() + + def network_connection_failed(self, message: str) -> None: + # This must run on the main GUI thread + if self.connection_check_message: + self.connection_check_message.close() + QtWidgets.QMessageBox.critical( + None, translate("AddonsInstaller", "Connection failed"), message + ) def launch(self) -> None: """Shows the Addon Manager UI""" @@ -281,7 +321,7 @@ class CommandAddonManager: self.packageList.itemSelected.connect(self.table_row_activated) self.packageList.setEnabled(False) self.packageDetails.execute.connect(self.executemacro) - self.packageDetails.install.connect(self.install) + self.packageDetails.install.connect(self.resolve_dependencies) self.packageDetails.uninstall.connect(self.remove) self.packageDetails.update.connect(self.update) self.packageDetails.back.connect(self.on_buttonBack_clicked) @@ -305,6 +345,9 @@ class CommandAddonManager: # set the label text to start with self.show_information(translate("AddonsInstaller", "Loading addon information")) + if hasattr(self, "connection_check_message") and self.connection_check_message: + self.connection_check_message.close() + # rock 'n roll!!! self.dialog.exec_() @@ -439,6 +482,7 @@ class CommandAddonManager: self.update_cache = False pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") pref.SetString("LastCacheUpdate", date.today().isoformat()) + self.packageList.item_filter.invalidateFilter() def get_cache_file_name(self, file: str) -> str: cache_path = FreeCAD.getUserCachePath() @@ -561,6 +605,12 @@ class CommandAddonManager: def on_buttonUpdateCache_clicked(self) -> None: self.update_cache = True + cache_path = FreeCAD.getUserCachePath() + am_path = os.path.join(cache_path, "AddonManager") + try: + shutil.rmtree(am_path, onerror=self.remove_readonly) + except Exception: + pass self.startup() def on_package_updated(self, repo: AddonManagerRepo) -> None: @@ -715,6 +765,176 @@ class CommandAddonManager: self.item_model.append_item(repo) + def resolve_dependencies(self, repo: AddonManagerRepo) -> None: + if not repo: + return + + deps = AddonManagerRepo.Dependencies() + repo_name_dict = dict() + for r in self.item_model.repos: + repo_name_dict[repo.name] = r + repo_name_dict[repo.display_name] = r + repo.walk_dependency_tree(repo_name_dict, deps) + + FreeCAD.Console.PrintLog("The following Workbenches are required:\n") + for addon in deps.unrecognized_addons: + FreeCAD.Console.PrintLog(addon + "\n") + + FreeCAD.Console.PrintLog("The following addons are required:\n") + for addon in deps.required_external_addons: + FreeCAD.Console.PrintLog(addon + "\n") + + FreeCAD.Console.PrintLog("The following Python modules are required:\n") + for pyreq in deps.python_required: + FreeCAD.Console.PrintLog(pyreq + "\n") + + FreeCAD.Console.PrintLog("The following Python modules are optional:\n") + for pyreq in deps.python_optional: + FreeCAD.Console.PrintLog(pyreq + "\n") + + missing_external_addons = [] + for dep in deps.required_external_addons: + if dep.update_status == AddonManagerRepo.UpdateStatus.NOT_INSTALLED: + missing_external_addons.append(dep) + + # Now check the loaded addons to see if we are missing an internal workbench: + wbs = FreeCADGui.listWorkbenches() + missing_wbs = [] + for dep in deps.unrecognized_addons: + if dep not in wbs and dep + "Workbench" not in wbs: + missing_wbs.append(dep) + + # Check the Python dependencies: + missing_python_requirements = [] + for py_dep in deps.python_required: + try: + __import__(py_dep) + except ImportError: + missing_python_requirements.append(py_dep) + + missing_python_optionals = [] + for py_dep in deps.python_optional: + try: + __import__(py_dep) + except ImportError: + missing_python_optionals.append(py_dep) + + # Possible cases + # 1) Missing required FreeCAD workbenches. Unrecoverable failure, needs a new version of FreeCAD installation. + # 2) Missing required external AddOn(s). List for the user and ask for permission to install them. + # 3) Missing required Python modules. List for the user and ask for permission to attempt installation. + # 4) Missing optional Python modules. User can choose from the list to attempt to install any or all. + # Option 1 is standalone, and simply causes failure to install. Other options can be combined and are + # presented through a dialog box with options. + + addon = repo.display_name if repo.display_name else repo.name + if missing_wbs: + if len(missing_wbs) == 1: + name = missing_wbs[0] + message = translate( + "AddonsInstaller", + f"Installing {addon} requires '{name}', which is not installed in your copy of FreeCAD.", + ) + else: + message = translate( + "AddonsInstaller", + f"Installing {addon} requires the following workbenches, which are not installed in your copy of FreeCAD:\n", + ) + for wb in missing_wbs: + message += " - " + wb + "\n" + QtWidgets.QMessageBox.critical( + self.dialog, + translate("AddonsInstaller", "Missing Requirement"), + message, + QtWidgets.QMessageBox.Cancel, + ) + elif ( + missing_external_addons + or missing_python_requirements + or missing_python_optionals + ): + self.dependency_dialog = FreeCADGui.PySideUic.loadUi( + os.path.join( + os.path.dirname(__file__), "dependency_resolution_dialog.ui" + ) + ) + missing_external_addons.sort() + missing_python_requirements.sort() + missing_python_optionals.sort() + missing_python_optionals = [ + option + for option in missing_python_optionals + if option not in missing_python_requirements + ] + + for addon in missing_external_addons: + self.dependency_dialog.listWidgetAddons.addItem(addon) + for mod in missing_python_requirements: + self.dependency_dialog.listWidgetPythonRequired.addItem(mod) + for mod in missing_python_optionals: + item = QtWidgets.QListWidgetItem(mod) + item.setFlags(Qt.ItemIsUserCheckable) + self.dependency_dialog.listWidgetPythonOptional.addItem(item) + + # For now, we don't offer to automatically install the dependencies + # self.dependency_dialog.buttonBox.button( + # QtWidgets.QDialogButtonBox.Yes + # ).clicked.connect(lambda: self.dependency_dialog_yes_clicked(repo)) + self.dependency_dialog.buttonBox.button( + QtWidgets.QDialogButtonBox.Ignore + ).clicked.connect(lambda: self.dependency_dialog_ignore_clicked(repo)) + self.dependency_dialog.exec() + else: + self.install(repo) + + def dependency_dialog_yes_clicked(self, repo: AddonManagerRepo) -> None: + # Get the lists out of the dialog: + addons = [] + for row in range(self.dependency_dialog.listWidgetAddons.count()): + item = self.dependency_dialog.listWidgetAddons.item(row) + name = item.text() + for repo in self.item_model.repos: + if repo.name == name or repo.display_name == name: + addons.append(repo) + + python_required = [] + for row in range(self.dependency_dialog.listWidgetPythonRequired.count()): + item = self.dependency_dialog.listWidgetPythonRequired.item(row) + python_required.append(item.text()) + + python_optional = [] + for row in range(self.dependency_dialog.listWidgetPythonOptional.count()): + item = self.dependency_dialog.listWidgetPythonOptional.item(row) + if item.checked(): + python_optional.append(item.text()) + + self.dependency_installation_worker = DependencyInstallationWorker( + addons, python_required, python_optional + ) + self.dependency_installation_worker.finished.connect(lambda: self.install(repo)) + self.dependency_installation_dialog = QtWidgets.QMessageBox( + QtWidgets.QMessageBox.Information, + translate("AddonsInstaller", "Installing dependencies"), + translate("AddonsInstaller", "Installing dependencies") + "...", + QtWidgets.QMessageBox.Cancel, + self.dialog, + ) + self.dependency_installation_dialog.rejected.connect( + self.cancel_dependency_installation + ) + self.dependency_installation_dialog.show() + self.dependency_installation_worker.start() + + def dependency_dialog_ignore_clicked(self, repo: AddonManagerRepo) -> None: + self.install(repo) + + def cancel_dependency_installation(self) -> None: + self.dependency_installation_worker.finished.disconnect( + lambda: self.install(repo) + ) + self.dependency_installation_worker.requestInterruption() + self.dependency_installation_dialog.hide() + def install(self, repo: AddonManagerRepo) -> None: """installs or updates a workbench, macro, or package""" @@ -722,6 +942,9 @@ class CommandAddonManager: if self.install_worker.isRunning(): return + if hasattr(self, "dependency_installation_dialog"): + self.dependency_installation_dialog.hide() + if not repo: return @@ -859,32 +1082,61 @@ class CommandAddonManager: def on_update_all_completed(self) -> None: self.hide_progress_widgets() + + def get_package_list( + message: str, repos: List[AddonManagerRepo], threshold: int + ): + """To ensure that the list doesn't get too long for the dialog, cut it off at some threshold""" + num_updates = len(repos) + if num_updates < threshold: + result = "".join([repo.name + "\n" for repo in repos]) + else: + result = translate( + "AddonsInstaller", f"{num_updates} total, see Report view for list" + ) + for repo in repos: + FreeCAD.Console.PrintMessage(f"{message}: {repo.name}\n") + return result + if not self.subupdates_failed: message = ( translate( "AddonsInstaller", - "All packages were successfully updated. Packages:", + "All packages were successfully updated", ) - + "\n" + + ": \n" + ) + message += get_package_list( + translate("AddonsInstaller", "Succeeded"), self.subupdates_succeeded, 15 ) - message += "".join([repo.name + "\n" for repo in self.subupdates_succeeded]) elif not self.subupdates_succeeded: message = ( - translate("AddonsInstaller", "All packages updates failed. Packages:") - + "\n" + translate("AddonsInstaller", "All packages updates failed:") + "\n" + ) + message += get_package_list( + translate("AddonsInstaller", "Failed"), self.subupdates_failed, 15 ) - message += "".join([repo.name + "\n" for repo in self.subupdates_failed]) else: message = ( translate( "AddonsInstaller", - "Some packages updates failed. Successful packages:", + "Some packages updates failed.", ) - + "\n" + + "\n\n" + + translate( + "AddonsInstaller", + "Succeeded", + ) + + ":\n" + ) + message += get_package_list( + translate("AddonsInstaller", "Succeeded"), self.subupdates_succeeded, 8 + ) + message += "\n\n" + message += translate("AddonsInstaller", "Failed") + ":\n" + message += get_package_list( + translate("AddonsInstaller", "Failed"), self.subupdates_failed, 8 ) - message += "".join([repo.name + "\n" for repo in self.subupdates_succeeded]) - message += translate("AddonsInstaller", "Failed packages:") + "\n" - message += "".join([repo.name + "\n" for repo in self.subupdates_failed]) for installed_repo in self.subupdates_succeeded: self.restart_required = True diff --git a/src/Mod/AddonManager/AddonManagerRepo.py b/src/Mod/AddonManager/AddonManagerRepo.py index 5dfbb3439f..c51fa4ad36 100644 --- a/src/Mod/AddonManager/AddonManagerRepo.py +++ b/src/Mod/AddonManager/AddonManagerRepo.py @@ -23,10 +23,12 @@ import FreeCAD import os -from typing import Dict +from typing import Dict, Set, List from addonmanager_macro import Macro +translate = FreeCAD.Qt.translate + class AddonManagerRepo: "Encapsulate information about a FreeCAD addon" @@ -70,6 +72,19 @@ class AddonManagerRepo: elif self.value == 4: return "Restart required" + class Dependencies: + def __init__(self): + self.required_external_addons = dict() + self.blockers = dict() + self.replaces = dict() + self.unrecognized_addons: Set[str] = set() + self.python_required: Set[str] = set() + self.python_optional: Set[str] = set() + + class ResolutionFailed(RuntimeError): + def __init__(self, msg): + super().__init__(msg) + def __init__(self, name: str, url: str, status: UpdateStatus, branch: str): self.name = name.strip() self.display_name = self.name @@ -93,6 +108,15 @@ class AddonManagerRepo: self.updated_timestamp = None self.installed_version = None + # Each repo is also a node in a directed dependency graph (referenced by name so + # they cen be serialized): + self.requires: Set[str] = set() + self.blocks: Set[str] = set() + + # And maintains a list of required and optional Python dependencies + self.python_requires: Set[str] = set() + self.python_optional: Set[str] = set() + def __str__(self) -> str: result = f"FreeCAD {self.repo_type}\n" result += f"Name: {self.name}\n" @@ -143,6 +167,13 @@ class AddonManagerRepo: ) if os.path.isfile(cached_package_xml_file): instance.load_metadata_file(cached_package_xml_file) + + if "requires" in cache_dict: + instance.requires = set(cache_dict["requires"]) + instance.blocks = set(cache_dict["blocks"]) + instance.python_requires = set(cache_dict["python_requires"]) + instance.python_optional = set(cache_dict["python_optional"]) + return instance def to_cache(self) -> Dict: @@ -159,6 +190,10 @@ class AddonManagerRepo: "python2": self.python2, "obsolete": self.obsolete, "rejected": self.rejected, + "requires": list(self.requires), + "blocks": list(self.blocks), + "python_requires": list(self.python_requires), + "python_optional": list(self.python_optional), } def load_metadata_file(self, file: str) -> None: @@ -251,3 +286,20 @@ class AddonManagerRepo: ) return self.cached_icon_filename + + def walk_dependency_tree(self, all_repos, deps): + """Compute the total dependency tree for this repo (recursive)""" + + deps.python_required |= self.python_requires + deps.python_optional |= self.python_optional + for dep in self.requires: + if dep in all_repos: + if not dep in deps.required: + deps.required_external_addons.append(all_repos[dep]) + all_repos[dep].walk_dependency_tree(all_repos, deps) + else: + # Maybe this is an internal workbench, just store its name + deps.unrecognized_addons.add(dep) + for dep in self.blocks: + if dep in all_repos: + deps.blockers[dep] = all_repos[dep] diff --git a/src/Mod/AddonManager/CMakeLists.txt b/src/Mod/AddonManager/CMakeLists.txt index 9ef2819ea1..1e5955f974 100644 --- a/src/Mod/AddonManager/CMakeLists.txt +++ b/src/Mod/AddonManager/CMakeLists.txt @@ -15,6 +15,7 @@ SET(AddonManager_SRCS AddonManagerOptions.ui first_run.ui compact_view.py + dependency_resolution_dialog.ui expanded_view.py package_list.py package_details.py diff --git a/src/Mod/AddonManager/addonmanager_metadata.py b/src/Mod/AddonManager/addonmanager_metadata.py index 59981863b1..b16149bbda 100644 --- a/src/Mod/AddonManager/addonmanager_metadata.py +++ b/src/Mod/AddonManager/addonmanager_metadata.py @@ -22,8 +22,8 @@ import FreeCAD -import tempfile import os +import io import hashlib from typing import Dict, List @@ -36,34 +36,17 @@ from AddonManagerRepo import AddonManagerRepo translate = FreeCAD.Qt.translate -class MetadataDownloadWorker(QObject): - """A worker for downloading package.xml and associated icon(s) - - To use, instantiate an object of this class and call the start_fetch() function - with a QNetworkAccessManager. It is expected that many of these objects will all - be created and associated with the same QNAM, which will then handle the actual - asynchronous downloads in some Qt-defined number of threads. To monitor progress - you should connect to the QNAM's "finished" signal, and ensure it is called the - number of times you expect based on how many workers you have enqueued. - - """ - +class DownloadWorker(QObject): updated = QtCore.Signal(AddonManagerRepo) - def __init__(self, parent, repo: AddonManagerRepo, index: Dict[str, str]): + def __init__(self, parent, url: str): """repo is an AddonManagerRepo object, and index is a dictionary of SHA1 hashes of the package.xml files in the cache""" super().__init__(parent) - self.repo = repo - self.index = index - self.store = os.path.join( - FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata" - ) - self.last_sha1 = "" - self.url = self.repo.metadata_url + self.url = url def start_fetch(self, network_manager: QtNetwork.QNetworkAccessManager): - """Asynchronously begin the network access. Intended as a set-and-forget black box for downloading metadata.""" + """Asynchronously begin the network access. Intended as a set-and-forget black box for downloading files.""" self.request = QtNetwork.QNetworkRequest(QtCore.QUrl(self.url)) self.request.setAttribute( @@ -92,11 +75,35 @@ class MetadataDownloadWorker(QObject): for error in errors: FreeCAD.Console.PrintWarning(error) + +class MetadataDownloadWorker(DownloadWorker): + """A worker for downloading package.xml and associated icon(s) + + To use, instantiate an object of this class and call the start_fetch() function + with a QNetworkAccessManager. It is expected that many of these objects will all + be created and associated with the same QNAM, which will then handle the actual + asynchronous downloads in some Qt-defined number of threads. To monitor progress + you should connect to the QNAM's "finished" signal, and ensure it is called the + number of times you expect based on how many workers you have enqueued. + + """ + + def __init__(self, parent, repo: AddonManagerRepo, index: Dict[str, str]): + """repo is an AddonManagerRepo object, and index is a dictionary of SHA1 hashes of the package.xml files in the cache""" + + super().__init__(parent, repo.metadata_url) + self.repo = repo + self.index = index + self.store = os.path.join( + FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata" + ) + self.last_sha1 = "" + def resolve_fetch(self): """Called when the data fetch completed, either with an error, or if it found the metadata file""" if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError: - FreeCAD.Console.PrintLog(f"Found a metadata file for {self.repo.name}\n") + FreeCAD.Console.PrintLog(f"Found a package.xml file for {self.repo.name}\n") self.repo.repo_type = AddonManagerRepo.RepoType.PACKAGE new_xml = self.fetch_task.readAll() hasher = hashlib.sha1() @@ -179,3 +186,70 @@ class MetadataDownloadWorker(QObject): icon_file.write(icon_data) self.repo.cached_icon_filename = cache_file self.updated.emit(self.repo) + + +class DependencyDownloadWorker(DownloadWorker): + """A worker for downloading metadata.txt""" + + def __init__(self, parent, repo: AddonManagerRepo): + super().__init__(parent, utils.construct_git_url(repo, "metadata.txt")) + self.repo = repo + + def resolve_fetch(self): + """Called when the data fetch completed, either with an error, or if it found the metadata file""" + + if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError: + FreeCAD.Console.PrintLog( + f"Found a metadata.txt file for {self.repo.name}\n" + ) + new_deps = self.fetch_task.readAll() + self.parse_file(new_deps.data().decode("utf8")) + elif ( + self.fetch_task.error() + == QtNetwork.QNetworkReply.NetworkError.ContentNotFoundError + ): + pass + elif ( + self.fetch_task.error() + == QtNetwork.QNetworkReply.NetworkError.OperationCanceledError + ): + pass + else: + FreeCAD.Console.PrintWarning( + translate("AddonsInstaller", "Failed to connect to URL") + + f":\n{self.url}\n {self.fetch_task.error()}\n" + ) + + def parse_file(self, data: str) -> None: + f = io.StringIO(data) + while True: + line = f.readline() + if not line: + break + if line.startswith("workbenches="): + depswb = line.split("=")[1].split(",") + for wb in depswb: + wb_name = wb.strip() + if wb_name: + self.repo.requires.add(wb_name) + FreeCAD.Console.PrintLog( + f"{self.repo.display_name} requires FreeCAD Addon '{wb_name}'\n" + ) + + elif line.startswith("pylibs="): + depspy = line.split("=")[1].split(",") + for pl in depspy: + if pl.strip(): + self.repo.python_requires.add(pl.strip()) + FreeCAD.Console.PrintLog( + f"{self.repo.display_name} requires python package '{pl.strip()}'\n" + ) + elif line.startswith("optionalpylibs="): + opspy = line.split("=")[1].split(",") + for pl in opspy: + if pl.strip(): + self.repo.python_optional.add(pl.strip()) + FreeCAD.Console.PrintLog( + f"{self.repo.display_name} optionally imports python package '{pl.strip()}'\n" + ) + self.updated.emit(self.repo) diff --git a/src/Mod/AddonManager/addonmanager_utilities.py b/src/Mod/AddonManager/addonmanager_utilities.py index 1ab5fafc15..1bd520f8d7 100644 --- a/src/Mod/AddonManager/addonmanager_utilities.py +++ b/src/Mod/AddonManager/addonmanager_utilities.py @@ -294,6 +294,43 @@ def fix_relative_links(text, base_url): return new_text +def repair_git_repo(repo_url: str, clone_dir: str) -> None: + # Repair addon installed with raw download by adding the .git + # directory to it + try: + bare_repo = git.Repo.clone_from( + repo_url, clone_dir + os.sep + ".git", bare=True + ) + with bare_repo.config_writer() as cw: + cw.set("core", "bare", False) + except AttributeError: + FreeCAD.Console.PrintLog( + translate( + "AddonsInstaller", + "Outdated GitPython detected, consider upgrading with pip.", + ) + + "\n" + ) + cw = bare_repo.config_writer() + cw.set("core", "bare", False) + del cw + except Exception as e: + FreeCAD.Console.PrintWarning( + translate("AddonsInstaller", "Failed to repair missing .git directory") + + "\n" + ) + FreeCAD.Console.PrintWarning( + translate("AddonsInstaller", "Repository URL") + f": {repo_url}\n" + ) + FreeCAD.Console.PrintWarning( + translate("AddonsInstaller", "Clone directory") + f": {clone_dir}\n" + ) + FreeCAD.Console.PrintWarning(e) + return + repo = git.Repo(clone_dir) + repo.head.reset("--hard") + + def warning_color_string() -> str: """A shade of red, adapted to darkmode if possible. Targets a minimum 7:1 contrast ratio.""" diff --git a/src/Mod/AddonManager/addonmanager_workers.py b/src/Mod/AddonManager/addonmanager_workers.py index d0752c9444..00efeb6a2c 100644 --- a/src/Mod/AddonManager/addonmanager_workers.py +++ b/src/Mod/AddonManager/addonmanager_workers.py @@ -32,6 +32,8 @@ import threading import queue import io import time +import subprocess +import sys from datetime import datetime from typing import Union, List @@ -44,7 +46,7 @@ if FreeCAD.GuiUp: import addonmanager_utilities as utils from addonmanager_macro import Macro -from addonmanager_metadata import MetadataDownloadWorker +from addonmanager_metadata import MetadataDownloadWorker, DependencyDownloadWorker from AddonManagerRepo import AddonManagerRepo translate = FreeCAD.Qt.translate @@ -95,12 +97,52 @@ NOMARKDOWN = False # for debugging purposes, set this to True to disable Markdo """Multithread workers for the Addon Manager""" +class ConnectionChecker(QtCore.QThread): + + success = QtCore.Signal() + failure = QtCore.Signal(str) + + def __init__(self): + QtCore.QThread.__init__(self) + + def run(self): + FreeCAD.Console.PrintLog( + translate("AddonsInstaller", "Checking network connection...\n") + ) + url = "https://api.github.com/zen" + request = utils.urlopen(url) + if QtCore.QThread.currentThread().isInterruptionRequested(): + return + if not request: + self.failure.emit( + translate( + "AddonsInstaller", + "Unable to connect to GitHub: check your internet connection and proxy settings and try again.", + ) + ) + return + result = request.read() + if QtCore.QThread.currentThread().isInterruptionRequested(): + return + if not result: + self.failure.emit( + translate( + "AddonsInstaller", + "Unable to read data from GitHub: check your internet connection and proxy settings and try again.", + ) + ) + return + + result = result.decode("utf8") + FreeCAD.Console.PrintLog(f"GitHub's zen message response: {result}\n") + self.success.emit() + + class UpdateWorker(QtCore.QThread): """This worker updates the list of available workbenches""" status_message = QtCore.Signal(str) addon_repo = QtCore.Signal(object) - done = QtCore.Signal() def __init__(self): @@ -190,8 +232,6 @@ class UpdateWorker(QtCore.QThread): "https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/.gitmodules" ) if not u: - self.done.emit() - self.stop = True return p = u.read() if isinstance(p, bytes): @@ -206,7 +246,7 @@ class UpdateWorker(QtCore.QThread): ), p, ) - for name, path, url, _, branch in p: + for name, _, url, _, branch in p: if self.current_thread.isInterruptionRequested(): return if name in package_names: @@ -240,14 +280,9 @@ class UpdateWorker(QtCore.QThread): translate("AddonsInstaller", "Workbenches list was updated.") ) - if not self.current_thread.isInterruptionRequested(): - self.done.emit() - self.stop = True - class LoadPackagesFromCacheWorker(QtCore.QThread): addon_repo = QtCore.Signal(object) - done = QtCore.Signal() def __init__(self, cache_file: str): QtCore.QThread.__init__(self) @@ -281,7 +316,6 @@ class LoadPackagesFromCacheWorker(QtCore.QThread): ) pass self.addon_repo.emit(repo) - self.done.emit() class LoadMacrosFromCacheWorker(QtCore.QThread): @@ -309,7 +343,6 @@ class CheckWorkbenchesForUpdatesWorker(QtCore.QThread): update_status = QtCore.Signal(AddonManagerRepo) progress_made = QtCore.Signal(int, int) - done = QtCore.Signal() def __init__(self, repos: List[AddonManagerRepo]): @@ -319,14 +352,10 @@ class CheckWorkbenchesForUpdatesWorker(QtCore.QThread): def run(self): if NOGIT or not have_git: - self.done.emit() - self.stop = True return self.current_thread = QtCore.QThread.currentThread() self.basedir = FreeCAD.getUserAppDataDir() self.moddir = self.basedir + os.sep + "Mod" - upds = [] - gitpython_warning = False count = 1 for repo in self.repos: if self.current_thread.isInterruptionRequested(): @@ -341,39 +370,14 @@ class CheckWorkbenchesForUpdatesWorker(QtCore.QThread): elif repo.repo_type == AddonManagerRepo.RepoType.PACKAGE: self.check_package(repo) - self.stop = True - self.done.emit() - def check_workbench(self, wb): - gitpython_warning = False if not have_git or NOGIT: return clonedir = self.moddir + os.sep + wb.name if os.path.exists(clonedir): # mark as already installed AND already checked for updates if not os.path.exists(clonedir + os.sep + ".git"): - # Repair addon installed with raw download - bare_repo = git.Repo.clone_from( - wb.url, clonedir + os.sep + ".git", bare=True - ) - try: - with bare_repo.config_writer() as cw: - cw.set("core", "bare", False) - except AttributeError: - if not gitpython_warning: - FreeCAD.Console.PrintWarning( - translate( - "AddonsInstaller", - "Outdated GitPython detected, consider upgrading with pip.", - ) - + "\n" - ) - gitpython_warning = True - cw = bare_repo.config_writer() - cw.set("core", "bare", False) - del cw - repo = git.Repo(clonedir) - repo.head.reset("--hard") + utils.repair_git_repo(wb.url, clonedir) gitrepo = git.Git(clonedir) try: gitrepo.fetch() @@ -430,7 +434,7 @@ class CheckWorkbenchesForUpdatesWorker(QtCore.QThread): AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE ) self.update_status.emit(package) - except Exception as e: + except Exception: FreeCAD.Console.PrintWarning( translate( "AddonsInstaller", @@ -499,7 +503,6 @@ class FillMacroListWorker(QtCore.QThread): add_macro_signal = QtCore.Signal(object) status_message_signal = QtCore.Signal(str) progress_made = QtCore.Signal(int, int) - done = QtCore.Signal() def __init__(self, repo_dir): @@ -525,7 +528,7 @@ class FillMacroListWorker(QtCore.QThread): self.status_message_signal.emit( translate( "AddonInstaller", - "Retrieving macros from FreeCAD/FreeCAD-Macros Git repository", + "Retrieving macros from FreeCAD wiki", ) ) self.retrieve_macros_from_wiki() @@ -536,8 +539,6 @@ class FillMacroListWorker(QtCore.QThread): self.status_message_signal.emit( translate("AddonsInstaller", "Done locating macros.") ) - self.stop = True - self.done.emit() def retrieve_macros_from_git(self): """Retrieve macros from FreeCAD-macros.git @@ -557,6 +558,10 @@ class FillMacroListWorker(QtCore.QThread): try: if os.path.exists(self.repo_dir): + if not os.path.exists(os.path.join(self.repo_dir, ".git")): + utils.repair_git_repo( + "https://github.com/FreeCAD/FreeCAD-macros.git", self.repo_dir + ) gitrepo = git.Git(self.repo_dir) gitrepo.pull() else: @@ -688,7 +693,7 @@ class CacheMacroCode(QtCore.QThread): time.sleep(0.1) # Make sure all of our child threads have fully exited: - for i, worker in enumerate(self.workers): + for worker in self.workers: worker.wait(50) if not worker.isFinished(): FreeCAD.Console.PrintError( @@ -773,7 +778,6 @@ class ShowWorker(QtCore.QThread): status_message = QtCore.Signal(str) readme_updated = QtCore.Signal(str) update_status = QtCore.Signal(AddonManagerRepo) - done = QtCore.Signal() def __init__(self, repo, cache_path): @@ -843,7 +847,6 @@ class ShowWorker(QtCore.QThread): # fall back to the description text u = utils.urlopen(url) if not u: - self.stop = True return p = u.read() if isinstance(p, bytes): @@ -871,27 +874,7 @@ class ShowWorker(QtCore.QThread): ) if os.path.exists(clonedir): if not os.path.exists(clonedir + os.sep + ".git"): - # Repair addon installed with raw download - bare_repo = git.Repo.clone_from( - repo.url, clonedir + os.sep + ".git", bare=True - ) - try: - with bare_repo.config_writer() as cw: - cw.set("core", "bare", False) - except AttributeError: - FreeCAD.Console.PrintWarning( - translate( - "AddonsInstaller", - "Outdated GitPython detected, " - "consider upgrading with pip.", - ) - + "\n" - ) - cw = bare_repo.config_writer() - cw.set("core", "bare", False) - del cw - repo = git.Repo(clonedir) - repo.head.reset("--hard") + utils.repair_git_repo(self.repo.url, clonedir) gitrepo = git.Git(clonedir) gitrepo.fetch() if "git pull" in gitrepo.status(): @@ -955,8 +938,6 @@ class ShowWorker(QtCore.QThread): self.readme_updated.emit(label) if QtCore.QThread.currentThread().isInterruptionRequested(): return - self.done.emit() - self.stop = True def stopImageLoading(self): "this stops the image loading process and allow the thread to terminate earlier" @@ -972,7 +953,6 @@ class ShowWorker(QtCore.QThread): imagepaths = re.findall(' None: if NOGIT or not have_git: @@ -1161,27 +1136,7 @@ class InstallWorkbenchWorker(QtCore.QThread): + "\n" ) if not os.path.exists(clonedir + os.sep + ".git"): - # Repair addon installed with raw download by adding the .git - # directory to it - bare_repo = git.Repo.clone_from( - self.repo.url, clonedir + os.sep + ".git", bare=True - ) - try: - with bare_repo.config_writer() as cw: - cw.set("core", "bare", False) - except AttributeError: - FreeCAD.Console.PrintLog( - translate( - "AddonsInstaller", - "Outdated GitPython detected, consider " "upgrading with pip.", - ) - + "\n" - ) - cw = bare_repo.config_writer() - cw.set("core", "bare", False) - del cw - repo = git.Repo(clonedir) - repo.head.reset("--hard") + utils.repair_git_repo(self.repo.url, clonedir) repo = git.Git(clonedir) try: repo.pull() @@ -1210,36 +1165,31 @@ class InstallWorkbenchWorker(QtCore.QThread): def run_git_clone(self, clonedir: str) -> None: self.status_message.emit("Checking module dependencies...") - depsok, answer = self.check_python_dependencies(self.repo.url) - if depsok: - if str(self.repo.name) in py2only: - FreeCAD.Console.PrintWarning( - translate( - "AddonsInstaller", - "You are installing a Python 2 workbench on " - "a system running Python 3 - ", - ) - + str(self.repo.name) - + "\n" + if str(self.repo.name) in py2only: + FreeCAD.Console.PrintWarning( + translate( + "AddonsInstaller", + "You are installing a Python 2 workbench on " + "a system running Python 3 - ", ) - self.status_message.emit("Cloning module...") - repo = git.Repo.clone_from(self.repo.url, clonedir) - - # Make sure to clone all the submodules as well - if repo.submodules: - repo.submodule_update(recursive=True) - - if self.repo.branch in repo.heads: - repo.heads[self.repo.branch].checkout() - - answer = translate( - "AddonsInstaller", - "Workbench successfully installed. Please restart " - "FreeCAD to apply the changes.", + + str(self.repo.name) + + "\n" ) - else: - self.failure.emit(self.repo, answer) - return + self.status_message.emit("Cloning module...") + repo = git.Repo.clone_from(self.repo.url, clonedir) + + # Make sure to clone all the submodules as well + if repo.submodules: + repo.submodule_update(recursive=True) + + if self.repo.branch in repo.heads: + repo.heads[self.repo.branch].checkout() + + answer = translate( + "AddonsInstaller", + "Workbench successfully installed. Please restart " + "FreeCAD to apply the changes.", + ) if self.repo.repo_type == AddonManagerRepo.RepoType.WORKBENCH: # symlink any macro contained in the module to the macros folder @@ -1270,98 +1220,6 @@ class InstallWorkbenchWorker(QtCore.QThread): self.update_metadata() self.success.emit(self.repo, answer) - def check_python_dependencies(self, baseurl: str) -> Union[bool, str]: - """checks if the repo contains a metadata.txt and check its contents""" - - ok = True - message = "" - depsurl = baseurl.replace("github.com", "raw.githubusercontent.com") - if not depsurl.endswith("/"): - depsurl += "/" - depsurl += "master/metadata.txt" - try: - mu = utils.urlopen(depsurl) - except Exception: - return True, translate( - "AddonsInstaller", - "No metadata.txt found, cannot evaluate Python dependencies", - ) - if mu: - # metadata.txt found - depsfile = mu.read() - mu.close() - - # urllib2 gives us a bytelike object instead of a string. Have to - # consider that - try: - depsfile = depsfile.decode("utf-8") - except AttributeError: - pass - - deps = depsfile.split("\n") - for line in deps: - if line.startswith("workbenches="): - depswb = line.split("=")[1].split(",") - for wb in depswb: - if wb.strip(): - if not wb.strip() in FreeCADGui.listWorkbenches().keys(): - if ( - not wb.strip() + "Workbench" - in FreeCADGui.listWorkbenches().keys() - ): - ok = False - message += ( - translate( - "AddonsInstaller", "Missing workbench" - ) - + ": " - + wb - + ", " - ) - elif line.startswith("pylibs="): - depspy = line.split("=")[1].split(",") - for pl in depspy: - if pl.strip(): - try: - __import__(pl.strip()) - except ImportError: - ok = False - message += ( - translate( - "AddonsInstaller", "Missing python module" - ) - + ": " - + pl - + ", " - ) - elif line.startswith("optionalpylibs="): - opspy = line.split("=")[1].split(",") - for pl in opspy: - if pl.strip(): - try: - __import__(pl.strip()) - except ImportError: - message += translate( - "AddonsInstaller", - "Missing optional python module (doesn't prevent installing)", - ) - message += ": " + pl + ", " - if message and (not ok): - final_message = translate( - "AddonsInstaller", - "Some errors were found that prevent installation of this workbench", - ) - final_message += ": " + message + ". " - final_message += translate( - "AddonsInstaller", "Please install the missing components first." - ) - message = final_message - return ok, message - - def check_package_dependencies(self): - # TODO: Use the dependencies set in the package.xml metadata - pass - def run_zip(self, zipdir: str) -> None: "downloads and unzip a zip version from a git repo" @@ -1454,6 +1312,67 @@ class InstallWorkbenchWorker(QtCore.QThread): self.repo.updated_timestamp = datetime.now().timestamp() +class DependencyInstallationWorker(QtCore.QThread): + """Install dependencies: not yet implemented, DO NOT CALL""" + + def __init__(self, addons, python_required, python_optional): + QtCore.QThread.__init__(self) + self.addons = addons + self.python_required = python_required + self.python_optional = python_optional + + def run(self): + + for repo in self.addons: + if QtCore.QThread.currentThread().isInterruptionRequested(): + return + worker = InstallWorkbenchWorker(repo) + # Don't bother with a separate thread for this right now, just run it here: + FreeCAD.Console.PrintMessage(f"Pretending to install {repo.name}") + time.sleep(3) + continue + # worker.run() + + if self.python_required or self.python_optional: + # See if we have pip available: + try: + subprocess.check_call(["pip", "--version"]) + except subprocess.CalledProcessError as e: + FreeCAD.Console.PrintError( + translate( + "AddonsInstaller", "Failed to execute pip. Returned error was:" + ) + + f"\n{e.output}" + ) + return + + for pymod in self.python_required: + if QtCore.QThread.currentThread().isInterruptionRequested(): + return + FreeCAD.Console.PrintMessage(f"Pretending to install {pymod}") + time.sleep(3) + continue + # subprocess.check_call(["pip", "install", pymod]) + + for pymod in self.python_optional: + if QtCore.QThread.currentThread().isInterruptionRequested(): + return + try: + FreeCAD.Console.PrintMessage(f"Pretending to install {pymod}") + time.sleep(3) + continue + # subprocess.check_call([sys.executable, "-m", "pip", "install", pymod]) + except subprocess.CalledProcessError as e: + FreeCAD.Console.PrintError( + translate( + "AddonsInstaller", + "Failed to install option dependency {pymod}. Returned error was:", + ) + + f"\n{e.output}" + ) + # This is not fatal, we can just continue without it + + class CheckSingleWorker(QtCore.QThread): """Worker to check for updates for a single addon""" @@ -1491,7 +1410,6 @@ class UpdateMetadataCacheWorker(QtCore.QThread): status_message = QtCore.Signal(str) progress_made = QtCore.Signal(int, int) - done = QtCore.Signal() package_updated = QtCore.Signal(AddonManagerRepo) class AtomicCounter(object): @@ -1546,11 +1464,18 @@ class UpdateMetadataCacheWorker(QtCore.QThread): self.downloaders = [] for repo in self.repos: if repo.metadata_url: + # package.xml downloader = MetadataDownloadWorker(None, repo, self.index) downloader.start_fetch(download_queue) downloader.updated.connect(self.on_updated) self.downloaders.append(downloader) + # metadata.txt + downloader = DependencyDownloadWorker(None, repo) + downloader.start_fetch(download_queue) + downloader.updated.connect(self.on_updated) + self.downloaders.append(downloader) + # Run a local event loop until we've processed all of the downloads: # this is local to this thread, and does not affect the main event loop ui_updater = QtCore.QTimer() @@ -1575,16 +1500,13 @@ class UpdateMetadataCacheWorker(QtCore.QThread): # Update and serialize the updated index, overwriting whatever was # there before for downloader in self.downloaders: - self.index[downloader.repo.name] = downloader.last_sha1 + if hasattr(downloader, "last_sha1"): + self.index[downloader.repo.name] = downloader.last_sha1 if not os.path.exists(store): os.makedirs(store) with open(index_file, "w") as f: json.dump(self.index, f, indent=" ") - # Signal completion to our parent thread - self.done.emit() - self.stop = True - def on_finished(self, reply): # Called by the QNetworkAccessManager's sub-threads when a fetch # process completed (in any state) @@ -1614,7 +1536,7 @@ if have_git and not NOGIT: def update( self, - op_code: int, + _: int, cur_count: Union[str, float], max_count: Union[str, float, None] = None, message: str = "", @@ -1632,7 +1554,6 @@ class UpdateAllWorker(QtCore.QThread): status_message = QtCore.Signal(str) success = QtCore.Signal(AddonManagerRepo) failure = QtCore.Signal(AddonManagerRepo) - done = QtCore.Signal() def __init__(self, repos): super().__init__() @@ -1666,11 +1587,9 @@ class UpdateAllWorker(QtCore.QThread): self.repo_queue.join() # Make sure all of our child threads have fully exited: - for i, worker in enumerate(workers): + for worker in workers: worker.wait() - self.done.emit() - def on_success(self, repo: AddonManagerRepo) -> None: self.progress_made.emit( len(self.repos) - self.repo_queue.qsize(), len(self.repos) @@ -1714,12 +1633,10 @@ class UpdateSingleWorker(QtCore.QThread): FreeCAD.getUserCachePath(), "AddonManager", "MacroCache" ) os.makedirs(cache_path, exist_ok=True) - install_succeeded, errors = repo.macro.install(cache_path) + install_succeeded, _ = repo.macro.install(cache_path) if install_succeeded: - install_succeeded, errors = repo.macro.install( - FreeCAD.getUserMacroDir(True) - ) + install_succeeded, _ = repo.macro.install(FreeCAD.getUserMacroDir(True)) utils.update_macro_installation_details(repo) if install_succeeded: diff --git a/src/Mod/AddonManager/dependency_resolution_dialog.ui b/src/Mod/AddonManager/dependency_resolution_dialog.ui new file mode 100644 index 0000000000..45803721c0 --- /dev/null +++ b/src/Mod/AddonManager/dependency_resolution_dialog.ui @@ -0,0 +1,117 @@ + + + DependencyResolutionDialog + + + Qt::WindowModal + + + + 0 + 0 + 455 + 200 + + + + Resolve Dependencies + + + + + + This Addon has the following required and optional dependencies. You must install them before this Addon can be used. + + + true + + + + + + + + + FreeCAD Addons + + + + + + + + + + + + Required Python modules + + + + + + + + + + + + Optional Python modules + + + + + + + + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ignore + + + + + + + + + buttonBox + accepted() + DependencyResolutionDialog + accept() + + + 248 + 254 + + + 157 + 274 + + + + + buttonBox + rejected() + DependencyResolutionDialog + reject() + + + 316 + 260 + + + 286 + 274 + + + + + diff --git a/src/Mod/AddonManager/package_list.py b/src/Mod/AddonManager/package_list.py index 247f5bac91..24040873e8 100644 --- a/src/Mod/AddonManager/package_list.py +++ b/src/Mod/AddonManager/package_list.py @@ -658,8 +658,6 @@ class Ui_PackageList(object): self.listPackages.setEditTriggers(QAbstractItemView.NoEditTriggers) self.listPackages.setProperty("showDropIndicator", False) self.listPackages.setSelectionMode(QAbstractItemView.NoSelection) - self.listPackages.setLayoutMode(QListView.Batched) - self.listPackages.setBatchSize(15) self.listPackages.setResizeMode(QListView.Adjust) self.listPackages.setUniformItemSizes(False) self.listPackages.setAlternatingRowColors(True)