From 768a0f086fecafb7cec6fcefcb532021b0d8cc4c Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sun, 10 Oct 2021 14:40:02 -0500 Subject: [PATCH] Addon Manager: Rework backend to use package.xml This shifts to use the model-view-controller pattern for the list of addons, and moves to using a full model class rather than an indexed array for the data storage and management. This enables much more information to be stored as part of the new AddonManagerRepo data type. It now wraps the Macro class for macros, supports Preference Packs, and provides access to the Metadata object. --- src/Mod/AddonManager/AddonManager.py | 1109 +++++++++++------ src/Mod/AddonManager/AddonManager.ui | 246 +++- src/Mod/AddonManager/AddonManagerRepo.py | 164 +++ src/Mod/AddonManager/CMakeLists.txt | 2 + src/Mod/AddonManager/addonmanager_macro.py | 114 +- src/Mod/AddonManager/addonmanager_metadata.py | 148 +++ .../AddonManager/addonmanager_utilities.py | 172 +-- src/Mod/AddonManager/addonmanager_workers.py | 1083 ++++++++++------ src/Mod/Start/StartPage/LoadNew.py | 2 +- 9 files changed, 2166 insertions(+), 874 deletions(-) create mode 100644 src/Mod/AddonManager/AddonManagerRepo.py create mode 100644 src/Mod/AddonManager/addonmanager_metadata.py diff --git a/src/Mod/AddonManager/AddonManager.py b/src/Mod/AddonManager/AddonManager.py index 083bec0aa3..e4c72c835e 100644 --- a/src/Mod/AddonManager/AddonManager.py +++ b/src/Mod/AddonManager/AddonManager.py @@ -4,6 +4,7 @@ #*************************************************************************** #* * #* Copyright (c) 2015 Yorik van Havre * +#* Copyright (c) 2021 Chris Hennes * #* * #* This program is free software; you can redistribute it and/or modify * #* it under the terms of the GNU Lesser General Public License (LGPL) * @@ -23,53 +24,62 @@ #* * #*************************************************************************** -from __future__ import print_function - import os import shutil import stat import tempfile +from typing import Dict, Union -from PySide import QtGui, QtCore +from PySide2 import QtGui, QtCore, QtWidgets import FreeCADGui from addonmanager_utilities import translate # this needs to be as is for pylupdate from addonmanager_workers import * import addonmanager_utilities as utils import AddonManager_rc +from AddonManagerRepo import AddonManagerRepo __title__ = "FreeCAD Addon Manager Module" -__author__ = "Yorik van Havre", "Jonathan Wiedemann", "Kurt Kremitzki" -__url__ = "http://www.freecadweb.org" +__author__ = "Yorik van Havre", "Jonathan Wiedemann", "Kurt Kremitzki", "Chris Hennes" +__url__ = "http://www.freecad.org" """ FreeCAD Addon Manager Module -It will fetch its contents from https://github.com/FreeCAD/FreeCAD-addons +Fetches various types of addons from a variety of sources. Built-in sources are: +* https://github.com/FreeCAD/FreeCAD-addons +* https://github.com/FreeCAD/FreeCAD-macros +* https://wiki.freecad.org/ + +Additional git sources may be configure via user preferences. + You need a working internet connection, and optionally the GitPython package installed. """ # \defgroup ADDONMANAGER AddonManager # \ingroup ADDONMANAGER -# \brief The Addon Manager allows to install workbenches and macros made by users +# \brief The Addon Manager allows users to install workbenches and macros made by other users # @{ def QT_TRANSLATE_NOOP(ctx, txt): return txt - class CommandAddonManager: """The main Addon Manager class and FreeCAD command""" - def GetResources(self): + workers = ["update_worker", "check_worker", "show_worker", + "showmacro_worker", "macro_worker", "install_worker", + "update_metadata_cache_worker", "update_all_worker"] + + def GetResources(self) -> Dict[str,str]: return {"Pixmap": "AddonManager", "MenuText": QT_TRANSLATE_NOOP("Std_AddonMgr", "&Addon manager"), - "ToolTip": QT_TRANSLATE_NOOP("Std_AddonMgr", "Manage external workbenches and macros"), + "ToolTip": QT_TRANSLATE_NOOP("Std_AddonMgr", "Manage external workbenches, macros, and preference packs"), "Group": "Tools"} - def Activated(self): + def Activated(self) -> None: # display first use dialog if needed readWarningParameter = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") @@ -77,23 +87,23 @@ class CommandAddonManager: newReadWarningParameter = FreeCAD.ParamGet("User parameter:Plugins/addonsRepository") readWarning |= newReadWarningParameter.GetBool("readWarning", False) if not readWarning: - if (QtGui.QMessageBox.warning(None, + if (QtWidgets.QMessageBox.warning(None, "FreeCAD", translate("AddonsInstaller", "The addons that can be installed here are not " "officially part of FreeCAD, and are not reviewed " "by the FreeCAD team. Make sure you know what you " "are installing!"), - QtGui.QMessageBox.Cancel | - QtGui.QMessageBox.Ok) != - QtGui.QMessageBox.StandardButton.Cancel): + QtWidgets.QMessageBox.Cancel | + QtWidgets.QMessageBox.Ok) != + QtWidgets.QMessageBox.StandardButton.Cancel): readWarningParameter.SetBool("readWarning", True) readWarning = True if readWarning: self.launch() - def launch(self): + def launch(self) -> None: """Shows the Addon Manager UI""" # create the dialog @@ -101,22 +111,10 @@ class CommandAddonManager: "AddonManager.ui")) # cleanup the leftovers from previous runs - self.repos = [] - self.macros = [] self.macro_repo_dir = tempfile.mkdtemp() - self.doUpdate = [] + self.packages_with_updates = [] self.addon_removed = False - for worker in ["update_worker", "check_worker", "show_worker", - "showmacro_worker", "macro_worker", "install_worker"]: - if hasattr(self, worker): - thread = getattr(self, worker) - if thread: - if thread.isFinished(): - setattr(self, worker, None) - self.dialog.tabWidget.setCurrentIndex(0) - # these 2 settings to prevent loading an addon description on start (let the user click something first) - self.firsttime = True - self.firstmacro = True + self.cleanup_workers() # restore window geometry and splitter state from stored state pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") @@ -127,36 +125,62 @@ class CommandAddonManager: sr = pref.GetInt("SplitterRight", 274) self.dialog.splitter.setSizes([sl, sr]) + # Set up the listing of packages using the model-view-controller architecture + self.item_model = PackageListItemModel() + self.item_delegate = PackageListIconDelegate() + self.item_filter = PackageListFilter() + self.item_filter.setSourceModel(self.item_model) + self.dialog.tablePackages.setModel (self.item_filter) + self.dialog.tablePackages.setItemDelegate(self.item_delegate) + header = self.dialog.tablePackages.horizontalHeader() + header.setSectionResizeMode(0, QtWidgets.QHeaderView.Fixed) + header.resizeSection(0,20) + header.setSectionResizeMode(1, QtWidgets.QHeaderView.Stretch) + header.setSectionResizeMode(2, QtWidgets.QHeaderView.ResizeToContents) + self.dialog.tablePackages.sortByColumn(1, QtCore.Qt.AscendingOrder) # Default to sorting alphabetically by name + # set nice icons to everything, by theme with fallback to FreeCAD icons self.dialog.setWindowIcon(QtGui.QIcon(":/icons/AddonManager.svg")) - self.dialog.buttonExecute.setIcon(QtGui.QIcon.fromTheme("execute", QtGui.QIcon(":/icons/button_valid.svg"))) self.dialog.buttonUninstall.setIcon(QtGui.QIcon.fromTheme("cancel", QtGui.QIcon(":/icons/edit_Cancel.svg"))) self.dialog.buttonInstall.setIcon(QtGui.QIcon.fromTheme("download", QtGui.QIcon(":/icons/edit_OK.svg"))) self.dialog.buttonUpdateAll.setIcon(QtGui.QIcon(":/icons/button_valid.svg")) self.dialog.buttonConfigure.setIcon(QtGui.QIcon(":/icons/preferences-system.svg")) self.dialog.buttonClose.setIcon(QtGui.QIcon.fromTheme("close", QtGui.QIcon(":/icons/process-stop.svg"))) - self.dialog.tabWidget.setTabIcon(0, QtGui.QIcon.fromTheme("folder", QtGui.QIcon(":/icons/folder.svg"))) - self.dialog.tabWidget.setTabIcon(1, QtGui.QIcon(":/icons/applications-python.svg")) # enable/disable stuff - self.dialog.buttonExecute.setEnabled(False) self.dialog.buttonUninstall.setEnabled(False) self.dialog.buttonInstall.setEnabled(False) self.dialog.buttonUpdateAll.setEnabled(False) + self.dialog.buttonExecute.hide() + self.dialog.labelFilterValidity.hide() + + # Hide package-related interface elements until needed + self.dialog.labelPackageName.hide() + self.dialog.labelVersion.hide() + self.dialog.labelMaintainer.hide() + self.dialog.labelIcon.hide() + self.dialog.labelDescription.hide() + self.dialog.labelUrl.hide() + self.dialog.labelUrlType.hide() + self.dialog.labelContents.hide() # connect slots - self.dialog.buttonExecute.clicked.connect(self.executemacro) self.dialog.rejected.connect(self.reject) self.dialog.buttonInstall.clicked.connect(self.install) self.dialog.buttonUninstall.clicked.connect(self.remove) - self.dialog.buttonUpdateAll.clicked.connect(self.apply_updates) - self.dialog.listWorkbenches.currentRowChanged.connect(self.show) - self.dialog.tabWidget.currentChanged.connect(self.switchtab) - self.dialog.listMacros.currentRowChanged.connect(self.show_macro) + self.dialog.buttonUpdateAll.clicked.connect(self.update_all) + self.dialog.comboPackageType.currentIndexChanged.connect(self.update_type_filter) + self.dialog.lineEditFilter.textChanged.connect(self.update_text_filter) self.dialog.buttonConfigure.clicked.connect(self.show_config) self.dialog.buttonClose.clicked.connect(self.dialog.reject) + self.dialog.buttonExecute.clicked.connect(self.executemacro) + self.dialog.tablePackages.selectionModel().currentRowChanged.connect(self.table_row_selected) + self.dialog.tablePackages.setEnabled(False) - # allow to open links in browser + # Show "Workbenches" to start with + self.dialog.comboPackageType.setCurrentIndex(1) + + # allow links to open in browser self.dialog.description.setOpenLinks(True) self.dialog.description.setOpenExternalLinks(True) @@ -164,13 +188,42 @@ class CommandAddonManager: mw = FreeCADGui.getMainWindow() self.dialog.move(mw.frameGeometry().topLeft() + mw.rect().center() - self.dialog.rect().center()) - # populate the list - self.update() + # set info for the progress bar: + self.number_of_progress_regions = 4 + self.current_progress_region = 0 + self.dialog.progressBar.setMaximum (100) + + # populate the table + self.populate_packages_table() + + # set the label text to start with + self.show_information(translate("AddonInstaller", "Loading addon information")) # rock 'n roll!!! self.dialog.exec_() - def reject(self): + def cleanup_workers(self) -> None: + """ Ensure that no workers are running by explicitly asking them to stop, and terminating them if they don't """ + for worker in self.workers: + if hasattr(self, worker): + thread = getattr(self, worker) + if thread: + if not thread.isFinished(): + thread.requestInterruption() + thread.wait(QtCore.QDeadlineTimer(250)) + if not thread.isFinished(): + thread.terminate() # Highly undesirable, hopefully the thread obeyed the request to interrupt + thread.wait() + + def wait_on_other_workers(self) -> None: + for worker in self.workers: + if hasattr(self, worker): + thread = getattr(self, worker) + if thread: + if not thread.isFinished(): + thread.wait() + + def reject(self) -> None: """called when the window has been closed""" # save window geometry and splitter state for next use @@ -182,20 +235,30 @@ class CommandAddonManager: # ensure all threads are finished before closing oktoclose = True - for worker in ["update_worker", "check_worker", "show_worker", "showmacro_worker", - "macro_worker", "install_worker"]: + for worker in self.workers: if hasattr(self, worker): thread = getattr(self, worker) if thread: if not thread.isFinished(): + thread.requestInterruption() oktoclose = False + if not oktoclose: + oktoclose = True + for worker in self.workers: + if hasattr(self, worker): + thread = getattr(self, worker) + if thread: + thread.wait(QtCore.QDeadlineTimer(50)) # 50ms to wrap up whatever loop iteration it was on + if not thread.isFinished(): + thread.terminate() # Highly undesirable, hopefully the thread obeyed the request to interrupt + thread.wait() # all threads have finished if oktoclose: if ((hasattr(self, "install_worker") and self.install_worker) or (hasattr(self, "addon_removed") and self.addon_removed)): # display restart dialog - m = QtGui.QMessageBox() + m = QtWidgets.QMessageBox() m.setWindowTitle(translate("AddonsInstaller", "Addon manager")) m.setWindowIcon(QtGui.QIcon(":/icons/AddonManager.svg")) m.setText(translate("AddonsInstaller", @@ -204,8 +267,8 @@ class CommandAddonManager: m.setIcon(m.Warning) m.setStandardButtons(m.Ok | m.Cancel) m.setDefaultButton(m.Cancel) - okBtn = m.button(QtGui.QMessageBox.StandardButton.Ok) - cancelBtn = m.button(QtGui.QMessageBox.StandardButton.Cancel) + okBtn = m.button(QtWidgets.QMessageBox.StandardButton.Ok) + cancelBtn = m.button(QtWidgets.QMessageBox.StandardButton.Cancel) okBtn.setText(translate("AddonsInstaller","Restart now")) cancelBtn.setText(translate("AddonsInstaller","Restart later")) ret = m.exec_() @@ -217,381 +280,512 @@ class CommandAddonManager: shutil.rmtree(self.macro_repo_dir, onerror=self.remove_readonly) except Exception: pass + else: + FreeCAD.Console.PrintWarning("Could not terminate sub-threads in Addon Manager.\n") + self.cleanup_workers() - return True + def populate_packages_table(self) -> None: + """ Downloads the available packages listings and populates the table - def update(self): - """updates the list of workbenches""" + This proceeds in four stages: first, the main GitHub repository is queried for a list of possible + addons. Each addon is specified as a git submodule with name and branch information. The actual specific + commit ID of the submodule (as listed on Github) is ignored. Any extra repositories specified by the + user are appended to this list. - self.dialog.listWorkbenches.clear() - self.dialog.buttonExecute.setEnabled(False) - self.repos = [] + Second, the list of macros is downloaded from the FreeCAD/FreeCAD-macros repository and the wiki + + Third, each of these items is queried for a package.xml metadata file. If that file exists it is + downloaded, cached, and any icons that it references are also downloaded and cached. + + Finally, for workbenches that are not contained within a package (e.g. they provide no metadata), an + additional git query is made to see if an update is available. Macros are checked for file changes. + + Each of these stages is launched in a separate thread to ensure that the UI remains responsive, and + the operation can be cancelled. + + """ + + self.item_model.clear() + self.current_progress_region += 1 self.update_worker = UpdateWorker() - self.update_worker.info_label.connect(self.show_information) + self.update_worker.status_message.connect(self.show_information) self.update_worker.addon_repo.connect(self.add_addon_repo) - self.update_worker.progressbar_show.connect(self.show_progress_bar) - self.update_worker.done.connect(self.check_updates) + self.update_progress_bar(10,100) + self.update_worker.done.connect(self.populate_macros) # Link to step 2 self.update_worker.start() - def check_updates(self): - "checks every installed addon for available updates" - + def populate_macros(self) -> None: + self.current_progress_region += 1 + self.macro_worker = FillMacroListWorker(self.macro_repo_dir) + self.macro_worker.status_message_signal.connect(self.show_information) + self.macro_worker.progress_made.connect(self.update_progress_bar) + self.macro_worker.add_macro_signal.connect(self.add_addon_repo) + self.macro_worker.done.connect(self.update_metadata_cache) # Link to step 3 + self.macro_worker.done.connect(lambda : self.dialog.tablePackages.setEnabled(True)) + self.macro_worker.start() + + def update_metadata_cache(self) -> None: + self.current_progress_region += 1 pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") - if pref.GetBool("AutoCheck", False) and not self.doUpdate: + if pref.GetBool("AutoFetchMetadata", True): + self.update_metadata_cache_worker = UpdateMetadataCacheWorker(self.item_model.repos) + self.update_metadata_cache_worker.status_message.connect(self.show_information) + self.update_metadata_cache_worker.done.connect(self.check_updates) # Link to step 4 + self.update_metadata_cache_worker.progress_made.connect(self.update_progress_bar) + self.update_metadata_cache_worker.package_updated.connect(self.on_package_updated) + self.update_metadata_cache_worker.start() + else: + self.check_updates() + + def on_package_updated(self, repo:AddonManagerRepo) -> None: + """Called when the named package has either new metadata or a new icon (or both)""" + + cache_path = os.path.join(FreeCAD.getUserAppDataDir(), "AddonManager", "PackageMetadata", repo.name) + icon_filename = repo.metadata.Icon + icon_path = os.path.join(cache_path, icon_filename) + if os.path.isfile(icon_path): + addonicon = QtGui.QIcon(icon_path) + repo.icon = addonicon + self.item_model.reload_item(repo) + + + def check_updates(self) -> None: + "checks every installed addon for available updates" + + self.current_progress_region += 1 + pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") + autocheck = pref.GetBool("AutoCheck", False) + if not autocheck: + return + if not self.packages_with_updates: if hasattr(self, "check_worker"): thread = self.check_worker if thread: if not thread.isFinished(): return self.dialog.buttonUpdateAll.setText(translate("AddonsInstaller", "Checking for updates...")) - self.check_worker = CheckWBWorker(self.repos) - self.check_worker.mark.connect(self.mark) - self.check_worker.enable.connect(self.enable_updates) - self.check_worker.addon_repos.connect(self.update_repos) + self.check_worker = CheckWorkbenchesForUpdatesWorker(self.item_model.repos) + self.check_worker.done.connect(self.hide_progress_widgets) + self.check_worker.progress_made.connect(self.update_progress_bar) + self.check_worker.update_status.connect(self.status_updated) self.check_worker.start() + self.enable_updates(len(self.packages_with_updates)) - def apply_updates(self): - """apply all available updates""" + def status_updated(self, repo:str, status:AddonManagerRepo.UpdateStatus) -> None: + self.item_model.update_item_status(repo.name, status) + if status == AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE: + self.packages_with_updates.append(repo) + self.enable_updates(len(self.packages_with_updates)) - if self.doUpdate: - self.install(self.doUpdate) - self.dialog.buttonUpdateAll.setEnabled(False) - - def enable_updates(self, num): + def enable_updates(self, number_of_updates:int) -> None: """enables the update button""" - if num: + if number_of_updates: self.dialog.buttonUpdateAll.setText(translate("AddonsInstaller", "Apply") + - " " + str(num) + " " + + " " + str(number_of_updates) + " " + translate("AddonsInstaller", "update(s)")) self.dialog.buttonUpdateAll.setEnabled(True) else: - self.dialog.buttonUpdateAll.setText(translate("AddonsInstaller", "No update available")) + self.dialog.buttonUpdateAll.setText(translate("AddonsInstaller", "No updates available")) self.dialog.buttonUpdateAll.setEnabled(False) - def add_addon_repo(self, addon_repo): + def add_addon_repo(self, addon_repo:AddonManagerRepo) -> None: """adds a workbench to the list""" + + if addon_repo.icon is None or addon_repo.icon.isNull(): + addon_repo.icon = self.get_icon(addon_repo) + for repo in self.item_model.repos: + if repo.name == addon_repo.name: + FreeCAD.Console.PrintLog(f"Possible duplicate addon: ignoring second addition of {addon_repo.name}\n") + return + self.item_model.append_item(addon_repo) - self.repos.append(addon_repo) - addonicon = self.get_icon(addon_repo[0]) - if addon_repo[2] > 0: - item = QtGui.QListWidgetItem(addonicon, - str(addon_repo[0]) + - str(" (" + - translate("AddonsInstaller", "Installed") + - ")")) - item.setForeground(QtGui.QBrush(QtGui.QColor(0, 182, 41))) - self.dialog.listWorkbenches.addItem(item) - else: - self.dialog.listWorkbenches.addItem(QtGui.QListWidgetItem(addonicon, str(addon_repo[0]))) - - def get_icon(self, repo): + def get_icon(self, repo:AddonManagerRepo, update:bool=False) -> QtGui.QIcon: """returns an icon for a repo""" - path = ":/icons/" + repo + "_workbench_icon.svg" + if not update and repo.icon and not repo.icon.isNull(): + return repo.icon + + path = ":/icons/" + repo.name.replace(" ", "_") + if repo.repo_type == AddonManagerRepo.RepoType.WORKBENCH: + path += "_workbench_icon.svg" + default_icon = QtGui.QIcon(":/icons/document-package.svg") + elif repo.repo_type == AddonManagerRepo.RepoType.MACRO: + path += "_macro_icon.svg" + default_icon = QtGui.QIcon(":/icons/document-python.svg") + elif repo.repo_type == AddonManagerRepo.RepoType.PACKAGE: + # The cache might not have been downloaded yet, check to see if it's there... + if repo.cached_icon_filename and os.path.isfile(repo.cached_icon_filename): + path = repo.cached_icon_filename + elif repo.contains_workbench(): + path += "_workbench_icon.svg" + default_icon = QtGui.QIcon(":/icons/document-package.svg") + elif repo.contains_macro(): + path += "_macro_icon.svg" + default_icon = QtGui.QIcon(":/icons/document-python.svg") + else: + default_icon = QtGui.QIcon(":/icons/document-package.svg") + if QtCore.QFile.exists(path): addonicon = QtGui.QIcon(path) else: - addonicon = QtGui.QIcon(":/icons/document-package.svg") - if addonicon.isNull(): - addonicon = QtGui.QIcon(":/icons/document-package.svg") + addonicon = default_icon + repo.icon = addonicon + return addonicon - def show_information(self, label): - """shows text in the information pane""" + def table_row_selected(self, current:QtCore.QModelIndex, previous:QtCore.QModelIndex) -> None: + """a new row was selected, show the relevant data""" + + if not current.isValid(): + self.selected_repo = None + return - self.dialog.description.setText(label) - if self.dialog.listWorkbenches.isVisible(): - self.dialog.listWorkbenches.setFocus() + source_selection = self.item_filter.mapToSource (current) + self.selected_repo = self.item_model.repos[source_selection.row()] + if self.selected_repo.repo_type == AddonManagerRepo.RepoType.MACRO: + self.show_macro(self.selected_repo) + self.dialog.buttonExecute.show() + elif self.selected_repo.repo_type == AddonManagerRepo.RepoType.WORKBENCH: + self.show_workbench(self.selected_repo) + self.dialog.buttonExecute.hide() + elif self.selected_repo.repo_type == AddonManagerRepo.RepoType.PACKAGE: + self.show_package(self.selected_repo) + self.dialog.buttonExecute.hide() + if self.selected_repo.update_status == AddonManagerRepo.UpdateStatus.NOT_INSTALLED: + self.dialog.buttonInstall.setEnabled(True) + self.dialog.buttonUninstall.setEnabled(False) + self.dialog.buttonInstall.setText(translate("AddonsInstaller", "Install selected")) + elif self.selected_repo.update_status == AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE: + self.dialog.buttonInstall.setEnabled(False) + self.dialog.buttonUninstall.setEnabled(True) + self.dialog.buttonInstall.setText(translate("AddonsInstaller", "Already updated")) + elif self.selected_repo.update_status == AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE: + self.dialog.buttonInstall.setEnabled(True) + self.dialog.buttonUninstall.setEnabled(True) + self.dialog.buttonInstall.setText(translate("AddonsInstaller", "Update selected")) + elif self.selected_repo.update_status == AddonManagerRepo.UpdateStatus.UNCHECKED: + self.dialog.buttonInstall.setEnabled(False) + self.dialog.buttonUninstall.setEnabled(True) + self.dialog.buttonInstall.setText(translate("AddonsInstaller", "Checking status...")) + + def show_package_widgets(self, show:bool) -> None: + """ Show or hide the widgets related to packages with a package.xml metadata file """ + + # Always rebuild the urlGrid, remove all previous items + for i in reversed(range(self.dialog.urlGrid.rowCount())): + if self.dialog.urlGrid.itemAtPosition(i,0): + self.dialog.urlGrid.itemAtPosition(i,0).widget().setParent(None) + if self.dialog.urlGrid.itemAtPosition(i,1): + self.dialog.urlGrid.itemAtPosition(i,1).widget().setParent(None) + + if show: + # Show all the package-related widgets: + self.dialog.labelPackageName.show() + self.dialog.labelVersion.show() + self.dialog.labelMaintainer.show() + self.dialog.labelIcon.show() + self.dialog.labelDescription.show() + self.dialog.labelContents.show() else: - self.dialog.listMacros.setFocus() + # Hide all the package-related widgets: + self.dialog.labelPackageName.hide() + self.dialog.labelVersion.hide() + self.dialog.labelMaintainer.hide() + self.dialog.labelIcon.hide() + self.dialog.labelDescription.hide() + self.dialog.labelContents.hide() - def show(self, idx): + def show_information(self, message:str) -> None: + """shows generic text in the information pane""" + + self.dialog.labelStatusInfo.show() + self.dialog.labelStatusInfo.setText(message) + + def show_workbench(self, repo:AddonManagerRepo) -> None: """loads information of a given workbench""" - # this function is triggered also when the list is populated, prevent that here - if idx == 0 and self.firsttime: - self.dialog.listWorkbenches.setCurrentRow(-1) - self.firsttime = False - return + self.cleanup_workers() + self.show_package_widgets(False) + self.show_worker = ShowWorker(repo) + self.show_worker.status_message.connect(self.show_information) + self.show_worker.description_updated.connect(lambda desc: self.dialog.description.setText(desc)) + self.show_worker.addon_repos.connect(self.append_to_repos_list) + self.show_worker.done.connect(lambda : self.dialog.labelStatusInfo.hide()) + self.show_worker.start() - if self.repos and idx >= 0: - if hasattr(self, "show_worker"): - # kill existing show worker (might still be busy loading images...) - if self.show_worker: - self.show_worker.exit() - self.show_worker.stopImageLoading() - # wait until the thread stops - while True: - if self.show_worker.isFinished(): - break - self.show_worker = ShowWorker(self.repos, idx) - self.show_worker.info_label.connect(self.show_information) - self.show_worker.addon_repos.connect(self.update_repos) - self.show_worker.progressbar_show.connect(self.show_progress_bar) - self.show_worker.start() - self.dialog.buttonInstall.setEnabled(True) - self.dialog.buttonUninstall.setEnabled(True) + def show_package(self, repo:AddonManagerRepo) -> None: + """ Show the details for a package (a repo with a package.xml metadata file) """ - def show_macro(self, idx): + self.cleanup_workers() + self.show_package_widgets(True) + + # Name + self.dialog.labelPackageName.setText(f"

{repo.metadata.Name}

") + + # Description + self.dialog.labelDescription.setText(repo.metadata.Description) + + # Version + self.dialog.labelVersion.setText(f"

v{repo.metadata.Version}

") + + # Maintainers and authors + maintainers = "" + for maintainer in repo.metadata.Maintainer: + maintainers += translate("AddonsInstaller","Maintainer") + f": {maintainer['name']} <{maintainer['email']}>\n" + if len(repo.metadata.Author) > 0: + for author in repo.metadata.Author: + maintainers += translate("AddonsInstaller","Author") + f": {author['name']} <{author['email']}>\n" + self.dialog.labelMaintainer.setText(maintainers) + + # Main package icon + if not repo.icon or repo.icon.isNull(): + icon = self.get_icon(repo, update=True) + self.item_model.update_item_icon(repo.name, icon) + self.dialog.labelIcon.setPixmap(repo.icon.pixmap(QtCore.QSize(64,64))) + + # Urls + urls = repo.metadata.Urls + ui = FreeCADGui.UiLoader() + for row, url in enumerate(urls): + location = url["location"] + url_type = url["type"] + url_type_string = translate("AddonsInstaller","Other URL") + if url_type == "website": + url_type_string = translate("AddonsInstaller", "Website") + elif url_type == "repository": + url_type_string = translate("AddonsInstaller", "Repository") + elif url_type == "bugtracker": + url_type_string = translate("AddonsInstaller", "Bug tracker") + elif url_type == "readme": + url_type_string = translate("AddonsInstaller", "Readme") + elif url_type == "documentation": + url_type_string = translate("AddonsInstaller", "Documentation") + self.dialog.urlGrid.addWidget(QtWidgets.QLabel(url_type_string), row, 0) + ui=FreeCADGui.UiLoader() + url_label=ui.createWidget("Gui::UrlLabel") + url_label.setText(location) + url_label.setUrl(location) + self.dialog.urlGrid.addWidget(url_label, row, 1) + + # Package contents: + content_string = "" + for name,item_list in repo.metadata.Content.items(): + if name == "preferencepack": + content_type = translate("AddonsInstaller","Preference Packs") + elif name == "workbench": + content_type = translate("AddonsInstaller","Workbenches") + elif name == "macro": + content_type = translate("AddonsInstaller","Macros") + else: + content_type = translate("AddonsInstaller","Other content") + ": " + name + content_string += f"

{content_type}

" + content_string += "
    " + for item in item_list: + content_string += f"
  • {item.Name} – {item.Description}
    • " + content_string += "
" + self.dialog.description.setText(content_string) + + def show_macro(self, repo:AddonManagerRepo) -> None: """loads information of a given macro""" - # this function is triggered when the list is populated, prevent that here - if idx == 0 and self.firstmacro: - self.dialog.listMacros.setCurrentRow(-1) - self.firstmacro = False - return + self.cleanup_workers() + self.show_package_widgets(False) + self.showmacro_worker = GetMacroDetailsWorker(repo) + self.showmacro_worker.status_message.connect(self.show_information) + self.showmacro_worker.description_updated.connect(lambda desc: self.dialog.description.setText(desc)) + self.showmacro_worker.done.connect(lambda : self.dialog.labelStatusInfo.hide()) + self.showmacro_worker.start() - if self.macros and idx >= 0: - if hasattr(self, "showmacro_worker"): - if self.showmacro_worker: - if not self.showmacro_worker.isFinished(): - self.showmacro_worker.exit() - if not self.showmacro_worker.isFinished(): - return - self.showmacro_worker = GetMacroDetailsWorker(self.macros[idx]) - self.showmacro_worker.info_label.connect(self.show_information) - self.showmacro_worker.progressbar_show.connect(self.show_progress_bar) - self.showmacro_worker.start() - self.dialog.buttonInstall.setEnabled(True) - self.dialog.buttonUninstall.setEnabled(True) - if self.macros[idx].is_installed(): - self.dialog.buttonExecute.setEnabled(True) - else: - self.dialog.buttonExecute.setEnabled(False) - - def switchtab(self, idx): - """does what needs to be done when switching tabs""" - - if idx == 1: - if not self.macros: - self.dialog.listMacros.clear() - self.macros = [] - self.macro_worker = FillMacroListWorker(self.macro_repo_dir) - self.macro_worker.add_macro_signal.connect(self.add_macro) - self.macro_worker.info_label_signal.connect(self.show_information) - self.macro_worker.progressbar_show.connect(self.show_progress_bar) - self.macro_worker.start() - self.dialog.listMacros.setCurrentRow(0) - - def update_repos(self, repos): + def append_to_repos_list(self, repo:AddonManagerRepo) -> None: """this function allows threads to update the main list of workbenches""" - self.repos = repos + self.item_model.append_item(repo) - def add_macro(self, macro): - """adds a macro to the list""" + def install(self) -> None: + """installs or updates a workbench, macro, or package""" - if macro.name: - if macro in self.macros: - # The macro is already in the list of macros. - old_macro = self.macros[self.macros.index(macro)] - utils.update_macro_details(old_macro, macro) + if hasattr(self, "install_worker") and self.install_worker: + if self.install_worker.isRunning(): + return + + if not hasattr(self, "selected_repo"): + FreeCAD.Console.PrintWarning ("Internal error: no selected repo\n") + return + + repo = self.selected_repo + + if not repo: + return + + if repo.repo_type == AddonManagerRepo.RepoType.WORKBENCH or repo.repo_type == AddonManagerRepo.RepoType.PACKAGE: + self.install_worker = InstallWorkbenchWorker(repo) + self.install_worker.status_message.connect(self.show_information) + self.current_progress_region = 1 + self.number_of_progress_regions = 1 + self.install_worker.progress_made.connect(self.update_progress_bar) + self.install_worker.success.connect(self.on_package_installed) + self.install_worker.failure.connect(self.on_installation_failed) + self.install_worker.start() + elif repo.repo_type == AddonManagerRepo.RepoType.MACRO: + macro = repo.macro + + # To try to ensure atomicity, test the installation into a temp directory first, + # and assume if that worked we have good odds of the real installation working + failed = False + with tempfile.TemporaryDirectory() as dir: + temp_install_succeeded = macro.install(dir) + if not temp_install_succeeded: + failed = True + + if not failed: + failed = macro.install(self.macro_repo_dir) + + if not failed: + message = translate("AddonsInstaller", + "Macro successfully installed. The macro is " + "now available from the Macros dialog.") + self.on_package_installed (repo, message) else: - self.macros.append(macro) - path = ":/icons/" + macro.name.replace(" ", "_") + "_macro_icon.svg" - if QtCore.QFile.exists(path): - addonicon = QtGui.QIcon(path) - else: - addonicon = QtGui.QIcon(":/icons/document-python.svg") - if addonicon.isNull(): - addonicon = QtGui.QIcon(":/icons/document-python.svg") - if macro.is_installed(): - item = QtGui.QListWidgetItem(addonicon, macro.name + str(" (Installed)")) - item.setForeground(QtGui.QBrush(QtGui.QColor(0, 182, 41))) - self.dialog.listMacros.addItem(item) - else: - self.dialog.listMacros.addItem(QtGui.QListWidgetItem(addonicon, macro.name)) + message = translate("AddonsInstaller", "Installation of macro failed. See console for failure details.") + self.on_installation_failed (repo, message) - def install(self, repos=None): - """installs a workbench or macro""" + def update_all(self) -> None: + """ Asynchronously apply all available updates: individual failures are noted, but do not stop other updates """ - if self.dialog.tabWidget.currentIndex() == 0: - # Tab "Workbenches". - idx = None - if repos: - idx = [] - for repo in repos: - for i, r in enumerate(self.repos): - if r[0] == repo: - idx.append(i) - else: - idx = self.dialog.listWorkbenches.currentRow() - if idx is not None: - if hasattr(self, "install_worker") and self.install_worker: - if self.install_worker.isRunning(): - return - self.install_worker = InstallWorker(self.repos, idx) - self.install_worker.info_label.connect(self.show_information) - self.install_worker.progressbar_show.connect(self.show_progress_bar) - self.install_worker.mark_recompute.connect(self.mark_recompute) - self.install_worker.start() + if hasattr(self, "update_all_worker") and self.update_all_worker: + if self.update_all_worker.isRunning(): + return - elif self.dialog.tabWidget.currentIndex() == 1: - # Tab "Macros". - macro = self.macros[self.dialog.listMacros.currentRow()] - if utils.install_macro(macro, self.macro_repo_dir): - self.dialog.description.setText(translate("AddonsInstaller", - "Macro successfully installed. The macro is " - "now available from the Macros dialog.")) - else: - self.dialog.description.setText(translate("AddonsInstaller", "Unable to install")) + self.subupdates_succeeded = [] + self.subupdates_failed = [] + + self.current_progress_region = 1 + self.number_of_progress_regions = 1 + self.update_all_worker = UpdateAllWorker(self.packages_with_updates) + self.update_all_worker.progress_made.connect(self.update_progress_bar) + self.update_all_worker.status_message.connect(self.show_information) + self.update_all_worker.success.connect(lambda repo : self.subupdates_succeeded.append(repo)) + self.update_all_worker.failure.connect(lambda repo : self.subupdates_failed.append(repo)) + self.update_all_worker.done.connect(self.on_update_all_completed) + self.update_all_worker.start() - def show_progress_bar(self, state): - """shows or hides the progress bar""" - - if state: - self.dialog.tabWidget.setEnabled(False) - self.dialog.buttonInstall.setEnabled(False) - self.dialog.buttonUninstall.setEnabled(False) - self.dialog.progressBar.show() + def on_update_all_completed(self) -> None: + #self.show_progress_bar(False) + if not self.subupdates_failed: + message = translate ("AddonsInstaller", "All packages were successfully updated. Packages:") + "\n" + message += ''.join([repo.name + "\n" for repo in self.subupdates_succeeded]) + elif not self.subupdates_succeeded: + message = translate ("AddonsInstaller", "All packages updates failed. Packages:") + "\n" + message += ''.join([repo.name + "\n" for repo in self.subupdates_failed]) else: - self.dialog.progressBar.hide() - self.dialog.tabWidget.setEnabled(True) - if not (self.firsttime and self.firstmacro): - self.dialog.buttonInstall.setEnabled(True) - self.dialog.buttonUninstall.setEnabled(True) - if self.dialog.listWorkbenches.isVisible(): - self.dialog.listWorkbenches.setFocus() - else: - self.dialog.listMacros.setFocus() + message = translate ("AddonsInstaller", "Some packages updates failed. Successful packages:") + "\n" + message += ''.join([repo.name + "\n" for repo in self.subupdates_succeeded]) + message += translate ("AddonsInstaller", "Failed packages:") + "\n" + message += ''.join([repo.name + "\n" for repo in self.subupdates_failed]) - def executemacro(self): + for installed_repo in self.subupdates_succeeded: + for requested_repo in self.packages_with_updates: + if installed_repo.name == requested_repo.name: + self.packages_with_updates.remove(installed_repo) + break + self.enable_updates(len(self.packages_with_updates)) + QtWidgets.QMessageBox.information(None, + translate("AddonsInstaller", "Update report"), + message, + QtWidgets.QMessageBox.Close) + + def hide_progress_widgets(self) -> None: + """ hides the progress bar and related widgets""" + + self.dialog.labelStatusInfo.hide() + self.dialog.progressBar.hide() + self.dialog.lineEditFilter.setFocus() + + def update_progress_bar(self, current_value:int, max_value:int) -> None: + """ Update the progress bar, showing it if it's hidden """ + + self.dialog.progressBar.show() + region_size = 100 / self.number_of_progress_regions + value = (self.current_progress_region-1)*region_size + (current_value / max_value / self.number_of_progress_regions)*region_size + self.dialog.progressBar.setValue(value) + + def on_package_installed(self, repo:AddonManagerRepo, message:str) -> None: + QtWidgets.QMessageBox.information(None, + translate("AddonsInstaller", "Installation succeeded"), + message, + QtWidgets.QMessageBox.Close) + self.dialog.progressBar.hide() + self.table_row_selected(self.dialog.tablePackages.selectionModel().selectedIndexes()[0], QtCore.QModelIndex()) + if repo.contains_workbench(): + self.item_model.update_item_status(repo.name, AddonManagerRepo.UpdateStatus.PENDING_RESTART) + else: + self.item_model.update_item_status(repo.name, AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE) + + def on_installation_failed(self, _:AddonManagerRepo, message:str) -> None: + QtWidgets.QMessageBox.warning(None, + translate("AddonsInstaller", "Installation failed"), + message, + QtWidgets.QMessageBox.Close) + self.dialog.progressBar.hide() + + def executemacro(self) -> None: """executes a selected macro""" - if self.dialog.tabWidget.currentIndex() == 1: - # Tab "Macros". - macro = self.macros[self.dialog.listMacros.currentRow()] - if not macro.is_installed(): - # Macro not installed, nothing to do. - return - macro_path = os.path.join(FreeCAD.getUserMacroDir(True), macro.filename) - if os.path.exists(macro_path): - macro_path = macro_path.replace("\\", "/") + macro = self.selected_repo.macro + if not macro or not macro.code: + return - FreeCADGui.open(str(macro_path)) - self.dialog.hide() - FreeCADGui.SendMsgToActiveView("Run") + if macro.is_installed(): + macro_path = os.path.join(self.macro_repo_dir,macro.filename) + FreeCADGui.open(str(macro_path)) + self.dialog.hide() + FreeCADGui.SendMsgToActiveView("Run") else: - self.dialog.buttonExecute.setEnabled(False) - - def remove_readonly(self, func, path, _): + with tempfile.TemporaryDirectory() as dir: + temp_install_succeeded = macro.install(dir) + if not temp_install_succeeded: + message = translate("AddonsInstaller", "Execution of macro failed. See console for failure details.") + self.on_installation_failed (self.selected_repo, message) + return + else: + macro_path = os.path.join(dir,macro.filename) + FreeCADGui.open(str(macro_path)) + self.dialog.hide() + FreeCADGui.SendMsgToActiveView("Run") + def remove_readonly(self, func, path, _) -> None: """Remove a read-only file.""" os.chmod(path, stat.S_IWRITE) func(path) - def remove(self): + def remove(self) -> None: """uninstalls a macro or workbench""" - if self.dialog.tabWidget.currentIndex() == 0: - # Tab "Workbenches". - idx = self.dialog.listWorkbenches.currentRow() + if self.selected_repo.repo_type == AddonManagerRepo.RepoType.WORKBENCH or \ + self.selected_repo.repo_type == AddonManagerRepo.RepoType.PACKAGE: basedir = FreeCAD.getUserAppDataDir() moddir = basedir + os.sep + "Mod" - clonedir = moddir + os.sep + self.repos[idx][0] + clonedir = moddir + os.sep + self.selected_repo.name if os.path.exists(clonedir): shutil.rmtree(clonedir, onerror=self.remove_readonly) self.dialog.description.setText(translate("AddonsInstaller", - "Addon successfully removed. Please restart FreeCAD")) + "Addon successfully removed. Please restart FreeCAD.")) + self.item_model.update_item_status(self.selected_repo.name, AddonManagerRepo.UpdateStatus.NOT_INSTALLED) + self.addon_removed = True # A value to trigger the restart message on dialog close else: - self.dialog.description.setText(translate("AddonsInstaller", "Unable to remove this addon")) + self.dialog.description.setText(translate("AddonsInstaller", "Unable to remove this addon with the Addon Manager.")) - elif self.dialog.tabWidget.currentIndex() == 1: - # Tab "Macros". - macro = self.macros[self.dialog.listMacros.currentRow()] - if utils.remove_macro(macro): + elif self.selected_repo.repo_type == AddonManagerRepo.RepoType.MACRO: + macro = self.selected_repo.macro + if macro.remove(): self.dialog.description.setText(translate("AddonsInstaller", "Macro successfully removed.")) + self.item_model.update_item_status(self.selected_repo.name, AddonManagerRepo.UpdateStatus.NOT_INSTALLED) else: self.dialog.description.setText(translate("AddonsInstaller", "Macro could not be removed.")) - self.update_status(soft=True) - self.addon_removed = True # A value to trigger the restart message on dialog close - def mark_recompute(self, addon): - """marks an addon in the list as installed but needs recompute""" - - for i in range(self.dialog.listWorkbenches.count()): - txt = self.dialog.listWorkbenches.item(i).text().strip() - if txt.endswith(" ("+translate("AddonsInstaller", "Installed")+")"): - txt = txt[:-12] - elif txt.endswith(" ("+translate("AddonsInstaller", "Update available")+")"): - txt = txt[:-19] - if txt == addon: - self.dialog.listWorkbenches.item(i).setText(txt + " (" + - translate("AddonsInstaller", - "Restart required") + - ")") - self.dialog.listWorkbenches.item(i).setIcon(QtGui.QIcon(":/icons/edit-undo.svg")) - - def update_status(self, soft=False): - """Updates the list of workbenches/macros. If soft is true, items - are not recreated (and therefore display text isn't triggered)" - """ - - moddir = FreeCAD.getUserAppDataDir() + os.sep + "Mod" - if soft: - for i in range(self.dialog.listWorkbenches.count()): - txt = self.dialog.listWorkbenches.item(i).text().strip() - ext = "" - if txt.endswith(" ("+translate("AddonsInstaller", "Installed")+")"): - txt = txt[:-12] - ext = " ("+translate("AddonsInstaller", "Installed")+")" - elif txt.endswith(" ("+translate("AddonsInstaller", "Update available")+")"): - txt = txt[:-19] - ext = " ("+translate("AddonsInstaller", "Update available")+")" - elif txt.endswith(" ("+translate("AddonsInstaller", "Restart required")+")"): - txt = txt[:-19] - ext = " ("+translate("AddonsInstaller", "Restart required")+")" - if os.path.exists(os.path.join(moddir, txt)): - self.dialog.listWorkbenches.item(i).setText(txt+ext) - else: - self.dialog.listWorkbenches.item(i).setText(txt) - self.dialog.listWorkbenches.item(i).setIcon(self.get_icon(txt)) - for i in range(self.dialog.listMacros.count()): - txt = self.dialog.listMacros.item(i).text().strip() - if txt.endswith(" ("+translate("AddonsInstaller", "Installed")+")"): - txt = txt[:-12] - elif txt.endswith(" ("+translate("AddonsInstaller", "Update available")+")"): - txt = txt[:-19] - if os.path.exists(os.path.join(moddir, txt)): - self.dialog.listMacros.item(i).setText(txt+ext) - else: - self.dialog.listMacros.item(i).setText(txt) - self.dialog.listMacros.item(i).setIcon(QtGui.QIcon(":/icons/document-python.svg")) - else: - self.dialog.listWorkbenches.clear() - self.dialog.listMacros.clear() - for wb in self.repos: - if os.path.exists(os.path.join(moddir, wb[0])): - self.dialog.listWorkbenches.addItem( - QtGui.QListWidgetItem(QtGui.QIcon(":/icons/button_valid.svg"), - str(wb[0]) + " (" + - translate("AddonsInstaller", "Installed") + ")")) - wb[2] = 1 - else: - self.dialog.listWorkbenches.addItem( - QtGui.QListWidgetItem(QtGui.QIcon(":/icons/document-python.svg"), str(wb[0]))) - wb[2] = 0 - for macro in self.macros: - if macro.is_installed(): - self.dialog.listMacros.addItem(item) - else: - self.dialog.listMacros.addItem( - QtGui.QListWidgetItem(QtGui.QIcon(":/icons/document-python.svg"), macro.name)) - - def mark(self, repo): - """mark a workbench as updatable""" - - for i in range(self.dialog.listWorkbenches.count()): - w = self.dialog.listWorkbenches.item(i) - if (w.text() == str(repo)) or w.text().startswith(str(repo)+" "): - w.setText(str(repo) + str(" ("+translate("AddonsInstaller", "Update available")+")")) - w.setForeground(QtGui.QBrush(QtGui.QColor(182, 90, 0))) - if repo not in self.doUpdate: - self.doUpdate.append(repo) - - def show_config(self): + def show_config(self) -> None: """shows the configuration dialog""" self.config = FreeCADGui.PySideUic.loadUi(os.path.join(os.path.dirname(__file__), "AddonManagerOptions.ui")) @@ -621,13 +815,226 @@ class CommandAddonManager: pref.SetBool("UserProxyCheck", self.config.radioButtonUserProxy.isChecked()) pref.SetString("ProxyUrl", self.config.userProxy.toPlainText()) + -def check_updates(addon_name, callback): - """Checks for updates for a given addon""" + def update_type_filter(self, type_filter:int) -> None: + """hide/show rows corresponding to the type filter + + type_filter is an integer: 0 for all, 1 for workbenches, 2 for macros, and 3 for preference packs + + """ + self.item_filter.setPackageFilter(type_filter) + - oname = "update_checker_"+addon_name - setattr(FreeCAD, oname, CheckSingleWorker(addon_name)) - getattr(FreeCAD, oname).updateAvailable.connect(callback) - getattr(FreeCAD, oname).start() + def update_text_filter(self, text_filter:str) -> None: + """filter name and description by the regex specified by text_filter""" + if text_filter: + test_regex = QtCore.QRegularExpression(text_filter) + if test_regex.isValid(): + self.dialog.labelFilterValidity.setToolTip(translate("AddonInstaller","Filter is valid")) + icon = QtGui.QIcon.fromTheme("ok", QtGui.QIcon(":/icons/edit_OK.svg")) + self.dialog.labelFilterValidity.setPixmap(icon.pixmap(16,16)) + else: + self.dialog.labelFilterValidity.setToolTip(translate("AddonInstaller","Filter regular expression is invalid")) + icon = QtGui.QIcon.fromTheme("cancel", QtGui.QIcon(":/icons/edit_Cancel.svg")) + self.dialog.labelFilterValidity.setPixmap(icon.pixmap(16,16)) + self.dialog.labelFilterValidity.show() + else: + self.dialog.labelFilterValidity.hide() + self.item_filter.setFilterRegularExpression(text_filter) + + +class PackageListItemModel(QtCore.QAbstractTableModel): + + repos = [] + write_lock = threading.Lock() + + DataAccessRole = QtCore.Qt.UserRole + StatusUpdateRole = QtCore.Qt.UserRole + 1 + IconUpdateRole = QtCore.Qt.UserRole + 2 + + def __init__(self) -> None: + QtCore.QAbstractTableModel.__init__(self) + + def rowCount(self, parent:QtCore.QModelIndex=QtCore.QModelIndex()) -> int: + if parent.isValid(): + return 0 + return len(self.repos) + + def columnCount(self, parent:QtCore.QModelIndex=QtCore.QModelIndex()) -> int: + if parent.isValid(): + return 0 + return 3 # Icon, Name, Status + + def data(self, index:QtCore.QModelIndex, role:int=QtCore.Qt.DisplayRole) -> Union[QtGui.QIcon,str]: + if not index.isValid(): + return None + row = index.row() + column = index.column() + if role == QtCore.Qt.DisplayRole: + if row >= len(self.repos): + return None + if column == 1: + return self.repos[row].name if self.repos[row].metadata is None else self.repos[row].metadata.Name + elif column == 2: + if self.repos[row].update_status == AddonManagerRepo.UpdateStatus.UNCHECKED: + return translate("AddonsInstaller","Installed") + elif self.repos[row].update_status == AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE: + return translate("AddonsInstaller","Up-to-date") + elif self.repos[row].update_status == AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE: + return translate("AddonsInstaller","Update available") + elif self.repos[row].update_status == AddonManagerRepo.UpdateStatus.PENDING_RESTART: + return translate("AddonsInstaller","Restart required") + else: + return None + else: + return None + elif role == QtCore.Qt.DecorationRole: + if column == 0: + return self.repos[row].icon + elif role == QtCore.Qt.ToolTipRole: + tooltip = "" + if self.repos[row].repo_type == AddonManagerRepo.RepoType.PACKAGE: + tooltip = f"Package '{self.repos[row].name}'" + # TODO add more info from Metadata + elif self.repos[row].repo_type == AddonManagerRepo.RepoType.WORKBENCH: + tooltip = f"Workbench '{self.repos[row].name}'" + elif self.repos[row].repo_type == AddonManagerRepo.RepoType.MACRO: + tooltip = f"Macro '{self.repos[row].name}'" + return tooltip + elif role == QtCore.Qt.TextAlignmentRole: + return QtCore.Qt.AlignLeft | QtCore.Qt.AlignTop + elif role == PackageListItemModel.DataAccessRole: + return self.repos[row] + + def headerData(self, section, orientation, role=QtCore.Qt.DisplayRole): + if role == QtCore.Qt.DisplayRole: + if orientation == QtCore.Qt.Horizontal: + if section == 0: + return None + elif section == 1: + return translate("AddonsInstaller", "Name") + elif section == 2: + return translate("AddonsInstaller", "Status") + else: + return None + + def setData(self, index:QtCore.QModelIndex, value, role=QtCore.Qt.EditRole) -> None: + """ Set the data for this row. The column of the index is ignored. """ + + row = index.row() + self.write_lock.acquire() + if role == PackageListItemModel.StatusUpdateRole: + self.repos[row].update_status = value + self.dataChanged.emit(self.index(row,2), self.index(row,2), [PackageListItemModel.StatusUpdateRole]) + elif role == PackageListItemModel.IconUpdateRole: + self.repos[row].icon = value + self.dataChanged.emit(self.index(row,0), self.index(row,0), [PackageListItemModel.IconUpdateRole]) + self.write_lock.release() + + def append_item(self, repo:AddonManagerRepo) -> None: + if repo in self.repos: + # Cowardly refuse to insert the same repo a second time + return + self.write_lock.acquire() + self.beginInsertRows(QtCore.QModelIndex(), self.rowCount(), self.rowCount()) + self.repos.append(repo) + self.endInsertRows() + self.write_lock.release() + + def clear(self) -> None: + if self.rowCount() > 0: + self.write_lock.acquire() + self.beginRemoveRows(QtCore.QModelIndex(), 0, self.rowCount()-1) + self.repos = [] + self.endRemoveRows() + self.write_lock.release() + + def update_item_status(self, name:str, status:AddonManagerRepo.UpdateStatus) -> None: + for row,item in enumerate(self.repos): + if item.name == name: + self.setData(self.index(row,0), status, PackageListItemModel.StatusUpdateRole) + return + + def update_item_icon(self, name:str, icon:QtGui.QIcon) -> None: + for row,item in enumerate(self.repos): + if item.name == name: + self.setData(self.index(row,0), icon, PackageListItemModel.IconUpdateRole) + return + + def reload_item(self,repo:AddonManagerRepo) -> None: + for index,item in enumerate(self.repos): + if item.name == repo.name: + self.write_lock.acquire() + self.repos[index] = repo + self.write_lock.release() + return + + +class PackageListIconDelegate(QtWidgets.QStyledItemDelegate): + """ A delegate to ensure proper alignment of the icon in the table cells """ + + def paint(self, painter, option, index): + if (index.column() == 0): + option.decorationAlignment = QtCore.Qt.AlignTop | QtCore.Qt.AlignHCenter + super().paint(painter, option, index) + + +class PackageListFilter(QtCore.QSortFilterProxyModel): + """ Handle filtering the item list on various criteria """ + + def __init__(self): + super().__init__() + self.package_type = 0 # Default to showing everything + self.setSortCaseSensitivity(QtCore.Qt.CaseInsensitive) + + def setPackageFilter(self, type:int) -> None: # 0=All, 1=Workbenches, 2=Macros, 3=Preference Packs + self.package_type = type + self.invalidateFilter() + + def lessThan(self, left, right) -> bool: + l = self.sourceModel().data(left,PackageListItemModel.DataAccessRole) + r = self.sourceModel().data(right,PackageListItemModel.DataAccessRole) + + if left.column() == 0: # Icon + return False + elif left.column() == 1: # Name + lname = l.name if l.metadata is None else l.metadata.Name + rname = r.name if r.metadata is None else r.metadata.Name + return lname.lower() < rname.lower() + elif left.column() == 2: # Status + return l.update_status < r.update_status + + def filterAcceptsRow(self, row, parent=QtCore.QModelIndex()): + index = self.sourceModel().createIndex(row, 0) + data = self.sourceModel().data(index,PackageListItemModel.DataAccessRole) + if self.package_type == 1: + if not data.contains_workbench(): + return False + elif self.package_type == 2: + if not data.contains_macro(): + return False + elif self.package_type == 3: + if not data.contains_preference_pack(): + return False + + name = data.name if data.metadata is None else data.metadata.Name + desc = data.description if not data.metadata else data.metadata.Description + re = self.filterRegularExpression() + if re.isValid(): + re.setPatternOptions(QtCore.QRegularExpression.CaseInsensitiveOption) + if re.match(name).hasMatch(): + return True + if re.match(desc).hasMatch(): + return True + return False + else: + return False + + def sort(self, column, order): + if column == 0: # Icons + return + else: + super().sort(column, order) # @} diff --git a/src/Mod/AddonManager/AddonManager.ui b/src/Mod/AddonManager/AddonManager.ui index 04b65a8281..5c3e08a1f2 100644 --- a/src/Mod/AddonManager/AddonManager.ui +++ b/src/Mod/AddonManager/AddonManager.ui @@ -28,40 +28,105 @@ - - - 0 + + + + + Show packages containing: + + + + + + + + All + + + + + Workbenches + + + + + Macros + + + + + Preference Packs + + + + + + + + + + + + Filter + + + true + + + + + + + OK + + + + + + + + + QAbstractItemView::NoEditTriggers - - - Workbenches - - - - - - - - - - Macros - - - - - - - - - Executes the selected macro, if installed - - - Execute - - - - - + + false + + + QAbstractItemView::SingleSelection + + + QAbstractItemView::SelectRows + + + + 16 + 16 + + + + false + + + true + + + false + + + 16 + + + false + + + 12 + + + 16 + + + false + @@ -95,13 +160,125 @@ + + + + + + + 0 + 0 + + + + + 64 + 64 + + + + + 64 + 64 + + + + Icon + + + + + + + 0 + + + QLayout::SetDefaultConstraint + + + + + <h1>Package Name</h1> + + + + + + + <em>Version</em> + + + + + + + Maintainer + + + + + + + + + + + Description + + + Qt::PlainText + + + true + + + + + + + + + URL Type + + + + + + + Url + + + + + + + + + Package contents: + + + + + + + Run Macro + + + + + + + labelStatusInfo + + + @@ -109,6 +286,9 @@ 24 + + false + Downloading info... diff --git a/src/Mod/AddonManager/AddonManagerRepo.py b/src/Mod/AddonManager/AddonManagerRepo.py new file mode 100644 index 0000000000..44b60fa67d --- /dev/null +++ b/src/Mod/AddonManager/AddonManagerRepo.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- +#*************************************************************************** +#* * +#* Copyright (c) 2021 Chris Hennes * +#* * +#* This program is free software; you can redistribute it and/or modify * +#* it under the terms of the GNU Lesser General Public License (LGPL) * +#* as published by the Free Software Foundation; either version 2 of * +#* the License, or (at your option) any later version. * +#* for detail see the LICENCE text file. * +#* * +#* This program is distributed in the hope that it will be useful, * +#* but WITHOUT ANY WARRANTY; without even the implied warranty of * +#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * +#* GNU Library General Public License for more details. * +#* * +#* You should have received a copy of the GNU Library General Public * +#* License along with this program; if not, write to the Free Software * +#* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * +#* USA * +#* * +#*************************************************************************** + +import FreeCAD + +import os + +from addonmanager_macro import Macro + +class AddonManagerRepo: + "Encapsulate information about a FreeCAD addon" + + from enum import Enum + + class RepoType(Enum): + WORKBENCH = 1 + MACRO = 2 + PACKAGE = 3 + + def __str__(self) ->str : + if self.value == 1: + return "Workbench" + elif self.value == 2: + return "Macro" + elif self.value == 3: + return "Package" + + class UpdateStatus(Enum): + NOT_INSTALLED = 0 + UNCHECKED = 1 + NO_UPDATE_AVAILABLE = 2 + UPDATE_AVAILABLE = 3 + PENDING_RESTART = 4 + + def __lt__(self, other): + if self.__class__ is other.__class__: + return self.value < other.value + return NotImplemented + + def __str__(self) ->str : + if self.value == 0: + return "Not installed" + elif self.value == 1: + return "Unchecked" + elif self.value == 2: + return "No update available" + elif self.value == 3: + return "Update available" + elif self.value == 4: + return "Restart required" + + def __init__ (self, name:str, url:str, status:UpdateStatus, branch:str): + self.name = name + self.url = url + self.branch = branch + self.update_status = status + self.repo_type = AddonManagerRepo.RepoType.WORKBENCH + self.description = None + from addonmanager_utilities import construct_git_url + self.metadata_url = "" if not self.url else construct_git_url(self, "package.xml") + self.metadata = None + self.icon = None + self.cached_icon_filename = "" + self.macro = None # Bridge to Gaël Écorchard's macro management class + + def __str__ (self) -> str: + result = f"FreeCAD {self.repo_type}\n" + result += f"Name: {self.name}\n" + result += f"URL: {self.url}\n" + result += "Has metadata\n" if self.metadata is not None else "No metadata found\n" + if self.macro is not None: + result += "Has linked Macro object\n" + return result + + @classmethod + def from_macro (self, macro:Macro): + if macro.is_installed(): + status = AddonManagerRepo.UpdateStatus.UNCHECKED + else: + status = AddonManagerRepo.UpdateStatus.NOT_INSTALLED + instance = AddonManagerRepo(macro.name, macro.url, status, "master") + instance.macro = macro + instance.repo_type = AddonManagerRepo.RepoType.MACRO + instance.description = macro.desc + return instance + + def contains_workbench(self) -> bool: + """ Determine if this package contains (or is) a workbench """ + + if self.repo_type == AddonManagerRepo.RepoType.WORKBENCH: + return True + elif self.repo_type == AddonManagerRepo.RepoType.PACKAGE: + content = self.metadata.Content + return "workbench" in content + else: + return False + + def contains_macro(self) -> bool: + """ Determine if this package contains (or is) a macro """ + + if self.repo_type == AddonManagerRepo.RepoType.MACRO: + return True + elif self.repo_type == AddonManagerRepo.RepoType.PACKAGE: + content = self.metadata.Content + return "macro" in content + else: + return False + + def contains_preference_pack(self) -> bool: + """ Determine if this package contains a preference pack """ + + if self.repo_type == AddonManagerRepo.RepoType.PACKAGE: + content = self.metadata.Content + return "preferencepack" in content + else: + return False + + def get_cached_icon_filename(self) ->str: + """ Get the filename for the locally-cached copy of the icon """ + + if self.cached_icon_filename: + return self.cached_icon_filename + + real_icon = self.metadata.Icon + if not real_icon: + # If there is no icon set for the entire package, see if there are any workbenches, which + # are required to have icons, and grab the first one we find: + content = self.metadata.Content + if "workbench" in content: + wb = content["workbench"][0] + if wb.Icon: + if wb.Subdirectory: + subdir = wb.Subdirectory + else: + subdir = wb.Name + real_icon = subdir + wb.Icon + + real_icon = real_icon.replace("/", os.path.sep) # Required path separator in the metadata.xml file to local separator + + _, file_extension = os.path.splitext(real_icon) + store = os.path.join(FreeCAD.getUserAppDataDir(), "AddonManager", "PackageMetadata") + self.cached_icon_filename = os.path.join(store, self.name, "cached_icon"+file_extension) + + return self.cached_icon_filename \ No newline at end of file diff --git a/src/Mod/AddonManager/CMakeLists.txt b/src/Mod/AddonManager/CMakeLists.txt index 23549f18c6..fa84f324fb 100644 --- a/src/Mod/AddonManager/CMakeLists.txt +++ b/src/Mod/AddonManager/CMakeLists.txt @@ -6,7 +6,9 @@ SET(AddonManager_SRCS Init.py InitGui.py AddonManager.py + AddonManagerRepo.py addonmanager_macro.py + addonmanager_metadata.py addonmanager_utilities.py addonmanager_workers.py AddonManager.ui diff --git a/src/Mod/AddonManager/addonmanager_macro.py b/src/Mod/AddonManager/addonmanager_macro.py index 45cb7cafe8..88481da068 100644 --- a/src/Mod/AddonManager/addonmanager_macro.py +++ b/src/Mod/AddonManager/addonmanager_macro.py @@ -24,12 +24,14 @@ import os import re import sys - +import codecs +import shutil import FreeCAD from addonmanager_utilities import translate from addonmanager_utilities import urlopen +from addonmanager_utilities import remove_directory_if_empty try: from HTMLParser import HTMLParser @@ -39,10 +41,10 @@ except ImportError: # @package AddonManager_macro # \ingroup ADDONMANAGER -# \brief Unified handler for FreeCAD macros that can be obtained from different sources +# \brief Unified handler for FreeCAD macros that can be obtained from +# different sources # @{ - class Macro(object): """This class provides a unified way to handle macros coming from different sources""" @@ -70,12 +72,11 @@ class Macro(object): def is_installed(self): if self.on_git and not self.src_filename: return False - return (os.path.exists(os.path.join(FreeCAD.getUserMacroDir(True), self.filename)) - or os.path.exists(os.path.join(FreeCAD.getUserMacroDir(True), "Macro_" + self.filename))) + return (os.path.exists(os.path.join(FreeCAD.getUserMacroDir(True), self.filename)) or os.path.exists(os.path.join(FreeCAD.getUserMacroDir(True), "Macro_" + self.filename))) def fill_details_from_file(self, filename): with open(filename) as f: - # Number of parsed fields of metadata. For now, __Comment__, + # Number of parsed fields of metadata. For now, __Comment__, # __Web__, __Version__, __Files__. number_of_required_fields = 4 re_desc = re.compile(r"^__Comment__\s*=\s*(['\"])(.*)\1") @@ -109,23 +110,24 @@ class Macro(object): code = "" u = urlopen(url) if u is None: - print("AddonManager: Debug: connection is lost (proxy setting changed?)", url) + FreeCAD.Console.PrintWarning("AddonManager: Debug: connection is lost (proxy setting changed?)", url, "\n") return p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): + if isinstance(p, bytes): p = p.decode("utf-8") u.close() - # check if the macro page has its code hosted elsewhere, download if needed + # check if the macro page has its code hosted elsewhere, download if + # needed if "rawcodeurl" in p: rawcodeurl = re.findall("rawcodeurl.*?href=\"(http.*?)\">", p) if rawcodeurl: rawcodeurl = rawcodeurl[0] u2 = urlopen(rawcodeurl) if u2 is None: - print("AddonManager: Debug: unable to open URL", rawcodeurl) + FreeCAD.Console.PrintWarning("AddonManager: Debug: unable to open URL", rawcodeurl, "\n") return # code = u2.read() - # github is slow to respond... We need to use this trick below + # github is slow to respond... We need to use this trick below response = "" block = 8192 # expected = int(u2.headers["content-length"]) @@ -134,7 +136,7 @@ class Macro(object): data = u2.read(block) if not data: break - if sys.version_info.major >= 3 and isinstance(data, bytes): + if isinstance(data, bytes): data = data.decode("utf-8") response += data if response: @@ -147,25 +149,101 @@ class Macro(object): code = sorted(code, key=len)[-1] code = code.replace("--endl--", "\n") # Clean HTML escape codes. - if sys.version_info.major < 3: - code = code.decode("utf8") code = unescape(code) code = code.replace(b"\xc2\xa0".decode("utf-8"), " ") - if sys.version_info.major < 3: - code = code.encode("utf8") else: - FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "Unable to fetch the code of this macro.")) + FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "Unable to fetch the code of this macro.") + "\n") desc = re.findall(r"(.*?)", p.replace("\n", " ")) if desc: desc = desc[0] else: FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "Unable to retrieve a description for this macro.")) + "Unable to retrieve a description for this macro.") + "\n") desc = "No description available" self.desc = desc self.url = url + if isinstance(code, list): + flat_code = "" + for chunk in code: + flat_code += chunk + code = flat_code self.code = code self.parsed = True + def install(self, macro_dir:str) -> bool: + """Install a macro and all its related files + + Returns True if the macro was installed correctly. + + Parameters + ---------- + - macro_dir: the directory to install into + """ + + if not self.code: + return False + if not os.path.isdir(macro_dir): + try: + os.makedirs(macro_dir) + except OSError: + FreeCAD.Console.PrintError(f"Failed to create {macro_dir}\n") + return False + macro_path = os.path.join(macro_dir, self.filename) + try: + with codecs.open(macro_path, 'w', 'utf-8') as macrofile: + macrofile.write(self.code) + except IOError: + FreeCAD.Console.PrintError(f"Failed to write {macro_path}\n") + return False + # Copy related files, which are supposed to be given relative to + # self.src_filename. + base_dir = os.path.dirname(self.src_filename) + for other_file in self.other_files: + dst_dir = os.path.join(macro_dir, os.path.dirname(other_file)) + if not os.path.isdir(dst_dir): + try: + os.makedirs(dst_dir) + except OSError: + FreeCAD.Console.PrintError(f"Failed to create {dst_dir}\n") + return False + src_file = os.path.join(base_dir, other_file) + dst_file = os.path.join(macro_dir, other_file) + try: + shutil.copy(src_file, dst_file) + except IOError: + FreeCAD.Console.PrintError(f"Failed to copy {src_file} to {dst_file}\n") + return False + + FreeCAD.Console.PrintMessage(f"Macro {self.name} was installed successfully.\n") + return True + + + def remove(self) -> bool: + """Remove a macro and all its related files + + Returns True if the macro was removed correctly. + """ + + if not self.is_installed(): + # Macro not installed, nothing to do. + return True + macro_dir = FreeCAD.getUserMacroDir(True) + macro_path = os.path.join(macro_dir, self.filename) + macro_path_with_macro_prefix = os.path.join(macro_dir, 'Macro_' + self.filename) + if os.path.exists(macro_path): + os.remove(macro_path) + elif os.path.exists(macro_path_with_macro_prefix): + os.remove(macro_path_with_macro_prefix) + # Remove related files, which are supposed to be given relative to + # self.src_filename. + for other_file in self.other_files: + dst_file = os.path.join(macro_dir, other_file) + try: + os.remove(dst_file) + remove_directory_if_empty(os.path.dirname(dst_file)) + except Exception: + FreeCAD.Console.PrintWarning(f"Failed to remove macro file '{dst_file}': it might not exist, or its permissions changed\n") + return True + # @} diff --git a/src/Mod/AddonManager/addonmanager_metadata.py b/src/Mod/AddonManager/addonmanager_metadata.py new file mode 100644 index 0000000000..94e290adeb --- /dev/null +++ b/src/Mod/AddonManager/addonmanager_metadata.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +#*************************************************************************** +#* * +#* Copyright (c) 2021 Chris Hennes * +#* * +#* This program is free software; you can redistribute it and/or modify * +#* it under the terms of the GNU Lesser General Public License (LGPL) * +#* as published by the Free Software Foundation; either version 2 of * +#* the License, or (at your option) any later version. * +#* for detail see the LICENCE text file. * +#* * +#* This program is distributed in the hope that it will be useful, * +#* but WITHOUT ANY WARRANTY; without even the implied warranty of * +#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * +#* GNU Library General Public License for more details. * +#* * +#* You should have received a copy of the GNU Library General Public * +#* License along with this program; if not, write to the Free Software * +#* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * +#* USA * +#* * +#*************************************************************************** + +import FreeCAD + +import git +import tempfile +import os +import hashlib + +from PySide2 import QtCore, QtNetwork +from PySide2.QtCore import QObject + +import addonmanager_utilities as utils +from AddonManagerRepo import AddonManagerRepo + +class MetadataDownloadWorker(QObject): + """A worker for downloading package.xml and associated icon(s) + + To use, instantiate an object of this class and call the start_fetch() function + with a QNetworkAccessManager. It is expected that many of these objects will all + be created and associated with the same QNAM, which will then handle the actual + asynchronous downloads in some Qt-defined number of threads. To monitor progress + you should connect to the QNAM's "finished" signal, and ensure it is called the + number of times you expect based on how many workers you have enqueued. + + """ + + updated = QtCore.Signal(AddonManagerRepo) + + def __init__(self, parent, repo, index): + "repo is an AddonManagerRepo object, and index is a dictionary of SHA1 hashes of the package.xml files in the cache" + + super().__init__(parent) + self.repo = repo + self.index = index + self.store = os.path.join(FreeCAD.getUserAppDataDir(), "AddonManager", "PackageMetadata") + self.last_sha1 = "" + self.url = self.repo.metadata_url + + def start_fetch(self, network_manager): + "Asynchronously begin the network access. Intended as a set-and-forget black box for downloading metadata." + self.request = QtNetwork.QNetworkRequest(QtCore.QUrl(self.url)) + self.request.setAttribute(QtNetwork.QNetworkRequest.RedirectPolicyAttribute, + QtNetwork.QNetworkRequest.UserVerifiedRedirectPolicy) + + self.fetch_task = network_manager.get(self.request) + self.fetch_task.finished.connect(self.resolve_fetch) + self.fetch_task.redirected.connect(self.on_redirect) + + def abort(self): + self.fetch_task.abort() + + def on_redirect(self, url): + # For now just blindly follow all redirects + self.fetch_task.redirectAllowed.emit() + + def resolve_fetch(self): + "Called when the data fetch completed, either with an error, or if it found the metadata file" + if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError: + FreeCAD.Console.PrintMessage(f"Found a metadata file for {self.repo.name}\n") + self.repo.repo_type = AddonManagerRepo.RepoType.PACKAGE + new_xml = self.fetch_task.readAll() + hasher = hashlib.sha1() + hasher.update(new_xml) + new_sha1 = hasher.hexdigest() + self.last_sha1 = new_sha1 + # Determine if we need to download the icon: only do that if the + # package.xml file changed (since + # a change in the version number will show up as a change in the + # SHA1, without having to actually + # read the metadata) + if self.repo.name in self.index: + cached_sha1 = self.index[self.repo.name] + if cached_sha1 != new_sha1: + self.update_local_copy(new_xml) + else: + # Assume that if the package.xml file didn't change, + # neither did the icon, so don't waste + # resources downloading it + xml_file = os.path.join(self.store, self.repo.name, "package.xml") + self.repo.metadata = FreeCAD.Metadata(xml_file) + else: + # There is no local copy yet, so we definitely have to update + # the cache + self.update_local_copy(new_xml) + + def update_local_copy(self, new_xml): + # We have to update the local copy of the metadata file and re-download + # the icon file + + name = self.repo.name + repo_url = self.repo.url + package_cache_directory = os.path.join(self.store, name) + if not os.path.exists(package_cache_directory): + os.makedirs(package_cache_directory) + new_xml_file = os.path.join(package_cache_directory, "package.xml") + with open(new_xml_file, "wb") as f: + f.write(new_xml) + metadata = FreeCAD.Metadata(new_xml_file) + self.repo.metadata = metadata + self.repo.repo_type = AddonManagerRepo.RepoType.PACKAGE + icon = metadata.Icon + + if not icon: + # If there is no icon set for the entire package, see if there are + # any workbenches, which + # are required to have icons, and grab the first one we find: + content = self.repo.metadata.Content + if "workbench" in content: + wb = content["workbench"][0] + if wb.Icon: + if wb.Subdirectory: + subdir = wb.Subdirectory + else: + subdir = wb.Name + self.repo.Icon = subdir + wb.Icon + icon = self.repo.Icon + + icon_url = utils.construct_git_url(self.repo, icon) + icon_stream = utils.urlopen(icon_url) + if icon and icon_stream and icon_url: + icon_data = icon_stream.read() + cache_file = self.repo.get_cached_icon_filename() + with open(cache_file, "wb") as icon_file: + icon_file.write(icon_data) + self.repo.cached_icon_filename = cache_file + self.updated.emit(self.repo) diff --git a/src/Mod/AddonManager/addonmanager_utilities.py b/src/Mod/AddonManager/addonmanager_utilities.py index 509e75c18c..1a7addec9f 100644 --- a/src/Mod/AddonManager/addonmanager_utilities.py +++ b/src/Mod/AddonManager/addonmanager_utilities.py @@ -27,12 +27,15 @@ import re import shutil import sys import ctypes +import tempfile +import ssl -import urllib.request as urllib2 +import urllib +from urllib.request import Request from urllib.error import URLError from urllib.parse import urlparse -from PySide import QtGui, QtCore +from PySide2 import QtGui, QtCore, QtWidgets import FreeCAD import FreeCADGui @@ -46,7 +49,9 @@ except ImportError: pass else: try: - ssl_ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) + #ssl_ctx = ssl.create_default_context(cafile=certifi.where()) + ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + #ssl_ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) except AttributeError: pass @@ -61,18 +66,17 @@ def translate(context, text, disambig=None): "Main translation function" try: - _encoding = QtGui.QApplication.UnicodeUTF8 + _encoding = QtWidgets.QApplication.UnicodeUTF8 except AttributeError: - return QtGui.QApplication.translate(context, text, disambig) + return QtWidgets.QApplication.translate(context, text, disambig) else: - return QtGui.QApplication.translate(context, text, disambig, _encoding) + return QtWidgets.QApplication.translate(context, text, disambig, _encoding) def symlink(source, link_name): "creates a symlink of a file, if possible" if os.path.exists(link_name) or os.path.lexists(link_name): - # print("macro already exists") pass else: os_symlink = getattr(os, "symlink", None) @@ -90,8 +94,8 @@ def symlink(source, link_name): raise ctypes.WinError() -def urlopen(url): - """Opens an url with urllib2""" +def urlopen(url:str): + """Opens an url with urllib and streams it to a temp file""" timeout = 5 @@ -101,25 +105,29 @@ def urlopen(url): proxies = {} else: if pref.GetBool("SystemProxyCheck", False): - proxy = urllib2.getproxies() + proxy = urllib.request.getproxies() proxies = {"http": proxy.get('http'), "https": proxy.get('http')} elif pref.GetBool("UserProxyCheck", False): proxy = pref.GetString("ProxyUrl", "") proxies = {"http": proxy, "https": proxy} if ssl_ctx: - handler = urllib2.HTTPSHandler(context=ssl_ctx) + handler = urllib.request.HTTPSHandler(context=ssl_ctx) else: handler = {} - proxy_support = urllib2.ProxyHandler(proxies) - opener = urllib2.build_opener(proxy_support, handler) - urllib2.install_opener(opener) + proxy_support = urllib.request.ProxyHandler(proxies) + opener = urllib.request.build_opener(proxy_support, handler) + urllib.request.install_opener(opener) # Url opening - req = urllib2.Request(url, + req = urllib.request.Request(url, headers={'User-Agent': "Magic Browser"}) try: - u = urllib2.urlopen(req, timeout=timeout) + u = urllib.request.urlopen(req, timeout=timeout) + + except URLError as e: + FreeCAD.Console.PrintError(f"Error loading {url}:\n {e.reason}\n") + return None except Exception: return None else: @@ -151,76 +159,7 @@ def update_macro_details(old_macro, new_macro): setattr(old_macro, attr, getattr(new_macro, attr)) -def install_macro(macro, macro_repo_dir): - """Install a macro and all its related files - Returns True if the macro was installed correctly. - - Parameters - ---------- - - macro: an addonmanager_macro.Macro instance - """ - - if not macro.code: - return False - macro_dir = FreeCAD.getUserMacroDir(True) - if not os.path.isdir(macro_dir): - try: - os.makedirs(macro_dir) - except OSError: - return False - macro_path = os.path.join(macro_dir, macro.filename) - try: - with codecs.open(macro_path, 'w', 'utf-8') as macrofile: - macrofile.write(macro.code) - except IOError: - return False - # Copy related files, which are supposed to be given relative to - # macro.src_filename. - base_dir = os.path.dirname(macro.src_filename) - for other_file in macro.other_files: - dst_dir = os.path.join(macro_dir, os.path.dirname(other_file)) - if not os.path.isdir(dst_dir): - try: - os.makedirs(dst_dir) - except OSError: - return False - src_file = os.path.join(base_dir, other_file) - dst_file = os.path.join(macro_dir, other_file) - try: - shutil.copy(src_file, dst_file) - except IOError: - return False - return True - - -def remove_macro(macro): - """Remove a macro and all its related files - - Returns True if the macro was removed correctly. - - Parameters - ---------- - - macro: an addonmanager_macro.Macro instance - """ - - if not macro.is_installed(): - # Macro not installed, nothing to do. - return True - macro_dir = FreeCAD.getUserMacroDir(True) - macro_path = os.path.join(macro_dir, macro.filename) - macro_path_with_macro_prefix = os.path.join(macro_dir, 'Macro_' + macro.filename) - if os.path.exists(macro_path): - os.remove(macro_path) - elif os.path.exists(macro_path_with_macro_prefix): - os.remove(macro_path_with_macro_prefix) - # Remove related files, which are supposed to be given relative to - # macro.src_filename. - for other_file in macro.other_files: - dst_file = os.path.join(macro_dir, other_file) - remove_directory_if_empty(os.path.dirname(dst_file)) - os.remove(dst_file) - return True def remove_directory_if_empty(dir): @@ -238,72 +177,81 @@ def remove_directory_if_empty(dir): def restart_freecad(): "Shuts down and restarts FreeCAD" - args = QtGui.QApplication.arguments()[1:] + args = QtWidgets.QApplication.arguments()[1:] if FreeCADGui.getMainWindow().close(): - QtCore.QProcess.startDetached(QtGui.QApplication.applicationFilePath(), args) + QtCore.QProcess.startDetached(QtWidgets.QApplication.applicationFilePath(), args) -def get_zip_url(baseurl): +def get_zip_url(repo): "Returns the location of a zip file from a repo, if available" - parsedUrl = urlparse(baseurl) + parsedUrl = urlparse(repo.url) if parsedUrl.netloc == "github.com": - return baseurl+"/archive/master.zip" + return f"{repo.url}/archive/{repo.branch}.zip" elif parsedUrl.netloc == "framagit.org" or parsedUrl.netloc == "gitlab.com": # https://framagit.org/freecad-france/mooc-workbench/-/archive/master/mooc-workbench-master.zip reponame = baseurl.strip("/").split("/")[-1] - return baseurl+"/-/archive/master/"+reponame+"-master.zip" + return f"{repo.url}/-/archive/{repo.branch}/{repo.name}-{repo.branch}.zip" else: - print("Debug: addonmanager_utilities.get_zip_url: Unknown git host:", parsedUrl.netloc) + FreeCAD.Console.PrintWarning("Debug: addonmanager_utilities.get_zip_url: Unknown git host:", parsedUrl.netloc) return None +def construct_git_url(repo, filename): + "Returns a direct download link to a file in an online Git repo: works with github, gitlab, and framagit" -def get_readme_url(url): - "Returns the location of a readme file" - - parsedUrl = urlparse(url) - if parsedUrl.netloc == "github.com" or parsedUrl.netloc == "framagit.com": - return url+"/raw/master/README.md" - elif parsedUrl.netloc == "gitlab.com": - return url+"/-/raw/master/README.md" + parsed_url = urlparse(repo.url) + if parsed_url.netloc == "github.com" or parsed_url.netloc == "framagit.com": + return f"{repo.url}/raw/{repo.branch}/{filename}" + elif parsed_url.netloc == "gitlab.com": + return f"{repo.url}/-/raw/{repo.branch}/{filename}" else: - print("Debug: addonmanager_utilities.get_readme_url: Unknown git host:", url) + FreeCAD.Console.PrintLog("Debug: addonmanager_utilities.construct_git_url: Unknown git host:", parsed_url.netloc) return None +def get_readme_url(repo): + "Returns the location of a readme file" -def get_desc_regex(url): + return construct_git_url(repo, "README.md") + +def get_metadata_url(url): + "Returns the location of a package.xml metadata file" + + return construct_git_url(repo, "package.xml") + + +def get_desc_regex(repo): """Returns a regex string that extracts a WB description to be displayed in the description panel of the Addon manager, if the README could not be found""" - parsedUrl = urlparse(url) + parsedUrl = urlparse(repo.url) if parsedUrl.netloc == "github.com": return r'' - print("Debug: addonmanager_utilities.get_desc_regex: Unknown git host:", url) + FreeCAD.Console.PrintWarning("Debug: addonmanager_utilities.get_desc_regex: Unknown git host:", repo.url) return None -def get_readme_html_url(url): +def get_readme_html_url(repo): """Returns the location of a html file containing readme""" - parsedUrl = urlparse(url) + parsedUrl = urlparse(repo.url) if parsedUrl.netloc == "github.com": - return url + "/blob/master/README.md" + return f"{repo.url}/blob/{repo.branch}/README.md" else: - print("Debug: addonmanager_utilities.get_readme_html_url: Unknown git host:", url) + FreeCAD.Console.PrintWarning("Debug: addonmanager_utilities.get_readme_html_url: Unknown git host:", repo.url) return None -def get_readme_regex(url): +def get_readme_regex(repo): """Return a regex string that extracts the contents to be displayed in the description panel of the Addon manager, from raw HTML data (the readme's html rendering usually)""" - parsedUrl = urlparse(url) + parsedUrl = urlparse(repo.url) if parsedUrl.netloc == "github.com": return "(.*?)" else: - print("Debug: addonmanager_utilities.get_readme_regex: Unknown git host:", url) + FreeCAD.Console.PrintWarning("Debug: addonmanager_utilities.get_readme_regex: Unknown git host:", repo.url) return None @@ -319,7 +267,7 @@ def fix_relative_links(text, base_url): if len(parts) < 2 or not re.match(r"^http|^www|^.+\.|^/", parts[0]): newlink = os.path.join(base_url, link.lstrip('./')) line = line.replace(link, newlink) - print("Debug: replaced " + link + " with " + newlink) + FreeCAD.Console.PrintLog("Debug: replaced " + link + " with " + newlink) new_text = new_text + '\n' + line return new_text diff --git a/src/Mod/AddonManager/addonmanager_workers.py b/src/Mod/AddonManager/addonmanager_workers.py index ba12996a78..8716375f91 100644 --- a/src/Mod/AddonManager/addonmanager_workers.py +++ b/src/Mod/AddonManager/addonmanager_workers.py @@ -2,6 +2,7 @@ #*************************************************************************** #* * #* Copyright (c) 2019 Yorik van Havre * +#* Copyright (c) 2021 Chris Hennes * #* * #* This program is free software; you can redistribute it and/or modify * #* it under the terms of the GNU Lesser General Public License (LGPL) * @@ -26,8 +27,13 @@ import re import shutil import sys import json +import tempfile +import hashlib +import threading +import queue +from typing import Union -from PySide import QtCore, QtGui +from PySide2 import QtCore, QtGui, QtWidgets, QtNetwork import FreeCAD if FreeCAD.GuiUp: @@ -36,11 +42,15 @@ if FreeCAD.GuiUp: import addonmanager_utilities as utils from addonmanager_utilities import translate # this needs to be as is for pylupdate from addonmanager_macro import Macro +from addonmanager_metadata import MetadataDownloadWorker +from AddonManagerRepo import AddonManagerRepo + have_git = False try: import git - # some versions of git module have no "Repo" class?? Bug #4072 module 'git' has no attribute 'Repo' + # some versions of git module have no "Repo" class?? Bug #4072 module + # 'git' has no attribute 'Repo' have_git = hasattr(git,"Repo") except ImportError: pass @@ -59,12 +69,7 @@ try: except ImportError: pass -try: - import StringIO as io - _stringio = io.StringIO -except ImportError: # StringIO is not available with python3 - import io - _stringio = io.BytesIO +from io import BytesIO # @package AddonManager_workers # \ingroup ADDONMANAGER @@ -81,18 +86,15 @@ obsolete = [] py2only = [] NOGIT = False # for debugging purposes, set this to True to always use http downloads - NOMARKDOWN = False # for debugging purposes, set this to True to disable Markdown lib - """Multithread workers for the Addon Manager""" class UpdateWorker(QtCore.QThread): """This worker updates the list of available workbenches""" - info_label = QtCore.Signal(str) + status_message = QtCore.Signal(str) addon_repo = QtCore.Signal(object) - progressbar_show = QtCore.Signal(bool) done = QtCore.Signal() def __init__(self): @@ -102,7 +104,7 @@ class UpdateWorker(QtCore.QThread): def run(self): "populates the list of addons" - self.progressbar_show.emit(True) + self.current_thread = QtCore.QThread.currentThread() # update info lists global obsolete, macros_blacklist, py2only @@ -120,102 +122,118 @@ class UpdateWorker(QtCore.QThread): if "py2only" in j and "Mod" in j["py2only"]: py2only = j["py2only"]["Mod"] else: - print("Debug: addon_flags.json not found") + FreeCAD.Console.PrintWarning("Debug: addon_flags.json not found\n") + + # Check the local package cache: + basedir = FreeCAD.getUserAppDataDir() + moddir = basedir + os.sep + "Mod" + cache_path = os.path.join(FreeCAD.getUserAppDataDir(), "AddonManager", "PackageMetadata") + package_names = [] + if os.path.isdir(cache_path): + for dir in os.listdir(cache_path): + if self.current_thread.isInterruptionRequested(): + return + dir_path = os.path.join(cache_path, dir) + if not os.path.isdir(dir_path): + continue + xml_cache = os.path.join(dir_path, "package.xml") + try: + meta = FreeCAD.Metadata(xml_cache) + except Exception: + FreeCAD.Console.PrintWarning(f"Failed to create Metadata from {xml_cache}\n") + continue + name = dir # Do not use metadata name here, we want to match the legacy fetch code below + url = None + branch = None + for meta_url in meta.Urls: + if meta_url["type"] == "repository": + url = meta_url["location"] + branch = meta_url["branch"] + break + addondir = moddir + os.sep + name + if os.path.exists(addondir) and os.listdir(addondir): + state = AddonManagerRepo.UpdateStatus.UNCHECKED + else: + state = AddonManagerRepo.UpdateStatus.NOT_INSTALLED + cached_package = AddonManagerRepo(name, url, state, branch) + cached_package.metadata = meta + cached_package.icon = QtGui.QIcon(cached_package.get_cached_icon_filename()) + cached_package.repo_type = AddonManagerRepo.RepoType.PACKAGE + cached_package.description = meta.Description + self.addon_repo.emit(cached_package) + package_names.append(name) u = utils.urlopen("https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/.gitmodules") if not u: - self.progressbar_show.emit(False) self.done.emit() self.stop = True return p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): + if isinstance(p, bytes): p = p.decode("utf-8") u.close() p = re.findall((r'(?m)\[submodule\s*"(?P.*)"\]\s*' r"path\s*=\s*(?P.+)\s*" - r"url\s*=\s*(?Phttps?://.*)"), p) - basedir = FreeCAD.getUserAppDataDir() - moddir = basedir + os.sep + "Mod" - repos = [] + r"url\s*=\s*(?Phttps?://.*)\s*" + r"(branch\s*=\s*(?P.*)\s*)?"), p) + # querying official addons - for name, path, url in p: - self.info_label.emit(name) + for name, path, url, _, branch in p: + if self.current_thread.isInterruptionRequested(): + return + if name in package_names: + # We've already got this info since it's a package + continue + if branch is None or len(branch) == 0: + branch = "master" url = url.split(".git")[0] addondir = moddir + os.sep + name if os.path.exists(addondir) and os.listdir(addondir): # make sure the folder exists and it contains files! - state = 1 + state = AddonManagerRepo.UpdateStatus.UNCHECKED else: - state = 0 - repos.append([name, url, state]) + state = AddonManagerRepo.UpdateStatus.NOT_INSTALLED + self.addon_repo.emit(AddonManagerRepo(name, url, state, branch)) # querying custom addons - customaddons = (FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") + addon_list = (FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") .GetString("CustomRepositories", "").split("\n")) - for url in customaddons: - if url: - name = url.split("/")[-1] + custom_addons = [] + for addon in addon_list: + if " " in addon: + addon_and_branch = addon.split(" ") + custom_addons.append({"url":addon_and_branch[0], "branch":addon_and_branch[1]}) + else: + custom_addons.append({"url":addon, "branch":"master"}) + for addon in custom_addons: + if self.current_thread.isInterruptionRequested(): + return + if addon and addon["url"]: + name = addon["url"].split("/")[-1] if name.lower().endswith(".git"): name = name[:-4] + if name in package_names: + # We already have this one cached since it's a package + continue addondir = moddir + os.sep + name - if not os.path.exists(addondir): - state = 0 + if os.path.exists(addondir) and os.listdir(addondir): + state = AddonManagerRepo.UpdateStatus.UNCHECKED else: - state = 1 - repos.append([name, url, state]) - if not repos: - self.info_label.emit(translate("AddonsInstaller", "Unable to download addon list.")) - else: - repos = sorted(repos, key=lambda s: s[0].lower()) - for repo in repos: - self.addon_repo.emit(repo) - self.info_label.emit(translate("AddonsInstaller", "Workbenches list was updated.")) - self.progressbar_show.emit(False) - self.done.emit() - self.stop = True + state = AddonManagerRepo.UpdateStatus.NOT_INSTALLED + self.addon_repo.emit(AddonManagerRepo(name, addon["url"], state, addon["branch"])) + self.status_message.emit(translate("AddonsInstaller", "Workbenches list was updated.")) + + if not self.current_thread.isInterruptionRequested(): + self.done.emit() + self.stop = True - -class InfoWorker(QtCore.QThread): - """This worker retrieves the description text of a workbench""" - - addon_repos = QtCore.Signal(object) - - def __init__(self): - - QtCore.QThread.__init__(self) - - def run(self): - - i = 0 - for repo in self.repos: - url = repo[1] - u = utils.urlopen(url) - if not u: - self.stop = True - return - p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): - p = p.decode("utf-8") - u.close() - desc = re.findall(' installed_metadata.Version: + self.update_status.emit(package, AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE) + else: + self.update_status.emit(package, AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE) + + + def check_macro(self, macro_wrapper): + # Make sure this macro has its code downloaded: + try: + if not macro_wrapper.macro.parsed and macro_wrapper.macro.on_git: + macro_wrapper.macro.fill_details_from_file(macro_wrapper.macro.src_filename) + if not macro_wrapper.macro.parsed and macro_wrapper.macro.on_wiki: + mac = macro_wrapper.macro.name.replace(" ", "_") + mac = mac.replace("&", "%26") + mac = mac.replace("+", "%2B") + url = "https://wiki.freecad.org/Macro_" + mac + macro_wrapper.macro.fill_details_from_wiki(url) + except Exception: + FreeCAD.Console.PrintWarning(f"Failed to fetch code for macro '{macro_wrapper.macro.name}'\n") + return + + hasher1 = hashlib.sha1() + hasher2 = hashlib.sha1() + hasher1.update(macro_wrapper.macro.code.encode("utf-8")) + new_sha1 = hasher1.hexdigest() + test_file_one = os.path.join(FreeCAD.getUserMacroDir(True), macro_wrapper.macro.filename) + test_file_two = os.path.join(FreeCAD.getUserMacroDir(True), "Macro_" + macro_wrapper.macro.filename) + if os.path.exists(test_file_one): + with open(test_file_one, "rb") as f: + contents = f.read() + hasher2.update(contents) + old_sha1 = hasher2.hexdigest() + elif os.path.exists(test_file_two): + with open(test_file_two, "rb") as f: + contents = f.read() + hasher2.update(contents) + old_sha1 = hasher2.hexdigest() + else: + return + if new_sha1 == old_sha1: + self.update_status.emit(macro_wrapper, AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE) + else: + self.update_status.emit(macro_wrapper, AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE) + class FillMacroListWorker(QtCore.QThread): """This worker populates the list of macros""" - add_macro_signal = QtCore.Signal(Macro) - info_label_signal = QtCore.Signal(str) - progressbar_show = QtCore.Signal(bool) + add_macro_signal = QtCore.Signal(object) + status_message_signal = QtCore.Signal(str) + progress_made = QtCore.Signal(int, int) + done = QtCore.Signal() def __init__(self, repo_dir): QtCore.QThread.__init__(self) self.repo_dir = repo_dir - self.macros = [] + self.repo_names = [] def run(self): """Populates the list of macros""" + + self.current_thread = QtCore.QThread.currentThread() - self.retrieve_macros_from_git() - self.retrieve_macros_from_wiki() - [self.add_macro_signal.emit(m) for m in sorted(self.macros, key=lambda m: m.name.lower())] - if self.macros: - self.info_label_signal.emit(translate("AddonsInstaller", "List of macros successfully retrieved.")) - self.progressbar_show.emit(False) + if not self.current_thread.isInterruptionRequested(): + self.status_message_signal.emit(translate("AddonInstaller","Retrieving macros from FreeCAD/FreeCAD-Macros Git repository")) + self.retrieve_macros_from_git() + + if not self.current_thread.isInterruptionRequested(): + self.status_message_signal.emit(translate("AddonInstaller","Retrieving macros from FreeCAD/FreeCAD-Macros Git repository")) + self.retrieve_macros_from_wiki() + + if self.current_thread.isInterruptionRequested(): + return + + self.status_message_signal.emit(translate("AddonsInstaller", "Done locating macros.")) self.stop = True + self.done.emit() def retrieve_macros_from_git(self): """Retrieve macros from FreeCAD-macros.git @@ -305,120 +396,129 @@ class FillMacroListWorker(QtCore.QThread): """ if not have_git: - self.info_label_signal.emit("GitPython not installed! Cannot retrieve macros from Git") + self.status_message_signal.emit("GitPython not installed! Cannot retrieve macros from Git") FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "GitPython not installed! Cannot retrieve macros from git")+"\n") + "GitPython not installed! Cannot retrieve macros from git") + "\n") return - self.info_label_signal.emit("Downloading list of macros from git...") try: + # TODO: someday see if the directory exists, and do a pull instead + # of a clone git.Repo.clone_from("https://github.com/FreeCAD/FreeCAD-macros.git", self.repo_dir) except Exception: FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "Something went wrong with the Git Macro Retrieval, " "possibly the Git executable is not in the path") + "\n") + n_files = 0 + for _, _, filenames in os.walk(self.repo_dir): + n_files += len(filenames) + counter = 0 for dirpath, _, filenames in os.walk(self.repo_dir): + self.progress_made.emit(counter, n_files) + counter += 1 + if self.current_thread.isInterruptionRequested(): + return if ".git" in dirpath: continue for filename in filenames: + if self.current_thread.isInterruptionRequested(): + return if filename.lower().endswith(".fcmacro"): macro = Macro(filename[:-8]) # Remove ".FCMacro". macro.on_git = True macro.src_filename = os.path.join(dirpath, filename) - self.macros.append(macro) + repo = AddonManagerRepo.from_macro(macro) + repo.url = "https://github.com/FreeCAD/FreeCAD-macros.git" + self.add_macro_signal.emit(repo) def retrieve_macros_from_wiki(self): """Retrieve macros from the wiki Read the wiki and emit a signal for each found macro. - Reads only the page https://wiki.freecadweb.org/Macros_recipes + Reads only the page https://wiki.freecad.org/Macros_recipes """ - self.info_label_signal.emit("Downloading list of macros from the FreeCAD wiki...") - self.progressbar_show.emit(True) - u = utils.urlopen("https://wiki.freecadweb.org/Macros_recipes") + u = utils.urlopen("https://wiki.freecad.org/Macros_recipes") if not u: FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "Appears to be an issue connecting to the Wiki, " - "therefore cannot retrieve Wiki macro list at this time") + "\n") + "There appears to be an issue connecting to the Wiki, " + "therefore FreeCAD cannot retrieve the Wiki macro list at this time") + "\n") return p = u.read() u.close() - if sys.version_info.major >= 3 and isinstance(p, bytes): + if isinstance(p, bytes): p = p.decode("utf-8") macros = re.findall('title="(Macro.*?)"', p) macros = [mac for mac in macros if ("translated" not in mac)] - for mac in macros: + macro_names = [] + for i, mac in enumerate(macros): + self.progress_made.emit(i, len(macros)) + if self.current_thread.isInterruptionRequested(): + return macname = mac[6:] # Remove "Macro ". macname = macname.replace("&", "&") - if (macname not in macros_blacklist) and ("recipes" not in macname.lower()): + if not macname: + continue + if (macname not in macros_blacklist) and ("recipes" not in macname.lower()) and (macname not in macro_names): + macro_names.append(macname) macro = Macro(macname) macro.on_wiki = True - self.macros.append(macro) + repo = AddonManagerRepo.from_macro(macro) + repo.url = "https://wiki.freecad.org/Macros_recipes" + self.add_macro_signal.emit(repo) class ShowWorker(QtCore.QThread): """This worker retrieves info of a given workbench""" - info_label = QtCore.Signal(str) + status_message = QtCore.Signal(str) + description_updated = QtCore.Signal(str) addon_repos = QtCore.Signal(object) - progressbar_show = QtCore.Signal(bool) + done = QtCore.Signal() - def __init__(self, repos, idx): - - # repos is a list of [name, url, installbit, descr] - # name : Addon name - # url : Addon repository location - # installbit: 0 = Addon is not installed - # 1 = Addon is installed - # 2 = Addon is installed and checked for available updates (none pending) - # 3 = Addon is installed and has a pending update - # descr : Addon description + def __init__(self, repo): QtCore.QThread.__init__(self) - self.repos = repos - self.idx = idx + self.repo = repo def run(self): - - self.progressbar_show.emit(True) - self.info_label.emit(translate("AddonsInstaller", "Retrieving description...")) - if len(self.repos[self.idx]) == 4: - desc = self.repos[self.idx][3] + self.status_message.emit(translate("AddonsInstaller", "Retrieving description...")) + if self.repo.description is not None: + desc = self.repo.description else: u = None - url = self.repos[self.idx][1] - self.info_label.emit(translate("AddonsInstaller", "Retrieving info from") + " " + str(url)) + url = self.repo.url + self.status_message.emit(translate("AddonsInstaller", "Retrieving info from") + " " + str(url)) desc = "" - regex = utils.get_readme_regex(url) + regex = utils.get_readme_regex(self.repo) if regex: # extract readme from html via regex - readmeurl = utils.get_readme_html_url(url) + readmeurl = utils.get_readme_html_url(self.repo) if not readmeurl: - print("Debug: README not found for", url) + FreeCAD.Console.PrintWarning(f"Debug: README not found for {url}\n") u = utils.urlopen(readmeurl) if not u: - print("Debug: README not found at", readmeurl) + FreeCAD.Console.PrintWarning(f"Debug: README not found at {readmeurl}\n") u = utils.urlopen(readmeurl) if u: p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): + if isinstance(p, bytes): p = p.decode("utf-8") u.close() readme = re.findall(regex, p, flags=re.MULTILINE | re.DOTALL) if readme: desc = readme[0] else: - print("Debug: README not found at", readmeurl) + FreeCAD.Console.PrintWarning(f"Debug: README not found at {readmeurl}\n") else: # convert raw markdown using lib - readmeurl = utils.get_readme_url(url) + readmeurl = utils.get_readme_url(self.repo) if not readmeurl: - print("Debug: README not found for", url) + FreeCAD.Console.PrintWarning(f"Debug: README not found for {url}\n") u = utils.urlopen(readmeurl) if u: p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): + if isinstance(p, bytes): p = p.decode("utf-8") u.close() desc = utils.fix_relative_links(p, readmeurl.rsplit("/README.md")[0]) @@ -435,38 +535,37 @@ class ShowWorker(QtCore.QThread): message += "
" + desc + "
" desc = message else: - print("Debug: README not found at", readmeurl) + FreeCAD.Console.PrintWarning("Debug: README not found at {readmeurl}\n") if desc == "": # fall back to the description text u = utils.urlopen(url) if not u: - self.progressbar_show.emit(False) self.stop = True return p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): + if isinstance(p, bytes): p = p.decode("utf-8") u.close() - descregex = utils.get_desc_regex(url) + descregex = utils.get_desc_regex(self.repo) if descregex: desc = re.findall(descregex, p) if desc: desc = desc[0] if not desc: desc = "Unable to retrieve addon description" - self.repos[self.idx].append(desc) - self.addon_repos.emit(self.repos) + self.repo.description = desc + self.addon_repos.emit(self.repo) # Addon is installed so lets check if it has an update - if self.repos[self.idx][2] == 1: + if self.repo.update_status == AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE: upd = False # checking for updates if not NOGIT and have_git: - repo = self.repos[self.idx] - clonedir = FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + repo[0] + repo = self.repo + clonedir = FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + repo.name if os.path.exists(clonedir): if not os.path.exists(clonedir + os.sep + ".git"): # Repair addon installed with raw download - bare_repo = git.Repo.clone_from(repo[1], clonedir + os.sep + ".git", bare=True) + bare_repo = git.Repo.clone_from(repo.url, clonedir + os.sep + ".git", bare=True) try: with bare_repo.config_writer() as cw: cw.set("core", "bare", False) @@ -492,9 +591,8 @@ class ShowWorker(QtCore.QThread): """ message += translate("AddonsInstaller", "An update is available for this addon.") message += "
" + desc + '

Addon repository: ' + self.repos[self.idx][1] + "" - # mark as already installed AND already checked for updates AND update is available - self.repos[self.idx][2] = 3 + message += self.repo.url + '">' + self.repo.url + "" + self.repo.update_status = AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE # If there isn't, indicate that this addon is already installed else: message = """ @@ -505,13 +603,24 @@ class ShowWorker(QtCore.QThread): message += translate("AddonsInstaller", "This addon is already installed.") message += "
" + desc message += '

Addon repository: ' + self.repos[self.idx][1] + "" - self.repos[self.idx][2] = 2 # mark as already installed AND already checked for updates + message += self.repo.url + '">' + self.repo.url + "" + self.repo.update_status = AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE # Let the user know the install path for this addon message += "
" + translate("AddonInstaller", "Installed location") + ": " - message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repos[self.idx][0] - self.addon_repos.emit(self.repos) - elif self.repos[self.idx][2] == 2: + message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repo.name + self.addon_repos.emit(self.repo) + elif self.repo.update_status == AddonManagerRepo.UpdateStatus.PENDING_RESTART: + message = """ +
+
+ +""" + message += translate("AddonsInstaller", "This addon has been updated, a restart is now required before it can be used.") + message += "
" + desc + '

Addon repository: ' + self.repo.url + "" + message += "
" + translate("AddonInstaller", "Installed location") + ": " + message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repo.name + elif self.repo.update_status == AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE: message = """

@@ -520,10 +629,10 @@ class ShowWorker(QtCore.QThread): message += translate("AddonsInstaller", "This addon is already installed.") message += "
" + desc message += '

Addon repository: ' + self.repos[self.idx][1] + "" + message += self.repo.url + '">' + self.repo.url + "" message += "
" + translate("AddonInstaller", "Installed location") + ": " - message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repos[self.idx][0] - elif self.repos[self.idx][2] == 3: + message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repo.name + elif self.repo.update_status == AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE: message = """

@@ -531,15 +640,15 @@ class ShowWorker(QtCore.QThread): """ message += translate("AddonsInstaller", "An update is available for this addon.") message += "
" + desc + '

Addon repository: ' + self.repos[self.idx][1] + "" + message += self.repo.url + '">' + self.repo.url + "" message += "
" + translate("AddonInstaller", "Installed location") + ": " - message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repos[self.idx][0] + message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repo.name else: message = desc + '

Addon repository: ' + self.repos[self.idx][1] + '' + message += self.repo.url + '">' + self.repo.url + '' # If the Addon is obsolete, let the user know through the Addon UI - if self.repos[self.idx][0] in obsolete: + if self.repo.name in obsolete: message = """
@@ -551,7 +660,7 @@ class ShowWorker(QtCore.QThread): "provides the same functionality.") + "
" + desc # If the Addon is Python 2 only, let the user know through the Addon UI - if self.repos[self.idx][0] in py2only: + if self.repo.name in py2only: message = """
@@ -563,12 +672,12 @@ class ShowWorker(QtCore.QThread): "likely result in errors at startup or while in use.") message += "
" + desc - self.info_label.emit(message) - self.progressbar_show.emit(False) + self.description_updated.emit(message) self.mustLoadImages = True - label = self.loadImages(message, self.repos[self.idx][1], self.repos[self.idx][0]) + label = self.loadImages(message, self.repo.url, self.repo.name) if label: - self.info_label.emit(label) + self.description_updated.emit(label) + self.done.emit() self.stop = True def stopImageLoading(self): @@ -579,7 +688,7 @@ class ShowWorker(QtCore.QThread): def loadImages(self, message, url, wbName): "checks if the given page contains images and downloads them" - # QTextBrowser cannot display online images. So we download them + # QTextBrowser cannot display online images. So we download them # here, and replace the image link in the html code with the # downloaded version @@ -590,8 +699,10 @@ class ShowWorker(QtCore.QThread): if not os.path.exists(store): os.makedirs(store) for path in imagepaths: + if QtCore.QThread.currentThread().isInterruptionRequested(): + return message if not self.mustLoadImages: - return None + return message origpath = path if "?" in path: # remove everything after the ? @@ -603,19 +714,20 @@ class ShowWorker(QtCore.QThread): storename = os.path.join(store, name) if len(storename) >= 260: remainChars = 259 - (len(store) + len(wbName) + 1) - storename = os.path.join(store, wbName+name[-remainChars:]) + storename = os.path.join(store, wbName + name[-remainChars:]) if not os.path.exists(storename): try: u = utils.urlopen(path) imagedata = u.read() u.close() except Exception: - print("AddonManager: Debug: Error retrieving image from", path) + FreeCAD.Console.PrintLog("AddonManager: Debug: Error retrieving image from", path) else: try: f = open(storename, "wb") except OSError: - # ecryptfs (and probably not only ecryptfs) has lower length limit for path + # ecryptfs (and probably not only ecryptfs) has + # lower length limit for path storename = storename[-140:] f = open(storename, "wb") f.write(imagedata) @@ -628,8 +740,7 @@ class ShowWorker(QtCore.QThread): QtCore.Qt.KeepAspectRatio, QtCore.Qt.FastTransformation)) pix.save(storename, "jpeg", 100) - message = message.replace("src=\""+origpath, "src=\"file:///"+storename.replace("\\", "/")) - # print(message) + message = message.replace("src=\"" + origpath, "src=\"file:///" + storename.replace("\\", "/")) return message return None @@ -637,151 +748,158 @@ class ShowWorker(QtCore.QThread): class GetMacroDetailsWorker(QtCore.QThread): """Retrieve the macro details for a macro""" - info_label = QtCore.Signal(str) - progressbar_show = QtCore.Signal(bool) + status_message = QtCore.Signal(str) + description_updated = QtCore.Signal(str) + done = QtCore.Signal() - def __init__(self, macro): + def __init__(self, repo): QtCore.QThread.__init__(self) - self.macro = macro + self.macro = repo.macro def run(self): - self.progressbar_show.emit(True) - self.info_label.emit(translate("AddonsInstaller", "Retrieving description...")) + self.status_message.emit(translate("AddonsInstaller", "Retrieving macro description...")) if not self.macro.parsed and self.macro.on_git: - self.info_label.emit(translate("AddonsInstaller", "Retrieving info from git")) + self.status_message.emit(translate("AddonsInstaller", "Retrieving info from git")) self.macro.fill_details_from_file(self.macro.src_filename) if not self.macro.parsed and self.macro.on_wiki: - self.info_label.emit(translate("AddonsInstaller", "Retrieving info from wiki")) + self.status_message.emit(translate("AddonsInstaller", "Retrieving info from wiki")) mac = self.macro.name.replace(" ", "_") mac = mac.replace("&", "%26") mac = mac.replace("+", "%2B") - url = "https://wiki.freecadweb.org/Macro_" + mac + url = "https://wiki.freecad.org/Macro_" + mac self.macro.fill_details_from_wiki(url) if self.macro.is_installed(): - already_installed_msg = ('' - + translate("AddonsInstaller", "This macro is already installed.") - + '
') + already_installed_msg = ('' + translate("AddonsInstaller", "This macro is already installed.") + '
') else: already_installed_msg = "" - message = (already_installed_msg - + "

"+self.macro.name+"

" - + self.macro.desc - + "

Macro location: " - + self.macro.url - + "") - self.info_label.emit(message) - self.progressbar_show.emit(False) + message = (already_installed_msg + "

" + self.macro.name + "

" + self.macro.desc + "

Macro location: " + self.macro.url + "") + self.description_updated.emit(message) + self.done.emit() self.stop = True -class InstallWorker(QtCore.QThread): +class InstallWorkbenchWorker(QtCore.QThread): "This worker installs a workbench" - info_label = QtCore.Signal(str) - progressbar_show = QtCore.Signal(bool) - mark_recompute = QtCore.Signal(str) + status_message = QtCore.Signal(str) + progress_made = QtCore.Signal(int, int) + success = QtCore.Signal(AddonManagerRepo, str) + failure = QtCore.Signal(AddonManagerRepo, str) - def __init__(self, repos, idx): + def __init__(self, repo:AddonManagerRepo): QtCore.QThread.__init__(self) - self.idx = idx - self.repos = repos + self.repo = repo + if have_git: + self.git_progress = GitProgressMonitor() + # TODO: What is wrong with these? + #self.git_progress.progress_made.connect(self.progress_made.emit) + #self.git_progress.info_message.connect(self.status_message.emit) def run(self): "installs or updates the selected addon" + + if not self.repo: + return if not have_git: - self.info_label.emit("GitPython not found.") FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "GitPython not found. Using standard download instead.") + "\n") + "GitPython not found. Using ZIP file download instead.") + "\n") if not have_zip: - self.info_label.emit("no zip support.") FreeCAD.Console.PrintError(translate("AddonsInstaller", "Your version of python doesn't appear to support ZIP " "files. Unable to proceed.") + "\n") return - if not isinstance(self.idx, list): - self.idx = [self.idx] - for idx in self.idx: - if idx < 0: - return - if not self.repos: - return - basedir = FreeCAD.getUserAppDataDir() - moddir = basedir + os.sep + "Mod" - if not os.path.exists(moddir): - os.makedirs(moddir) - clonedir = moddir + os.sep + self.repos[idx][0] - self.progressbar_show.emit(True) - if os.path.exists(clonedir): - self.info_label.emit("Updating module...") - if sys.version_info.major > 2 and str(self.repos[idx][0]) in py2only: - FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "User requested updating a Python 2 workbench on " - "a system running Python 3 - ") + - str(self.repos[idx][0])+"\n") - if have_git: - if not os.path.exists(clonedir + os.sep + ".git"): - # Repair addon installed with raw download - bare_repo = git.Repo.clone_from(self.repos[idx][1], clonedir + os.sep + ".git", bare=True) - try: - with bare_repo.config_writer() as cw: - cw.set("core", "bare", False) - except AttributeError: - FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "Outdated GitPython detected, consider " - "upgrading with pip.") + "\n") - cw = bare_repo.config_writer() - cw.set("core", "bare", False) - del cw - repo = git.Repo(clonedir) - repo.head.reset("--hard") - repo = git.Git(clonedir) - try: - answer = repo.pull() + "\n\n" + translate("AddonsInstaller", - "Workbench successfully updated. " - "Please restart FreeCAD to apply the changes.") - except Exception: - print("Error updating module", self.repos[idx][1], " - Please fix manually") - answer = repo.status() - print(answer) - else: - # Update the submodules for this repository - repo_sms = git.Repo(clonedir) - for submodule in repo_sms.submodules: - submodule.update(init=True, recursive=True) - else: - answer = self.download(self.repos[idx][1], clonedir) + "\n\n" - answer += translate("AddonsInstaller", - "Workbench successfully updated. Please " - "restart FreeCAD to apply the changes.") - else: - self.info_label.emit("Checking module dependencies...") - depsok, answer = self.check_dependencies(self.repos[idx][1]) - if depsok: - if sys.version_info.major > 2 and str(self.repos[idx][0]) in py2only: - FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "User requested installing a Python 2 " - "workbench on a system running Python 3 - ") + - str(self.repos[idx][0]) + "\n") - if have_git: - self.info_label.emit("Cloning module...") - repo = git.Repo.clone_from(self.repos[idx][1], clonedir) + basedir = FreeCAD.getUserAppDataDir() + moddir = basedir + os.sep + "Mod" + if not os.path.exists(moddir): + os.makedirs(moddir) + target_dir = moddir + os.sep + self.repo.name - # Make sure to clone all the submodules as well - if repo.submodules: - repo.submodule_update(recursive=True) - else: - self.info_label.emit("Downloading module...") - self.download(self.repos[idx][1], clonedir) - answer = translate("AddonsInstaller", - "Workbench successfully installed. Please restart " - "FreeCAD to apply the changes.") + if have_git: + self.run_git(target_dir) + else: + self.run_zip(target_dir) + + self.stop = True + + def run_git(self, clonedir:str) -> None: + + if os.path.exists(clonedir): + self.run_git_update(clonedir) + else: + self.run_git_clone(clonedir) + + def run_git_update(self, clonedir:str) -> None: + self.status_message.emit("Updating module...") + if str(self.repo.name) in py2only: + FreeCAD.Console.PrintWarning(translate("AddonsInstaller", + "User requested updating a Python 2 workbench on " + "a system running Python 3 - ") + str(self.repo.name) + "\n") + if not os.path.exists(clonedir + os.sep + ".git"): + # Repair addon installed with raw download by adding the .git + # directory to it + bare_repo = git.Repo.clone_from(self.repo.url, clonedir + os.sep + ".git", bare=True) + try: + with bare_repo.config_writer() as cw: + cw.set("core", "bare", False) + except AttributeError: + FreeCAD.Console.PrintWarning(translate("AddonsInstaller", + "Outdated GitPython detected, consider " + "upgrading with pip.") + "\n") + cw = bare_repo.config_writer() + cw.set("core", "bare", False) + del cw + repo = git.Repo(clonedir) + repo.head.reset("--hard") + repo = git.Git(clonedir) + try: + repo.pull() + answer = translate("AddonsInstaller","Workbench successfully updated. " + "Please restart FreeCAD to apply the changes.") + except Exception: + answer = translate("AddonsInstaller", "Error updating module ") + \ + self.repo.name + " - " + \ + translate("AddonsInstaller", "Please fix manually") + answer += repo.status() + self.failure.emit(self.repo, answer) + else: + # Update the submodules for this repository + repo_sms = git.Repo(clonedir) + self.status_message.emit("Updating submodules...") + for submodule in repo_sms.submodules: + submodule.update(init=True, recursive=True) + self.success.emit(self.repo, answer) + + def run_git_clone(self, clonedir:str) -> None: + self.status_message.emit("Checking module dependencies...") + depsok, answer = self.check_python_dependencies(self.repo.url) + if depsok: + if str(self.repo.name) in py2only: + FreeCAD.Console.PrintWarning(translate("AddonsInstaller", + "User requested installing a Python 2 " + "workbench on a system running Python 3 - ") + str(self.repo.name) + "\n") + self.status_message.emit("Cloning module...") + repo = git.Repo.clone_from(self.repo.url, clonedir) + + # Make sure to clone all the submodules as well + if repo.submodules: + repo.submodule_update(recursive=True) + + if self.repo.branch in repo.heads: + repo.heads[self.repo.branch].checkout() + + answer = translate("AddonsInstaller", + "Workbench successfully installed. Please restart " + "FreeCAD to apply the changes.") + else: + self.emit.failure(self.repo, answer) + return + + if self.repo.repo_type == AddonManagerRepo.RepoType.WORKBENCH: # symlink any macro contained in the module to the macros folder macro_dir = FreeCAD.getUserMacroDir(True) if not os.path.exists(macro_dir): @@ -789,21 +907,16 @@ class InstallWorker(QtCore.QThread): if os.path.exists(clonedir): for f in os.listdir(clonedir): if f.lower().endswith(".fcmacro"): - print("copying macro:", f) utils.symlink(os.path.join(clonedir, f), os.path.join(macro_dir, f)) - FreeCAD.ParamGet('User parameter:Plugins/' + - self.repos[idx][0]).SetString("destination", clonedir) + FreeCAD.ParamGet('User parameter:Plugins/' + self.repo.name).SetString("destination", clonedir) answer += "\n\n" + translate("AddonsInstaller", - "A macro has been installed and is available " - "under Macro -> Macros menu") + "A macro has been installed and is available " + "under Macro -> Macros menu") answer += ":\n" + f + "" - self.progressbar_show.emit(False) - self.info_label.emit(answer) - self.mark_recompute.emit(self.repos[idx][0]) - self.stop = True + self.success.emit(self.repo, answer) - def check_dependencies(self, baseurl): - "checks if the repo contains a metadata.txt and check its contents" + def check_python_dependencies(self, baseurl:str) -> [bool,str]: + """checks if the repo contains a metadata.txt and check its contents""" ok = True message = "" @@ -817,7 +930,8 @@ class InstallWorker(QtCore.QThread): depsfile = mu.read() mu.close() - # urllib2 gives us a bytelike object instead of a string. Have to consider that + # urllib2 gives us a bytelike object instead of a string. Have to + # consider that try: depsfile = depsfile.decode("utf-8") except AttributeError: @@ -830,7 +944,7 @@ class InstallWorker(QtCore.QThread): for wb in depswb: if wb.strip(): if not wb.strip() in FreeCADGui.listWorkbenches().keys(): - if not wb.strip()+"Workbench" in FreeCADGui.listWorkbenches().keys(): + if not wb.strip() + "Workbench" in FreeCADGui.listWorkbenches().keys(): ok = False message += translate("AddonsInstaller", "Missing workbench") + ": " + wb + ", " elif line.startswith("pylibs="): @@ -858,40 +972,66 @@ class InstallWorker(QtCore.QThread): message += translate("AddonsInstaller", "Please install the missing components first.") return ok, message - def download(self, baseurl, clonedir): + def check_package_dependencies(self): + # TODO: Use the dependencies set in the package.xml metadata + pass + + def run_zip(self, zipdir:str) -> None: "downloads and unzip a zip version from a git repo" bakdir = None - if os.path.exists(clonedir): - bakdir = clonedir+".bak" + if os.path.exists(zipdir): + bakdir = zipdir + ".bak" if os.path.exists(bakdir): shutil.rmtree(bakdir) - os.rename(clonedir, bakdir) - os.makedirs(clonedir) - zipurl = utils.get_zip_url(baseurl) + os.rename(zipdir, bakdir) + os.makedirs(zipdir) + zipurl = utils.get_zip_url(self.repo) if not zipurl: - return translate("AddonsInstaller", "Error: Unable to locate zip from") + " " + baseurl + self.failure.emit(self.repo, translate("AddonsInstaller", "Error: Unable to locate zip from") + " " + self.repo.name) + return try: - print("Downloading "+zipurl) u = utils.urlopen(zipurl) except Exception: - return translate("AddonsInstaller", "Error: Unable to download") + " " + zipurl + self.failure.emit(self.repo, translate("AddonsInstaller", "Error: Unable to download") + " " + zipurl) + return if not u: - return translate("AddonsInstaller", "Error: Unable to download") + " " + zipurl - zfile = _stringio() - zfile.write(u.read()) - zfile = zipfile.ZipFile(zfile) + self.failure.emit(self.repo, translate("AddonsInstaller", "Error: Unable to download") + " " + zipurl) + return + + data_size = u.headers['content-length'] + + current_thread = QtCore.QThread.currentThread() + if data_size and data_size > 5 * 1024 * 1024: + # Use an on-disk file and track download progress, if the zipfile + # is over 5mb + with tempfile.NamedTemporaryFile(delete=False) as temp: + bytes_to_read = 16 * 1024 + bytes_read = 0 + while True: + if current_thread.isInterruptionRequested(): + return + chunk = u.read(bytes_to_read) + if not chunk: + break + bytes_read += bytes_to_read + temp.write(chunk) + self.progress_made.emit(bytes_read, data_size) + zfile = zipfile.ZipFile(temp) + else: + zfile = io.BytesIO() + zfile.write(u.read()) + zfile = zipfile.ZipFile(zfile) master = zfile.namelist()[0] # github will put everything in a subfolder - zfile.extractall(clonedir) + zfile.extractall(zipdir) u.close() zfile.close() - for filename in os.listdir(clonedir+os.sep+master): - shutil.move(clonedir+os.sep+master+os.sep+filename, clonedir+os.sep+filename) - os.rmdir(clonedir+os.sep+master) + for filename in os.listdir(zipdir + os.sep + master): + shutil.move(zipdir + os.sep + master + os.sep + filename, zipdir + os.sep + filename) + os.rmdir(zipdir + os.sep + master) if bakdir: shutil.rmtree(bakdir) - return translate("AddonsInstaller", "Successfully installed") + " " + zipurl - + self.success.emit(self.repo, translate("AddonsInstaller", "Successfully installed") + " " + zipurl) class CheckSingleWorker(QtCore.QThread): """Worker to check for updates for a single addon""" @@ -907,7 +1047,7 @@ class CheckSingleWorker(QtCore.QThread): if not have_git: return - FreeCAD.Console.PrintLog("Checking for available updates of the "+self.name+" addon\n") + FreeCAD.Console.PrintLog("Checking for available updates of the " + self.name + " addon\n") addondir = os.path.join(FreeCAD.getUserAppDataDir(), "Mod", self.name) if os.path.exists(addondir): if os.path.exists(addondir + os.sep + ".git"): @@ -918,8 +1058,233 @@ class CheckSingleWorker(QtCore.QThread): self.updateAvailable.emit(True) return except Exception: - # can fail for any number of reasons, ex. not being online + # can fail for any number of reasons, ex. not being online pass self.updateAvailable.emit(False) -# @} +class UpdateMetadataCacheWorker(QtCore.QThread): + "Scan through all available packages and see if our local copy of package.xml needs to be updated" + + status_message = QtCore.Signal(str) + progress_made = QtCore.Signal(int, int) + done = QtCore.Signal() + package_updated = QtCore.Signal(AddonManagerRepo) + + class AtomicCounter(object): + def __init__(self,start=0): + self.lock = threading.Lock() + self.count = start + def set(self,new_value): + with self.lock: + self.count = new_value + def get(self): + with self.lock: + return self.count + def increment(self): + with self.lock: + self.count += 1 + def decrement(self): + with self.lock: + self.count -= 1 + + def __init__(self, repos): + + QtCore.QThread.__init__(self) + self.repos = repos + self.counter = UpdateMetadataCacheWorker.AtomicCounter() + + def run(self): + if not have_git: + return + current_thread = QtCore.QThread.currentThread() + self.num_downloads_required = len(self.repos) + self.progress_made.emit(0, self.num_downloads_required) + self.status_message.emit(translate("AddonsInstaller", "Retrieving package metadata...")) + store = os.path.join(FreeCAD.getUserAppDataDir(), "AddonManager", "PackageMetadata") + index_file = os.path.join(store,"index.json") + self.index = {} + if os.path.isfile(index_file): + with open(index_file, "r") as f: + index_string = f.read() + self.index = json.loads(index_string) + + download_queue = QtNetwork.QNetworkAccessManager() # Must be created on this thread + download_queue.finished.connect(self.on_finished) + + self.downloaders = [] + for repo in self.repos: + downloader = MetadataDownloadWorker(None, repo, self.index) + downloader.start_fetch(download_queue) + downloader.updated.connect(self.on_updated) + self.downloaders.append(downloader) + + # Run a local event loop until we've processed all of the downloads: + # this is local + # to this thread, and does not affect the main event loop + ui_updater = QtCore.QTimer() + ui_updater.timeout.connect(self.send_ui_update) + ui_updater.start(100) # Send an update back to the main thread every 100ms + self.num_downloads_required = len(self.repos) + self.num_downloads_completed = UpdateMetadataCacheWorker.AtomicCounter() + while True: + if current_thread.isInterruptionRequested(): + for downloader in self.downloaders: + downloader.abort() + QtCore.QCoreApplication.processEvents() + if self.num_downloads_completed.get() == self.num_downloads_required: + break + + if current_thread.isInterruptionRequested(): + FreeCAD.Console.PrintMessage("Bailing out of downloads") + return + + # Update and serialize the updated index, overwriting whatever was + # there before + for downloader in self.downloaders: + self.index[downloader.repo.name] = downloader.last_sha1 + if not os.path.exists(store): + os.makedirs(store) + with open(index_file, "w") as f: + json.dump(self.index, f, indent=" ") + + # Signal completion to our parent thread + self.done.emit() + self.stop = True + + def on_finished(self, reply): + # Called by the QNetworkAccessManager's sub-threads when a fetch + # process completed (in any state) + self.num_downloads_completed.increment() + + def on_updated(self, repo): + # Called if this repo got new metadata and/or a new icon + self.package_updated.emit(repo) + + def send_ui_update(self): + self.progress_made.emit(self.num_downloads_completed.get(), self.num_downloads_required) + + +class GitProgressMonitor(git.RemoteProgress): + """ An object that receives git progress updates and transforms them into Qt signals """ + + progress_made = QtCore.Signal(int, int) + info_message = QtCore.Signal(str) + + def __init__(self): + super().__init__() + + def update(self, op_code: int, cur_count: Union[str, float], max_count: Union[str, float, None]=None, message: str='') -> None: + if max_count: + self.progress_made.emit(int(cur_count), int(max_count)) + if message: + self.info_message.emit(message) + + +class UpdateAllWorker(QtCore.QThread): + """ Update all listed packages, of any kind """ + + progress_made = QtCore.Signal(int, int) + status_message = QtCore.Signal(str) + success = QtCore.Signal(AddonManagerRepo) + failure = QtCore.Signal(AddonManagerRepo) + done = QtCore.Signal() + + def __init__(self, repos): + super().__init__() + self.repos = repos + + def run (self): + self.progress_made.emit(0,len(self.repos)) + self.repo_queue = queue.Queue() + current_thread = QtCore.QThread.currentThread() + for repo in self.repos: + self.repo_queue.put(repo) + + # Following the QNetworkAccessManager model, we'll spawn six threads to process these requests in parallel: + workers = [] + for _ in range(6): + worker = UpdateSingleWorker(self.repo_queue) + worker.success.connect(self.on_success) + worker.failure.connect(self.on_failure) + worker.start() + workers.append(worker) + + while not self.repo_queue.empty(): + if current_thread.isInterruptionRequested(): + for worker in workers: + worker.requestInterruption() + worker.wait() + return + # Ensure our signals propagate out by running an internal thread-local event loop + QtCore.QCoreApplication.processEvents() + + self.repo_queue.join() + + # Make sure all of our child threads have fully exited: + for i, worker in enumerate(workers): + worker.wait() + + self.done.emit() + + def on_success(self, repo:AddonManagerRepo) -> None: + self.progress_made.emit(self.repo_queue.qsize(), len(self.repos)) + self.success.emit(repo) + + def on_failure(self, repo:AddonManagerRepo) -> None: + self.progress_made.emit(self.repo_queue.qsize(), len(self.repos)) + self.failure.emit(repo) + +class UpdateSingleWorker(QtCore.QThread): + success = QtCore.Signal(AddonManagerRepo) + failure = QtCore.Signal(AddonManagerRepo) + + def __init__(self,repo_queue:queue.Queue): + super().__init__() + self.repo_queue = repo_queue + + def run(self): + current_thread = QtCore.QThread.currentThread() + while True: + if current_thread.isInterruptionRequested(): + return + try: + repo = self.repo_queue.get_nowait() + except queue.Empty: + return + if repo.repo_type == AddonManagerRepo.RepoType.MACRO: + self.update_macro(repo) + else: + self.update_package(repo) + self.repo_queue.task_done() + + + def update_macro(self, repo:AddonManagerRepo): + """ Updating a macro happens in this function, in the current thread """ + + with tempfile.TemporaryDirectory() as dir: + temp_install_succeeded = macro.install(dir) + if not temp_install_succeeded: + failed = True + + if not failed: + failed = macro.install(self.macro_repo_dir) + + if not failed: + self.success.emit(repo) + else: + self.failure.emit(repo) + + def update_package(self, repo:AddonManagerRepo): + """ Updating a package re-uses the package installation worker, so actually spawns another thread that we block on """ + + worker = InstallWorkbenchWorker(repo) + worker.success.connect(lambda repo,_:self.success.emit(repo)) + worker.failure.connect(lambda repo,_:self.failure.emit(repo)) + worker.start() + while True: + # Ensure our signals propagate out by running an internal thread-local event loop + QtCore.QCoreApplication.processEvents() + if not worker.isRunning(): + break + +# @} \ No newline at end of file diff --git a/src/Mod/Start/StartPage/LoadNew.py b/src/Mod/Start/StartPage/LoadNew.py index e4434d81e2..bf0040ceba 100644 --- a/src/Mod/Start/StartPage/LoadNew.py +++ b/src/Mod/Start/StartPage/LoadNew.py @@ -27,4 +27,4 @@ FreeCAD.newDocument() FreeCADGui.activeDocument().activeView().viewDefaultOrientation() from StartPage import StartPage -StartPage.postStart() +StartPage.postStart() \ No newline at end of file