diff --git a/src/Mod/AddonManager/AddonManager.py b/src/Mod/AddonManager/AddonManager.py index 083bec0aa3..e4c72c835e 100644 --- a/src/Mod/AddonManager/AddonManager.py +++ b/src/Mod/AddonManager/AddonManager.py @@ -4,6 +4,7 @@ #*************************************************************************** #* * #* Copyright (c) 2015 Yorik van Havre * +#* Copyright (c) 2021 Chris Hennes * #* * #* This program is free software; you can redistribute it and/or modify * #* it under the terms of the GNU Lesser General Public License (LGPL) * @@ -23,53 +24,62 @@ #* * #*************************************************************************** -from __future__ import print_function - import os import shutil import stat import tempfile +from typing import Dict, Union -from PySide import QtGui, QtCore +from PySide2 import QtGui, QtCore, QtWidgets import FreeCADGui from addonmanager_utilities import translate # this needs to be as is for pylupdate from addonmanager_workers import * import addonmanager_utilities as utils import AddonManager_rc +from AddonManagerRepo import AddonManagerRepo __title__ = "FreeCAD Addon Manager Module" -__author__ = "Yorik van Havre", "Jonathan Wiedemann", "Kurt Kremitzki" -__url__ = "http://www.freecadweb.org" +__author__ = "Yorik van Havre", "Jonathan Wiedemann", "Kurt Kremitzki", "Chris Hennes" +__url__ = "http://www.freecad.org" """ FreeCAD Addon Manager Module -It will fetch its contents from https://github.com/FreeCAD/FreeCAD-addons +Fetches various types of addons from a variety of sources. Built-in sources are: +* https://github.com/FreeCAD/FreeCAD-addons +* https://github.com/FreeCAD/FreeCAD-macros +* https://wiki.freecad.org/ + +Additional git sources may be configure via user preferences. + You need a working internet connection, and optionally the GitPython package installed. """ # \defgroup ADDONMANAGER AddonManager # \ingroup ADDONMANAGER -# \brief The Addon Manager allows to install workbenches and macros made by users +# \brief The Addon Manager allows users to install workbenches and macros made by other users # @{ def QT_TRANSLATE_NOOP(ctx, txt): return txt - class CommandAddonManager: """The main Addon Manager class and FreeCAD command""" - def GetResources(self): + workers = ["update_worker", "check_worker", "show_worker", + "showmacro_worker", "macro_worker", "install_worker", + "update_metadata_cache_worker", "update_all_worker"] + + def GetResources(self) -> Dict[str,str]: return {"Pixmap": "AddonManager", "MenuText": QT_TRANSLATE_NOOP("Std_AddonMgr", "&Addon manager"), - "ToolTip": QT_TRANSLATE_NOOP("Std_AddonMgr", "Manage external workbenches and macros"), + "ToolTip": QT_TRANSLATE_NOOP("Std_AddonMgr", "Manage external workbenches, macros, and preference packs"), "Group": "Tools"} - def Activated(self): + def Activated(self) -> None: # display first use dialog if needed readWarningParameter = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") @@ -77,23 +87,23 @@ class CommandAddonManager: newReadWarningParameter = FreeCAD.ParamGet("User parameter:Plugins/addonsRepository") readWarning |= newReadWarningParameter.GetBool("readWarning", False) if not readWarning: - if (QtGui.QMessageBox.warning(None, + if (QtWidgets.QMessageBox.warning(None, "FreeCAD", translate("AddonsInstaller", "The addons that can be installed here are not " "officially part of FreeCAD, and are not reviewed " "by the FreeCAD team. Make sure you know what you " "are installing!"), - QtGui.QMessageBox.Cancel | - QtGui.QMessageBox.Ok) != - QtGui.QMessageBox.StandardButton.Cancel): + QtWidgets.QMessageBox.Cancel | + QtWidgets.QMessageBox.Ok) != + QtWidgets.QMessageBox.StandardButton.Cancel): readWarningParameter.SetBool("readWarning", True) readWarning = True if readWarning: self.launch() - def launch(self): + def launch(self) -> None: """Shows the Addon Manager UI""" # create the dialog @@ -101,22 +111,10 @@ class CommandAddonManager: "AddonManager.ui")) # cleanup the leftovers from previous runs - self.repos = [] - self.macros = [] self.macro_repo_dir = tempfile.mkdtemp() - self.doUpdate = [] + self.packages_with_updates = [] self.addon_removed = False - for worker in ["update_worker", "check_worker", "show_worker", - "showmacro_worker", "macro_worker", "install_worker"]: - if hasattr(self, worker): - thread = getattr(self, worker) - if thread: - if thread.isFinished(): - setattr(self, worker, None) - self.dialog.tabWidget.setCurrentIndex(0) - # these 2 settings to prevent loading an addon description on start (let the user click something first) - self.firsttime = True - self.firstmacro = True + self.cleanup_workers() # restore window geometry and splitter state from stored state pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") @@ -127,36 +125,62 @@ class CommandAddonManager: sr = pref.GetInt("SplitterRight", 274) self.dialog.splitter.setSizes([sl, sr]) + # Set up the listing of packages using the model-view-controller architecture + self.item_model = PackageListItemModel() + self.item_delegate = PackageListIconDelegate() + self.item_filter = PackageListFilter() + self.item_filter.setSourceModel(self.item_model) + self.dialog.tablePackages.setModel (self.item_filter) + self.dialog.tablePackages.setItemDelegate(self.item_delegate) + header = self.dialog.tablePackages.horizontalHeader() + header.setSectionResizeMode(0, QtWidgets.QHeaderView.Fixed) + header.resizeSection(0,20) + header.setSectionResizeMode(1, QtWidgets.QHeaderView.Stretch) + header.setSectionResizeMode(2, QtWidgets.QHeaderView.ResizeToContents) + self.dialog.tablePackages.sortByColumn(1, QtCore.Qt.AscendingOrder) # Default to sorting alphabetically by name + # set nice icons to everything, by theme with fallback to FreeCAD icons self.dialog.setWindowIcon(QtGui.QIcon(":/icons/AddonManager.svg")) - self.dialog.buttonExecute.setIcon(QtGui.QIcon.fromTheme("execute", QtGui.QIcon(":/icons/button_valid.svg"))) self.dialog.buttonUninstall.setIcon(QtGui.QIcon.fromTheme("cancel", QtGui.QIcon(":/icons/edit_Cancel.svg"))) self.dialog.buttonInstall.setIcon(QtGui.QIcon.fromTheme("download", QtGui.QIcon(":/icons/edit_OK.svg"))) self.dialog.buttonUpdateAll.setIcon(QtGui.QIcon(":/icons/button_valid.svg")) self.dialog.buttonConfigure.setIcon(QtGui.QIcon(":/icons/preferences-system.svg")) self.dialog.buttonClose.setIcon(QtGui.QIcon.fromTheme("close", QtGui.QIcon(":/icons/process-stop.svg"))) - self.dialog.tabWidget.setTabIcon(0, QtGui.QIcon.fromTheme("folder", QtGui.QIcon(":/icons/folder.svg"))) - self.dialog.tabWidget.setTabIcon(1, QtGui.QIcon(":/icons/applications-python.svg")) # enable/disable stuff - self.dialog.buttonExecute.setEnabled(False) self.dialog.buttonUninstall.setEnabled(False) self.dialog.buttonInstall.setEnabled(False) self.dialog.buttonUpdateAll.setEnabled(False) + self.dialog.buttonExecute.hide() + self.dialog.labelFilterValidity.hide() + + # Hide package-related interface elements until needed + self.dialog.labelPackageName.hide() + self.dialog.labelVersion.hide() + self.dialog.labelMaintainer.hide() + self.dialog.labelIcon.hide() + self.dialog.labelDescription.hide() + self.dialog.labelUrl.hide() + self.dialog.labelUrlType.hide() + self.dialog.labelContents.hide() # connect slots - self.dialog.buttonExecute.clicked.connect(self.executemacro) self.dialog.rejected.connect(self.reject) self.dialog.buttonInstall.clicked.connect(self.install) self.dialog.buttonUninstall.clicked.connect(self.remove) - self.dialog.buttonUpdateAll.clicked.connect(self.apply_updates) - self.dialog.listWorkbenches.currentRowChanged.connect(self.show) - self.dialog.tabWidget.currentChanged.connect(self.switchtab) - self.dialog.listMacros.currentRowChanged.connect(self.show_macro) + self.dialog.buttonUpdateAll.clicked.connect(self.update_all) + self.dialog.comboPackageType.currentIndexChanged.connect(self.update_type_filter) + self.dialog.lineEditFilter.textChanged.connect(self.update_text_filter) self.dialog.buttonConfigure.clicked.connect(self.show_config) self.dialog.buttonClose.clicked.connect(self.dialog.reject) + self.dialog.buttonExecute.clicked.connect(self.executemacro) + self.dialog.tablePackages.selectionModel().currentRowChanged.connect(self.table_row_selected) + self.dialog.tablePackages.setEnabled(False) - # allow to open links in browser + # Show "Workbenches" to start with + self.dialog.comboPackageType.setCurrentIndex(1) + + # allow links to open in browser self.dialog.description.setOpenLinks(True) self.dialog.description.setOpenExternalLinks(True) @@ -164,13 +188,42 @@ class CommandAddonManager: mw = FreeCADGui.getMainWindow() self.dialog.move(mw.frameGeometry().topLeft() + mw.rect().center() - self.dialog.rect().center()) - # populate the list - self.update() + # set info for the progress bar: + self.number_of_progress_regions = 4 + self.current_progress_region = 0 + self.dialog.progressBar.setMaximum (100) + + # populate the table + self.populate_packages_table() + + # set the label text to start with + self.show_information(translate("AddonInstaller", "Loading addon information")) # rock 'n roll!!! self.dialog.exec_() - def reject(self): + def cleanup_workers(self) -> None: + """ Ensure that no workers are running by explicitly asking them to stop, and terminating them if they don't """ + for worker in self.workers: + if hasattr(self, worker): + thread = getattr(self, worker) + if thread: + if not thread.isFinished(): + thread.requestInterruption() + thread.wait(QtCore.QDeadlineTimer(250)) + if not thread.isFinished(): + thread.terminate() # Highly undesirable, hopefully the thread obeyed the request to interrupt + thread.wait() + + def wait_on_other_workers(self) -> None: + for worker in self.workers: + if hasattr(self, worker): + thread = getattr(self, worker) + if thread: + if not thread.isFinished(): + thread.wait() + + def reject(self) -> None: """called when the window has been closed""" # save window geometry and splitter state for next use @@ -182,20 +235,30 @@ class CommandAddonManager: # ensure all threads are finished before closing oktoclose = True - for worker in ["update_worker", "check_worker", "show_worker", "showmacro_worker", - "macro_worker", "install_worker"]: + for worker in self.workers: if hasattr(self, worker): thread = getattr(self, worker) if thread: if not thread.isFinished(): + thread.requestInterruption() oktoclose = False + if not oktoclose: + oktoclose = True + for worker in self.workers: + if hasattr(self, worker): + thread = getattr(self, worker) + if thread: + thread.wait(QtCore.QDeadlineTimer(50)) # 50ms to wrap up whatever loop iteration it was on + if not thread.isFinished(): + thread.terminate() # Highly undesirable, hopefully the thread obeyed the request to interrupt + thread.wait() # all threads have finished if oktoclose: if ((hasattr(self, "install_worker") and self.install_worker) or (hasattr(self, "addon_removed") and self.addon_removed)): # display restart dialog - m = QtGui.QMessageBox() + m = QtWidgets.QMessageBox() m.setWindowTitle(translate("AddonsInstaller", "Addon manager")) m.setWindowIcon(QtGui.QIcon(":/icons/AddonManager.svg")) m.setText(translate("AddonsInstaller", @@ -204,8 +267,8 @@ class CommandAddonManager: m.setIcon(m.Warning) m.setStandardButtons(m.Ok | m.Cancel) m.setDefaultButton(m.Cancel) - okBtn = m.button(QtGui.QMessageBox.StandardButton.Ok) - cancelBtn = m.button(QtGui.QMessageBox.StandardButton.Cancel) + okBtn = m.button(QtWidgets.QMessageBox.StandardButton.Ok) + cancelBtn = m.button(QtWidgets.QMessageBox.StandardButton.Cancel) okBtn.setText(translate("AddonsInstaller","Restart now")) cancelBtn.setText(translate("AddonsInstaller","Restart later")) ret = m.exec_() @@ -217,381 +280,512 @@ class CommandAddonManager: shutil.rmtree(self.macro_repo_dir, onerror=self.remove_readonly) except Exception: pass + else: + FreeCAD.Console.PrintWarning("Could not terminate sub-threads in Addon Manager.\n") + self.cleanup_workers() - return True + def populate_packages_table(self) -> None: + """ Downloads the available packages listings and populates the table - def update(self): - """updates the list of workbenches""" + This proceeds in four stages: first, the main GitHub repository is queried for a list of possible + addons. Each addon is specified as a git submodule with name and branch information. The actual specific + commit ID of the submodule (as listed on Github) is ignored. Any extra repositories specified by the + user are appended to this list. - self.dialog.listWorkbenches.clear() - self.dialog.buttonExecute.setEnabled(False) - self.repos = [] + Second, the list of macros is downloaded from the FreeCAD/FreeCAD-macros repository and the wiki + + Third, each of these items is queried for a package.xml metadata file. If that file exists it is + downloaded, cached, and any icons that it references are also downloaded and cached. + + Finally, for workbenches that are not contained within a package (e.g. they provide no metadata), an + additional git query is made to see if an update is available. Macros are checked for file changes. + + Each of these stages is launched in a separate thread to ensure that the UI remains responsive, and + the operation can be cancelled. + + """ + + self.item_model.clear() + self.current_progress_region += 1 self.update_worker = UpdateWorker() - self.update_worker.info_label.connect(self.show_information) + self.update_worker.status_message.connect(self.show_information) self.update_worker.addon_repo.connect(self.add_addon_repo) - self.update_worker.progressbar_show.connect(self.show_progress_bar) - self.update_worker.done.connect(self.check_updates) + self.update_progress_bar(10,100) + self.update_worker.done.connect(self.populate_macros) # Link to step 2 self.update_worker.start() - def check_updates(self): - "checks every installed addon for available updates" - + def populate_macros(self) -> None: + self.current_progress_region += 1 + self.macro_worker = FillMacroListWorker(self.macro_repo_dir) + self.macro_worker.status_message_signal.connect(self.show_information) + self.macro_worker.progress_made.connect(self.update_progress_bar) + self.macro_worker.add_macro_signal.connect(self.add_addon_repo) + self.macro_worker.done.connect(self.update_metadata_cache) # Link to step 3 + self.macro_worker.done.connect(lambda : self.dialog.tablePackages.setEnabled(True)) + self.macro_worker.start() + + def update_metadata_cache(self) -> None: + self.current_progress_region += 1 pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") - if pref.GetBool("AutoCheck", False) and not self.doUpdate: + if pref.GetBool("AutoFetchMetadata", True): + self.update_metadata_cache_worker = UpdateMetadataCacheWorker(self.item_model.repos) + self.update_metadata_cache_worker.status_message.connect(self.show_information) + self.update_metadata_cache_worker.done.connect(self.check_updates) # Link to step 4 + self.update_metadata_cache_worker.progress_made.connect(self.update_progress_bar) + self.update_metadata_cache_worker.package_updated.connect(self.on_package_updated) + self.update_metadata_cache_worker.start() + else: + self.check_updates() + + def on_package_updated(self, repo:AddonManagerRepo) -> None: + """Called when the named package has either new metadata or a new icon (or both)""" + + cache_path = os.path.join(FreeCAD.getUserAppDataDir(), "AddonManager", "PackageMetadata", repo.name) + icon_filename = repo.metadata.Icon + icon_path = os.path.join(cache_path, icon_filename) + if os.path.isfile(icon_path): + addonicon = QtGui.QIcon(icon_path) + repo.icon = addonicon + self.item_model.reload_item(repo) + + + def check_updates(self) -> None: + "checks every installed addon for available updates" + + self.current_progress_region += 1 + pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") + autocheck = pref.GetBool("AutoCheck", False) + if not autocheck: + return + if not self.packages_with_updates: if hasattr(self, "check_worker"): thread = self.check_worker if thread: if not thread.isFinished(): return self.dialog.buttonUpdateAll.setText(translate("AddonsInstaller", "Checking for updates...")) - self.check_worker = CheckWBWorker(self.repos) - self.check_worker.mark.connect(self.mark) - self.check_worker.enable.connect(self.enable_updates) - self.check_worker.addon_repos.connect(self.update_repos) + self.check_worker = CheckWorkbenchesForUpdatesWorker(self.item_model.repos) + self.check_worker.done.connect(self.hide_progress_widgets) + self.check_worker.progress_made.connect(self.update_progress_bar) + self.check_worker.update_status.connect(self.status_updated) self.check_worker.start() + self.enable_updates(len(self.packages_with_updates)) - def apply_updates(self): - """apply all available updates""" + def status_updated(self, repo:str, status:AddonManagerRepo.UpdateStatus) -> None: + self.item_model.update_item_status(repo.name, status) + if status == AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE: + self.packages_with_updates.append(repo) + self.enable_updates(len(self.packages_with_updates)) - if self.doUpdate: - self.install(self.doUpdate) - self.dialog.buttonUpdateAll.setEnabled(False) - - def enable_updates(self, num): + def enable_updates(self, number_of_updates:int) -> None: """enables the update button""" - if num: + if number_of_updates: self.dialog.buttonUpdateAll.setText(translate("AddonsInstaller", "Apply") + - " " + str(num) + " " + + " " + str(number_of_updates) + " " + translate("AddonsInstaller", "update(s)")) self.dialog.buttonUpdateAll.setEnabled(True) else: - self.dialog.buttonUpdateAll.setText(translate("AddonsInstaller", "No update available")) + self.dialog.buttonUpdateAll.setText(translate("AddonsInstaller", "No updates available")) self.dialog.buttonUpdateAll.setEnabled(False) - def add_addon_repo(self, addon_repo): + def add_addon_repo(self, addon_repo:AddonManagerRepo) -> None: """adds a workbench to the list""" + + if addon_repo.icon is None or addon_repo.icon.isNull(): + addon_repo.icon = self.get_icon(addon_repo) + for repo in self.item_model.repos: + if repo.name == addon_repo.name: + FreeCAD.Console.PrintLog(f"Possible duplicate addon: ignoring second addition of {addon_repo.name}\n") + return + self.item_model.append_item(addon_repo) - self.repos.append(addon_repo) - addonicon = self.get_icon(addon_repo[0]) - if addon_repo[2] > 0: - item = QtGui.QListWidgetItem(addonicon, - str(addon_repo[0]) + - str(" (" + - translate("AddonsInstaller", "Installed") + - ")")) - item.setForeground(QtGui.QBrush(QtGui.QColor(0, 182, 41))) - self.dialog.listWorkbenches.addItem(item) - else: - self.dialog.listWorkbenches.addItem(QtGui.QListWidgetItem(addonicon, str(addon_repo[0]))) - - def get_icon(self, repo): + def get_icon(self, repo:AddonManagerRepo, update:bool=False) -> QtGui.QIcon: """returns an icon for a repo""" - path = ":/icons/" + repo + "_workbench_icon.svg" + if not update and repo.icon and not repo.icon.isNull(): + return repo.icon + + path = ":/icons/" + repo.name.replace(" ", "_") + if repo.repo_type == AddonManagerRepo.RepoType.WORKBENCH: + path += "_workbench_icon.svg" + default_icon = QtGui.QIcon(":/icons/document-package.svg") + elif repo.repo_type == AddonManagerRepo.RepoType.MACRO: + path += "_macro_icon.svg" + default_icon = QtGui.QIcon(":/icons/document-python.svg") + elif repo.repo_type == AddonManagerRepo.RepoType.PACKAGE: + # The cache might not have been downloaded yet, check to see if it's there... + if repo.cached_icon_filename and os.path.isfile(repo.cached_icon_filename): + path = repo.cached_icon_filename + elif repo.contains_workbench(): + path += "_workbench_icon.svg" + default_icon = QtGui.QIcon(":/icons/document-package.svg") + elif repo.contains_macro(): + path += "_macro_icon.svg" + default_icon = QtGui.QIcon(":/icons/document-python.svg") + else: + default_icon = QtGui.QIcon(":/icons/document-package.svg") + if QtCore.QFile.exists(path): addonicon = QtGui.QIcon(path) else: - addonicon = QtGui.QIcon(":/icons/document-package.svg") - if addonicon.isNull(): - addonicon = QtGui.QIcon(":/icons/document-package.svg") + addonicon = default_icon + repo.icon = addonicon + return addonicon - def show_information(self, label): - """shows text in the information pane""" + def table_row_selected(self, current:QtCore.QModelIndex, previous:QtCore.QModelIndex) -> None: + """a new row was selected, show the relevant data""" + + if not current.isValid(): + self.selected_repo = None + return - self.dialog.description.setText(label) - if self.dialog.listWorkbenches.isVisible(): - self.dialog.listWorkbenches.setFocus() + source_selection = self.item_filter.mapToSource (current) + self.selected_repo = self.item_model.repos[source_selection.row()] + if self.selected_repo.repo_type == AddonManagerRepo.RepoType.MACRO: + self.show_macro(self.selected_repo) + self.dialog.buttonExecute.show() + elif self.selected_repo.repo_type == AddonManagerRepo.RepoType.WORKBENCH: + self.show_workbench(self.selected_repo) + self.dialog.buttonExecute.hide() + elif self.selected_repo.repo_type == AddonManagerRepo.RepoType.PACKAGE: + self.show_package(self.selected_repo) + self.dialog.buttonExecute.hide() + if self.selected_repo.update_status == AddonManagerRepo.UpdateStatus.NOT_INSTALLED: + self.dialog.buttonInstall.setEnabled(True) + self.dialog.buttonUninstall.setEnabled(False) + self.dialog.buttonInstall.setText(translate("AddonsInstaller", "Install selected")) + elif self.selected_repo.update_status == AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE: + self.dialog.buttonInstall.setEnabled(False) + self.dialog.buttonUninstall.setEnabled(True) + self.dialog.buttonInstall.setText(translate("AddonsInstaller", "Already updated")) + elif self.selected_repo.update_status == AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE: + self.dialog.buttonInstall.setEnabled(True) + self.dialog.buttonUninstall.setEnabled(True) + self.dialog.buttonInstall.setText(translate("AddonsInstaller", "Update selected")) + elif self.selected_repo.update_status == AddonManagerRepo.UpdateStatus.UNCHECKED: + self.dialog.buttonInstall.setEnabled(False) + self.dialog.buttonUninstall.setEnabled(True) + self.dialog.buttonInstall.setText(translate("AddonsInstaller", "Checking status...")) + + def show_package_widgets(self, show:bool) -> None: + """ Show or hide the widgets related to packages with a package.xml metadata file """ + + # Always rebuild the urlGrid, remove all previous items + for i in reversed(range(self.dialog.urlGrid.rowCount())): + if self.dialog.urlGrid.itemAtPosition(i,0): + self.dialog.urlGrid.itemAtPosition(i,0).widget().setParent(None) + if self.dialog.urlGrid.itemAtPosition(i,1): + self.dialog.urlGrid.itemAtPosition(i,1).widget().setParent(None) + + if show: + # Show all the package-related widgets: + self.dialog.labelPackageName.show() + self.dialog.labelVersion.show() + self.dialog.labelMaintainer.show() + self.dialog.labelIcon.show() + self.dialog.labelDescription.show() + self.dialog.labelContents.show() else: - self.dialog.listMacros.setFocus() + # Hide all the package-related widgets: + self.dialog.labelPackageName.hide() + self.dialog.labelVersion.hide() + self.dialog.labelMaintainer.hide() + self.dialog.labelIcon.hide() + self.dialog.labelDescription.hide() + self.dialog.labelContents.hide() - def show(self, idx): + def show_information(self, message:str) -> None: + """shows generic text in the information pane""" + + self.dialog.labelStatusInfo.show() + self.dialog.labelStatusInfo.setText(message) + + def show_workbench(self, repo:AddonManagerRepo) -> None: """loads information of a given workbench""" - # this function is triggered also when the list is populated, prevent that here - if idx == 0 and self.firsttime: - self.dialog.listWorkbenches.setCurrentRow(-1) - self.firsttime = False - return + self.cleanup_workers() + self.show_package_widgets(False) + self.show_worker = ShowWorker(repo) + self.show_worker.status_message.connect(self.show_information) + self.show_worker.description_updated.connect(lambda desc: self.dialog.description.setText(desc)) + self.show_worker.addon_repos.connect(self.append_to_repos_list) + self.show_worker.done.connect(lambda : self.dialog.labelStatusInfo.hide()) + self.show_worker.start() - if self.repos and idx >= 0: - if hasattr(self, "show_worker"): - # kill existing show worker (might still be busy loading images...) - if self.show_worker: - self.show_worker.exit() - self.show_worker.stopImageLoading() - # wait until the thread stops - while True: - if self.show_worker.isFinished(): - break - self.show_worker = ShowWorker(self.repos, idx) - self.show_worker.info_label.connect(self.show_information) - self.show_worker.addon_repos.connect(self.update_repos) - self.show_worker.progressbar_show.connect(self.show_progress_bar) - self.show_worker.start() - self.dialog.buttonInstall.setEnabled(True) - self.dialog.buttonUninstall.setEnabled(True) + def show_package(self, repo:AddonManagerRepo) -> None: + """ Show the details for a package (a repo with a package.xml metadata file) """ - def show_macro(self, idx): + self.cleanup_workers() + self.show_package_widgets(True) + + # Name + self.dialog.labelPackageName.setText(f"

{repo.metadata.Name}

") + + # Description + self.dialog.labelDescription.setText(repo.metadata.Description) + + # Version + self.dialog.labelVersion.setText(f"

v{repo.metadata.Version}

") + + # Maintainers and authors + maintainers = "" + for maintainer in repo.metadata.Maintainer: + maintainers += translate("AddonsInstaller","Maintainer") + f": {maintainer['name']} <{maintainer['email']}>\n" + if len(repo.metadata.Author) > 0: + for author in repo.metadata.Author: + maintainers += translate("AddonsInstaller","Author") + f": {author['name']} <{author['email']}>\n" + self.dialog.labelMaintainer.setText(maintainers) + + # Main package icon + if not repo.icon or repo.icon.isNull(): + icon = self.get_icon(repo, update=True) + self.item_model.update_item_icon(repo.name, icon) + self.dialog.labelIcon.setPixmap(repo.icon.pixmap(QtCore.QSize(64,64))) + + # Urls + urls = repo.metadata.Urls + ui = FreeCADGui.UiLoader() + for row, url in enumerate(urls): + location = url["location"] + url_type = url["type"] + url_type_string = translate("AddonsInstaller","Other URL") + if url_type == "website": + url_type_string = translate("AddonsInstaller", "Website") + elif url_type == "repository": + url_type_string = translate("AddonsInstaller", "Repository") + elif url_type == "bugtracker": + url_type_string = translate("AddonsInstaller", "Bug tracker") + elif url_type == "readme": + url_type_string = translate("AddonsInstaller", "Readme") + elif url_type == "documentation": + url_type_string = translate("AddonsInstaller", "Documentation") + self.dialog.urlGrid.addWidget(QtWidgets.QLabel(url_type_string), row, 0) + ui=FreeCADGui.UiLoader() + url_label=ui.createWidget("Gui::UrlLabel") + url_label.setText(location) + url_label.setUrl(location) + self.dialog.urlGrid.addWidget(url_label, row, 1) + + # Package contents: + content_string = "" + for name,item_list in repo.metadata.Content.items(): + if name == "preferencepack": + content_type = translate("AddonsInstaller","Preference Packs") + elif name == "workbench": + content_type = translate("AddonsInstaller","Workbenches") + elif name == "macro": + content_type = translate("AddonsInstaller","Macros") + else: + content_type = translate("AddonsInstaller","Other content") + ": " + name + content_string += f"

{content_type}

" + content_string += "" + self.dialog.description.setText(content_string) + + def show_macro(self, repo:AddonManagerRepo) -> None: """loads information of a given macro""" - # this function is triggered when the list is populated, prevent that here - if idx == 0 and self.firstmacro: - self.dialog.listMacros.setCurrentRow(-1) - self.firstmacro = False - return + self.cleanup_workers() + self.show_package_widgets(False) + self.showmacro_worker = GetMacroDetailsWorker(repo) + self.showmacro_worker.status_message.connect(self.show_information) + self.showmacro_worker.description_updated.connect(lambda desc: self.dialog.description.setText(desc)) + self.showmacro_worker.done.connect(lambda : self.dialog.labelStatusInfo.hide()) + self.showmacro_worker.start() - if self.macros and idx >= 0: - if hasattr(self, "showmacro_worker"): - if self.showmacro_worker: - if not self.showmacro_worker.isFinished(): - self.showmacro_worker.exit() - if not self.showmacro_worker.isFinished(): - return - self.showmacro_worker = GetMacroDetailsWorker(self.macros[idx]) - self.showmacro_worker.info_label.connect(self.show_information) - self.showmacro_worker.progressbar_show.connect(self.show_progress_bar) - self.showmacro_worker.start() - self.dialog.buttonInstall.setEnabled(True) - self.dialog.buttonUninstall.setEnabled(True) - if self.macros[idx].is_installed(): - self.dialog.buttonExecute.setEnabled(True) - else: - self.dialog.buttonExecute.setEnabled(False) - - def switchtab(self, idx): - """does what needs to be done when switching tabs""" - - if idx == 1: - if not self.macros: - self.dialog.listMacros.clear() - self.macros = [] - self.macro_worker = FillMacroListWorker(self.macro_repo_dir) - self.macro_worker.add_macro_signal.connect(self.add_macro) - self.macro_worker.info_label_signal.connect(self.show_information) - self.macro_worker.progressbar_show.connect(self.show_progress_bar) - self.macro_worker.start() - self.dialog.listMacros.setCurrentRow(0) - - def update_repos(self, repos): + def append_to_repos_list(self, repo:AddonManagerRepo) -> None: """this function allows threads to update the main list of workbenches""" - self.repos = repos + self.item_model.append_item(repo) - def add_macro(self, macro): - """adds a macro to the list""" + def install(self) -> None: + """installs or updates a workbench, macro, or package""" - if macro.name: - if macro in self.macros: - # The macro is already in the list of macros. - old_macro = self.macros[self.macros.index(macro)] - utils.update_macro_details(old_macro, macro) + if hasattr(self, "install_worker") and self.install_worker: + if self.install_worker.isRunning(): + return + + if not hasattr(self, "selected_repo"): + FreeCAD.Console.PrintWarning ("Internal error: no selected repo\n") + return + + repo = self.selected_repo + + if not repo: + return + + if repo.repo_type == AddonManagerRepo.RepoType.WORKBENCH or repo.repo_type == AddonManagerRepo.RepoType.PACKAGE: + self.install_worker = InstallWorkbenchWorker(repo) + self.install_worker.status_message.connect(self.show_information) + self.current_progress_region = 1 + self.number_of_progress_regions = 1 + self.install_worker.progress_made.connect(self.update_progress_bar) + self.install_worker.success.connect(self.on_package_installed) + self.install_worker.failure.connect(self.on_installation_failed) + self.install_worker.start() + elif repo.repo_type == AddonManagerRepo.RepoType.MACRO: + macro = repo.macro + + # To try to ensure atomicity, test the installation into a temp directory first, + # and assume if that worked we have good odds of the real installation working + failed = False + with tempfile.TemporaryDirectory() as dir: + temp_install_succeeded = macro.install(dir) + if not temp_install_succeeded: + failed = True + + if not failed: + failed = macro.install(self.macro_repo_dir) + + if not failed: + message = translate("AddonsInstaller", + "Macro successfully installed. The macro is " + "now available from the Macros dialog.") + self.on_package_installed (repo, message) else: - self.macros.append(macro) - path = ":/icons/" + macro.name.replace(" ", "_") + "_macro_icon.svg" - if QtCore.QFile.exists(path): - addonicon = QtGui.QIcon(path) - else: - addonicon = QtGui.QIcon(":/icons/document-python.svg") - if addonicon.isNull(): - addonicon = QtGui.QIcon(":/icons/document-python.svg") - if macro.is_installed(): - item = QtGui.QListWidgetItem(addonicon, macro.name + str(" (Installed)")) - item.setForeground(QtGui.QBrush(QtGui.QColor(0, 182, 41))) - self.dialog.listMacros.addItem(item) - else: - self.dialog.listMacros.addItem(QtGui.QListWidgetItem(addonicon, macro.name)) + message = translate("AddonsInstaller", "Installation of macro failed. See console for failure details.") + self.on_installation_failed (repo, message) - def install(self, repos=None): - """installs a workbench or macro""" + def update_all(self) -> None: + """ Asynchronously apply all available updates: individual failures are noted, but do not stop other updates """ - if self.dialog.tabWidget.currentIndex() == 0: - # Tab "Workbenches". - idx = None - if repos: - idx = [] - for repo in repos: - for i, r in enumerate(self.repos): - if r[0] == repo: - idx.append(i) - else: - idx = self.dialog.listWorkbenches.currentRow() - if idx is not None: - if hasattr(self, "install_worker") and self.install_worker: - if self.install_worker.isRunning(): - return - self.install_worker = InstallWorker(self.repos, idx) - self.install_worker.info_label.connect(self.show_information) - self.install_worker.progressbar_show.connect(self.show_progress_bar) - self.install_worker.mark_recompute.connect(self.mark_recompute) - self.install_worker.start() + if hasattr(self, "update_all_worker") and self.update_all_worker: + if self.update_all_worker.isRunning(): + return - elif self.dialog.tabWidget.currentIndex() == 1: - # Tab "Macros". - macro = self.macros[self.dialog.listMacros.currentRow()] - if utils.install_macro(macro, self.macro_repo_dir): - self.dialog.description.setText(translate("AddonsInstaller", - "Macro successfully installed. The macro is " - "now available from the Macros dialog.")) - else: - self.dialog.description.setText(translate("AddonsInstaller", "Unable to install")) + self.subupdates_succeeded = [] + self.subupdates_failed = [] + + self.current_progress_region = 1 + self.number_of_progress_regions = 1 + self.update_all_worker = UpdateAllWorker(self.packages_with_updates) + self.update_all_worker.progress_made.connect(self.update_progress_bar) + self.update_all_worker.status_message.connect(self.show_information) + self.update_all_worker.success.connect(lambda repo : self.subupdates_succeeded.append(repo)) + self.update_all_worker.failure.connect(lambda repo : self.subupdates_failed.append(repo)) + self.update_all_worker.done.connect(self.on_update_all_completed) + self.update_all_worker.start() - def show_progress_bar(self, state): - """shows or hides the progress bar""" - - if state: - self.dialog.tabWidget.setEnabled(False) - self.dialog.buttonInstall.setEnabled(False) - self.dialog.buttonUninstall.setEnabled(False) - self.dialog.progressBar.show() + def on_update_all_completed(self) -> None: + #self.show_progress_bar(False) + if not self.subupdates_failed: + message = translate ("AddonsInstaller", "All packages were successfully updated. Packages:") + "\n" + message += ''.join([repo.name + "\n" for repo in self.subupdates_succeeded]) + elif not self.subupdates_succeeded: + message = translate ("AddonsInstaller", "All packages updates failed. Packages:") + "\n" + message += ''.join([repo.name + "\n" for repo in self.subupdates_failed]) else: - self.dialog.progressBar.hide() - self.dialog.tabWidget.setEnabled(True) - if not (self.firsttime and self.firstmacro): - self.dialog.buttonInstall.setEnabled(True) - self.dialog.buttonUninstall.setEnabled(True) - if self.dialog.listWorkbenches.isVisible(): - self.dialog.listWorkbenches.setFocus() - else: - self.dialog.listMacros.setFocus() + message = translate ("AddonsInstaller", "Some packages updates failed. Successful packages:") + "\n" + message += ''.join([repo.name + "\n" for repo in self.subupdates_succeeded]) + message += translate ("AddonsInstaller", "Failed packages:") + "\n" + message += ''.join([repo.name + "\n" for repo in self.subupdates_failed]) - def executemacro(self): + for installed_repo in self.subupdates_succeeded: + for requested_repo in self.packages_with_updates: + if installed_repo.name == requested_repo.name: + self.packages_with_updates.remove(installed_repo) + break + self.enable_updates(len(self.packages_with_updates)) + QtWidgets.QMessageBox.information(None, + translate("AddonsInstaller", "Update report"), + message, + QtWidgets.QMessageBox.Close) + + def hide_progress_widgets(self) -> None: + """ hides the progress bar and related widgets""" + + self.dialog.labelStatusInfo.hide() + self.dialog.progressBar.hide() + self.dialog.lineEditFilter.setFocus() + + def update_progress_bar(self, current_value:int, max_value:int) -> None: + """ Update the progress bar, showing it if it's hidden """ + + self.dialog.progressBar.show() + region_size = 100 / self.number_of_progress_regions + value = (self.current_progress_region-1)*region_size + (current_value / max_value / self.number_of_progress_regions)*region_size + self.dialog.progressBar.setValue(value) + + def on_package_installed(self, repo:AddonManagerRepo, message:str) -> None: + QtWidgets.QMessageBox.information(None, + translate("AddonsInstaller", "Installation succeeded"), + message, + QtWidgets.QMessageBox.Close) + self.dialog.progressBar.hide() + self.table_row_selected(self.dialog.tablePackages.selectionModel().selectedIndexes()[0], QtCore.QModelIndex()) + if repo.contains_workbench(): + self.item_model.update_item_status(repo.name, AddonManagerRepo.UpdateStatus.PENDING_RESTART) + else: + self.item_model.update_item_status(repo.name, AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE) + + def on_installation_failed(self, _:AddonManagerRepo, message:str) -> None: + QtWidgets.QMessageBox.warning(None, + translate("AddonsInstaller", "Installation failed"), + message, + QtWidgets.QMessageBox.Close) + self.dialog.progressBar.hide() + + def executemacro(self) -> None: """executes a selected macro""" - if self.dialog.tabWidget.currentIndex() == 1: - # Tab "Macros". - macro = self.macros[self.dialog.listMacros.currentRow()] - if not macro.is_installed(): - # Macro not installed, nothing to do. - return - macro_path = os.path.join(FreeCAD.getUserMacroDir(True), macro.filename) - if os.path.exists(macro_path): - macro_path = macro_path.replace("\\", "/") + macro = self.selected_repo.macro + if not macro or not macro.code: + return - FreeCADGui.open(str(macro_path)) - self.dialog.hide() - FreeCADGui.SendMsgToActiveView("Run") + if macro.is_installed(): + macro_path = os.path.join(self.macro_repo_dir,macro.filename) + FreeCADGui.open(str(macro_path)) + self.dialog.hide() + FreeCADGui.SendMsgToActiveView("Run") else: - self.dialog.buttonExecute.setEnabled(False) - - def remove_readonly(self, func, path, _): + with tempfile.TemporaryDirectory() as dir: + temp_install_succeeded = macro.install(dir) + if not temp_install_succeeded: + message = translate("AddonsInstaller", "Execution of macro failed. See console for failure details.") + self.on_installation_failed (self.selected_repo, message) + return + else: + macro_path = os.path.join(dir,macro.filename) + FreeCADGui.open(str(macro_path)) + self.dialog.hide() + FreeCADGui.SendMsgToActiveView("Run") + def remove_readonly(self, func, path, _) -> None: """Remove a read-only file.""" os.chmod(path, stat.S_IWRITE) func(path) - def remove(self): + def remove(self) -> None: """uninstalls a macro or workbench""" - if self.dialog.tabWidget.currentIndex() == 0: - # Tab "Workbenches". - idx = self.dialog.listWorkbenches.currentRow() + if self.selected_repo.repo_type == AddonManagerRepo.RepoType.WORKBENCH or \ + self.selected_repo.repo_type == AddonManagerRepo.RepoType.PACKAGE: basedir = FreeCAD.getUserAppDataDir() moddir = basedir + os.sep + "Mod" - clonedir = moddir + os.sep + self.repos[idx][0] + clonedir = moddir + os.sep + self.selected_repo.name if os.path.exists(clonedir): shutil.rmtree(clonedir, onerror=self.remove_readonly) self.dialog.description.setText(translate("AddonsInstaller", - "Addon successfully removed. Please restart FreeCAD")) + "Addon successfully removed. Please restart FreeCAD.")) + self.item_model.update_item_status(self.selected_repo.name, AddonManagerRepo.UpdateStatus.NOT_INSTALLED) + self.addon_removed = True # A value to trigger the restart message on dialog close else: - self.dialog.description.setText(translate("AddonsInstaller", "Unable to remove this addon")) + self.dialog.description.setText(translate("AddonsInstaller", "Unable to remove this addon with the Addon Manager.")) - elif self.dialog.tabWidget.currentIndex() == 1: - # Tab "Macros". - macro = self.macros[self.dialog.listMacros.currentRow()] - if utils.remove_macro(macro): + elif self.selected_repo.repo_type == AddonManagerRepo.RepoType.MACRO: + macro = self.selected_repo.macro + if macro.remove(): self.dialog.description.setText(translate("AddonsInstaller", "Macro successfully removed.")) + self.item_model.update_item_status(self.selected_repo.name, AddonManagerRepo.UpdateStatus.NOT_INSTALLED) else: self.dialog.description.setText(translate("AddonsInstaller", "Macro could not be removed.")) - self.update_status(soft=True) - self.addon_removed = True # A value to trigger the restart message on dialog close - def mark_recompute(self, addon): - """marks an addon in the list as installed but needs recompute""" - - for i in range(self.dialog.listWorkbenches.count()): - txt = self.dialog.listWorkbenches.item(i).text().strip() - if txt.endswith(" ("+translate("AddonsInstaller", "Installed")+")"): - txt = txt[:-12] - elif txt.endswith(" ("+translate("AddonsInstaller", "Update available")+")"): - txt = txt[:-19] - if txt == addon: - self.dialog.listWorkbenches.item(i).setText(txt + " (" + - translate("AddonsInstaller", - "Restart required") + - ")") - self.dialog.listWorkbenches.item(i).setIcon(QtGui.QIcon(":/icons/edit-undo.svg")) - - def update_status(self, soft=False): - """Updates the list of workbenches/macros. If soft is true, items - are not recreated (and therefore display text isn't triggered)" - """ - - moddir = FreeCAD.getUserAppDataDir() + os.sep + "Mod" - if soft: - for i in range(self.dialog.listWorkbenches.count()): - txt = self.dialog.listWorkbenches.item(i).text().strip() - ext = "" - if txt.endswith(" ("+translate("AddonsInstaller", "Installed")+")"): - txt = txt[:-12] - ext = " ("+translate("AddonsInstaller", "Installed")+")" - elif txt.endswith(" ("+translate("AddonsInstaller", "Update available")+")"): - txt = txt[:-19] - ext = " ("+translate("AddonsInstaller", "Update available")+")" - elif txt.endswith(" ("+translate("AddonsInstaller", "Restart required")+")"): - txt = txt[:-19] - ext = " ("+translate("AddonsInstaller", "Restart required")+")" - if os.path.exists(os.path.join(moddir, txt)): - self.dialog.listWorkbenches.item(i).setText(txt+ext) - else: - self.dialog.listWorkbenches.item(i).setText(txt) - self.dialog.listWorkbenches.item(i).setIcon(self.get_icon(txt)) - for i in range(self.dialog.listMacros.count()): - txt = self.dialog.listMacros.item(i).text().strip() - if txt.endswith(" ("+translate("AddonsInstaller", "Installed")+")"): - txt = txt[:-12] - elif txt.endswith(" ("+translate("AddonsInstaller", "Update available")+")"): - txt = txt[:-19] - if os.path.exists(os.path.join(moddir, txt)): - self.dialog.listMacros.item(i).setText(txt+ext) - else: - self.dialog.listMacros.item(i).setText(txt) - self.dialog.listMacros.item(i).setIcon(QtGui.QIcon(":/icons/document-python.svg")) - else: - self.dialog.listWorkbenches.clear() - self.dialog.listMacros.clear() - for wb in self.repos: - if os.path.exists(os.path.join(moddir, wb[0])): - self.dialog.listWorkbenches.addItem( - QtGui.QListWidgetItem(QtGui.QIcon(":/icons/button_valid.svg"), - str(wb[0]) + " (" + - translate("AddonsInstaller", "Installed") + ")")) - wb[2] = 1 - else: - self.dialog.listWorkbenches.addItem( - QtGui.QListWidgetItem(QtGui.QIcon(":/icons/document-python.svg"), str(wb[0]))) - wb[2] = 0 - for macro in self.macros: - if macro.is_installed(): - self.dialog.listMacros.addItem(item) - else: - self.dialog.listMacros.addItem( - QtGui.QListWidgetItem(QtGui.QIcon(":/icons/document-python.svg"), macro.name)) - - def mark(self, repo): - """mark a workbench as updatable""" - - for i in range(self.dialog.listWorkbenches.count()): - w = self.dialog.listWorkbenches.item(i) - if (w.text() == str(repo)) or w.text().startswith(str(repo)+" "): - w.setText(str(repo) + str(" ("+translate("AddonsInstaller", "Update available")+")")) - w.setForeground(QtGui.QBrush(QtGui.QColor(182, 90, 0))) - if repo not in self.doUpdate: - self.doUpdate.append(repo) - - def show_config(self): + def show_config(self) -> None: """shows the configuration dialog""" self.config = FreeCADGui.PySideUic.loadUi(os.path.join(os.path.dirname(__file__), "AddonManagerOptions.ui")) @@ -621,13 +815,226 @@ class CommandAddonManager: pref.SetBool("UserProxyCheck", self.config.radioButtonUserProxy.isChecked()) pref.SetString("ProxyUrl", self.config.userProxy.toPlainText()) + -def check_updates(addon_name, callback): - """Checks for updates for a given addon""" + def update_type_filter(self, type_filter:int) -> None: + """hide/show rows corresponding to the type filter + + type_filter is an integer: 0 for all, 1 for workbenches, 2 for macros, and 3 for preference packs + + """ + self.item_filter.setPackageFilter(type_filter) + - oname = "update_checker_"+addon_name - setattr(FreeCAD, oname, CheckSingleWorker(addon_name)) - getattr(FreeCAD, oname).updateAvailable.connect(callback) - getattr(FreeCAD, oname).start() + def update_text_filter(self, text_filter:str) -> None: + """filter name and description by the regex specified by text_filter""" + if text_filter: + test_regex = QtCore.QRegularExpression(text_filter) + if test_regex.isValid(): + self.dialog.labelFilterValidity.setToolTip(translate("AddonInstaller","Filter is valid")) + icon = QtGui.QIcon.fromTheme("ok", QtGui.QIcon(":/icons/edit_OK.svg")) + self.dialog.labelFilterValidity.setPixmap(icon.pixmap(16,16)) + else: + self.dialog.labelFilterValidity.setToolTip(translate("AddonInstaller","Filter regular expression is invalid")) + icon = QtGui.QIcon.fromTheme("cancel", QtGui.QIcon(":/icons/edit_Cancel.svg")) + self.dialog.labelFilterValidity.setPixmap(icon.pixmap(16,16)) + self.dialog.labelFilterValidity.show() + else: + self.dialog.labelFilterValidity.hide() + self.item_filter.setFilterRegularExpression(text_filter) + + +class PackageListItemModel(QtCore.QAbstractTableModel): + + repos = [] + write_lock = threading.Lock() + + DataAccessRole = QtCore.Qt.UserRole + StatusUpdateRole = QtCore.Qt.UserRole + 1 + IconUpdateRole = QtCore.Qt.UserRole + 2 + + def __init__(self) -> None: + QtCore.QAbstractTableModel.__init__(self) + + def rowCount(self, parent:QtCore.QModelIndex=QtCore.QModelIndex()) -> int: + if parent.isValid(): + return 0 + return len(self.repos) + + def columnCount(self, parent:QtCore.QModelIndex=QtCore.QModelIndex()) -> int: + if parent.isValid(): + return 0 + return 3 # Icon, Name, Status + + def data(self, index:QtCore.QModelIndex, role:int=QtCore.Qt.DisplayRole) -> Union[QtGui.QIcon,str]: + if not index.isValid(): + return None + row = index.row() + column = index.column() + if role == QtCore.Qt.DisplayRole: + if row >= len(self.repos): + return None + if column == 1: + return self.repos[row].name if self.repos[row].metadata is None else self.repos[row].metadata.Name + elif column == 2: + if self.repos[row].update_status == AddonManagerRepo.UpdateStatus.UNCHECKED: + return translate("AddonsInstaller","Installed") + elif self.repos[row].update_status == AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE: + return translate("AddonsInstaller","Up-to-date") + elif self.repos[row].update_status == AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE: + return translate("AddonsInstaller","Update available") + elif self.repos[row].update_status == AddonManagerRepo.UpdateStatus.PENDING_RESTART: + return translate("AddonsInstaller","Restart required") + else: + return None + else: + return None + elif role == QtCore.Qt.DecorationRole: + if column == 0: + return self.repos[row].icon + elif role == QtCore.Qt.ToolTipRole: + tooltip = "" + if self.repos[row].repo_type == AddonManagerRepo.RepoType.PACKAGE: + tooltip = f"Package '{self.repos[row].name}'" + # TODO add more info from Metadata + elif self.repos[row].repo_type == AddonManagerRepo.RepoType.WORKBENCH: + tooltip = f"Workbench '{self.repos[row].name}'" + elif self.repos[row].repo_type == AddonManagerRepo.RepoType.MACRO: + tooltip = f"Macro '{self.repos[row].name}'" + return tooltip + elif role == QtCore.Qt.TextAlignmentRole: + return QtCore.Qt.AlignLeft | QtCore.Qt.AlignTop + elif role == PackageListItemModel.DataAccessRole: + return self.repos[row] + + def headerData(self, section, orientation, role=QtCore.Qt.DisplayRole): + if role == QtCore.Qt.DisplayRole: + if orientation == QtCore.Qt.Horizontal: + if section == 0: + return None + elif section == 1: + return translate("AddonsInstaller", "Name") + elif section == 2: + return translate("AddonsInstaller", "Status") + else: + return None + + def setData(self, index:QtCore.QModelIndex, value, role=QtCore.Qt.EditRole) -> None: + """ Set the data for this row. The column of the index is ignored. """ + + row = index.row() + self.write_lock.acquire() + if role == PackageListItemModel.StatusUpdateRole: + self.repos[row].update_status = value + self.dataChanged.emit(self.index(row,2), self.index(row,2), [PackageListItemModel.StatusUpdateRole]) + elif role == PackageListItemModel.IconUpdateRole: + self.repos[row].icon = value + self.dataChanged.emit(self.index(row,0), self.index(row,0), [PackageListItemModel.IconUpdateRole]) + self.write_lock.release() + + def append_item(self, repo:AddonManagerRepo) -> None: + if repo in self.repos: + # Cowardly refuse to insert the same repo a second time + return + self.write_lock.acquire() + self.beginInsertRows(QtCore.QModelIndex(), self.rowCount(), self.rowCount()) + self.repos.append(repo) + self.endInsertRows() + self.write_lock.release() + + def clear(self) -> None: + if self.rowCount() > 0: + self.write_lock.acquire() + self.beginRemoveRows(QtCore.QModelIndex(), 0, self.rowCount()-1) + self.repos = [] + self.endRemoveRows() + self.write_lock.release() + + def update_item_status(self, name:str, status:AddonManagerRepo.UpdateStatus) -> None: + for row,item in enumerate(self.repos): + if item.name == name: + self.setData(self.index(row,0), status, PackageListItemModel.StatusUpdateRole) + return + + def update_item_icon(self, name:str, icon:QtGui.QIcon) -> None: + for row,item in enumerate(self.repos): + if item.name == name: + self.setData(self.index(row,0), icon, PackageListItemModel.IconUpdateRole) + return + + def reload_item(self,repo:AddonManagerRepo) -> None: + for index,item in enumerate(self.repos): + if item.name == repo.name: + self.write_lock.acquire() + self.repos[index] = repo + self.write_lock.release() + return + + +class PackageListIconDelegate(QtWidgets.QStyledItemDelegate): + """ A delegate to ensure proper alignment of the icon in the table cells """ + + def paint(self, painter, option, index): + if (index.column() == 0): + option.decorationAlignment = QtCore.Qt.AlignTop | QtCore.Qt.AlignHCenter + super().paint(painter, option, index) + + +class PackageListFilter(QtCore.QSortFilterProxyModel): + """ Handle filtering the item list on various criteria """ + + def __init__(self): + super().__init__() + self.package_type = 0 # Default to showing everything + self.setSortCaseSensitivity(QtCore.Qt.CaseInsensitive) + + def setPackageFilter(self, type:int) -> None: # 0=All, 1=Workbenches, 2=Macros, 3=Preference Packs + self.package_type = type + self.invalidateFilter() + + def lessThan(self, left, right) -> bool: + l = self.sourceModel().data(left,PackageListItemModel.DataAccessRole) + r = self.sourceModel().data(right,PackageListItemModel.DataAccessRole) + + if left.column() == 0: # Icon + return False + elif left.column() == 1: # Name + lname = l.name if l.metadata is None else l.metadata.Name + rname = r.name if r.metadata is None else r.metadata.Name + return lname.lower() < rname.lower() + elif left.column() == 2: # Status + return l.update_status < r.update_status + + def filterAcceptsRow(self, row, parent=QtCore.QModelIndex()): + index = self.sourceModel().createIndex(row, 0) + data = self.sourceModel().data(index,PackageListItemModel.DataAccessRole) + if self.package_type == 1: + if not data.contains_workbench(): + return False + elif self.package_type == 2: + if not data.contains_macro(): + return False + elif self.package_type == 3: + if not data.contains_preference_pack(): + return False + + name = data.name if data.metadata is None else data.metadata.Name + desc = data.description if not data.metadata else data.metadata.Description + re = self.filterRegularExpression() + if re.isValid(): + re.setPatternOptions(QtCore.QRegularExpression.CaseInsensitiveOption) + if re.match(name).hasMatch(): + return True + if re.match(desc).hasMatch(): + return True + return False + else: + return False + + def sort(self, column, order): + if column == 0: # Icons + return + else: + super().sort(column, order) # @} diff --git a/src/Mod/AddonManager/AddonManager.ui b/src/Mod/AddonManager/AddonManager.ui index 04b65a8281..5c3e08a1f2 100644 --- a/src/Mod/AddonManager/AddonManager.ui +++ b/src/Mod/AddonManager/AddonManager.ui @@ -28,40 +28,105 @@ - - - 0 + + + + + Show packages containing: + + + + + + + + All + + + + + Workbenches + + + + + Macros + + + + + Preference Packs + + + + + + + + + + + + Filter + + + true + + + + + + + OK + + + + + + + + + QAbstractItemView::NoEditTriggers - - - Workbenches - - - - - - - - - - Macros - - - - - - - - - Executes the selected macro, if installed - - - Execute - - - - - + + false + + + QAbstractItemView::SingleSelection + + + QAbstractItemView::SelectRows + + + + 16 + 16 + + + + false + + + true + + + false + + + 16 + + + false + + + 12 + + + 16 + + + false + @@ -95,13 +160,125 @@ + + + + + + + 0 + 0 + + + + + 64 + 64 + + + + + 64 + 64 + + + + Icon + + + + + + + 0 + + + QLayout::SetDefaultConstraint + + + + + <h1>Package Name</h1> + + + + + + + <em>Version</em> + + + + + + + Maintainer + + + + + + + + + + + Description + + + Qt::PlainText + + + true + + + + + + + + + URL Type + + + + + + + Url + + + + + + + + + Package contents: + + + + + + + Run Macro + + + + + + + labelStatusInfo + + + @@ -109,6 +286,9 @@ 24 + + false + Downloading info... diff --git a/src/Mod/AddonManager/AddonManagerRepo.py b/src/Mod/AddonManager/AddonManagerRepo.py new file mode 100644 index 0000000000..44b60fa67d --- /dev/null +++ b/src/Mod/AddonManager/AddonManagerRepo.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- +#*************************************************************************** +#* * +#* Copyright (c) 2021 Chris Hennes * +#* * +#* This program is free software; you can redistribute it and/or modify * +#* it under the terms of the GNU Lesser General Public License (LGPL) * +#* as published by the Free Software Foundation; either version 2 of * +#* the License, or (at your option) any later version. * +#* for detail see the LICENCE text file. * +#* * +#* This program is distributed in the hope that it will be useful, * +#* but WITHOUT ANY WARRANTY; without even the implied warranty of * +#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * +#* GNU Library General Public License for more details. * +#* * +#* You should have received a copy of the GNU Library General Public * +#* License along with this program; if not, write to the Free Software * +#* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * +#* USA * +#* * +#*************************************************************************** + +import FreeCAD + +import os + +from addonmanager_macro import Macro + +class AddonManagerRepo: + "Encapsulate information about a FreeCAD addon" + + from enum import Enum + + class RepoType(Enum): + WORKBENCH = 1 + MACRO = 2 + PACKAGE = 3 + + def __str__(self) ->str : + if self.value == 1: + return "Workbench" + elif self.value == 2: + return "Macro" + elif self.value == 3: + return "Package" + + class UpdateStatus(Enum): + NOT_INSTALLED = 0 + UNCHECKED = 1 + NO_UPDATE_AVAILABLE = 2 + UPDATE_AVAILABLE = 3 + PENDING_RESTART = 4 + + def __lt__(self, other): + if self.__class__ is other.__class__: + return self.value < other.value + return NotImplemented + + def __str__(self) ->str : + if self.value == 0: + return "Not installed" + elif self.value == 1: + return "Unchecked" + elif self.value == 2: + return "No update available" + elif self.value == 3: + return "Update available" + elif self.value == 4: + return "Restart required" + + def __init__ (self, name:str, url:str, status:UpdateStatus, branch:str): + self.name = name + self.url = url + self.branch = branch + self.update_status = status + self.repo_type = AddonManagerRepo.RepoType.WORKBENCH + self.description = None + from addonmanager_utilities import construct_git_url + self.metadata_url = "" if not self.url else construct_git_url(self, "package.xml") + self.metadata = None + self.icon = None + self.cached_icon_filename = "" + self.macro = None # Bridge to Gaël Écorchard's macro management class + + def __str__ (self) -> str: + result = f"FreeCAD {self.repo_type}\n" + result += f"Name: {self.name}\n" + result += f"URL: {self.url}\n" + result += "Has metadata\n" if self.metadata is not None else "No metadata found\n" + if self.macro is not None: + result += "Has linked Macro object\n" + return result + + @classmethod + def from_macro (self, macro:Macro): + if macro.is_installed(): + status = AddonManagerRepo.UpdateStatus.UNCHECKED + else: + status = AddonManagerRepo.UpdateStatus.NOT_INSTALLED + instance = AddonManagerRepo(macro.name, macro.url, status, "master") + instance.macro = macro + instance.repo_type = AddonManagerRepo.RepoType.MACRO + instance.description = macro.desc + return instance + + def contains_workbench(self) -> bool: + """ Determine if this package contains (or is) a workbench """ + + if self.repo_type == AddonManagerRepo.RepoType.WORKBENCH: + return True + elif self.repo_type == AddonManagerRepo.RepoType.PACKAGE: + content = self.metadata.Content + return "workbench" in content + else: + return False + + def contains_macro(self) -> bool: + """ Determine if this package contains (or is) a macro """ + + if self.repo_type == AddonManagerRepo.RepoType.MACRO: + return True + elif self.repo_type == AddonManagerRepo.RepoType.PACKAGE: + content = self.metadata.Content + return "macro" in content + else: + return False + + def contains_preference_pack(self) -> bool: + """ Determine if this package contains a preference pack """ + + if self.repo_type == AddonManagerRepo.RepoType.PACKAGE: + content = self.metadata.Content + return "preferencepack" in content + else: + return False + + def get_cached_icon_filename(self) ->str: + """ Get the filename for the locally-cached copy of the icon """ + + if self.cached_icon_filename: + return self.cached_icon_filename + + real_icon = self.metadata.Icon + if not real_icon: + # If there is no icon set for the entire package, see if there are any workbenches, which + # are required to have icons, and grab the first one we find: + content = self.metadata.Content + if "workbench" in content: + wb = content["workbench"][0] + if wb.Icon: + if wb.Subdirectory: + subdir = wb.Subdirectory + else: + subdir = wb.Name + real_icon = subdir + wb.Icon + + real_icon = real_icon.replace("/", os.path.sep) # Required path separator in the metadata.xml file to local separator + + _, file_extension = os.path.splitext(real_icon) + store = os.path.join(FreeCAD.getUserAppDataDir(), "AddonManager", "PackageMetadata") + self.cached_icon_filename = os.path.join(store, self.name, "cached_icon"+file_extension) + + return self.cached_icon_filename \ No newline at end of file diff --git a/src/Mod/AddonManager/CMakeLists.txt b/src/Mod/AddonManager/CMakeLists.txt index 23549f18c6..fa84f324fb 100644 --- a/src/Mod/AddonManager/CMakeLists.txt +++ b/src/Mod/AddonManager/CMakeLists.txt @@ -6,7 +6,9 @@ SET(AddonManager_SRCS Init.py InitGui.py AddonManager.py + AddonManagerRepo.py addonmanager_macro.py + addonmanager_metadata.py addonmanager_utilities.py addonmanager_workers.py AddonManager.ui diff --git a/src/Mod/AddonManager/addonmanager_macro.py b/src/Mod/AddonManager/addonmanager_macro.py index 45cb7cafe8..88481da068 100644 --- a/src/Mod/AddonManager/addonmanager_macro.py +++ b/src/Mod/AddonManager/addonmanager_macro.py @@ -24,12 +24,14 @@ import os import re import sys - +import codecs +import shutil import FreeCAD from addonmanager_utilities import translate from addonmanager_utilities import urlopen +from addonmanager_utilities import remove_directory_if_empty try: from HTMLParser import HTMLParser @@ -39,10 +41,10 @@ except ImportError: # @package AddonManager_macro # \ingroup ADDONMANAGER -# \brief Unified handler for FreeCAD macros that can be obtained from different sources +# \brief Unified handler for FreeCAD macros that can be obtained from +# different sources # @{ - class Macro(object): """This class provides a unified way to handle macros coming from different sources""" @@ -70,12 +72,11 @@ class Macro(object): def is_installed(self): if self.on_git and not self.src_filename: return False - return (os.path.exists(os.path.join(FreeCAD.getUserMacroDir(True), self.filename)) - or os.path.exists(os.path.join(FreeCAD.getUserMacroDir(True), "Macro_" + self.filename))) + return (os.path.exists(os.path.join(FreeCAD.getUserMacroDir(True), self.filename)) or os.path.exists(os.path.join(FreeCAD.getUserMacroDir(True), "Macro_" + self.filename))) def fill_details_from_file(self, filename): with open(filename) as f: - # Number of parsed fields of metadata. For now, __Comment__, + # Number of parsed fields of metadata. For now, __Comment__, # __Web__, __Version__, __Files__. number_of_required_fields = 4 re_desc = re.compile(r"^__Comment__\s*=\s*(['\"])(.*)\1") @@ -109,23 +110,24 @@ class Macro(object): code = "" u = urlopen(url) if u is None: - print("AddonManager: Debug: connection is lost (proxy setting changed?)", url) + FreeCAD.Console.PrintWarning("AddonManager: Debug: connection is lost (proxy setting changed?)", url, "\n") return p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): + if isinstance(p, bytes): p = p.decode("utf-8") u.close() - # check if the macro page has its code hosted elsewhere, download if needed + # check if the macro page has its code hosted elsewhere, download if + # needed if "rawcodeurl" in p: rawcodeurl = re.findall("rawcodeurl.*?href=\"(http.*?)\">", p) if rawcodeurl: rawcodeurl = rawcodeurl[0] u2 = urlopen(rawcodeurl) if u2 is None: - print("AddonManager: Debug: unable to open URL", rawcodeurl) + FreeCAD.Console.PrintWarning("AddonManager: Debug: unable to open URL", rawcodeurl, "\n") return # code = u2.read() - # github is slow to respond... We need to use this trick below + # github is slow to respond... We need to use this trick below response = "" block = 8192 # expected = int(u2.headers["content-length"]) @@ -134,7 +136,7 @@ class Macro(object): data = u2.read(block) if not data: break - if sys.version_info.major >= 3 and isinstance(data, bytes): + if isinstance(data, bytes): data = data.decode("utf-8") response += data if response: @@ -147,25 +149,101 @@ class Macro(object): code = sorted(code, key=len)[-1] code = code.replace("--endl--", "\n") # Clean HTML escape codes. - if sys.version_info.major < 3: - code = code.decode("utf8") code = unescape(code) code = code.replace(b"\xc2\xa0".decode("utf-8"), " ") - if sys.version_info.major < 3: - code = code.encode("utf8") else: - FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "Unable to fetch the code of this macro.")) + FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "Unable to fetch the code of this macro.") + "\n") desc = re.findall(r"(.*?)", p.replace("\n", " ")) if desc: desc = desc[0] else: FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "Unable to retrieve a description for this macro.")) + "Unable to retrieve a description for this macro.") + "\n") desc = "No description available" self.desc = desc self.url = url + if isinstance(code, list): + flat_code = "" + for chunk in code: + flat_code += chunk + code = flat_code self.code = code self.parsed = True + def install(self, macro_dir:str) -> bool: + """Install a macro and all its related files + + Returns True if the macro was installed correctly. + + Parameters + ---------- + - macro_dir: the directory to install into + """ + + if not self.code: + return False + if not os.path.isdir(macro_dir): + try: + os.makedirs(macro_dir) + except OSError: + FreeCAD.Console.PrintError(f"Failed to create {macro_dir}\n") + return False + macro_path = os.path.join(macro_dir, self.filename) + try: + with codecs.open(macro_path, 'w', 'utf-8') as macrofile: + macrofile.write(self.code) + except IOError: + FreeCAD.Console.PrintError(f"Failed to write {macro_path}\n") + return False + # Copy related files, which are supposed to be given relative to + # self.src_filename. + base_dir = os.path.dirname(self.src_filename) + for other_file in self.other_files: + dst_dir = os.path.join(macro_dir, os.path.dirname(other_file)) + if not os.path.isdir(dst_dir): + try: + os.makedirs(dst_dir) + except OSError: + FreeCAD.Console.PrintError(f"Failed to create {dst_dir}\n") + return False + src_file = os.path.join(base_dir, other_file) + dst_file = os.path.join(macro_dir, other_file) + try: + shutil.copy(src_file, dst_file) + except IOError: + FreeCAD.Console.PrintError(f"Failed to copy {src_file} to {dst_file}\n") + return False + + FreeCAD.Console.PrintMessage(f"Macro {self.name} was installed successfully.\n") + return True + + + def remove(self) -> bool: + """Remove a macro and all its related files + + Returns True if the macro was removed correctly. + """ + + if not self.is_installed(): + # Macro not installed, nothing to do. + return True + macro_dir = FreeCAD.getUserMacroDir(True) + macro_path = os.path.join(macro_dir, self.filename) + macro_path_with_macro_prefix = os.path.join(macro_dir, 'Macro_' + self.filename) + if os.path.exists(macro_path): + os.remove(macro_path) + elif os.path.exists(macro_path_with_macro_prefix): + os.remove(macro_path_with_macro_prefix) + # Remove related files, which are supposed to be given relative to + # self.src_filename. + for other_file in self.other_files: + dst_file = os.path.join(macro_dir, other_file) + try: + os.remove(dst_file) + remove_directory_if_empty(os.path.dirname(dst_file)) + except Exception: + FreeCAD.Console.PrintWarning(f"Failed to remove macro file '{dst_file}': it might not exist, or its permissions changed\n") + return True + # @} diff --git a/src/Mod/AddonManager/addonmanager_metadata.py b/src/Mod/AddonManager/addonmanager_metadata.py new file mode 100644 index 0000000000..94e290adeb --- /dev/null +++ b/src/Mod/AddonManager/addonmanager_metadata.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +#*************************************************************************** +#* * +#* Copyright (c) 2021 Chris Hennes * +#* * +#* This program is free software; you can redistribute it and/or modify * +#* it under the terms of the GNU Lesser General Public License (LGPL) * +#* as published by the Free Software Foundation; either version 2 of * +#* the License, or (at your option) any later version. * +#* for detail see the LICENCE text file. * +#* * +#* This program is distributed in the hope that it will be useful, * +#* but WITHOUT ANY WARRANTY; without even the implied warranty of * +#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * +#* GNU Library General Public License for more details. * +#* * +#* You should have received a copy of the GNU Library General Public * +#* License along with this program; if not, write to the Free Software * +#* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * +#* USA * +#* * +#*************************************************************************** + +import FreeCAD + +import git +import tempfile +import os +import hashlib + +from PySide2 import QtCore, QtNetwork +from PySide2.QtCore import QObject + +import addonmanager_utilities as utils +from AddonManagerRepo import AddonManagerRepo + +class MetadataDownloadWorker(QObject): + """A worker for downloading package.xml and associated icon(s) + + To use, instantiate an object of this class and call the start_fetch() function + with a QNetworkAccessManager. It is expected that many of these objects will all + be created and associated with the same QNAM, which will then handle the actual + asynchronous downloads in some Qt-defined number of threads. To monitor progress + you should connect to the QNAM's "finished" signal, and ensure it is called the + number of times you expect based on how many workers you have enqueued. + + """ + + updated = QtCore.Signal(AddonManagerRepo) + + def __init__(self, parent, repo, index): + "repo is an AddonManagerRepo object, and index is a dictionary of SHA1 hashes of the package.xml files in the cache" + + super().__init__(parent) + self.repo = repo + self.index = index + self.store = os.path.join(FreeCAD.getUserAppDataDir(), "AddonManager", "PackageMetadata") + self.last_sha1 = "" + self.url = self.repo.metadata_url + + def start_fetch(self, network_manager): + "Asynchronously begin the network access. Intended as a set-and-forget black box for downloading metadata." + self.request = QtNetwork.QNetworkRequest(QtCore.QUrl(self.url)) + self.request.setAttribute(QtNetwork.QNetworkRequest.RedirectPolicyAttribute, + QtNetwork.QNetworkRequest.UserVerifiedRedirectPolicy) + + self.fetch_task = network_manager.get(self.request) + self.fetch_task.finished.connect(self.resolve_fetch) + self.fetch_task.redirected.connect(self.on_redirect) + + def abort(self): + self.fetch_task.abort() + + def on_redirect(self, url): + # For now just blindly follow all redirects + self.fetch_task.redirectAllowed.emit() + + def resolve_fetch(self): + "Called when the data fetch completed, either with an error, or if it found the metadata file" + if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError: + FreeCAD.Console.PrintMessage(f"Found a metadata file for {self.repo.name}\n") + self.repo.repo_type = AddonManagerRepo.RepoType.PACKAGE + new_xml = self.fetch_task.readAll() + hasher = hashlib.sha1() + hasher.update(new_xml) + new_sha1 = hasher.hexdigest() + self.last_sha1 = new_sha1 + # Determine if we need to download the icon: only do that if the + # package.xml file changed (since + # a change in the version number will show up as a change in the + # SHA1, without having to actually + # read the metadata) + if self.repo.name in self.index: + cached_sha1 = self.index[self.repo.name] + if cached_sha1 != new_sha1: + self.update_local_copy(new_xml) + else: + # Assume that if the package.xml file didn't change, + # neither did the icon, so don't waste + # resources downloading it + xml_file = os.path.join(self.store, self.repo.name, "package.xml") + self.repo.metadata = FreeCAD.Metadata(xml_file) + else: + # There is no local copy yet, so we definitely have to update + # the cache + self.update_local_copy(new_xml) + + def update_local_copy(self, new_xml): + # We have to update the local copy of the metadata file and re-download + # the icon file + + name = self.repo.name + repo_url = self.repo.url + package_cache_directory = os.path.join(self.store, name) + if not os.path.exists(package_cache_directory): + os.makedirs(package_cache_directory) + new_xml_file = os.path.join(package_cache_directory, "package.xml") + with open(new_xml_file, "wb") as f: + f.write(new_xml) + metadata = FreeCAD.Metadata(new_xml_file) + self.repo.metadata = metadata + self.repo.repo_type = AddonManagerRepo.RepoType.PACKAGE + icon = metadata.Icon + + if not icon: + # If there is no icon set for the entire package, see if there are + # any workbenches, which + # are required to have icons, and grab the first one we find: + content = self.repo.metadata.Content + if "workbench" in content: + wb = content["workbench"][0] + if wb.Icon: + if wb.Subdirectory: + subdir = wb.Subdirectory + else: + subdir = wb.Name + self.repo.Icon = subdir + wb.Icon + icon = self.repo.Icon + + icon_url = utils.construct_git_url(self.repo, icon) + icon_stream = utils.urlopen(icon_url) + if icon and icon_stream and icon_url: + icon_data = icon_stream.read() + cache_file = self.repo.get_cached_icon_filename() + with open(cache_file, "wb") as icon_file: + icon_file.write(icon_data) + self.repo.cached_icon_filename = cache_file + self.updated.emit(self.repo) diff --git a/src/Mod/AddonManager/addonmanager_utilities.py b/src/Mod/AddonManager/addonmanager_utilities.py index 509e75c18c..1a7addec9f 100644 --- a/src/Mod/AddonManager/addonmanager_utilities.py +++ b/src/Mod/AddonManager/addonmanager_utilities.py @@ -27,12 +27,15 @@ import re import shutil import sys import ctypes +import tempfile +import ssl -import urllib.request as urllib2 +import urllib +from urllib.request import Request from urllib.error import URLError from urllib.parse import urlparse -from PySide import QtGui, QtCore +from PySide2 import QtGui, QtCore, QtWidgets import FreeCAD import FreeCADGui @@ -46,7 +49,9 @@ except ImportError: pass else: try: - ssl_ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) + #ssl_ctx = ssl.create_default_context(cafile=certifi.where()) + ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + #ssl_ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) except AttributeError: pass @@ -61,18 +66,17 @@ def translate(context, text, disambig=None): "Main translation function" try: - _encoding = QtGui.QApplication.UnicodeUTF8 + _encoding = QtWidgets.QApplication.UnicodeUTF8 except AttributeError: - return QtGui.QApplication.translate(context, text, disambig) + return QtWidgets.QApplication.translate(context, text, disambig) else: - return QtGui.QApplication.translate(context, text, disambig, _encoding) + return QtWidgets.QApplication.translate(context, text, disambig, _encoding) def symlink(source, link_name): "creates a symlink of a file, if possible" if os.path.exists(link_name) or os.path.lexists(link_name): - # print("macro already exists") pass else: os_symlink = getattr(os, "symlink", None) @@ -90,8 +94,8 @@ def symlink(source, link_name): raise ctypes.WinError() -def urlopen(url): - """Opens an url with urllib2""" +def urlopen(url:str): + """Opens an url with urllib and streams it to a temp file""" timeout = 5 @@ -101,25 +105,29 @@ def urlopen(url): proxies = {} else: if pref.GetBool("SystemProxyCheck", False): - proxy = urllib2.getproxies() + proxy = urllib.request.getproxies() proxies = {"http": proxy.get('http'), "https": proxy.get('http')} elif pref.GetBool("UserProxyCheck", False): proxy = pref.GetString("ProxyUrl", "") proxies = {"http": proxy, "https": proxy} if ssl_ctx: - handler = urllib2.HTTPSHandler(context=ssl_ctx) + handler = urllib.request.HTTPSHandler(context=ssl_ctx) else: handler = {} - proxy_support = urllib2.ProxyHandler(proxies) - opener = urllib2.build_opener(proxy_support, handler) - urllib2.install_opener(opener) + proxy_support = urllib.request.ProxyHandler(proxies) + opener = urllib.request.build_opener(proxy_support, handler) + urllib.request.install_opener(opener) # Url opening - req = urllib2.Request(url, + req = urllib.request.Request(url, headers={'User-Agent': "Magic Browser"}) try: - u = urllib2.urlopen(req, timeout=timeout) + u = urllib.request.urlopen(req, timeout=timeout) + + except URLError as e: + FreeCAD.Console.PrintError(f"Error loading {url}:\n {e.reason}\n") + return None except Exception: return None else: @@ -151,76 +159,7 @@ def update_macro_details(old_macro, new_macro): setattr(old_macro, attr, getattr(new_macro, attr)) -def install_macro(macro, macro_repo_dir): - """Install a macro and all its related files - Returns True if the macro was installed correctly. - - Parameters - ---------- - - macro: an addonmanager_macro.Macro instance - """ - - if not macro.code: - return False - macro_dir = FreeCAD.getUserMacroDir(True) - if not os.path.isdir(macro_dir): - try: - os.makedirs(macro_dir) - except OSError: - return False - macro_path = os.path.join(macro_dir, macro.filename) - try: - with codecs.open(macro_path, 'w', 'utf-8') as macrofile: - macrofile.write(macro.code) - except IOError: - return False - # Copy related files, which are supposed to be given relative to - # macro.src_filename. - base_dir = os.path.dirname(macro.src_filename) - for other_file in macro.other_files: - dst_dir = os.path.join(macro_dir, os.path.dirname(other_file)) - if not os.path.isdir(dst_dir): - try: - os.makedirs(dst_dir) - except OSError: - return False - src_file = os.path.join(base_dir, other_file) - dst_file = os.path.join(macro_dir, other_file) - try: - shutil.copy(src_file, dst_file) - except IOError: - return False - return True - - -def remove_macro(macro): - """Remove a macro and all its related files - - Returns True if the macro was removed correctly. - - Parameters - ---------- - - macro: an addonmanager_macro.Macro instance - """ - - if not macro.is_installed(): - # Macro not installed, nothing to do. - return True - macro_dir = FreeCAD.getUserMacroDir(True) - macro_path = os.path.join(macro_dir, macro.filename) - macro_path_with_macro_prefix = os.path.join(macro_dir, 'Macro_' + macro.filename) - if os.path.exists(macro_path): - os.remove(macro_path) - elif os.path.exists(macro_path_with_macro_prefix): - os.remove(macro_path_with_macro_prefix) - # Remove related files, which are supposed to be given relative to - # macro.src_filename. - for other_file in macro.other_files: - dst_file = os.path.join(macro_dir, other_file) - remove_directory_if_empty(os.path.dirname(dst_file)) - os.remove(dst_file) - return True def remove_directory_if_empty(dir): @@ -238,72 +177,81 @@ def remove_directory_if_empty(dir): def restart_freecad(): "Shuts down and restarts FreeCAD" - args = QtGui.QApplication.arguments()[1:] + args = QtWidgets.QApplication.arguments()[1:] if FreeCADGui.getMainWindow().close(): - QtCore.QProcess.startDetached(QtGui.QApplication.applicationFilePath(), args) + QtCore.QProcess.startDetached(QtWidgets.QApplication.applicationFilePath(), args) -def get_zip_url(baseurl): +def get_zip_url(repo): "Returns the location of a zip file from a repo, if available" - parsedUrl = urlparse(baseurl) + parsedUrl = urlparse(repo.url) if parsedUrl.netloc == "github.com": - return baseurl+"/archive/master.zip" + return f"{repo.url}/archive/{repo.branch}.zip" elif parsedUrl.netloc == "framagit.org" or parsedUrl.netloc == "gitlab.com": # https://framagit.org/freecad-france/mooc-workbench/-/archive/master/mooc-workbench-master.zip reponame = baseurl.strip("/").split("/")[-1] - return baseurl+"/-/archive/master/"+reponame+"-master.zip" + return f"{repo.url}/-/archive/{repo.branch}/{repo.name}-{repo.branch}.zip" else: - print("Debug: addonmanager_utilities.get_zip_url: Unknown git host:", parsedUrl.netloc) + FreeCAD.Console.PrintWarning("Debug: addonmanager_utilities.get_zip_url: Unknown git host:", parsedUrl.netloc) return None +def construct_git_url(repo, filename): + "Returns a direct download link to a file in an online Git repo: works with github, gitlab, and framagit" -def get_readme_url(url): - "Returns the location of a readme file" - - parsedUrl = urlparse(url) - if parsedUrl.netloc == "github.com" or parsedUrl.netloc == "framagit.com": - return url+"/raw/master/README.md" - elif parsedUrl.netloc == "gitlab.com": - return url+"/-/raw/master/README.md" + parsed_url = urlparse(repo.url) + if parsed_url.netloc == "github.com" or parsed_url.netloc == "framagit.com": + return f"{repo.url}/raw/{repo.branch}/{filename}" + elif parsed_url.netloc == "gitlab.com": + return f"{repo.url}/-/raw/{repo.branch}/{filename}" else: - print("Debug: addonmanager_utilities.get_readme_url: Unknown git host:", url) + FreeCAD.Console.PrintLog("Debug: addonmanager_utilities.construct_git_url: Unknown git host:", parsed_url.netloc) return None +def get_readme_url(repo): + "Returns the location of a readme file" -def get_desc_regex(url): + return construct_git_url(repo, "README.md") + +def get_metadata_url(url): + "Returns the location of a package.xml metadata file" + + return construct_git_url(repo, "package.xml") + + +def get_desc_regex(repo): """Returns a regex string that extracts a WB description to be displayed in the description panel of the Addon manager, if the README could not be found""" - parsedUrl = urlparse(url) + parsedUrl = urlparse(repo.url) if parsedUrl.netloc == "github.com": return r'' - print("Debug: addonmanager_utilities.get_desc_regex: Unknown git host:", url) + FreeCAD.Console.PrintWarning("Debug: addonmanager_utilities.get_desc_regex: Unknown git host:", repo.url) return None -def get_readme_html_url(url): +def get_readme_html_url(repo): """Returns the location of a html file containing readme""" - parsedUrl = urlparse(url) + parsedUrl = urlparse(repo.url) if parsedUrl.netloc == "github.com": - return url + "/blob/master/README.md" + return f"{repo.url}/blob/{repo.branch}/README.md" else: - print("Debug: addonmanager_utilities.get_readme_html_url: Unknown git host:", url) + FreeCAD.Console.PrintWarning("Debug: addonmanager_utilities.get_readme_html_url: Unknown git host:", repo.url) return None -def get_readme_regex(url): +def get_readme_regex(repo): """Return a regex string that extracts the contents to be displayed in the description panel of the Addon manager, from raw HTML data (the readme's html rendering usually)""" - parsedUrl = urlparse(url) + parsedUrl = urlparse(repo.url) if parsedUrl.netloc == "github.com": return "(.*?)" else: - print("Debug: addonmanager_utilities.get_readme_regex: Unknown git host:", url) + FreeCAD.Console.PrintWarning("Debug: addonmanager_utilities.get_readme_regex: Unknown git host:", repo.url) return None @@ -319,7 +267,7 @@ def fix_relative_links(text, base_url): if len(parts) < 2 or not re.match(r"^http|^www|^.+\.|^/", parts[0]): newlink = os.path.join(base_url, link.lstrip('./')) line = line.replace(link, newlink) - print("Debug: replaced " + link + " with " + newlink) + FreeCAD.Console.PrintLog("Debug: replaced " + link + " with " + newlink) new_text = new_text + '\n' + line return new_text diff --git a/src/Mod/AddonManager/addonmanager_workers.py b/src/Mod/AddonManager/addonmanager_workers.py index ba12996a78..8716375f91 100644 --- a/src/Mod/AddonManager/addonmanager_workers.py +++ b/src/Mod/AddonManager/addonmanager_workers.py @@ -2,6 +2,7 @@ #*************************************************************************** #* * #* Copyright (c) 2019 Yorik van Havre * +#* Copyright (c) 2021 Chris Hennes * #* * #* This program is free software; you can redistribute it and/or modify * #* it under the terms of the GNU Lesser General Public License (LGPL) * @@ -26,8 +27,13 @@ import re import shutil import sys import json +import tempfile +import hashlib +import threading +import queue +from typing import Union -from PySide import QtCore, QtGui +from PySide2 import QtCore, QtGui, QtWidgets, QtNetwork import FreeCAD if FreeCAD.GuiUp: @@ -36,11 +42,15 @@ if FreeCAD.GuiUp: import addonmanager_utilities as utils from addonmanager_utilities import translate # this needs to be as is for pylupdate from addonmanager_macro import Macro +from addonmanager_metadata import MetadataDownloadWorker +from AddonManagerRepo import AddonManagerRepo + have_git = False try: import git - # some versions of git module have no "Repo" class?? Bug #4072 module 'git' has no attribute 'Repo' + # some versions of git module have no "Repo" class?? Bug #4072 module + # 'git' has no attribute 'Repo' have_git = hasattr(git,"Repo") except ImportError: pass @@ -59,12 +69,7 @@ try: except ImportError: pass -try: - import StringIO as io - _stringio = io.StringIO -except ImportError: # StringIO is not available with python3 - import io - _stringio = io.BytesIO +from io import BytesIO # @package AddonManager_workers # \ingroup ADDONMANAGER @@ -81,18 +86,15 @@ obsolete = [] py2only = [] NOGIT = False # for debugging purposes, set this to True to always use http downloads - NOMARKDOWN = False # for debugging purposes, set this to True to disable Markdown lib - """Multithread workers for the Addon Manager""" class UpdateWorker(QtCore.QThread): """This worker updates the list of available workbenches""" - info_label = QtCore.Signal(str) + status_message = QtCore.Signal(str) addon_repo = QtCore.Signal(object) - progressbar_show = QtCore.Signal(bool) done = QtCore.Signal() def __init__(self): @@ -102,7 +104,7 @@ class UpdateWorker(QtCore.QThread): def run(self): "populates the list of addons" - self.progressbar_show.emit(True) + self.current_thread = QtCore.QThread.currentThread() # update info lists global obsolete, macros_blacklist, py2only @@ -120,102 +122,118 @@ class UpdateWorker(QtCore.QThread): if "py2only" in j and "Mod" in j["py2only"]: py2only = j["py2only"]["Mod"] else: - print("Debug: addon_flags.json not found") + FreeCAD.Console.PrintWarning("Debug: addon_flags.json not found\n") + + # Check the local package cache: + basedir = FreeCAD.getUserAppDataDir() + moddir = basedir + os.sep + "Mod" + cache_path = os.path.join(FreeCAD.getUserAppDataDir(), "AddonManager", "PackageMetadata") + package_names = [] + if os.path.isdir(cache_path): + for dir in os.listdir(cache_path): + if self.current_thread.isInterruptionRequested(): + return + dir_path = os.path.join(cache_path, dir) + if not os.path.isdir(dir_path): + continue + xml_cache = os.path.join(dir_path, "package.xml") + try: + meta = FreeCAD.Metadata(xml_cache) + except Exception: + FreeCAD.Console.PrintWarning(f"Failed to create Metadata from {xml_cache}\n") + continue + name = dir # Do not use metadata name here, we want to match the legacy fetch code below + url = None + branch = None + for meta_url in meta.Urls: + if meta_url["type"] == "repository": + url = meta_url["location"] + branch = meta_url["branch"] + break + addondir = moddir + os.sep + name + if os.path.exists(addondir) and os.listdir(addondir): + state = AddonManagerRepo.UpdateStatus.UNCHECKED + else: + state = AddonManagerRepo.UpdateStatus.NOT_INSTALLED + cached_package = AddonManagerRepo(name, url, state, branch) + cached_package.metadata = meta + cached_package.icon = QtGui.QIcon(cached_package.get_cached_icon_filename()) + cached_package.repo_type = AddonManagerRepo.RepoType.PACKAGE + cached_package.description = meta.Description + self.addon_repo.emit(cached_package) + package_names.append(name) u = utils.urlopen("https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/.gitmodules") if not u: - self.progressbar_show.emit(False) self.done.emit() self.stop = True return p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): + if isinstance(p, bytes): p = p.decode("utf-8") u.close() p = re.findall((r'(?m)\[submodule\s*"(?P.*)"\]\s*' r"path\s*=\s*(?P.+)\s*" - r"url\s*=\s*(?Phttps?://.*)"), p) - basedir = FreeCAD.getUserAppDataDir() - moddir = basedir + os.sep + "Mod" - repos = [] + r"url\s*=\s*(?Phttps?://.*)\s*" + r"(branch\s*=\s*(?P.*)\s*)?"), p) + # querying official addons - for name, path, url in p: - self.info_label.emit(name) + for name, path, url, _, branch in p: + if self.current_thread.isInterruptionRequested(): + return + if name in package_names: + # We've already got this info since it's a package + continue + if branch is None or len(branch) == 0: + branch = "master" url = url.split(".git")[0] addondir = moddir + os.sep + name if os.path.exists(addondir) and os.listdir(addondir): # make sure the folder exists and it contains files! - state = 1 + state = AddonManagerRepo.UpdateStatus.UNCHECKED else: - state = 0 - repos.append([name, url, state]) + state = AddonManagerRepo.UpdateStatus.NOT_INSTALLED + self.addon_repo.emit(AddonManagerRepo(name, url, state, branch)) # querying custom addons - customaddons = (FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") + addon_list = (FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") .GetString("CustomRepositories", "").split("\n")) - for url in customaddons: - if url: - name = url.split("/")[-1] + custom_addons = [] + for addon in addon_list: + if " " in addon: + addon_and_branch = addon.split(" ") + custom_addons.append({"url":addon_and_branch[0], "branch":addon_and_branch[1]}) + else: + custom_addons.append({"url":addon, "branch":"master"}) + for addon in custom_addons: + if self.current_thread.isInterruptionRequested(): + return + if addon and addon["url"]: + name = addon["url"].split("/")[-1] if name.lower().endswith(".git"): name = name[:-4] + if name in package_names: + # We already have this one cached since it's a package + continue addondir = moddir + os.sep + name - if not os.path.exists(addondir): - state = 0 + if os.path.exists(addondir) and os.listdir(addondir): + state = AddonManagerRepo.UpdateStatus.UNCHECKED else: - state = 1 - repos.append([name, url, state]) - if not repos: - self.info_label.emit(translate("AddonsInstaller", "Unable to download addon list.")) - else: - repos = sorted(repos, key=lambda s: s[0].lower()) - for repo in repos: - self.addon_repo.emit(repo) - self.info_label.emit(translate("AddonsInstaller", "Workbenches list was updated.")) - self.progressbar_show.emit(False) - self.done.emit() - self.stop = True + state = AddonManagerRepo.UpdateStatus.NOT_INSTALLED + self.addon_repo.emit(AddonManagerRepo(name, addon["url"], state, addon["branch"])) + self.status_message.emit(translate("AddonsInstaller", "Workbenches list was updated.")) + + if not self.current_thread.isInterruptionRequested(): + self.done.emit() + self.stop = True - -class InfoWorker(QtCore.QThread): - """This worker retrieves the description text of a workbench""" - - addon_repos = QtCore.Signal(object) - - def __init__(self): - - QtCore.QThread.__init__(self) - - def run(self): - - i = 0 - for repo in self.repos: - url = repo[1] - u = utils.urlopen(url) - if not u: - self.stop = True - return - p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): - p = p.decode("utf-8") - u.close() - desc = re.findall(' installed_metadata.Version: + self.update_status.emit(package, AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE) + else: + self.update_status.emit(package, AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE) + + + def check_macro(self, macro_wrapper): + # Make sure this macro has its code downloaded: + try: + if not macro_wrapper.macro.parsed and macro_wrapper.macro.on_git: + macro_wrapper.macro.fill_details_from_file(macro_wrapper.macro.src_filename) + if not macro_wrapper.macro.parsed and macro_wrapper.macro.on_wiki: + mac = macro_wrapper.macro.name.replace(" ", "_") + mac = mac.replace("&", "%26") + mac = mac.replace("+", "%2B") + url = "https://wiki.freecad.org/Macro_" + mac + macro_wrapper.macro.fill_details_from_wiki(url) + except Exception: + FreeCAD.Console.PrintWarning(f"Failed to fetch code for macro '{macro_wrapper.macro.name}'\n") + return + + hasher1 = hashlib.sha1() + hasher2 = hashlib.sha1() + hasher1.update(macro_wrapper.macro.code.encode("utf-8")) + new_sha1 = hasher1.hexdigest() + test_file_one = os.path.join(FreeCAD.getUserMacroDir(True), macro_wrapper.macro.filename) + test_file_two = os.path.join(FreeCAD.getUserMacroDir(True), "Macro_" + macro_wrapper.macro.filename) + if os.path.exists(test_file_one): + with open(test_file_one, "rb") as f: + contents = f.read() + hasher2.update(contents) + old_sha1 = hasher2.hexdigest() + elif os.path.exists(test_file_two): + with open(test_file_two, "rb") as f: + contents = f.read() + hasher2.update(contents) + old_sha1 = hasher2.hexdigest() + else: + return + if new_sha1 == old_sha1: + self.update_status.emit(macro_wrapper, AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE) + else: + self.update_status.emit(macro_wrapper, AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE) + class FillMacroListWorker(QtCore.QThread): """This worker populates the list of macros""" - add_macro_signal = QtCore.Signal(Macro) - info_label_signal = QtCore.Signal(str) - progressbar_show = QtCore.Signal(bool) + add_macro_signal = QtCore.Signal(object) + status_message_signal = QtCore.Signal(str) + progress_made = QtCore.Signal(int, int) + done = QtCore.Signal() def __init__(self, repo_dir): QtCore.QThread.__init__(self) self.repo_dir = repo_dir - self.macros = [] + self.repo_names = [] def run(self): """Populates the list of macros""" + + self.current_thread = QtCore.QThread.currentThread() - self.retrieve_macros_from_git() - self.retrieve_macros_from_wiki() - [self.add_macro_signal.emit(m) for m in sorted(self.macros, key=lambda m: m.name.lower())] - if self.macros: - self.info_label_signal.emit(translate("AddonsInstaller", "List of macros successfully retrieved.")) - self.progressbar_show.emit(False) + if not self.current_thread.isInterruptionRequested(): + self.status_message_signal.emit(translate("AddonInstaller","Retrieving macros from FreeCAD/FreeCAD-Macros Git repository")) + self.retrieve_macros_from_git() + + if not self.current_thread.isInterruptionRequested(): + self.status_message_signal.emit(translate("AddonInstaller","Retrieving macros from FreeCAD/FreeCAD-Macros Git repository")) + self.retrieve_macros_from_wiki() + + if self.current_thread.isInterruptionRequested(): + return + + self.status_message_signal.emit(translate("AddonsInstaller", "Done locating macros.")) self.stop = True + self.done.emit() def retrieve_macros_from_git(self): """Retrieve macros from FreeCAD-macros.git @@ -305,120 +396,129 @@ class FillMacroListWorker(QtCore.QThread): """ if not have_git: - self.info_label_signal.emit("GitPython not installed! Cannot retrieve macros from Git") + self.status_message_signal.emit("GitPython not installed! Cannot retrieve macros from Git") FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "GitPython not installed! Cannot retrieve macros from git")+"\n") + "GitPython not installed! Cannot retrieve macros from git") + "\n") return - self.info_label_signal.emit("Downloading list of macros from git...") try: + # TODO: someday see if the directory exists, and do a pull instead + # of a clone git.Repo.clone_from("https://github.com/FreeCAD/FreeCAD-macros.git", self.repo_dir) except Exception: FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "Something went wrong with the Git Macro Retrieval, " "possibly the Git executable is not in the path") + "\n") + n_files = 0 + for _, _, filenames in os.walk(self.repo_dir): + n_files += len(filenames) + counter = 0 for dirpath, _, filenames in os.walk(self.repo_dir): + self.progress_made.emit(counter, n_files) + counter += 1 + if self.current_thread.isInterruptionRequested(): + return if ".git" in dirpath: continue for filename in filenames: + if self.current_thread.isInterruptionRequested(): + return if filename.lower().endswith(".fcmacro"): macro = Macro(filename[:-8]) # Remove ".FCMacro". macro.on_git = True macro.src_filename = os.path.join(dirpath, filename) - self.macros.append(macro) + repo = AddonManagerRepo.from_macro(macro) + repo.url = "https://github.com/FreeCAD/FreeCAD-macros.git" + self.add_macro_signal.emit(repo) def retrieve_macros_from_wiki(self): """Retrieve macros from the wiki Read the wiki and emit a signal for each found macro. - Reads only the page https://wiki.freecadweb.org/Macros_recipes + Reads only the page https://wiki.freecad.org/Macros_recipes """ - self.info_label_signal.emit("Downloading list of macros from the FreeCAD wiki...") - self.progressbar_show.emit(True) - u = utils.urlopen("https://wiki.freecadweb.org/Macros_recipes") + u = utils.urlopen("https://wiki.freecad.org/Macros_recipes") if not u: FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "Appears to be an issue connecting to the Wiki, " - "therefore cannot retrieve Wiki macro list at this time") + "\n") + "There appears to be an issue connecting to the Wiki, " + "therefore FreeCAD cannot retrieve the Wiki macro list at this time") + "\n") return p = u.read() u.close() - if sys.version_info.major >= 3 and isinstance(p, bytes): + if isinstance(p, bytes): p = p.decode("utf-8") macros = re.findall('title="(Macro.*?)"', p) macros = [mac for mac in macros if ("translated" not in mac)] - for mac in macros: + macro_names = [] + for i, mac in enumerate(macros): + self.progress_made.emit(i, len(macros)) + if self.current_thread.isInterruptionRequested(): + return macname = mac[6:] # Remove "Macro ". macname = macname.replace("&", "&") - if (macname not in macros_blacklist) and ("recipes" not in macname.lower()): + if not macname: + continue + if (macname not in macros_blacklist) and ("recipes" not in macname.lower()) and (macname not in macro_names): + macro_names.append(macname) macro = Macro(macname) macro.on_wiki = True - self.macros.append(macro) + repo = AddonManagerRepo.from_macro(macro) + repo.url = "https://wiki.freecad.org/Macros_recipes" + self.add_macro_signal.emit(repo) class ShowWorker(QtCore.QThread): """This worker retrieves info of a given workbench""" - info_label = QtCore.Signal(str) + status_message = QtCore.Signal(str) + description_updated = QtCore.Signal(str) addon_repos = QtCore.Signal(object) - progressbar_show = QtCore.Signal(bool) + done = QtCore.Signal() - def __init__(self, repos, idx): - - # repos is a list of [name, url, installbit, descr] - # name : Addon name - # url : Addon repository location - # installbit: 0 = Addon is not installed - # 1 = Addon is installed - # 2 = Addon is installed and checked for available updates (none pending) - # 3 = Addon is installed and has a pending update - # descr : Addon description + def __init__(self, repo): QtCore.QThread.__init__(self) - self.repos = repos - self.idx = idx + self.repo = repo def run(self): - - self.progressbar_show.emit(True) - self.info_label.emit(translate("AddonsInstaller", "Retrieving description...")) - if len(self.repos[self.idx]) == 4: - desc = self.repos[self.idx][3] + self.status_message.emit(translate("AddonsInstaller", "Retrieving description...")) + if self.repo.description is not None: + desc = self.repo.description else: u = None - url = self.repos[self.idx][1] - self.info_label.emit(translate("AddonsInstaller", "Retrieving info from") + " " + str(url)) + url = self.repo.url + self.status_message.emit(translate("AddonsInstaller", "Retrieving info from") + " " + str(url)) desc = "" - regex = utils.get_readme_regex(url) + regex = utils.get_readme_regex(self.repo) if regex: # extract readme from html via regex - readmeurl = utils.get_readme_html_url(url) + readmeurl = utils.get_readme_html_url(self.repo) if not readmeurl: - print("Debug: README not found for", url) + FreeCAD.Console.PrintWarning(f"Debug: README not found for {url}\n") u = utils.urlopen(readmeurl) if not u: - print("Debug: README not found at", readmeurl) + FreeCAD.Console.PrintWarning(f"Debug: README not found at {readmeurl}\n") u = utils.urlopen(readmeurl) if u: p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): + if isinstance(p, bytes): p = p.decode("utf-8") u.close() readme = re.findall(regex, p, flags=re.MULTILINE | re.DOTALL) if readme: desc = readme[0] else: - print("Debug: README not found at", readmeurl) + FreeCAD.Console.PrintWarning(f"Debug: README not found at {readmeurl}\n") else: # convert raw markdown using lib - readmeurl = utils.get_readme_url(url) + readmeurl = utils.get_readme_url(self.repo) if not readmeurl: - print("Debug: README not found for", url) + FreeCAD.Console.PrintWarning(f"Debug: README not found for {url}\n") u = utils.urlopen(readmeurl) if u: p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): + if isinstance(p, bytes): p = p.decode("utf-8") u.close() desc = utils.fix_relative_links(p, readmeurl.rsplit("/README.md")[0]) @@ -435,38 +535,37 @@ class ShowWorker(QtCore.QThread): message += "
" + desc + "
" desc = message else: - print("Debug: README not found at", readmeurl) + FreeCAD.Console.PrintWarning("Debug: README not found at {readmeurl}\n") if desc == "": # fall back to the description text u = utils.urlopen(url) if not u: - self.progressbar_show.emit(False) self.stop = True return p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): + if isinstance(p, bytes): p = p.decode("utf-8") u.close() - descregex = utils.get_desc_regex(url) + descregex = utils.get_desc_regex(self.repo) if descregex: desc = re.findall(descregex, p) if desc: desc = desc[0] if not desc: desc = "Unable to retrieve addon description" - self.repos[self.idx].append(desc) - self.addon_repos.emit(self.repos) + self.repo.description = desc + self.addon_repos.emit(self.repo) # Addon is installed so lets check if it has an update - if self.repos[self.idx][2] == 1: + if self.repo.update_status == AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE: upd = False # checking for updates if not NOGIT and have_git: - repo = self.repos[self.idx] - clonedir = FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + repo[0] + repo = self.repo + clonedir = FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + repo.name if os.path.exists(clonedir): if not os.path.exists(clonedir + os.sep + ".git"): # Repair addon installed with raw download - bare_repo = git.Repo.clone_from(repo[1], clonedir + os.sep + ".git", bare=True) + bare_repo = git.Repo.clone_from(repo.url, clonedir + os.sep + ".git", bare=True) try: with bare_repo.config_writer() as cw: cw.set("core", "bare", False) @@ -492,9 +591,8 @@ class ShowWorker(QtCore.QThread): """ message += translate("AddonsInstaller", "An update is available for this addon.") message += "
" + desc + '

Addon repository: ' + self.repos[self.idx][1] + "" - # mark as already installed AND already checked for updates AND update is available - self.repos[self.idx][2] = 3 + message += self.repo.url + '">' + self.repo.url + "" + self.repo.update_status = AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE # If there isn't, indicate that this addon is already installed else: message = """ @@ -505,13 +603,24 @@ class ShowWorker(QtCore.QThread): message += translate("AddonsInstaller", "This addon is already installed.") message += "
" + desc message += '

Addon repository: ' + self.repos[self.idx][1] + "" - self.repos[self.idx][2] = 2 # mark as already installed AND already checked for updates + message += self.repo.url + '">' + self.repo.url + "" + self.repo.update_status = AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE # Let the user know the install path for this addon message += "
" + translate("AddonInstaller", "Installed location") + ": " - message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repos[self.idx][0] - self.addon_repos.emit(self.repos) - elif self.repos[self.idx][2] == 2: + message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repo.name + self.addon_repos.emit(self.repo) + elif self.repo.update_status == AddonManagerRepo.UpdateStatus.PENDING_RESTART: + message = """ +
+
+ +""" + message += translate("AddonsInstaller", "This addon has been updated, a restart is now required before it can be used.") + message += "
" + desc + '

Addon repository: ' + self.repo.url + "" + message += "
" + translate("AddonInstaller", "Installed location") + ": " + message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repo.name + elif self.repo.update_status == AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE: message = """

@@ -520,10 +629,10 @@ class ShowWorker(QtCore.QThread): message += translate("AddonsInstaller", "This addon is already installed.") message += "
" + desc message += '

Addon repository: ' + self.repos[self.idx][1] + "" + message += self.repo.url + '">' + self.repo.url + "" message += "
" + translate("AddonInstaller", "Installed location") + ": " - message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repos[self.idx][0] - elif self.repos[self.idx][2] == 3: + message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repo.name + elif self.repo.update_status == AddonManagerRepo.UpdateStatus.UPDATE_AVAILABLE: message = """

@@ -531,15 +640,15 @@ class ShowWorker(QtCore.QThread): """ message += translate("AddonsInstaller", "An update is available for this addon.") message += "
" + desc + '

Addon repository: ' + self.repos[self.idx][1] + "" + message += self.repo.url + '">' + self.repo.url + "" message += "
" + translate("AddonInstaller", "Installed location") + ": " - message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repos[self.idx][0] + message += FreeCAD.getUserAppDataDir() + os.sep + "Mod" + os.sep + self.repo.name else: message = desc + '

Addon repository: ' + self.repos[self.idx][1] + '' + message += self.repo.url + '">' + self.repo.url + '' # If the Addon is obsolete, let the user know through the Addon UI - if self.repos[self.idx][0] in obsolete: + if self.repo.name in obsolete: message = """
@@ -551,7 +660,7 @@ class ShowWorker(QtCore.QThread): "provides the same functionality.") + "
" + desc # If the Addon is Python 2 only, let the user know through the Addon UI - if self.repos[self.idx][0] in py2only: + if self.repo.name in py2only: message = """
@@ -563,12 +672,12 @@ class ShowWorker(QtCore.QThread): "likely result in errors at startup or while in use.") message += "
" + desc - self.info_label.emit(message) - self.progressbar_show.emit(False) + self.description_updated.emit(message) self.mustLoadImages = True - label = self.loadImages(message, self.repos[self.idx][1], self.repos[self.idx][0]) + label = self.loadImages(message, self.repo.url, self.repo.name) if label: - self.info_label.emit(label) + self.description_updated.emit(label) + self.done.emit() self.stop = True def stopImageLoading(self): @@ -579,7 +688,7 @@ class ShowWorker(QtCore.QThread): def loadImages(self, message, url, wbName): "checks if the given page contains images and downloads them" - # QTextBrowser cannot display online images. So we download them + # QTextBrowser cannot display online images. So we download them # here, and replace the image link in the html code with the # downloaded version @@ -590,8 +699,10 @@ class ShowWorker(QtCore.QThread): if not os.path.exists(store): os.makedirs(store) for path in imagepaths: + if QtCore.QThread.currentThread().isInterruptionRequested(): + return message if not self.mustLoadImages: - return None + return message origpath = path if "?" in path: # remove everything after the ? @@ -603,19 +714,20 @@ class ShowWorker(QtCore.QThread): storename = os.path.join(store, name) if len(storename) >= 260: remainChars = 259 - (len(store) + len(wbName) + 1) - storename = os.path.join(store, wbName+name[-remainChars:]) + storename = os.path.join(store, wbName + name[-remainChars:]) if not os.path.exists(storename): try: u = utils.urlopen(path) imagedata = u.read() u.close() except Exception: - print("AddonManager: Debug: Error retrieving image from", path) + FreeCAD.Console.PrintLog("AddonManager: Debug: Error retrieving image from", path) else: try: f = open(storename, "wb") except OSError: - # ecryptfs (and probably not only ecryptfs) has lower length limit for path + # ecryptfs (and probably not only ecryptfs) has + # lower length limit for path storename = storename[-140:] f = open(storename, "wb") f.write(imagedata) @@ -628,8 +740,7 @@ class ShowWorker(QtCore.QThread): QtCore.Qt.KeepAspectRatio, QtCore.Qt.FastTransformation)) pix.save(storename, "jpeg", 100) - message = message.replace("src=\""+origpath, "src=\"file:///"+storename.replace("\\", "/")) - # print(message) + message = message.replace("src=\"" + origpath, "src=\"file:///" + storename.replace("\\", "/")) return message return None @@ -637,151 +748,158 @@ class ShowWorker(QtCore.QThread): class GetMacroDetailsWorker(QtCore.QThread): """Retrieve the macro details for a macro""" - info_label = QtCore.Signal(str) - progressbar_show = QtCore.Signal(bool) + status_message = QtCore.Signal(str) + description_updated = QtCore.Signal(str) + done = QtCore.Signal() - def __init__(self, macro): + def __init__(self, repo): QtCore.QThread.__init__(self) - self.macro = macro + self.macro = repo.macro def run(self): - self.progressbar_show.emit(True) - self.info_label.emit(translate("AddonsInstaller", "Retrieving description...")) + self.status_message.emit(translate("AddonsInstaller", "Retrieving macro description...")) if not self.macro.parsed and self.macro.on_git: - self.info_label.emit(translate("AddonsInstaller", "Retrieving info from git")) + self.status_message.emit(translate("AddonsInstaller", "Retrieving info from git")) self.macro.fill_details_from_file(self.macro.src_filename) if not self.macro.parsed and self.macro.on_wiki: - self.info_label.emit(translate("AddonsInstaller", "Retrieving info from wiki")) + self.status_message.emit(translate("AddonsInstaller", "Retrieving info from wiki")) mac = self.macro.name.replace(" ", "_") mac = mac.replace("&", "%26") mac = mac.replace("+", "%2B") - url = "https://wiki.freecadweb.org/Macro_" + mac + url = "https://wiki.freecad.org/Macro_" + mac self.macro.fill_details_from_wiki(url) if self.macro.is_installed(): - already_installed_msg = ('' - + translate("AddonsInstaller", "This macro is already installed.") - + '
') + already_installed_msg = ('' + translate("AddonsInstaller", "This macro is already installed.") + '
') else: already_installed_msg = "" - message = (already_installed_msg - + "

"+self.macro.name+"

" - + self.macro.desc - + "

Macro location: " - + self.macro.url - + "") - self.info_label.emit(message) - self.progressbar_show.emit(False) + message = (already_installed_msg + "

" + self.macro.name + "

" + self.macro.desc + "

Macro location: " + self.macro.url + "") + self.description_updated.emit(message) + self.done.emit() self.stop = True -class InstallWorker(QtCore.QThread): +class InstallWorkbenchWorker(QtCore.QThread): "This worker installs a workbench" - info_label = QtCore.Signal(str) - progressbar_show = QtCore.Signal(bool) - mark_recompute = QtCore.Signal(str) + status_message = QtCore.Signal(str) + progress_made = QtCore.Signal(int, int) + success = QtCore.Signal(AddonManagerRepo, str) + failure = QtCore.Signal(AddonManagerRepo, str) - def __init__(self, repos, idx): + def __init__(self, repo:AddonManagerRepo): QtCore.QThread.__init__(self) - self.idx = idx - self.repos = repos + self.repo = repo + if have_git: + self.git_progress = GitProgressMonitor() + # TODO: What is wrong with these? + #self.git_progress.progress_made.connect(self.progress_made.emit) + #self.git_progress.info_message.connect(self.status_message.emit) def run(self): "installs or updates the selected addon" + + if not self.repo: + return if not have_git: - self.info_label.emit("GitPython not found.") FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "GitPython not found. Using standard download instead.") + "\n") + "GitPython not found. Using ZIP file download instead.") + "\n") if not have_zip: - self.info_label.emit("no zip support.") FreeCAD.Console.PrintError(translate("AddonsInstaller", "Your version of python doesn't appear to support ZIP " "files. Unable to proceed.") + "\n") return - if not isinstance(self.idx, list): - self.idx = [self.idx] - for idx in self.idx: - if idx < 0: - return - if not self.repos: - return - basedir = FreeCAD.getUserAppDataDir() - moddir = basedir + os.sep + "Mod" - if not os.path.exists(moddir): - os.makedirs(moddir) - clonedir = moddir + os.sep + self.repos[idx][0] - self.progressbar_show.emit(True) - if os.path.exists(clonedir): - self.info_label.emit("Updating module...") - if sys.version_info.major > 2 and str(self.repos[idx][0]) in py2only: - FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "User requested updating a Python 2 workbench on " - "a system running Python 3 - ") + - str(self.repos[idx][0])+"\n") - if have_git: - if not os.path.exists(clonedir + os.sep + ".git"): - # Repair addon installed with raw download - bare_repo = git.Repo.clone_from(self.repos[idx][1], clonedir + os.sep + ".git", bare=True) - try: - with bare_repo.config_writer() as cw: - cw.set("core", "bare", False) - except AttributeError: - FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "Outdated GitPython detected, consider " - "upgrading with pip.") + "\n") - cw = bare_repo.config_writer() - cw.set("core", "bare", False) - del cw - repo = git.Repo(clonedir) - repo.head.reset("--hard") - repo = git.Git(clonedir) - try: - answer = repo.pull() + "\n\n" + translate("AddonsInstaller", - "Workbench successfully updated. " - "Please restart FreeCAD to apply the changes.") - except Exception: - print("Error updating module", self.repos[idx][1], " - Please fix manually") - answer = repo.status() - print(answer) - else: - # Update the submodules for this repository - repo_sms = git.Repo(clonedir) - for submodule in repo_sms.submodules: - submodule.update(init=True, recursive=True) - else: - answer = self.download(self.repos[idx][1], clonedir) + "\n\n" - answer += translate("AddonsInstaller", - "Workbench successfully updated. Please " - "restart FreeCAD to apply the changes.") - else: - self.info_label.emit("Checking module dependencies...") - depsok, answer = self.check_dependencies(self.repos[idx][1]) - if depsok: - if sys.version_info.major > 2 and str(self.repos[idx][0]) in py2only: - FreeCAD.Console.PrintWarning(translate("AddonsInstaller", - "User requested installing a Python 2 " - "workbench on a system running Python 3 - ") + - str(self.repos[idx][0]) + "\n") - if have_git: - self.info_label.emit("Cloning module...") - repo = git.Repo.clone_from(self.repos[idx][1], clonedir) + basedir = FreeCAD.getUserAppDataDir() + moddir = basedir + os.sep + "Mod" + if not os.path.exists(moddir): + os.makedirs(moddir) + target_dir = moddir + os.sep + self.repo.name - # Make sure to clone all the submodules as well - if repo.submodules: - repo.submodule_update(recursive=True) - else: - self.info_label.emit("Downloading module...") - self.download(self.repos[idx][1], clonedir) - answer = translate("AddonsInstaller", - "Workbench successfully installed. Please restart " - "FreeCAD to apply the changes.") + if have_git: + self.run_git(target_dir) + else: + self.run_zip(target_dir) + + self.stop = True + + def run_git(self, clonedir:str) -> None: + + if os.path.exists(clonedir): + self.run_git_update(clonedir) + else: + self.run_git_clone(clonedir) + + def run_git_update(self, clonedir:str) -> None: + self.status_message.emit("Updating module...") + if str(self.repo.name) in py2only: + FreeCAD.Console.PrintWarning(translate("AddonsInstaller", + "User requested updating a Python 2 workbench on " + "a system running Python 3 - ") + str(self.repo.name) + "\n") + if not os.path.exists(clonedir + os.sep + ".git"): + # Repair addon installed with raw download by adding the .git + # directory to it + bare_repo = git.Repo.clone_from(self.repo.url, clonedir + os.sep + ".git", bare=True) + try: + with bare_repo.config_writer() as cw: + cw.set("core", "bare", False) + except AttributeError: + FreeCAD.Console.PrintWarning(translate("AddonsInstaller", + "Outdated GitPython detected, consider " + "upgrading with pip.") + "\n") + cw = bare_repo.config_writer() + cw.set("core", "bare", False) + del cw + repo = git.Repo(clonedir) + repo.head.reset("--hard") + repo = git.Git(clonedir) + try: + repo.pull() + answer = translate("AddonsInstaller","Workbench successfully updated. " + "Please restart FreeCAD to apply the changes.") + except Exception: + answer = translate("AddonsInstaller", "Error updating module ") + \ + self.repo.name + " - " + \ + translate("AddonsInstaller", "Please fix manually") + answer += repo.status() + self.failure.emit(self.repo, answer) + else: + # Update the submodules for this repository + repo_sms = git.Repo(clonedir) + self.status_message.emit("Updating submodules...") + for submodule in repo_sms.submodules: + submodule.update(init=True, recursive=True) + self.success.emit(self.repo, answer) + + def run_git_clone(self, clonedir:str) -> None: + self.status_message.emit("Checking module dependencies...") + depsok, answer = self.check_python_dependencies(self.repo.url) + if depsok: + if str(self.repo.name) in py2only: + FreeCAD.Console.PrintWarning(translate("AddonsInstaller", + "User requested installing a Python 2 " + "workbench on a system running Python 3 - ") + str(self.repo.name) + "\n") + self.status_message.emit("Cloning module...") + repo = git.Repo.clone_from(self.repo.url, clonedir) + + # Make sure to clone all the submodules as well + if repo.submodules: + repo.submodule_update(recursive=True) + + if self.repo.branch in repo.heads: + repo.heads[self.repo.branch].checkout() + + answer = translate("AddonsInstaller", + "Workbench successfully installed. Please restart " + "FreeCAD to apply the changes.") + else: + self.emit.failure(self.repo, answer) + return + + if self.repo.repo_type == AddonManagerRepo.RepoType.WORKBENCH: # symlink any macro contained in the module to the macros folder macro_dir = FreeCAD.getUserMacroDir(True) if not os.path.exists(macro_dir): @@ -789,21 +907,16 @@ class InstallWorker(QtCore.QThread): if os.path.exists(clonedir): for f in os.listdir(clonedir): if f.lower().endswith(".fcmacro"): - print("copying macro:", f) utils.symlink(os.path.join(clonedir, f), os.path.join(macro_dir, f)) - FreeCAD.ParamGet('User parameter:Plugins/' + - self.repos[idx][0]).SetString("destination", clonedir) + FreeCAD.ParamGet('User parameter:Plugins/' + self.repo.name).SetString("destination", clonedir) answer += "\n\n" + translate("AddonsInstaller", - "A macro has been installed and is available " - "under Macro -> Macros menu") + "A macro has been installed and is available " + "under Macro -> Macros menu") answer += ":\n" + f + "" - self.progressbar_show.emit(False) - self.info_label.emit(answer) - self.mark_recompute.emit(self.repos[idx][0]) - self.stop = True + self.success.emit(self.repo, answer) - def check_dependencies(self, baseurl): - "checks if the repo contains a metadata.txt and check its contents" + def check_python_dependencies(self, baseurl:str) -> [bool,str]: + """checks if the repo contains a metadata.txt and check its contents""" ok = True message = "" @@ -817,7 +930,8 @@ class InstallWorker(QtCore.QThread): depsfile = mu.read() mu.close() - # urllib2 gives us a bytelike object instead of a string. Have to consider that + # urllib2 gives us a bytelike object instead of a string. Have to + # consider that try: depsfile = depsfile.decode("utf-8") except AttributeError: @@ -830,7 +944,7 @@ class InstallWorker(QtCore.QThread): for wb in depswb: if wb.strip(): if not wb.strip() in FreeCADGui.listWorkbenches().keys(): - if not wb.strip()+"Workbench" in FreeCADGui.listWorkbenches().keys(): + if not wb.strip() + "Workbench" in FreeCADGui.listWorkbenches().keys(): ok = False message += translate("AddonsInstaller", "Missing workbench") + ": " + wb + ", " elif line.startswith("pylibs="): @@ -858,40 +972,66 @@ class InstallWorker(QtCore.QThread): message += translate("AddonsInstaller", "Please install the missing components first.") return ok, message - def download(self, baseurl, clonedir): + def check_package_dependencies(self): + # TODO: Use the dependencies set in the package.xml metadata + pass + + def run_zip(self, zipdir:str) -> None: "downloads and unzip a zip version from a git repo" bakdir = None - if os.path.exists(clonedir): - bakdir = clonedir+".bak" + if os.path.exists(zipdir): + bakdir = zipdir + ".bak" if os.path.exists(bakdir): shutil.rmtree(bakdir) - os.rename(clonedir, bakdir) - os.makedirs(clonedir) - zipurl = utils.get_zip_url(baseurl) + os.rename(zipdir, bakdir) + os.makedirs(zipdir) + zipurl = utils.get_zip_url(self.repo) if not zipurl: - return translate("AddonsInstaller", "Error: Unable to locate zip from") + " " + baseurl + self.failure.emit(self.repo, translate("AddonsInstaller", "Error: Unable to locate zip from") + " " + self.repo.name) + return try: - print("Downloading "+zipurl) u = utils.urlopen(zipurl) except Exception: - return translate("AddonsInstaller", "Error: Unable to download") + " " + zipurl + self.failure.emit(self.repo, translate("AddonsInstaller", "Error: Unable to download") + " " + zipurl) + return if not u: - return translate("AddonsInstaller", "Error: Unable to download") + " " + zipurl - zfile = _stringio() - zfile.write(u.read()) - zfile = zipfile.ZipFile(zfile) + self.failure.emit(self.repo, translate("AddonsInstaller", "Error: Unable to download") + " " + zipurl) + return + + data_size = u.headers['content-length'] + + current_thread = QtCore.QThread.currentThread() + if data_size and data_size > 5 * 1024 * 1024: + # Use an on-disk file and track download progress, if the zipfile + # is over 5mb + with tempfile.NamedTemporaryFile(delete=False) as temp: + bytes_to_read = 16 * 1024 + bytes_read = 0 + while True: + if current_thread.isInterruptionRequested(): + return + chunk = u.read(bytes_to_read) + if not chunk: + break + bytes_read += bytes_to_read + temp.write(chunk) + self.progress_made.emit(bytes_read, data_size) + zfile = zipfile.ZipFile(temp) + else: + zfile = io.BytesIO() + zfile.write(u.read()) + zfile = zipfile.ZipFile(zfile) master = zfile.namelist()[0] # github will put everything in a subfolder - zfile.extractall(clonedir) + zfile.extractall(zipdir) u.close() zfile.close() - for filename in os.listdir(clonedir+os.sep+master): - shutil.move(clonedir+os.sep+master+os.sep+filename, clonedir+os.sep+filename) - os.rmdir(clonedir+os.sep+master) + for filename in os.listdir(zipdir + os.sep + master): + shutil.move(zipdir + os.sep + master + os.sep + filename, zipdir + os.sep + filename) + os.rmdir(zipdir + os.sep + master) if bakdir: shutil.rmtree(bakdir) - return translate("AddonsInstaller", "Successfully installed") + " " + zipurl - + self.success.emit(self.repo, translate("AddonsInstaller", "Successfully installed") + " " + zipurl) class CheckSingleWorker(QtCore.QThread): """Worker to check for updates for a single addon""" @@ -907,7 +1047,7 @@ class CheckSingleWorker(QtCore.QThread): if not have_git: return - FreeCAD.Console.PrintLog("Checking for available updates of the "+self.name+" addon\n") + FreeCAD.Console.PrintLog("Checking for available updates of the " + self.name + " addon\n") addondir = os.path.join(FreeCAD.getUserAppDataDir(), "Mod", self.name) if os.path.exists(addondir): if os.path.exists(addondir + os.sep + ".git"): @@ -918,8 +1058,233 @@ class CheckSingleWorker(QtCore.QThread): self.updateAvailable.emit(True) return except Exception: - # can fail for any number of reasons, ex. not being online + # can fail for any number of reasons, ex. not being online pass self.updateAvailable.emit(False) -# @} +class UpdateMetadataCacheWorker(QtCore.QThread): + "Scan through all available packages and see if our local copy of package.xml needs to be updated" + + status_message = QtCore.Signal(str) + progress_made = QtCore.Signal(int, int) + done = QtCore.Signal() + package_updated = QtCore.Signal(AddonManagerRepo) + + class AtomicCounter(object): + def __init__(self,start=0): + self.lock = threading.Lock() + self.count = start + def set(self,new_value): + with self.lock: + self.count = new_value + def get(self): + with self.lock: + return self.count + def increment(self): + with self.lock: + self.count += 1 + def decrement(self): + with self.lock: + self.count -= 1 + + def __init__(self, repos): + + QtCore.QThread.__init__(self) + self.repos = repos + self.counter = UpdateMetadataCacheWorker.AtomicCounter() + + def run(self): + if not have_git: + return + current_thread = QtCore.QThread.currentThread() + self.num_downloads_required = len(self.repos) + self.progress_made.emit(0, self.num_downloads_required) + self.status_message.emit(translate("AddonsInstaller", "Retrieving package metadata...")) + store = os.path.join(FreeCAD.getUserAppDataDir(), "AddonManager", "PackageMetadata") + index_file = os.path.join(store,"index.json") + self.index = {} + if os.path.isfile(index_file): + with open(index_file, "r") as f: + index_string = f.read() + self.index = json.loads(index_string) + + download_queue = QtNetwork.QNetworkAccessManager() # Must be created on this thread + download_queue.finished.connect(self.on_finished) + + self.downloaders = [] + for repo in self.repos: + downloader = MetadataDownloadWorker(None, repo, self.index) + downloader.start_fetch(download_queue) + downloader.updated.connect(self.on_updated) + self.downloaders.append(downloader) + + # Run a local event loop until we've processed all of the downloads: + # this is local + # to this thread, and does not affect the main event loop + ui_updater = QtCore.QTimer() + ui_updater.timeout.connect(self.send_ui_update) + ui_updater.start(100) # Send an update back to the main thread every 100ms + self.num_downloads_required = len(self.repos) + self.num_downloads_completed = UpdateMetadataCacheWorker.AtomicCounter() + while True: + if current_thread.isInterruptionRequested(): + for downloader in self.downloaders: + downloader.abort() + QtCore.QCoreApplication.processEvents() + if self.num_downloads_completed.get() == self.num_downloads_required: + break + + if current_thread.isInterruptionRequested(): + FreeCAD.Console.PrintMessage("Bailing out of downloads") + return + + # Update and serialize the updated index, overwriting whatever was + # there before + for downloader in self.downloaders: + self.index[downloader.repo.name] = downloader.last_sha1 + if not os.path.exists(store): + os.makedirs(store) + with open(index_file, "w") as f: + json.dump(self.index, f, indent=" ") + + # Signal completion to our parent thread + self.done.emit() + self.stop = True + + def on_finished(self, reply): + # Called by the QNetworkAccessManager's sub-threads when a fetch + # process completed (in any state) + self.num_downloads_completed.increment() + + def on_updated(self, repo): + # Called if this repo got new metadata and/or a new icon + self.package_updated.emit(repo) + + def send_ui_update(self): + self.progress_made.emit(self.num_downloads_completed.get(), self.num_downloads_required) + + +class GitProgressMonitor(git.RemoteProgress): + """ An object that receives git progress updates and transforms them into Qt signals """ + + progress_made = QtCore.Signal(int, int) + info_message = QtCore.Signal(str) + + def __init__(self): + super().__init__() + + def update(self, op_code: int, cur_count: Union[str, float], max_count: Union[str, float, None]=None, message: str='') -> None: + if max_count: + self.progress_made.emit(int(cur_count), int(max_count)) + if message: + self.info_message.emit(message) + + +class UpdateAllWorker(QtCore.QThread): + """ Update all listed packages, of any kind """ + + progress_made = QtCore.Signal(int, int) + status_message = QtCore.Signal(str) + success = QtCore.Signal(AddonManagerRepo) + failure = QtCore.Signal(AddonManagerRepo) + done = QtCore.Signal() + + def __init__(self, repos): + super().__init__() + self.repos = repos + + def run (self): + self.progress_made.emit(0,len(self.repos)) + self.repo_queue = queue.Queue() + current_thread = QtCore.QThread.currentThread() + for repo in self.repos: + self.repo_queue.put(repo) + + # Following the QNetworkAccessManager model, we'll spawn six threads to process these requests in parallel: + workers = [] + for _ in range(6): + worker = UpdateSingleWorker(self.repo_queue) + worker.success.connect(self.on_success) + worker.failure.connect(self.on_failure) + worker.start() + workers.append(worker) + + while not self.repo_queue.empty(): + if current_thread.isInterruptionRequested(): + for worker in workers: + worker.requestInterruption() + worker.wait() + return + # Ensure our signals propagate out by running an internal thread-local event loop + QtCore.QCoreApplication.processEvents() + + self.repo_queue.join() + + # Make sure all of our child threads have fully exited: + for i, worker in enumerate(workers): + worker.wait() + + self.done.emit() + + def on_success(self, repo:AddonManagerRepo) -> None: + self.progress_made.emit(self.repo_queue.qsize(), len(self.repos)) + self.success.emit(repo) + + def on_failure(self, repo:AddonManagerRepo) -> None: + self.progress_made.emit(self.repo_queue.qsize(), len(self.repos)) + self.failure.emit(repo) + +class UpdateSingleWorker(QtCore.QThread): + success = QtCore.Signal(AddonManagerRepo) + failure = QtCore.Signal(AddonManagerRepo) + + def __init__(self,repo_queue:queue.Queue): + super().__init__() + self.repo_queue = repo_queue + + def run(self): + current_thread = QtCore.QThread.currentThread() + while True: + if current_thread.isInterruptionRequested(): + return + try: + repo = self.repo_queue.get_nowait() + except queue.Empty: + return + if repo.repo_type == AddonManagerRepo.RepoType.MACRO: + self.update_macro(repo) + else: + self.update_package(repo) + self.repo_queue.task_done() + + + def update_macro(self, repo:AddonManagerRepo): + """ Updating a macro happens in this function, in the current thread """ + + with tempfile.TemporaryDirectory() as dir: + temp_install_succeeded = macro.install(dir) + if not temp_install_succeeded: + failed = True + + if not failed: + failed = macro.install(self.macro_repo_dir) + + if not failed: + self.success.emit(repo) + else: + self.failure.emit(repo) + + def update_package(self, repo:AddonManagerRepo): + """ Updating a package re-uses the package installation worker, so actually spawns another thread that we block on """ + + worker = InstallWorkbenchWorker(repo) + worker.success.connect(lambda repo,_:self.success.emit(repo)) + worker.failure.connect(lambda repo,_:self.failure.emit(repo)) + worker.start() + while True: + # Ensure our signals propagate out by running an internal thread-local event loop + QtCore.QCoreApplication.processEvents() + if not worker.isRunning(): + break + +# @} \ No newline at end of file diff --git a/src/Mod/Start/StartPage/LoadNew.py b/src/Mod/Start/StartPage/LoadNew.py index e4434d81e2..bf0040ceba 100644 --- a/src/Mod/Start/StartPage/LoadNew.py +++ b/src/Mod/Start/StartPage/LoadNew.py @@ -27,4 +27,4 @@ FreeCAD.newDocument() FreeCADGui.activeDocument().activeView().viewDefaultOrientation() from StartPage import StartPage -StartPage.postStart() +StartPage.postStart() \ No newline at end of file