diff --git a/src/Mod/AddonManager/AddonManager.py b/src/Mod/AddonManager/AddonManager.py index ae8b6e887b..0eb185a123 100644 --- a/src/Mod/AddonManager/AddonManager.py +++ b/src/Mod/AddonManager/AddonManager.py @@ -24,6 +24,7 @@ # * * # *************************************************************************** +from inspect import indentsize import os import shutil import stat @@ -78,6 +79,7 @@ class CommandAddonManager: "update_worker", "check_worker", "show_worker", + "cache_macros_worker", "showmacro_worker", "macro_worker", "install_worker", @@ -262,7 +264,7 @@ class CommandAddonManager: ) # set info for the progress bar: - self.dialog.progressBar.setMaximum(100) + self.dialog.progressBar.setMaximum(1000) # begin populating the table in a set of sub-threads self.startup() @@ -383,6 +385,7 @@ class CommandAddonManager: self.activate_table_widgets, self.populate_macros, self.update_metadata_cache, + self.cache_macros, self.check_updates, ] self.current_progress_region = 0 @@ -410,7 +413,6 @@ class CommandAddonManager: def populate_packages_table(self) -> None: self.item_model.clear() - self.current_progress_region += 1 use_cache = not self.update_cache if use_cache: @@ -456,7 +458,7 @@ class CommandAddonManager: if hasattr(self, "package_cache"): package_cache_path = self.get_cache_file_name("package_cache.json") with open(package_cache_path, "w") as f: - f.write(json.dumps(self.package_cache)) + f.write(json.dumps(self.package_cache, indent=" ")) def activate_table_widgets(self) -> None: self.packageList.setEnabled(True) @@ -464,7 +466,6 @@ class CommandAddonManager: self.do_next_startup_phase() def populate_macros(self) -> None: - self.current_progress_region += 1 if self.update_cache or not os.path.isfile( self.get_cache_file_name("macro_cache.json") ): @@ -495,11 +496,10 @@ class CommandAddonManager: def write_macro_cache(self): macro_cache_path = self.get_cache_file_name("macro_cache.json") with open(macro_cache_path, "w") as f: - f.write(json.dumps(self.macro_cache)) + f.write(json.dumps(self.macro_cache, indent=" ")) self.macro_cache = [] def update_metadata_cache(self) -> None: - self.current_progress_region += 1 if self.update_cache: self.update_metadata_cache_worker = UpdateMetadataCacheWorker( self.item_model.repos @@ -532,10 +532,20 @@ class CommandAddonManager: repo.icon = self.get_icon(repo, update=True) self.item_model.reload_item(repo) + def cache_macros(self) -> None: + if self.update_cache: + self.cache_macros_worker = CacheMacroCode(self.item_model.repos) + self.cache_macros_worker.status_message.connect(self.show_information) + self.cache_macros_worker.update_macro.connect(self.on_package_updated) + self.cache_macros_worker.progress_made.connect(self.update_progress_bar) + self.cache_macros_worker.finished.connect(self.do_next_startup_phase) + self.cache_macros_worker.start() + else: + self.do_next_startup_phase() + def check_updates(self) -> None: "checks every installed addon for available updates" - self.current_progress_region += 1 pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") autocheck = pref.GetBool("AutoCheck", False) if not autocheck: @@ -645,6 +655,7 @@ class CommandAddonManager: """shows generic text in the information pane (which might be collapsed)""" self.dialog.labelStatusInfo.setText(message) + self.dialog.labelStatusInfo.repaint() def show_workbench(self, repo: AddonManagerRepo) -> None: self.packageList.hide() @@ -867,12 +878,20 @@ class CommandAddonManager: def update_progress_bar(self, current_value: int, max_value: int) -> None: """Update the progress bar, showing it if it's hidden""" + if current_value < 0: + FreeCAD.Console.PrintWarning( + f"Addon Manager: Internal error, current progress value is negative in region {self.current_progress_region}" + ) + self.show_progress_widgets() - region_size = 100 / self.number_of_progress_regions - value = (self.current_progress_region - 1) * region_size + ( - current_value / max_value / self.number_of_progress_regions - ) * region_size - self.dialog.progressBar.setValue(value) + region_size = 100.0 / self.number_of_progress_regions + completed_region_portion = (self.current_progress_region - 1) * region_size + current_region_portion = (float(current_value) / float(max_value)) * region_size + value = completed_region_portion + current_region_portion + self.dialog.progressBar.setValue( + value * 10 + ) # Out of 1000 segments, so it moves sort of smoothly + self.dialog.progressBar.repaint() def toggle_details(self) -> None: if self.dialog.labelStatusInfo.isHidden(): diff --git a/src/Mod/AddonManager/addonmanager_macro.py b/src/Mod/AddonManager/addonmanager_macro.py index d602dabd15..4fbcf96067 100644 --- a/src/Mod/AddonManager/addonmanager_macro.py +++ b/src/Mod/AddonManager/addonmanager_macro.py @@ -23,10 +23,10 @@ import os import re -import sys +import io import codecs import shutil -from typing import Dict, Union, List +from typing import Dict, Tuple, List import FreeCAD @@ -56,10 +56,12 @@ class Macro(object): self.on_wiki = False self.on_git = False self.desc = "" + self.comment = "" self.code = "" self.url = "" self.version = "" self.src_filename = "" + self.author = "" self.other_files = [] self.parsed = False @@ -93,37 +95,56 @@ class Macro(object): os.path.join(FreeCAD.getUserMacroDir(True), "Macro_" + self.filename) ) - def fill_details_from_file(self, filename): - with open(filename) as f: - # Number of parsed fields of metadata. For now, __Comment__, - # __Web__, __Version__, __Files__. - number_of_required_fields = 4 - re_desc = re.compile(r"^__Comment__\s*=\s*(['\"])(.*)\1") - re_url = re.compile(r"^__Web__\s*=\s*(['\"])(.*)\1") - re_version = re.compile(r"^__Version__\s*=\s*(['\"])(.*)\1") - re_files = re.compile(r"^__Files__\s*=\s*(['\"])(.*)\1") - for line in f.readlines(): - match = re.match(re_desc, line) - if match: - self.desc = match.group(2) - number_of_required_fields -= 1 - match = re.match(re_url, line) - if match: - self.url = match.group(2) - number_of_required_fields -= 1 - match = re.match(re_version, line) - if match: - self.version = match.group(2) - number_of_required_fields -= 1 - match = re.match(re_files, line) - if match: - self.other_files = [of.strip() for of in match.group(2).split(",")] - number_of_required_fields -= 1 - if number_of_required_fields <= 0: - break - f.seek(0) + def fill_details_from_file(self, filename: str) -> None: + with open(filename, errors="replace") as f: self.code = f.read() - self.parsed = True + self.fill_details_from_code(self.code) + + def fill_details_from_code(self, code: str) -> None: + # Number of parsed fields of metadata. Overrides anything set previously (the code is considered authoritative). + # For now: + # __Comment__ + # __Web__ + # __Version__ + # __Files__ + # __Author__ + number_of_fields = 5 + re_comment = re.compile( + r"^__Comment__\s*=\s*(['\"])(.*)\1", flags=re.IGNORECASE + ) + re_url = re.compile(r"^__Web__\s*=\s*(['\"])(.*)\1", flags=re.IGNORECASE) + re_version = re.compile( + r"^__Version__\s*=\s*(['\"])(.*)\1", flags=re.IGNORECASE + ) + re_files = re.compile(r"^__Files__\s*=\s*(['\"])(.*)\1", flags=re.IGNORECASE) + re_author = re.compile(r"^__Author__\s*=\s*(['\"])(.*)\1", flags=re.IGNORECASE) + + f = io.StringIO(code) + while f: + line = f.readline() + match = re.match(re_comment, line) + if match: + self.comment = match.group(2) + number_of_fields -= 1 + match = re.match(re_author, line) + if match: + self.author = match.group(2) + number_of_fields -= 1 + match = re.match(re_url, line) + if match: + self.url = match.group(2) + number_of_fields -= 1 + match = re.match(re_version, line) + if match: + self.version = match.group(2) + number_of_fields -= 1 + match = re.match(re_files, line) + if match: + self.other_files = [of.strip() for of in match.group(2).split(",")] + number_of_fields -= 1 + if number_of_fields <= 0: + break + self.parsed = True def fill_details_from_wiki(self, url): code = "" @@ -157,16 +178,9 @@ class Macro(object): + "\n" ) return - # code = u2.read() - # github is slow to respond... We need to use this trick below response = "" block = 8192 - # expected = int(u2.headers["content-length"]) - while True: - # print("expected:", expected, "got:", len(response)) - data = u2.read(block) - if not data: - break + while data := u2.read(block): if isinstance(data, bytes): data = data.decode("utf-8") response += data @@ -213,9 +227,9 @@ class Macro(object): flat_code += chunk code = flat_code self.code = code - self.parsed = True + self.fill_details_from_code(self.code) - def install(self, macro_dir: str) -> (bool, List[str]): + def install(self, macro_dir: str) -> Tuple[bool, List[str]]: """Install a macro and all its related files Returns True if the macro was installed correctly. diff --git a/src/Mod/AddonManager/addonmanager_utilities.py b/src/Mod/AddonManager/addonmanager_utilities.py index 532c46ec67..c7f516b9ef 100644 --- a/src/Mod/AddonManager/addonmanager_utilities.py +++ b/src/Mod/AddonManager/addonmanager_utilities.py @@ -29,11 +29,13 @@ import sys import ctypes import tempfile import ssl +from typing import Union import urllib from urllib.request import Request from urllib.error import URLError from urllib.parse import urlparse +from http.client import HTTPResponse from PySide2 import QtGui, QtCore, QtWidgets @@ -94,7 +96,7 @@ def symlink(source, link_name): raise ctypes.WinError() -def urlopen(url: str): +def urlopen(url: str) -> Union[None, HTTPResponse]: """Opens an url with urllib and streams it to a temp file""" timeout = 5 diff --git a/src/Mod/AddonManager/addonmanager_workers.py b/src/Mod/AddonManager/addonmanager_workers.py index 1460301b11..8a65498ae3 100644 --- a/src/Mod/AddonManager/addonmanager_workers.py +++ b/src/Mod/AddonManager/addonmanager_workers.py @@ -31,6 +31,7 @@ import hashlib import threading import queue import io +import time from datetime import datetime from typing import Union, List @@ -637,6 +638,128 @@ class FillMacroListWorker(QtCore.QThread): self.add_macro_signal.emit(repo) +class CacheMacroCode(QtCore.QThread): + """Download and cache the macro code, and parse its internal metadata""" + + status_message = QtCore.Signal(str) + update_macro = QtCore.Signal(AddonManagerRepo) + progress_made = QtCore.Signal(int, int) + + def __init__(self, repos: List[AddonManagerRepo]) -> None: + QtCore.QThread.__init__(self) + self.repos = repos + self.workers = [] + self.terminators = [] + self.lock = threading.Lock() + self.failed = [] + self.counter = 0 + + def run(self): + self.status_message.emit(translate("AddonsInstaller", "Caching macro code...")) + + self.repo_queue = queue.Queue() + current_thread = QtCore.QThread.currentThread() + num_macros = 0 + for repo in self.repos: + if repo.macro is not None: + self.repo_queue.put(repo) + num_macros += 1 + + # Emulate QNetworkAccessManager and spool up six connections: + for _ in range(6): + self.update_and_advance(None) + + while True: + if current_thread.isInterruptionRequested(): + for worker in self.workers: + worker.requestInterruption() + worker.wait(100) + if not worker.isFinished(): + # Kill it + worker.terminate() + return + # Ensure our signals propagate out by running an internal thread-local event loop + QtCore.QCoreApplication.processEvents() + with self.lock: + if self.counter >= num_macros: + break + time.sleep(0.1) + + # Make sure all of our child threads have fully exited: + for i, worker in enumerate(self.workers): + worker.wait(50) + if not worker.isFinished(): + FreeCAD.Console.PrintError( + f"Addon Manager: a worker process failed to complete while fetching {worker.macro.name}\n" + ) + worker.terminate() + + self.repo_queue.join() + for terminator in self.terminators: + if terminator and terminator.isActive(): + terminator.stop() + + FreeCAD.Console.PrintMessage( + f"Out of {num_macros} macros, {len(self.failed)} failed" + ) + + def update_and_advance(self, repo: AddonManagerRepo) -> None: + if repo is not None: + if repo.macro.name not in self.failed: + self.update_macro.emit(repo) + self.repo_queue.task_done() + with self.lock: + self.counter += 1 + + if QtCore.QThread.currentThread().isInterruptionRequested(): + return + + self.progress_made.emit( + len(self.repos) - self.repo_queue.qsize(), len(self.repos) + ) + + try: + next_repo = self.repo_queue.get_nowait() + worker = GetMacroDetailsWorker(next_repo) + worker.finished.connect(lambda: self.update_and_advance(next_repo)) + with self.lock: + self.workers.append(worker) + self.terminators.append( + QtCore.QTimer.singleShot(10000, lambda: self.terminate(worker)) + ) + self.status_message.emit( + translate( + "AddonsInstaller", + f"Getting metadata from macro {next_repo.macro.name}", + ) + ) + worker.start() + except queue.Empty: + pass + + def terminate(self, worker) -> None: + if not worker.isFinished(): + macro_name = worker.macro.name + FreeCAD.Console.PrintWarning( + translate( + "AddonsInstaller", + f"Timeout while fetching metadata for macro {macro_name}", + ) + + "\n" + ) + worker.requestInterruption() + worker.wait(100) + if worker.isRunning(): + worker.terminate() + worker.wait(50) + if worker.isRunning(): + FreeCAD.Console.PrintError( + f"Failed to kill process for macro {macro_name}!\n" + ) + with self.lock: + self.failed.append(macro_name) + + class ShowWorker(QtCore.QThread): """This worker retrieves info of a given workbench""" @@ -1545,11 +1668,15 @@ class UpdateAllWorker(QtCore.QThread): self.done.emit() def on_success(self, repo: AddonManagerRepo) -> None: - self.progress_made.emit(self.repo_queue.qsize(), len(self.repos)) + self.progress_made.emit( + len(self.repos) - self.repo_queue.qsize(), len(self.repos) + ) self.success.emit(repo) def on_failure(self, repo: AddonManagerRepo) -> None: - self.progress_made.emit(self.repo_queue.qsize(), len(self.repos)) + self.progress_made.emit( + len(self.repos) - self.repo_queue.qsize(), len(self.repos) + ) self.failure.emit(repo) diff --git a/src/Mod/AddonManager/package_list.py b/src/Mod/AddonManager/package_list.py index 17a2a78e08..2e7e677dee 100644 --- a/src/Mod/AddonManager/package_list.py +++ b/src/Mod/AddonManager/package_list.py @@ -358,6 +358,15 @@ class PackageListItemDelegate(QStyledItemDelegate): f"\n{maintainer['name']} <{maintainer['email']}>" ) self.widget.ui.labelMaintainer.setText(maintainers_string) + elif repo.macro and repo.macro.parsed: + if repo.macro.comment: + self.widget.ui.labelDescription.setText(repo.macro.comment) + elif repo.macro.desc: + comment, _, _ = repo.desc.partition("