kindred/create
1
1
Fork 0
Code Issues 37 Pull Requests Actions Packages Projects 9 Releases 2 Wiki Activity

Addon Manager: Implement simple macro metadata cache

Browse Source
This commit is contained in:
Chris Hennes
2021-12-31 15:10:57 -06:00
parent 7d748d958d
commit e450e50bb2
5 changed files with 228 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/Mod/AddonManager/AddonManager.py
View File

@@ -24,6 +24,7 @@
# * *
# ***************************************************************************
from inspect import indentsize
import os
import shutil
import stat
@@ -78,6 +79,7 @@ class CommandAddonManager:
"update_worker",
"check_worker",
"show_worker",
"cache_macros_worker",
"showmacro_worker",
"macro_worker",
"install_worker",
@@ -262,7 +264,7 @@ class CommandAddonManager:
)
# set info for the progress bar:
self.dialog.progressBar.setMaximum(100)
self.dialog.progressBar.setMaximum(1000)
# begin populating the table in a set of sub-threads
self.startup()
@@ -383,6 +385,7 @@ class CommandAddonManager:
self.activate_table_widgets,
self.populate_macros,
self.update_metadata_cache,
self.cache_macros,
self.check_updates,
]
self.current_progress_region = 0
@@ -410,7 +413,6 @@ class CommandAddonManager:
def populate_packages_table(self) -> None:
self.item_model.clear()
self.current_progress_region += 1
use_cache = not self.update_cache
if use_cache:
@@ -456,7 +458,7 @@ class CommandAddonManager:
if hasattr(self, "package_cache"):
package_cache_path = self.get_cache_file_name("package_cache.json")
with open(package_cache_path, "w") as f:
f.write(json.dumps(self.package_cache))
f.write(json.dumps(self.package_cache, indent=" "))
def activate_table_widgets(self) -> None:
self.packageList.setEnabled(True)
@@ -464,7 +466,6 @@ class CommandAddonManager:
self.do_next_startup_phase()
def populate_macros(self) -> None:
self.current_progress_region += 1
if self.update_cache or not os.path.isfile(
self.get_cache_file_name("macro_cache.json")
):
@@ -495,11 +496,10 @@ class CommandAddonManager:
def write_macro_cache(self):
macro_cache_path = self.get_cache_file_name("macro_cache.json")
with open(macro_cache_path, "w") as f:
f.write(json.dumps(self.macro_cache))
f.write(json.dumps(self.macro_cache, indent=" "))
self.macro_cache = []
def update_metadata_cache(self) -> None:
self.current_progress_region += 1
if self.update_cache:
self.update_metadata_cache_worker = UpdateMetadataCacheWorker(
self.item_model.repos
@@ -532,10 +532,20 @@ class CommandAddonManager:
repo.icon = self.get_icon(repo, update=True)
self.item_model.reload_item(repo)
def cache_macros(self) -> None:
if self.update_cache:
self.cache_macros_worker = CacheMacroCode(self.item_model.repos)
self.cache_macros_worker.status_message.connect(self.show_information)
self.cache_macros_worker.update_macro.connect(self.on_package_updated)
self.cache_macros_worker.progress_made.connect(self.update_progress_bar)
self.cache_macros_worker.finished.connect(self.do_next_startup_phase)
self.cache_macros_worker.start()
else:
self.do_next_startup_phase()
def check_updates(self) -> None:
"checks every installed addon for available updates"
self.current_progress_region += 1
pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons")
autocheck = pref.GetBool("AutoCheck", False)
if not autocheck:
@@ -645,6 +655,7 @@ class CommandAddonManager:
"""shows generic text in the information pane (which might be collapsed)"""
self.dialog.labelStatusInfo.setText(message)
self.dialog.labelStatusInfo.repaint()
def show_workbench(self, repo: AddonManagerRepo) -> None:
self.packageList.hide()
@@ -867,12 +878,20 @@ class CommandAddonManager:
def update_progress_bar(self, current_value: int, max_value: int) -> None:
"""Update the progress bar, showing it if it's hidden"""
if current_value < 0:
FreeCAD.Console.PrintWarning(
f"Addon Manager: Internal error, current progress value is negative in region {self.current_progress_region}"
)
self.show_progress_widgets()
region_size = 100 / self.number_of_progress_regions
value = (self.current_progress_region - 1) * region_size + (
current_value / max_value / self.number_of_progress_regions
) * region_size
self.dialog.progressBar.setValue(value)
region_size = 100.0 / self.number_of_progress_regions
completed_region_portion = (self.current_progress_region - 1) * region_size
current_region_portion = (float(current_value) / float(max_value)) * region_size
value = completed_region_portion + current_region_portion
self.dialog.progressBar.setValue(
value * 10
) # Out of 1000 segments, so it moves sort of smoothly
self.dialog.progressBar.repaint()
def toggle_details(self) -> None:
if self.dialog.labelStatusInfo.isHidden():

98
src/Mod/AddonManager/addonmanager_macro.py
View File

@@ -23,10 +23,10 @@
import os
import re
import sys
import io
import codecs
import shutil
from typing import Dict, Union, List
from typing import Dict, Tuple, List
import FreeCAD
@@ -56,10 +56,12 @@ class Macro(object):
self.on_wiki = False
self.on_git = False
self.desc = ""
self.comment = ""
self.code = ""
self.url = ""
self.version = ""
self.src_filename = ""
self.author = ""
self.other_files = []
self.parsed = False
@@ -93,37 +95,56 @@ class Macro(object):
os.path.join(FreeCAD.getUserMacroDir(True), "Macro_" + self.filename)
)
def fill_details_from_file(self, filename):
with open(filename) as f:
# Number of parsed fields of metadata. For now, __Comment__,
# __Web__, __Version__, __Files__.
number_of_required_fields = 4
re_desc = re.compile(r"^__Comment__\s*=\s*(['\"])(.*)\1")
re_url = re.compile(r"^__Web__\s*=\s*(['\"])(.*)\1")
re_version = re.compile(r"^__Version__\s*=\s*(['\"])(.*)\1")
re_files = re.compile(r"^__Files__\s*=\s*(['\"])(.*)\1")
for line in f.readlines():
match = re.match(re_desc, line)
if match:
self.desc = match.group(2)
number_of_required_fields -= 1
match = re.match(re_url, line)
if match:
self.url = match.group(2)
number_of_required_fields -= 1
match = re.match(re_version, line)
if match:
self.version = match.group(2)
number_of_required_fields -= 1
match = re.match(re_files, line)
if match:
self.other_files = [of.strip() for of in match.group(2).split(",")]
number_of_required_fields -= 1
if number_of_required_fields <= 0:
break
f.seek(0)
def fill_details_from_file(self, filename: str) -> None:
with open(filename, errors="replace") as f:
self.code = f.read()
self.parsed = True
self.fill_details_from_code(self.code)
def fill_details_from_code(self, code: str) -> None:
# Number of parsed fields of metadata. Overrides anything set previously (the code is considered authoritative).
# For now:
# __Comment__
# __Web__
# __Version__
# __Files__
# __Author__
number_of_fields = 5
re_comment = re.compile(
r"^__Comment__\s*=\s*(['\"])(.*)\1", flags=re.IGNORECASE
)
re_url = re.compile(r"^__Web__\s*=\s*(['\"])(.*)\1", flags=re.IGNORECASE)
re_version = re.compile(
r"^__Version__\s*=\s*(['\"])(.*)\1", flags=re.IGNORECASE
)
re_files = re.compile(r"^__Files__\s*=\s*(['\"])(.*)\1", flags=re.IGNORECASE)
re_author = re.compile(r"^__Author__\s*=\s*(['\"])(.*)\1", flags=re.IGNORECASE)
f = io.StringIO(code)
while f:
line = f.readline()
match = re.match(re_comment, line)
if match:
self.comment = match.group(2)
number_of_fields -= 1
match = re.match(re_author, line)
if match:
self.author = match.group(2)
number_of_fields -= 1
match = re.match(re_url, line)
if match:
self.url = match.group(2)
number_of_fields -= 1
match = re.match(re_version, line)
if match:
self.version = match.group(2)
number_of_fields -= 1
match = re.match(re_files, line)
if match:
self.other_files = [of.strip() for of in match.group(2).split(",")]
number_of_fields -= 1
if number_of_fields <= 0:
break
self.parsed = True
def fill_details_from_wiki(self, url):
code = ""
@@ -157,16 +178,9 @@ class Macro(object):
+ "\n"
)
return
# code = u2.read()
# github is slow to respond... We need to use this trick below
response = ""
block = 8192
# expected = int(u2.headers["content-length"])
while True:
# print("expected:", expected, "got:", len(response))
data = u2.read(block)
if not data:
break
while data := u2.read(block):
if isinstance(data, bytes):
data = data.decode("utf-8")
response += data
@@ -213,9 +227,9 @@ class Macro(object):
flat_code += chunk
code = flat_code
self.code = code
self.parsed = True
self.fill_details_from_code(self.code)
def install(self, macro_dir: str) -> (bool, List[str]):
def install(self, macro_dir: str) -> Tuple[bool, List[str]]:
"""Install a macro and all its related files
Returns True if the macro was installed correctly.

4
src/Mod/AddonManager/addonmanager_utilities.py
View File

@@ -29,11 +29,13 @@ import sys
import ctypes
import tempfile
import ssl
from typing import Union
import urllib
from urllib.request import Request
from urllib.error import URLError
from urllib.parse import urlparse
from http.client import HTTPResponse
from PySide2 import QtGui, QtCore, QtWidgets
@@ -94,7 +96,7 @@ def symlink(source, link_name):
raise ctypes.WinError()
def urlopen(url: str):
def urlopen(url: str) -> Union[None, HTTPResponse]:
"""Opens an url with urllib and streams it to a temp file"""
timeout = 5

131
src/Mod/AddonManager/addonmanager_workers.py
View File

@@ -31,6 +31,7 @@ import hashlib
import threading
import queue
import io
import time
from datetime import datetime
from typing import Union, List
@@ -637,6 +638,128 @@ class FillMacroListWorker(QtCore.QThread):
self.add_macro_signal.emit(repo)
class CacheMacroCode(QtCore.QThread):
"""Download and cache the macro code, and parse its internal metadata"""
status_message = QtCore.Signal(str)
update_macro = QtCore.Signal(AddonManagerRepo)
progress_made = QtCore.Signal(int, int)
def __init__(self, repos: List[AddonManagerRepo]) -> None:
QtCore.QThread.__init__(self)
self.repos = repos
self.workers = []
self.terminators = []
self.lock = threading.Lock()
self.failed = []
self.counter = 0
def run(self):
self.status_message.emit(translate("AddonsInstaller", "Caching macro code..."))
self.repo_queue = queue.Queue()
current_thread = QtCore.QThread.currentThread()
num_macros = 0
for repo in self.repos:
if repo.macro is not None:
self.repo_queue.put(repo)
num_macros += 1
# Emulate QNetworkAccessManager and spool up six connections:
for _ in range(6):
self.update_and_advance(None)
while True:
if current_thread.isInterruptionRequested():
for worker in self.workers:
worker.requestInterruption()
worker.wait(100)
if not worker.isFinished():
# Kill it
worker.terminate()
return
# Ensure our signals propagate out by running an internal thread-local event loop
QtCore.QCoreApplication.processEvents()
with self.lock:
if self.counter >= num_macros:
break
time.sleep(0.1)
# Make sure all of our child threads have fully exited:
for i, worker in enumerate(self.workers):
worker.wait(50)
if not worker.isFinished():
FreeCAD.Console.PrintError(
f"Addon Manager: a worker process failed to complete while fetching {worker.macro.name}\n"
)
worker.terminate()
self.repo_queue.join()
for terminator in self.terminators:
if terminator and terminator.isActive():
terminator.stop()
FreeCAD.Console.PrintMessage(
f"Out of {num_macros} macros, {len(self.failed)} failed"
)
def update_and_advance(self, repo: AddonManagerRepo) -> None:
if repo is not None:
if repo.macro.name not in self.failed:
self.update_macro.emit(repo)
self.repo_queue.task_done()
with self.lock:
self.counter += 1
if QtCore.QThread.currentThread().isInterruptionRequested():
return
self.progress_made.emit(
len(self.repos) - self.repo_queue.qsize(), len(self.repos)
)
try:
next_repo = self.repo_queue.get_nowait()
worker = GetMacroDetailsWorker(next_repo)
worker.finished.connect(lambda: self.update_and_advance(next_repo))
with self.lock:
self.workers.append(worker)
self.terminators.append(
QtCore.QTimer.singleShot(10000, lambda: self.terminate(worker))
)
self.status_message.emit(
translate(
"AddonsInstaller",
f"Getting metadata from macro {next_repo.macro.name}",
)
)
worker.start()
except queue.Empty:
pass
def terminate(self, worker) -> None:
if not worker.isFinished():
macro_name = worker.macro.name
FreeCAD.Console.PrintWarning(
translate(
"AddonsInstaller",
f"Timeout while fetching metadata for macro {macro_name}",
)
+ "\n"
)
worker.requestInterruption()
worker.wait(100)
if worker.isRunning():
worker.terminate()
worker.wait(50)
if worker.isRunning():
FreeCAD.Console.PrintError(
f"Failed to kill process for macro {macro_name}!\n"
)
with self.lock:
self.failed.append(macro_name)
class ShowWorker(QtCore.QThread):
"""This worker retrieves info of a given workbench"""
@@ -1545,11 +1668,15 @@ class UpdateAllWorker(QtCore.QThread):
self.done.emit()
def on_success(self, repo: AddonManagerRepo) -> None:
self.progress_made.emit(self.repo_queue.qsize(), len(self.repos))
self.progress_made.emit(
len(self.repos) - self.repo_queue.qsize(), len(self.repos)
)
self.success.emit(repo)
def on_failure(self, repo: AddonManagerRepo) -> None:
self.progress_made.emit(self.repo_queue.qsize(), len(self.repos))
self.progress_made.emit(
len(self.repos) - self.repo_queue.qsize(), len(self.repos)
)
self.failure.emit(repo)

9
src/Mod/AddonManager/package_list.py
View File

@@ -358,6 +358,15 @@ class PackageListItemDelegate(QStyledItemDelegate):
f"\n{maintainer['name']} <{maintainer['email']}>"
)
self.widget.ui.labelMaintainer.setText(maintainers_string)
elif repo.macro and repo.macro.parsed:
if repo.macro.comment:
self.widget.ui.labelDescription.setText(repo.macro.comment)
elif repo.macro.desc:
comment, _, _ = repo.desc.partition("<br")
self.widget.ui.labelDescription.setText(comment)
self.widget.ui.labelVersion.setText(repo.macro.version)
if self.displayStyle == ListDisplayStyle.EXPANDED:
self.widget.ui.labelMaintainer.setText(repo.macro.author)
else:
self.widget.ui.labelDescription.setText("")
self.widget.ui.labelVersion.setText("")