Addon Manager: Cleanup

This commit is contained in:
Chris Hennes
2022-07-27 10:48:29 -05:00
parent acbd4b1cd7
commit 514118192a
4 changed files with 180 additions and 96 deletions

View File

@@ -48,7 +48,7 @@ from install_to_toolbar import (
from manage_python_dependencies import (
check_for_python_package_updates,
CheckForPythonPackageUpdatesWorker,
PythonPackageManager
PythonPackageManager,
)
from NetworkManager import HAVE_QTNETWORK, InitializeNetworkManager
@@ -82,6 +82,7 @@ installed.
def QT_TRANSLATE_NOOP(ctx, txt):
return txt
ADDON_MANAGER_DEVELOPER_MODE = False
@@ -100,7 +101,7 @@ class CommandAddonManager:
"load_macro_metadata_worker",
"update_all_worker",
"dependency_installation_worker",
"check_for_python_package_updates_worker"
"check_for_python_package_updates_worker",
]
lock = threading.Lock()
@@ -109,7 +110,7 @@ class CommandAddonManager:
def __init__(self):
FreeCADGui.addPreferencePage(
os.path.join(os.path.dirname(__file__), "AddonManagerOptions.ui"),
translate("AddonsInstaller","Addon Manager"),
translate("AddonsInstaller", "Addon Manager"),
)
self.allowed_packages = set()
@@ -343,7 +344,9 @@ class CommandAddonManager:
self.dialog.buttonCheckForUpdates.clicked.connect(
lambda: self.force_check_updates(standalone=True)
)
self.dialog.buttonUpdateDependencies.clicked.connect(self.show_python_updates_dialog)
self.dialog.buttonUpdateDependencies.clicked.connect(
self.show_python_updates_dialog
)
self.packageList.itemSelected.connect(self.table_row_activated)
self.packageList.setEnabled(False)
self.packageDetails.execute.connect(self.executemacro)
@@ -577,7 +580,7 @@ class CommandAddonManager:
self.populate_macros,
self.update_metadata_cache,
self.check_updates,
self.check_python_updates
self.check_python_updates,
]
pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons")
if pref.GetBool("DownloadMacros", False):
@@ -868,7 +871,9 @@ class CommandAddonManager:
if not thread.isFinished():
self.do_next_startup_phase()
return
self.check_for_python_package_updates_worker = CheckForPythonPackageUpdatesWorker()
self.check_for_python_package_updates_worker = (
CheckForPythonPackageUpdatesWorker()
)
self.check_for_python_package_updates_worker.python_package_updates_available.connect(
lambda: self.dialog.buttonUpdateDependencies.show()
)
@@ -878,7 +883,7 @@ class CommandAddonManager:
self.check_for_python_package_updates_worker.start()
def show_python_updates_dialog(self) -> None:
if not hasattr(self,"manage_python_packages_dialog"):
if not hasattr(self, "manage_python_packages_dialog"):
self.manage_python_packages_dialog = PythonPackageManager()
self.manage_python_packages_dialog.show()
@@ -889,7 +894,7 @@ class CommandAddonManager:
addon_repo.icon = self.get_icon(addon_repo)
for repo in self.item_model.repos:
if repo.name == addon_repo.name:
#self.item_model.reload_item(repo) # If we want to have later additions supersede earlier
# self.item_model.reload_item(repo) # If we want to have later additions supersede earlier
return
self.item_model.append_item(addon_repo)
@@ -1041,20 +1046,26 @@ class CommandAddonManager:
]
def update_allowed_packages_list(self) -> None:
FreeCAD.Console.PrintLog("Attempting to fetch remote copy of ALLOWED_PYTHON_PACKAGES.txt...\n")
FreeCAD.Console.PrintLog(
"Attempting to fetch remote copy of ALLOWED_PYTHON_PACKAGES.txt...\n"
)
p = NetworkManager.AM_NETWORK_MANAGER.blocking_get(
"https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/ALLOWED_PYTHON_PACKAGES.txt"
)
if p:
FreeCAD.Console.PrintLog("Remote ALLOWED_PYTHON_PACKAGES.txt file located, overriding locally-installed copy\n")
FreeCAD.Console.PrintLog(
"Remote ALLOWED_PYTHON_PACKAGES.txt file located, overriding locally-installed copy\n"
)
p = p.data().decode("utf8")
lines = p.split("\n")
self.allowed_packages.clear() # Unset the locally-defined list
self.allowed_packages.clear() # Unset the locally-defined list
for line in lines:
if line and len(line) > 0 and line[0] != "#":
self.allowed_packages.add(line.strip())
else:
FreeCAD.Console.PrintLog("Could not fetch remote ALLOWED_PYTHON_PACKAGES.txt, using local copy\n")
FreeCAD.Console.PrintLog(
"Could not fetch remote ALLOWED_PYTHON_PACKAGES.txt, using local copy\n"
)
def handle_disallowed_python(self, python_required: List[str]) -> bool:
"""Determine if we are missing any required Python packages that are not in the allowed
@@ -1724,14 +1735,18 @@ class CommandAddonManager:
elif addon.repo_type == Addon.Kind.MACRO:
if addon.macro.parsed:
if len(addon.macro.icon) == 0 and len(addon.macro.xpm) == 0:
FreeCAD.Console.PrintLog(f"Macro '{addon.name}' does not have an icon\n")
FreeCAD.Console.PrintLog(
f"Macro '{addon.name}' does not have an icon\n"
)
else:
FreeCAD.Console.PrintLog(f"Addon '{addon.name}' does not have a package.xml file\n")
FreeCAD.Console.PrintLog(
f"Addon '{addon.name}' does not have a package.xml file\n"
)
FreeCAD.Console.PrintLog(f"-----------------------------------\n\n")
self.do_next_startup_phase()
def validate_package_xml(self, addon:Addon):
def validate_package_xml(self, addon: Addon):
if addon.metadata is None:
return
@@ -1743,20 +1758,28 @@ class CommandAddonManager:
# Top-level required elements
if not addon.metadata.Name or len(addon.metadata.Name) == 0:
errors.append(f"No top-level <name> element found, or <name> element is empty")
errors.append(
f"No top-level <name> element found, or <name> element is empty"
)
if not addon.metadata.Version or addon.metadata.Version == "0.0.0":
errors.append(f"No top-level <version> element found, or <version> element is invalid")
#if not addon.metadata.Date or len(addon.metadata.Date) == 0:
errors.append(
f"No top-level <version> element found, or <version> element is invalid"
)
# if not addon.metadata.Date or len(addon.metadata.Date) == 0:
# errors.append(f"No top-level <date> element found, or <date> element is invalid")
if not addon.metadata.Description or len(addon.metadata.Description) == 0:
errors.append(f"No top-level <description> element found, or <description> element is invalid")
errors.append(
f"No top-level <description> element found, or <description> element is invalid"
)
maintainers = addon.metadata.Maintainer
if len(maintainers) == 0:
errors.append(f"No top-level <maintainers> found, at least one is required")
for maintainer in maintainers:
if len(maintainer['email']) == 0:
errors.append(f"No email address specified for maintainer '{maintainer['name']}'")
if len(maintainer["email"]) == 0:
errors.append(
f"No email address specified for maintainer '{maintainer['name']}'"
)
licenses = addon.metadata.License
if len(licenses) == 0:
@@ -1764,7 +1787,9 @@ class CommandAddonManager:
urls = addon.metadata.Urls
if len(urls) == 0:
errors.append(f"No <url> elements found, at least a repo url must be provided")
errors.append(
f"No <url> elements found, at least a repo url must be provided"
)
else:
found_repo = False
found_readme = False
@@ -1772,23 +1797,31 @@ class CommandAddonManager:
if url["type"] == "repository":
found_repo = True
if len(url["branch"]) == 0:
errors.append("<repository> element is missing the 'branch' attribute")
errors.append(
"<repository> element is missing the 'branch' attribute"
)
elif url["type"] == "readme":
found_readme = True
location = url["location"]
p = NetworkManager.AM_NETWORK_MANAGER.blocking_get(location)
if not p:
errors.append(f"Could not access specified readme at {location}")
errors.append(
f"Could not access specified readme at {location}"
)
else:
p = p.data().decode("utf8")
if "<html" in p or "<!DOCTYPE html>" in p:
pass
else:
errors.append(f"Readme data found at {location} does not appear to be rendered HTML")
errors.append(
f"Readme data found at {location} does not appear to be rendered HTML"
)
if not found_repo:
errors.append("No repo url specified")
if not found_readme:
errors.append("No readme url specified (not required, but highly recommended)")
errors.append(
"No readme url specified (not required, but highly recommended)"
)
contents = addon.metadata.Content
if not contents or len(contents) == 0:
@@ -1801,20 +1834,22 @@ class CommandAddonManager:
if "workbench" in contents:
wb = contents["workbench"][0]
if wb.Icon:
missing_icon = False
missing_icon = False
if missing_icon:
errors.append(f"No <icon> element found, or <icon> element is invalid")
if "workbench" in contents:
for wb in contents["workbench"]:
errors.extend (self.validate_workbench_metadata(wb))
errors.extend(self.validate_workbench_metadata(wb))
if "preferencepack" in contents:
for wb in contents["preferencepack"]:
errors.extend (self.validate_preference_pack_metadata(wb))
errors.extend(self.validate_preference_pack_metadata(wb))
if len(errors) > 0:
FreeCAD.Console.PrintLog(f"Errors found in package.xml file for '{addon.name}'\n")
FreeCAD.Console.PrintLog(
f"Errors found in package.xml file for '{addon.name}'\n"
)
for error in errors:
FreeCAD.Console.PrintLog(f" * {error}\n")
@@ -1830,4 +1865,5 @@ class CommandAddonManager:
errors.append("No <name> specified for preference pack")
return errors
# @}

View File

@@ -347,6 +347,7 @@ def is_float(element: Any) -> bool:
# @}
def get_python_exe() -> str:
# Find Python. In preference order
# A) The value of the PythonExecutableForPip user preference
@@ -356,11 +357,7 @@ def get_python_exe() -> str:
# E) The result of an shutil search for your system's "python" executable
prefs = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons")
python_exe = prefs.GetString("PythonExecutableForPip", "Not set")
if (
not python_exe
or python_exe == "Not set"
or not os.path.exists(python_exe)
):
if not python_exe or python_exe == "Not set" or not os.path.exists(python_exe):
fc_dir = FreeCAD.getHomePath()
python_exe = os.path.join(fc_dir, "bin", "python3")
if "Windows" in platform.system():
@@ -382,4 +379,4 @@ def get_python_exe() -> str:
return ""
prefs.SetString("PythonExecutableForPip", python_exe)
return python_exe
return python_exe

View File

@@ -629,7 +629,12 @@ class FillMacroListWorker(QtCore.QThread):
try:
if os.path.exists(self.repo_dir):
if not os.path.exists(os.path.join(self.repo_dir, ".git")):
FreeCAD.Console.PrintWarning(translate("AddonsInstaller","Attempting to change non-git Macro setup to use git\n"))
FreeCAD.Console.PrintWarning(
translate(
"AddonsInstaller",
"Attempting to change non-git Macro setup to use git\n",
)
)
utils.repair_git_repo(
"https://github.com/FreeCAD/FreeCAD-macros.git", self.repo_dir
)
@@ -642,15 +647,15 @@ class FillMacroListWorker(QtCore.QThread):
except Exception as e:
FreeCAD.Console.PrintMessage(
translate(
"AddonsInstaller", "An error occurred updating macros from GitHub, trying clean checkout..."
"AddonsInstaller",
"An error occurred updating macros from GitHub, trying clean checkout...",
)
+ f":\n{e}\n"
)
FreeCAD.Console.PrintMessage(f"{self.repo_dir}\n")
FreeCAD.Console.PrintMessage(
translate(
"AddonsInstaller", "Attempting to do a clean checkout..."
) + "\n"
translate("AddonsInstaller", "Attempting to do a clean checkout...")
+ "\n"
)
try:
shutil.rmtree(self.repo_dir, onerror=self.remove_readonly)
@@ -658,15 +663,13 @@ class FillMacroListWorker(QtCore.QThread):
"https://github.com/FreeCAD/FreeCAD-macros.git", self.repo_dir
)
FreeCAD.Console.PrintMessage(
translate(
"AddonsInstaller", "Clean checkout succeeded"
)
+ "\n"
translate("AddonsInstaller", "Clean checkout succeeded") + "\n"
)
except Exception as e:
FreeCAD.Console.PrintWarning(
translate(
"AddonsInstaller", "Failed to update macros from GitHub -- try clearing the Addon Manager's cache."
"AddonsInstaller",
"Failed to update macros from GitHub -- try clearing the Addon Manager's cache.",
)
+ f":\n{str(e)}\n"
)
@@ -1646,7 +1649,9 @@ class UpdateAllWorker(QtCore.QThread):
current_thread = QtCore.QThread.currentThread()
for repo in self.repos:
self.repo_queue.put(repo)
FreeCAD.Console.PrintLog(f" UPDATER: Adding '{repo.name}' to update queue\n")
FreeCAD.Console.PrintLog(
f" UPDATER: Adding '{repo.name}' to update queue\n"
)
# The original design called for multiple update threads at the same time, but the updater
# itself is not thread-safe, so for the time being only spawn one update thread.
@@ -1676,14 +1681,18 @@ class UpdateAllWorker(QtCore.QThread):
worker.wait()
def on_success(self, repo: Addon) -> None:
FreeCAD.Console.PrintLog(f" UPDATER: Main thread received notice that worker successfully updated {repo.name}\n")
FreeCAD.Console.PrintLog(
f" UPDATER: Main thread received notice that worker successfully updated {repo.name}\n"
)
self.progress_made.emit(
len(self.repos) - self.repo_queue.qsize(), len(self.repos)
)
self.success.emit(repo)
def on_failure(self, repo: Addon) -> None:
FreeCAD.Console.PrintLog(f" UPDATER: Main thread received notice that worker failed to update {repo.name}\n")
FreeCAD.Console.PrintLog(
f" UPDATER: Main thread received notice that worker failed to update {repo.name}\n"
)
self.progress_made.emit(
len(self.repos) - self.repo_queue.qsize(), len(self.repos)
)
@@ -1702,13 +1711,19 @@ class UpdateSingleWorker(QtCore.QThread):
current_thread = QtCore.QThread.currentThread()
while True:
if current_thread.isInterruptionRequested():
FreeCAD.Console.PrintLog(f" UPDATER: Interruption requested, stopping all updates\n")
FreeCAD.Console.PrintLog(
f" UPDATER: Interruption requested, stopping all updates\n"
)
return
try:
repo = self.repo_queue.get_nowait()
FreeCAD.Console.PrintLog(f" UPDATER: Pulling {repo.name} from the update queue\n")
FreeCAD.Console.PrintLog(
f" UPDATER: Pulling {repo.name} from the update queue\n"
)
except queue.Empty:
FreeCAD.Console.PrintLog(f" UPDATER: Worker thread queue is empty, exiting thread\n")
FreeCAD.Console.PrintLog(
f" UPDATER: Worker thread queue is empty, exiting thread\n"
)
return
if repo.repo_type == Addon.Kind.MACRO:
FreeCAD.Console.PrintLog(f" UPDATER: Updating macro '{repo.name}'\n")
@@ -1717,8 +1732,9 @@ class UpdateSingleWorker(QtCore.QThread):
FreeCAD.Console.PrintLog(f" UPDATER: Updating addon '{repo.name}'\n")
self.update_package(repo)
self.repo_queue.task_done()
FreeCAD.Console.PrintLog(f" UPDATER: Worker thread completed action for '{repo.name}' and reported result to main thread\n")
FreeCAD.Console.PrintLog(
f" UPDATER: Worker thread completed action for '{repo.name}' and reported result to main thread\n"
)
def update_macro(self, repo: Addon):
"""Updating a macro happens in this function, in the current thread"""
@@ -1751,7 +1767,7 @@ class UpdateSingleWorker(QtCore.QThread):
if not worker.isRunning():
break
time.sleep(0.1) # Give the signal a moment to propagate to the other threads
time.sleep(0.1) # Give the signal a moment to propagate to the other threads
QtCore.QCoreApplication.processEvents()

View File

@@ -27,13 +27,13 @@ from typing import List, Dict
import os
import subprocess
from functools import partial, partialmethod
from functools import partial
translate = FreeCAD.Qt.translate
# For non-blocking update availability checking:
class CheckForPythonPackageUpdatesWorker(QtCore.QThread):
python_package_updates_available = QtCore.Signal()
def __init__(self):
@@ -44,12 +44,11 @@ class CheckForPythonPackageUpdatesWorker(QtCore.QThread):
if check_for_python_package_updates():
self.python_package_updates_available.emit()
def check_for_python_package_updates() -> bool:
vendor_path = os.path.join(
FreeCAD.getUserAppDataDir(), "AdditionalPythonPackages"
)
vendor_path = os.path.join(FreeCAD.getUserAppDataDir(), "AdditionalPythonPackages")
package_counter = 0
outdated_packages_stdout = call_pip(["list","-o","--path",vendor_path])
outdated_packages_stdout = call_pip(["list", "-o", "--path", vendor_path])
FreeCAD.Console.PrintLog("Output from pip -o:\n")
for line in outdated_packages_stdout:
if len(line) > 0:
@@ -57,6 +56,7 @@ def check_for_python_package_updates() -> bool:
FreeCAD.Console.PrintLog(f" {line}\n")
return package_counter > 0
def call_pip(args) -> List[str]:
python_exe = utils.get_python_exe()
pip_failed = False
@@ -64,7 +64,6 @@ def call_pip(args) -> List[str]:
try:
call_args = [python_exe, "-m", "pip", "--disable-pip-version-check"]
call_args.extend(args)
print(call_args)
proc = subprocess.run(
call_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
)
@@ -80,13 +79,11 @@ def call_pip(args) -> List[str]:
data = proc.stdout.decode()
result = data.split("\n")
else:
print(proc.stderr.decode())
raise Exception(proc.stderr.decode())
return result
class PythonPackageManager:
def __init__(self):
self.dlg = FreeCADGui.PySideUic.loadUi(
os.path.join(os.path.dirname(__file__), "PythonDependencyUpdateDialog.ui")
@@ -103,43 +100,71 @@ class PythonPackageManager:
self.dlg.exec()
def _create_list_from_pip(self):
all_packages_stdout = call_pip(["list","--path",self.vendor_path])
outdated_packages_stdout = call_pip(["list","-o","--path",self.vendor_path])
package_list = self._parse_pip_list_output(all_packages_stdout, outdated_packages_stdout)
self.dlg.buttonUpdateAll.clicked.connect(partial(self._update_all_packages, package_list))
all_packages_stdout = call_pip(["list", "--path", self.vendor_path])
outdated_packages_stdout = call_pip(["list", "-o", "--path", self.vendor_path])
package_list = self._parse_pip_list_output(
all_packages_stdout, outdated_packages_stdout
)
self.dlg.buttonUpdateAll.clicked.connect(
partial(self._update_all_packages, package_list)
)
self.dlg.tableWidget.setRowCount(len(package_list))
updateButtons = list()
counter = 0
update_counter = 0
self.dlg.tableWidget.setSortingEnabled(False)
for package_name, package_details in package_list.items():
self.dlg.tableWidget.setItem(counter,0,QtWidgets.QTableWidgetItem(package_name))
self.dlg.tableWidget.setItem(counter,1,QtWidgets.QTableWidgetItem(package_details["installed_version"]))
self.dlg.tableWidget.setItem(counter,2,QtWidgets.QTableWidgetItem(package_details["available_version"]))
self.dlg.tableWidget.setItem(
counter, 0, QtWidgets.QTableWidgetItem(package_name)
)
self.dlg.tableWidget.setItem(
counter,
1,
QtWidgets.QTableWidgetItem(package_details["installed_version"]),
)
self.dlg.tableWidget.setItem(
counter,
2,
QtWidgets.QTableWidgetItem(package_details["available_version"]),
)
if len(package_details["available_version"]) > 0:
updateButtons.append(QtWidgets.QPushButton(translate("AddonsInstaller","Update")))
updateButtons.append(
QtWidgets.QPushButton(translate("AddonsInstaller", "Update"))
)
updateButtons[-1].setIcon(QtGui.QIcon(":/icons/button_up.svg"))
updateButtons[-1].clicked.connect(partial(self._update_package,package_name))
self.dlg.tableWidget.setCellWidget(counter,3,updateButtons[-1])
updateButtons[-1].clicked.connect(
partial(self._update_package, package_name)
)
self.dlg.tableWidget.setCellWidget(counter, 3, updateButtons[-1])
update_counter += 1
else:
self.dlg.tableWidget.removeCellWidget(counter,3)
self.dlg.tableWidget.removeCellWidget(counter, 3)
counter += 1
self.dlg.tableWidget.setSortingEnabled(True)
self.dlg.tableWidget.horizontalHeader().setStretchLastSection(False)
self.dlg.tableWidget.horizontalHeader().setSectionResizeMode(0, QtWidgets.QHeaderView.Stretch)
self.dlg.tableWidget.horizontalHeader().setSectionResizeMode(1, QtWidgets.QHeaderView.ResizeToContents)
self.dlg.tableWidget.horizontalHeader().setSectionResizeMode(2, QtWidgets.QHeaderView.ResizeToContents)
self.dlg.tableWidget.horizontalHeader().setSectionResizeMode(3, QtWidgets.QHeaderView.ResizeToContents)
self.dlg.tableWidget.horizontalHeader().setSectionResizeMode(
0, QtWidgets.QHeaderView.Stretch
)
self.dlg.tableWidget.horizontalHeader().setSectionResizeMode(
1, QtWidgets.QHeaderView.ResizeToContents
)
self.dlg.tableWidget.horizontalHeader().setSectionResizeMode(
2, QtWidgets.QHeaderView.ResizeToContents
)
self.dlg.tableWidget.horizontalHeader().setSectionResizeMode(
3, QtWidgets.QHeaderView.ResizeToContents
)
if update_counter > 0:
self.dlg.buttonUpdateAll.setEnabled(True)
else:
self.dlg.buttonUpdateAll.setEnabled(False)
def _parse_pip_list_output(self, all_packages, outdated_packages) -> Dict[str,Dict[str,str]]:
def _parse_pip_list_output(
self, all_packages, outdated_packages
) -> Dict[str, Dict[str, str]]:
# All Packages output looks like this:
# Package Version
# ---------- -------
@@ -160,14 +185,13 @@ class PythonPackageManager:
skip_counter += 1
continue
entries = line.split()
if entries:
print(entries[0])
if len(entries) > 1:
package_name = entries[0]
installed_version = entries[1]
packages[package_name] = {"installed_version":installed_version,"available_version":""}
else:
print(line)
packages[package_name] = {
"installed_version": installed_version,
"available_version": "",
}
skip_counter = 0
for line in outdated_packages:
@@ -179,24 +203,35 @@ class PythonPackageManager:
package_name = entries[0]
installed_version = entries[1]
available_version = entries[2]
packages[package_name] = {"installed_version":installed_version,"available_version":available_version}
else:
print(line)
packages[package_name] = {
"installed_version": installed_version,
"available_version": available_version,
}
return packages
def _update_package(self, package_name) -> None:
for line in range(self.dlg.tableWidget.rowCount()):
if self.dlg.tableWidget.item(line,0).text() == package_name:
self.dlg.tableWidget.setItem(line,2,QtWidgets.QTableWidgetItem(translate("AddonsInstaller","Updating...")))
if self.dlg.tableWidget.item(line, 0).text() == package_name:
self.dlg.tableWidget.setItem(
line,
2,
QtWidgets.QTableWidgetItem(
translate("AddonsInstaller", "Updating...")
),
)
break
QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents, 50)
call_pip(["install","--upgrade",package_name,"--target",self.vendor_path])
call_pip(["install", "--upgrade", package_name, "--target", self.vendor_path])
self._create_list_from_pip()
QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents, 50)
def _update_all_packages(self, package_list) -> None:
for package_name, package_details in package_list.items():
if len(package_details["available_version"]) > 0 and package_details["available_version"] != package_details["installed_version"]:
self._update_package(package_name)
if (
len(package_details["available_version"]) > 0
and package_details["available_version"]
!= package_details["installed_version"]
):
self._update_package(package_name)