From e9d6a2c4a4edbced17434836b3fe7fc9ffb3928d Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Fri, 14 Jan 2022 11:52:31 -0600 Subject: [PATCH] Addon Manager: Create NetworkManager class To enable single-login authenticated proxy use, and simplified multi-threaded network accesses, this commit adds a new wrapper around a QNetworkAccessManager and includes a global instantiation of the class intended to exist for the lifetime of the program. This instance can be used to enqueue any number of network requests, which the manager will send out to the networking subsystem in an appropriate manner. --- src/Mod/AddonManager/AddonManager.py | 17 +- src/Mod/AddonManager/CMakeLists.txt | 2 +- src/Mod/AddonManager/NetworkManager.py | 552 +++++++++++++++++ src/Mod/AddonManager/addonmanager_macro.py | 33 +- src/Mod/AddonManager/addonmanager_metadata.py | 315 ---------- .../AddonManager/addonmanager_utilities.py | 61 +- src/Mod/AddonManager/addonmanager_workers.py | 585 ++++++++++-------- src/Mod/AddonManager/proxy_authentication.ui | 150 +++++ 8 files changed, 1055 insertions(+), 660 deletions(-) create mode 100644 src/Mod/AddonManager/NetworkManager.py delete mode 100644 src/Mod/AddonManager/addonmanager_metadata.py create mode 100644 src/Mod/AddonManager/proxy_authentication.ui diff --git a/src/Mod/AddonManager/AddonManager.py b/src/Mod/AddonManager/AddonManager.py index a87b5a6fc2..f2a27515f6 100644 --- a/src/Mod/AddonManager/AddonManager.py +++ b/src/Mod/AddonManager/AddonManager.py @@ -42,6 +42,8 @@ from package_list import PackageList, PackageListItemModel from package_details import PackageDetails from AddonManagerRepo import AddonManagerRepo +from NetworkManager import HAVE_QTNETWORK + __title__ = "FreeCAD Addon Manager Module" __author__ = "Yorik van Havre", "Jonathan Wiedemann", "Kurt Kremitzki", "Chris Hennes" __url__ = "http://www.freecad.org" @@ -228,9 +230,16 @@ class CommandAddonManager: # This must run on the main GUI thread if hasattr(self, "connection_check_message") and self.connection_check_message: self.connection_check_message.close() - QtWidgets.QMessageBox.critical( - None, translate("AddonsInstaller", "Connection failed"), message - ) + if HAVE_QTNETWORK: + QtWidgets.QMessageBox.critical( + None, translate("AddonsInstaller", "Connection failed"), message + ) + else: + QtWidgets.QMessageBox.critical( + None, + translate("AddonsInstaller", "Missing dependency"), + translate("AddonsInstaller", "Could not import QtNetwork -- see Report View for details. Addon Manager unavailable."), + ) def launch(self) -> None: """Shows the Addon Manager UI""" @@ -821,7 +830,7 @@ class CommandAddonManager: self.packageDetails.show_repo(selected_repo) def show_information(self, message: str) -> None: - """shows generic text in the information pane (which might be collapsed)""" + """shows generic text in the information pane""" self.dialog.labelStatusInfo.setText(message) self.dialog.labelStatusInfo.repaint() diff --git a/src/Mod/AddonManager/CMakeLists.txt b/src/Mod/AddonManager/CMakeLists.txt index 8b77d2846d..56a12e7983 100644 --- a/src/Mod/AddonManager/CMakeLists.txt +++ b/src/Mod/AddonManager/CMakeLists.txt @@ -8,7 +8,6 @@ SET(AddonManager_SRCS AddonManager.py AddonManagerRepo.py addonmanager_macro.py - addonmanager_metadata.py addonmanager_utilities.py addonmanager_workers.py AddonManager.ui @@ -18,6 +17,7 @@ SET(AddonManager_SRCS compact_view.py dependency_resolution_dialog.ui expanded_view.py + NetworkManager.py package_list.py package_details.py ) diff --git a/src/Mod/AddonManager/NetworkManager.py b/src/Mod/AddonManager/NetworkManager.py new file mode 100644 index 0000000000..5c9314bc1c --- /dev/null +++ b/src/Mod/AddonManager/NetworkManager.py @@ -0,0 +1,552 @@ +# -*- coding: utf-8 -*- + +# *************************************************************************** +# * * +# * Copyright (c) 2022 Chris Hennes * +# * * +# * This program is free software; you can redistribute it and/or modify * +# * it under the terms of the GNU Lesser General Public License (LGPL) * +# * as published by the Free Software Foundation; either version 2 of * +# * the License, or (at your option) any later version. * +# * for detail see the LICENSE text file. * +# * * +# * This program is distributed in the hope that it will be useful, * +# * but WITHOUT ANY WARRANTY; without even the implied warranty of * +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * +# * GNU Library General Public License for more details. * +# * * +# * You should have received a copy of the GNU Library General Public * +# * License along with this program; if not, write to the Free Software * +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * +# * USA * +# * * +# *************************************************************************** + + +############################################################################# +# +# ABOUT NETWORK MANAGER +# +# A wrapper around QNetworkAccessManager providing proxy-handling +# capabilities, and simplified access to submitting requests from any +# application thread. +# +# +# USAGE +# +# Once imported, this file provides access to a global object called +# AM_NETWORK_MANAGER. This is a QThread object running on its own thread, but +# designed to be interacted with from any other application thread. It +# provides two principal methods: submit_unmonitored_get() and +# submit_monitored_get(). Use the unmonitored version for small amounts of +# data (suitable for caching in RAM, and without a need to show a progress +# bar during download), and the monitored version for larger amounts of data. +# Both functions take a URL, and return an integer index. That index allows +# tracking of the completed request by attaching to the signals completed(), +# progress_made(), and progress_complete(). All three provide, as the first +# argument to the signal, the index of the request the signal refers to. +# Code attached to those signals should filter them to look for the indices +# of the requests they care about. Requests may complete in any order. +# +# A secondary blocking interface is also provided, for very short network +# accesses: the blocking_get() function blocks until the network transmission +# is complete, directly returning a QByteArray object with the received data. +# Do not run on the main GUI thread! + + +try: + import FreeCAD + + if FreeCAD.GuiUp: + import FreeCADGui + + HAVE_FREECAD = True + translate = FreeCAD.Qt.translate +except Exception: + # For standalone testing support working without the FreeCAD import + HAVE_FREECAD = False + +import threading +from PySide2 import QtCore + +import os +import queue +import itertools +import tempfile +from typing import Dict, List + +# This is the global instance of the NetworkManager that outside code +# should access +AM_NETWORK_MANAGER = None + +HAVE_QTNETWORK = True +try: + from PySide2 import QtNetwork +except Exception: + if HAVE_FREECAD: + FreeCAD.Console.PrintError( + translate( + "AddonsInstaller", + "Could not import QtNetwork -- it does not appear to be installed on your system. Please install the package 'python3-pyside2.qtnetwork' on your system and if possible contact your FreeCAD package maintainer to alert them to the missing dependency. The Addon Manager will not be available.", + ) + + "\n" + ) + else: + print( + "Could not import QtNetwork, unable to test this file. Try installing the python3-pyside2.qtnetwork package." + ) + exit(1) + HAVE_QTNETWORK = False + +if HAVE_QTNETWORK: + + class QueueItem: + def __init__( + self, index: int, request: QtNetwork.QNetworkRequest, track_progress: bool + ): + self.index = index + self.request = request + self.track_progress = track_progress + + class NetworkManager(QtCore.QThread): + """A single global instance of NetworkManager is instantiated and stored as + AM_NETWORK_MANAGER. Outside threads should send GET requests to this class by + calling the submit_unmonitored_request() or submit_monitored_request() function, + as needed. See the documentation of those functions for details.""" + + # Connect to complete for requests with no progress monitoring (e.g. small amounts of data) + completed = QtCore.Signal( + int, int, QtCore.QByteArray + ) # Index, http response code, received data (if any) + + # Connect to progress_made and progress_complete for large amounts of data, which get buffered into a temp file + # That temp file should be deleted when your code is done with it + progress_made = QtCore.Signal( + int, int, int + ) # Index, bytes read, total bytes (may be None) + + progress_complete = QtCore.Signal( + int, int, os.PathLike + ) # Index, http response code, filename + + def __init__(self): + super().__init__() + + # Set up the incoming request queue: requests get put on this queue, and the main event loop + # pulls them off and runs them. + self.counting_iterator = itertools.count() + self.queue = queue.Queue() + self.__last_started_index = 0 + self.__abort_when_found: List[int] = [] + self.replies: Dict[int, QtNetwork.QNetworkReply] = {} + self.file_buffers = {} + + # We support an arbitrary number of threads using synchronous GET calls: + self.synchronous_lock = threading.Lock() + self.synchronous_complete: Dict[int, bool] = {} + self.synchronous_result_data: Dict[int, QtCore.QByteArray] = {} + + def run(self): + """Do not call directly: use start() to begin the event loop on a new thread.""" + + # Create the QNAM on this thread: + self.thread = QtCore.QThread.currentThread() + self.QNAM = QtNetwork.QNetworkAccessManager() + self.QNAM.proxyAuthenticationRequired.connect(self.__authenticate_proxy) + self.QNAM.authenticationRequired.connect(self.__authenticate_resource) + + # A helper connection for our blocking interface + self.completed.connect(self.__synchronous_process_completion) + + # Set up the proxy, if necesssary: + noProxyCheck = True + systemProxyCheck = False + userProxyCheck = False + proxy_string = "" + if HAVE_FREECAD: + pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") + noProxyCheck = pref.GetBool("NoProxyCheck", noProxyCheck) + systemProxyCheck = pref.GetBool("SystemProxyCheck", systemProxyCheck) + userProxyCheck = pref.GetBool("UserProxyCheck", userProxyCheck) + proxy_string = pref.GetString("ProxyUrl", "") + else: + print("Please select a proxy type:") + print("1) No proxy") + print("2) Use system proxy settings") + print("3) Custom proxy settings") + result = input("Choice: ") + if result == "1": + pass + elif result == "2": + noProxyCheck = False + systemProxyCheck = True + elif result == "3": + noProxyCheck = False + userProxyCheck = True + proxy_string = input("Enter your proxy server (host:port): ") + else: + print(f"Got {result}, expected 1, 2, or 3.") + app.quit() + + if noProxyCheck: + pass + elif systemProxyCheck: + query = QtNetwork.QNetworkProxyQuery( + QtCore.QUrl("https://github.com/FreeCAD/FreeCAD") + ) + proxy = QtNetwork.QNetworkProxyFactory.systemProxyForQuery(query) + if proxy and proxy[0]: + self.QNAM.setProxy( + proxy[0] + ) # This may still be QNetworkProxy.NoProxy + elif userProxyCheck: + host, _, port_string = proxy_string.rpartition(":") + port = 0 if not port_string else int(port_string) + # For now assume an HttpProxy, but eventually this should be a parameter + proxy = QtNetwork.QNetworkProxy( + QtNetwork.QNetworkProxy.HttpProxy, host, port + ) + self.QNAM.setProxy(proxy) + + qnam_cache = QtCore.QStandardPaths.writableLocation(QtCore.QStandardPaths.CacheLocation); + os.makedirs(qnam_cache,exist_ok=True) + diskCache = QtNetwork.QNetworkDiskCache() + diskCache.setCacheDirectory(qnam_cache) + self.QNAM.setCache(diskCache) + + # Start an event loop + while True: + if QtCore.QThread.currentThread().isInterruptionRequested(): + # Support shutting down the entire thread, but this should be very rarely used. + # Callers should generally just call abort() on each network call they want to + # terminate, using requestInterruption() will terminate ALL network requests. + if not HAVE_FREECAD: + print( + "Shutting down all active network requests...", flush=True + ) + self.abort_all() + self.queue.join() + if not HAVE_FREECAD: + print("All requests terminated.", flush=True) + return + try: + item = self.queue.get_nowait() + if item: + if item.index in self.__abort_when_found: + self.__abort_when_found.remove(item.index) + continue # Do not do anything with this item, it's been aborted... + reply = self.QNAM.get(item.request) + + self.__last_started_index = item.index + reply.finished.connect(lambda i=item: self.__reply_finished(i)) + reply.redirected.connect( + lambda url, r=reply: self.__on_redirect(r, url) + ) + reply.sslErrors.connect(self.__on_ssl_error) + if item.track_progress: + reply.readyRead.connect( + lambda i=item.index: self.__data_incoming(i) + ) + reply.downloadProgress.connect( + lambda a, b, i=item.index: self.progress_made.emit( + i, a, b + ) + ) + self.replies[item.index] = reply + except queue.Empty: + pass + QtCore.QCoreApplication.processEvents() + + def submit_unmonitored_get(self, url: str) -> int: + """Adds this request to the queue, and returns an index that can be used by calling code + in conjunction with the completed() signal to handle the results of the call. All data is + kept in memory, and the completed() call includes a direct handle to the bytes returned. It + is not called until the data transfer has finished and the connection is closed.""" + + current_index = next(self.counting_iterator) # A thread-safe counter + # Use a queue because we can only put things on the QNAM from the main event loop thread + self.queue.put( + QueueItem( + current_index, self.__create_get_request(url), track_progress=False + ) + ) + return current_index + + def submit_monitored_get(self, url: str) -> int: + """Adds this request to the queue, and returns an index that can be used by calling code + in conjunction with the progress_made() and progress_completed() signals to handle the + results of the call. All data is cached to disk, and progress is reported periodically + as the underlying QNetworkReply reports its progress. The progress_completed() signal + contains a path to a temporary file with the stored data. Calling code should delete this + file when done with it (or move it into its final place, etc.).""" + + current_index = next(self.counting_iterator) # A thread-safe counter + # Use a queue because we can only put things on the QNAM from the main event loop thread + self.queue.put( + QueueItem( + current_index, self.__create_get_request(url), track_progress=True + ) + ) + return current_index + + def blocking_get(self, url: str) -> QtCore.QByteArray: + """Submits a GET request to the QNetworkAccessManager and block until it is complete""" + + current_index = next(self.counting_iterator) # A thread-safe counter + with self.synchronous_lock: + self.synchronous_complete[current_index] = False + + self.queue.put( + QueueItem( + current_index, self.__create_get_request(url), track_progress=False + ) + ) + while not self.synchronous_complete[current_index]: + if QtCore.QThread.currentThread().isInterruptionRequested(): + return None + QtCore.QCoreApplication.processEvents() + + with self.synchronous_lock: + self.synchronous_complete.pop(current_index) + return self.synchronous_result_data.pop(current_index) + + def __synchronous_process_completion( + self, index: int, code: int, data: QtCore.QByteArray + ) -> None: + with self.synchronous_lock: + if index in self.synchronous_complete: + if code == 200: + self.synchronous_result_data[index] = data + self.synchronous_complete[index] = True + + def __create_get_request(self, url: str) -> QtNetwork.QNetworkRequest: + request = QtNetwork.QNetworkRequest(QtCore.QUrl(url)) + request.setAttribute( + QtNetwork.QNetworkRequest.RedirectPolicyAttribute, + QtNetwork.QNetworkRequest.UserVerifiedRedirectPolicy, + ) + request.setAttribute( + QtNetwork.QNetworkRequest.CacheSaveControlAttribute, True + ) + request.setAttribute( + QtNetwork.QNetworkRequest.CacheLoadControlAttribute, + QtNetwork.QNetworkRequest.PreferCache, + ) + request.setAttribute( + QtNetwork.QNetworkRequest.BackgroundRequestAttribute, True + ) + return request + + def abort_all(self): + """Abort ALL network calls in progress, including clearing the queue""" + for reply in self.replies: + if reply.isRunning(): + reply.abort() + while True: + try: + self.queue.get() + self.queue.task_done() + except queue.Empty: + break + + def abort(self, index: int): + if index in self.replies and self.replies[index].isRunning(): + self.replies[index].abort() + elif index < self.__last_started_index: + # It's still in the queue. Mark it for later destruction. + self.__abort_when_found.append(index) + + def __authenticate_proxy( + self, + reply: QtNetwork.QNetworkProxy, + authenticator: QtNetwork.QAuthenticator, + ): + if HAVE_FREECAD and FreeCAD.GuiUp: + proxy_authentication = FreeCADGui.PySideUic.loadUi( + os.path.join(os.path.dirname(__file__), "proxy_authentication.ui") + ) + # Show the right labels, etc. + proxy_authentication.labelProxyAddress.setText( + f"{reply.hostName()}:{reply.port()}" + ) + if authenticator.realm(): + proxy_authentication.labelProxyRealm.setText(authenticator.realm()) + else: + proxy_authentication.labelProxyRealm.hide() + proxy_authentication.labelRealmCaption.hide() + result = proxy_authentication.exec() + if result == QtWidgets.QDialogButtonBox.Ok: + authenticator.setUser(proxy_authentication.lineEditUsername.text()) + authenticator.setPassword( + proxy_authentication.lineEditPassword.text() + ) + else: + username = input("Proxy username: ") + import getpass + + password = getpass.getpass() + authenticator.setUser(username) + authenticator.setPassword(password) + + def __authenticate_resource( + self, + _reply: QtNetwork.QNetworkReply, + _authenticator: QtNetwork.QAuthenticator, + ): + pass + + def __on_redirect(self, reply, _): + # For now just blindly follow all redirects + reply.redirectAllowed.emit() + + def __on_ssl_error(self, reply: str, errors: List[str]): + if HAVE_FREECAD: + FreeCAD.Console.PrintWarning( + translate("AddonsInstaller", "Error with encrypted connection") + + "\n:" + ) + FreeCAD.Console.PrintWarning(reply) + for error in errors: + FreeCAD.Console.PrintWarning(error) + else: + print("Error with encrypted connection") + for error in errors: + print(error) + + def __data_incoming(self, index: int): + reply = self.replies[index] + chunk_size = reply.bytesAvailable() + buffer = reply.read(chunk_size) + if not index in self.file_buffers: + f = tempfile.NamedTemporaryFile("wb", delete=False) + self.file_buffers[index] = f + else: + f = self.file_buffers[index] + f.write(buffer) + + def __reply_finished(self, item: QueueItem) -> None: + reply = self.replies.pop(item.index) + response_code = reply.attribute( + QtNetwork.QNetworkRequest.HttpStatusCodeAttribute + ) + self.queue.task_done() + if reply.error() == QtNetwork.QNetworkReply.NetworkError.NoError: + if item.track_progress: + f = self.file_buffers[item.index] + f.close() + self.progress_complete.emit(item.index, response_code, f.name) + else: + data = reply.readAll() + self.completed.emit(item.index, response_code, data) + else: + if item.track_progress: + self.progress_complete.emit(item.index, response_code, "") + else: + self.completed.emit(item.index, response_code, None) + +else: # HAVE_QTNETWORK is false: + + class NetworkManager(QtCore.QThread): + """A dummy class to enable an offline mode when the QtNetwork package is not yet installed""" + + completed = QtCore.Signal( + int, int, bytes + ) # Emitted as soon as the request is made, with a connection failed error + progress_made = QtCore.Signal( + int, int, int + ) # Never emitted, no progress is made here + progress_complete = QtCore.Signal( + int, int, os.PathLike + ) # Emitted as soon as the request is made, with a connection failed error + + def __init__(self): + super().__init__() + self.monitored_queue = queue.Queue() + self.unmonitored_queue = queue.Queue() + + def run(self): + while True: + try: + index = self.monitored_queue.get_nowait() + self.progress_complete.emit( + index, 418, "--ERR418--" + ) # Refuse to provide data + self.monitored_queue.task_done() + except queue.Empty: + pass + try: + index = self.unmonitored_queue.get_nowait() + self.completed.emit(index, 418, None) + self.unmonitored_queue.task_done() + except queue.Empty: + pass + if QtCore.QThread.currentThread().isInterruptionRequested(): + return + QtCore.QCoreApplication.processEvents() + + def submit_unmonitored_request(self, _) -> int: + current_index = next(itertools.count()) + self.unmonitored_queue.put(current_index) + return current_index + + def submit_monitored_request(self, _) -> int: + current_index = next(itertools.count()) + self.monitored_queue.put(current_index) + return current_index + + def blocking_get(self, _: str) -> QtCore.QByteArray: + return None + + def abort_all( + self, + ): + pass # Nothing to do + + def abort(self, _): + pass # Nothing to do + + +AM_NETWORK_MANAGER = NetworkManager() +AM_NETWORK_MANAGER.start() + + +if __name__ == "__main__": + + app = QtCore.QCoreApplication() + + count = 0 + + # For testing, create several network requests and send them off in quick succession: + # (Choose small downloads, no need for significant data) + urls = [ + "https://api.github.com/zen", + "http://climate.ok.gov/index.php/climate/rainfall_table/local_data", + "https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/AIANNHA/MapServer", + ] + + def handle_completion(index: int, code: int, data): + global count + if code == 200: + print( + f"For request {index+1}, response was {data.size()} bytes.", flush=True + ) + else: + print( + f"For request {index+1}, request failed with HTTP result code {code}", + flush=True, + ) + + count += 1 + if count >= len(urls): + print(f"Shutting down...", flush=True) + AM_NETWORK_MANAGER.requestInterruption() + AM_NETWORK_MANAGER.wait(5000) + app.quit() + + AM_NETWORK_MANAGER.completed.connect(handle_completion) + for url in urls: + AM_NETWORK_MANAGER.submit_unmonitored_get(url) + + app.exec_() + + print("Done with all requests.") diff --git a/src/Mod/AddonManager/addonmanager_macro.py b/src/Mod/AddonManager/addonmanager_macro.py index 73797ffaeb..4dfd1f8674 100644 --- a/src/Mod/AddonManager/addonmanager_macro.py +++ b/src/Mod/AddonManager/addonmanager_macro.py @@ -30,9 +30,10 @@ import time from typing import Dict, Tuple, List, Union import FreeCAD +from NetworkManager import AM_NETWORK_MANAGER + +translate = FreeCAD.Qt.translate -from addonmanager_utilities import translate -from addonmanager_utilities import urlopen from addonmanager_utilities import remove_directory_if_empty try: @@ -165,28 +166,25 @@ class Macro(object): def fill_details_from_wiki(self, url): code = "" - u = urlopen(url) - if u is None: + p = AM_NETWORK_MANAGER.blocking_get(url) + if not p: FreeCAD.Console.PrintWarning( translate( "AddonsInstaller", - f"Could not connect to {url} - check connection and proxy settings", + f"Unable to open macro wiki page at {url}", ) + "\n" ) return - p = u.read() - if isinstance(p, bytes): - p = p.decode("utf-8") - u.close() + p = p.data().decode("utf8") # check if the macro page has its code hosted elsewhere, download if # needed if "rawcodeurl" in p: rawcodeurl = re.findall('rawcodeurl.*?href="(http.*?)">', p) if rawcodeurl: rawcodeurl = rawcodeurl[0] - u2 = urlopen(rawcodeurl) - if u2 is None: + u2 = AM_NETWORK_MANAGER.blocking_get(rawcodeurl) + if not u2: FreeCAD.Console.PrintWarning( translate( "AddonsInstaller", @@ -195,18 +193,7 @@ class Macro(object): + "\n" ) return - response = "" - block = 8192 - while True: - data = u2.read(block) - if not data: - break - if isinstance(data, bytes): - data = data.decode("utf-8") - response += data - if response: - code = response - u2.close() + code = u2.data().decode("utf8") if not code: code = re.findall(r"
(.*?)
", p.replace("\n", "--endl--")) if code: diff --git a/src/Mod/AddonManager/addonmanager_metadata.py b/src/Mod/AddonManager/addonmanager_metadata.py deleted file mode 100644 index b6e8eebc46..0000000000 --- a/src/Mod/AddonManager/addonmanager_metadata.py +++ /dev/null @@ -1,315 +0,0 @@ -# *************************************************************************** -# * * -# * Copyright (c) 2021 Chris Hennes * -# * * -# * This program is free software; you can redistribute it and/or modify * -# * it under the terms of the GNU Lesser General Public License (LGPL) * -# * as published by the Free Software Foundation; either version 2 of * -# * the License, or (at your option) any later version. * -# * for detail see the LICENCE text file. * -# * * -# * This program is distributed in the hope that it will be useful, * -# * but WITHOUT ANY WARRANTY; without even the implied warranty of * -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * -# * GNU Library General Public License for more details. * -# * * -# * You should have received a copy of the GNU Library General Public * -# * License along with this program; if not, write to the Free Software * -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * -# * USA * -# * * -# *************************************************************************** - -import FreeCAD - -import os -import io -import hashlib -from typing import Dict, List, Set - -from PySide2 import QtCore, QtNetwork -from PySide2.QtCore import QObject - -import addonmanager_utilities as utils -from AddonManagerRepo import AddonManagerRepo - -translate = FreeCAD.Qt.translate - - -class DownloadWorker(QObject): - updated = QtCore.Signal(AddonManagerRepo) - - def __init__(self, parent, url: str): - """repo is an AddonManagerRepo object, and index is a dictionary of SHA1 hashes of the package.xml files in the cache""" - - super().__init__(parent) - self.url = url - - def start_fetch(self, network_manager: QtNetwork.QNetworkAccessManager): - """Asynchronously begin the network access. Intended as a set-and-forget black box for downloading files.""" - - self.request = QtNetwork.QNetworkRequest(QtCore.QUrl(self.url)) - self.request.setAttribute( - QtNetwork.QNetworkRequest.RedirectPolicyAttribute, - QtNetwork.QNetworkRequest.UserVerifiedRedirectPolicy, - ) - self.request.setAttribute( - QtNetwork.QNetworkRequest.CacheSaveControlAttribute, True - ) - self.request.setAttribute( - QtNetwork.QNetworkRequest.CacheLoadControlAttribute, - QtNetwork.QNetworkRequest.PreferCache, - ) - self.request.setAttribute( - QtNetwork.QNetworkRequest.BackgroundRequestAttribute, True - ) - - self.fetch_task = network_manager.get(self.request) - self.fetch_task.finished.connect(self.resolve_fetch) - self.fetch_task.redirected.connect(self.on_redirect) - self.fetch_task.sslErrors.connect(self.on_ssl_error) - - def abort(self): - if not self.fetch_task.isFinished(): - self.fetch_task.abort() - - def on_redirect(self, _): - # For now just blindly follow all redirects - self.fetch_task.redirectAllowed.emit() - - def on_ssl_error(self, reply: str, errors: List[str]): - FreeCAD.Console.PrintWarning( - translate("AddonsInstaller", "Error with encrypted connection") + "\n:" - ) - FreeCAD.Console.PrintWarning(reply) - for error in errors: - FreeCAD.Console.PrintWarning(error) - - -class MetadataDownloadWorker(DownloadWorker): - """A worker for downloading package.xml and associated icon(s) - - To use, instantiate an object of this class and call the start_fetch() function - with a QNetworkAccessManager. It is expected that many of these objects will all - be created and associated with the same QNAM, which will then handle the actual - asynchronous downloads in some Qt-defined number of threads. To monitor progress - you should connect to the QNAM's "finished" signal, and ensure it is called the - number of times you expect based on how many workers you have enqueued. - - """ - - def __init__(self, parent, repo: AddonManagerRepo, index: Dict[str, str]): - """repo is an AddonManagerRepo object, and index is a dictionary of SHA1 hashes of the package.xml files in the cache""" - - super().__init__(parent, repo.metadata_url) - self.repo = repo - self.index = index - self.store = os.path.join( - FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata" - ) - self.last_sha1 = "" - - def resolve_fetch(self): - """Called when the data fetch completed, either with an error, or if it found the metadata file""" - - if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError: - FreeCAD.Console.PrintLog(f"Found a package.xml file for {self.repo.name}\n") - self.repo.repo_type = AddonManagerRepo.RepoType.PACKAGE - new_xml = self.fetch_task.readAll() - hasher = hashlib.sha1() - hasher.update(new_xml.data()) - new_sha1 = hasher.hexdigest() - self.last_sha1 = new_sha1 - # Determine if we need to download the icon: only do that if the - # package.xml file changed (since - # a change in the version number will show up as a change in the - # SHA1, without having to actually - # read the metadata) - if self.repo.name in self.index: - cached_sha1 = self.index[self.repo.name] - if cached_sha1 != new_sha1: - self.update_local_copy(new_xml) - else: - # Assume that if the package.xml file didn't change, - # neither did the icon, so don't waste - # resources downloading it - xml_file = os.path.join(self.store, self.repo.name, "package.xml") - self.repo.metadata = FreeCAD.Metadata(xml_file) - else: - # There is no local copy yet, so we definitely have to update - # the cache - self.update_local_copy(new_xml) - elif ( - self.fetch_task.error() - == QtNetwork.QNetworkReply.NetworkError.ContentNotFoundError - ): - pass - elif ( - self.fetch_task.error() - == QtNetwork.QNetworkReply.NetworkError.OperationCanceledError - ): - pass - else: - FreeCAD.Console.PrintWarning( - translate("AddonsInstaller", "Failed to connect to URL") - + f":\n{self.url}\n {self.fetch_task.error()}\n" - ) - - def update_local_copy(self, new_xml): - # We have to update the local copy of the metadata file and re-download - # the icon file - - name = self.repo.name - repo_url = self.repo.url - package_cache_directory = os.path.join(self.store, name) - if not os.path.exists(package_cache_directory): - os.makedirs(package_cache_directory) - new_xml_file = os.path.join(package_cache_directory, "package.xml") - with open(new_xml_file, "wb") as f: - f.write(new_xml.data()) - metadata = FreeCAD.Metadata(new_xml_file) - self.repo.metadata = metadata - self.repo.repo_type = AddonManagerRepo.RepoType.PACKAGE - icon = metadata.Icon - - if not icon: - # If there is no icon set for the entire package, see if there are - # any workbenches, which are required to have icons, and grab the first - # one we find: - content = self.repo.metadata.Content - if "workbench" in content: - wb = content["workbench"][0] - if wb.Icon: - if wb.Subdirectory: - subdir = wb.Subdirectory - else: - subdir = wb.Name - self.repo.Icon = subdir + wb.Icon - icon = self.repo.Icon - - icon_url = utils.construct_git_url(self.repo, icon) - icon_stream = utils.urlopen(icon_url) - if icon and icon_stream and icon_url: - icon_data = icon_stream.read() - cache_file = self.repo.get_cached_icon_filename() - with open(cache_file, "wb") as icon_file: - icon_file.write(icon_data) - self.repo.cached_icon_filename = cache_file - self.updated.emit(self.repo) - - -class MetadataTxtDownloadWorker(DownloadWorker): - """A worker for downloading metadata.txt""" - - def __init__(self, parent, repo: AddonManagerRepo): - super().__init__(parent, utils.construct_git_url(repo, "metadata.txt")) - self.repo = repo - - def resolve_fetch(self): - """Called when the data fetch completed, either with an error, or if it found the metadata file""" - - if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError: - FreeCAD.Console.PrintLog( - f"Found a metadata.txt file for {self.repo.name}\n" - ) - new_deps = self.fetch_task.readAll() - self.parse_file(new_deps.data().decode("utf8")) - elif ( - self.fetch_task.error() - == QtNetwork.QNetworkReply.NetworkError.ContentNotFoundError - ): - pass - elif ( - self.fetch_task.error() - == QtNetwork.QNetworkReply.NetworkError.OperationCanceledError - ): - pass - else: - FreeCAD.Console.PrintWarning( - translate("AddonsInstaller", "Failed to connect to URL") - + f":\n{self.url}\n {self.fetch_task.error()}\n" - ) - - def parse_file(self, data: str) -> None: - f = io.StringIO(data) - while True: - line = f.readline() - if not line: - break - if line.startswith("workbenches="): - depswb = line.split("=")[1].split(",") - for wb in depswb: - wb_name = wb.strip() - if wb_name: - self.repo.requires.add(wb_name) - FreeCAD.Console.PrintLog( - f"{self.repo.display_name} requires FreeCAD Addon '{wb_name}'\n" - ) - - elif line.startswith("pylibs="): - depspy = line.split("=")[1].split(",") - for pl in depspy: - dep = pl.strip() - if dep: - self.repo.python_requires.add(dep) - FreeCAD.Console.PrintLog( - f"{self.repo.display_name} requires python package '{dep}'\n" - ) - - elif line.startswith("optionalpylibs="): - opspy = line.split("=")[1].split(",") - for pl in opspy: - dep = pl.strip() - if dep: - self.repo.python_optional.add(dep) - FreeCAD.Console.PrintLog( - f"{self.repo.display_name} optionally imports python package '{pl.strip()}'\n" - ) - self.updated.emit(self.repo) - - -class RequirementsTxtDownloadWorker(DownloadWorker): - """A worker for downloading requirements.txt""" - - def __init__(self, parent, repo: AddonManagerRepo): - super().__init__(parent, utils.construct_git_url(repo, "requirements.txt")) - self.repo = repo - - def resolve_fetch(self): - """Called when the data fetch completed, either with an error, or if it found the metadata file""" - - if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError: - FreeCAD.Console.PrintLog( - f"Found a requirements.txt file for {self.repo.name}\n" - ) - new_deps = self.fetch_task.readAll() - self.parse_file(new_deps.data().decode("utf8")) - elif ( - self.fetch_task.error() - == QtNetwork.QNetworkReply.NetworkError.ContentNotFoundError - ): - pass - elif ( - self.fetch_task.error() - == QtNetwork.QNetworkReply.NetworkError.OperationCanceledError - ): - pass - else: - FreeCAD.Console.PrintWarning( - translate("AddonsInstaller", "Failed to connect to URL") - + f":\n{self.url}\n {self.fetch_task.error()}\n" - ) - - def parse_file(self, data: str) -> None: - f = io.StringIO(data) - lines = f.readlines() - for line in lines: - break_chars = " <>=~!+#" - package = line - for n, c in enumerate(line): - if c in break_chars: - package = line[:n].strip() - break - if package: - self.repo.python_requires.add(package) - self.updated.emit(self.repo) diff --git a/src/Mod/AddonManager/addonmanager_utilities.py b/src/Mod/AddonManager/addonmanager_utilities.py index 43c09dfc6b..50b441d4f6 100644 --- a/src/Mod/AddonManager/addonmanager_utilities.py +++ b/src/Mod/AddonManager/addonmanager_utilities.py @@ -25,7 +25,7 @@ import os import re import ctypes import ssl -from typing import Union +from typing import Union, Optional import urllib from urllib.request import Request @@ -53,7 +53,6 @@ else: except AttributeError: pass - # @package AddonManager_utilities # \ingroup ADDONMANAGER # \brief Utilities to work across different platforms, providers and python versions @@ -72,7 +71,7 @@ def translate(context, text, disambig=None): def symlink(source, link_name): - "creates a symlink of a file, if possible" + """Creates a symlink of a file, if possible. Note that it fails on most modern Windows installations""" if os.path.exists(link_name) or os.path.lexists(link_name): pass @@ -81,6 +80,8 @@ def symlink(source, link_name): if callable(os_symlink): os_symlink(source, link_name) else: + # NOTE: This does not work on most normal Windows 10 and later installations, unless developer + # mode is turned on. Make sure to catch any exception thrown and have a fallback plan. csl = ctypes.windll.kernel32.CreateSymbolicLinkW csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32) csl.restype = ctypes.c_ubyte @@ -92,47 +93,6 @@ def symlink(source, link_name): raise ctypes.WinError() -def urlopen(url: str) -> Union[None, HTTPResponse]: - """Opens an url with urllib and streams it to a temp file""" - - timeout = 5 - - # Proxy an ssl configuration - pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") - if pref.GetBool("NoProxyCheck", True): - proxies = {} - else: - if pref.GetBool("SystemProxyCheck", False): - proxy = urllib.request.getproxies() - proxies = {"http": proxy.get("http"), "https": proxy.get("http")} - elif pref.GetBool("UserProxyCheck", False): - proxy = pref.GetString("ProxyUrl", "") - proxies = {"http": proxy, "https": proxy} - - if ssl_ctx: - handler = urllib.request.HTTPSHandler(context=ssl_ctx) - else: - handler = {} - proxy_support = urllib.request.ProxyHandler(proxies) - opener = urllib.request.build_opener(proxy_support, handler) - urllib.request.install_opener(opener) - - # Url opening - req = urllib.request.Request( - url, headers={"User-Agent": "Mozilla/5.0 Magic Browser"} - ) - try: - u = urllib.request.urlopen(req, timeout=timeout) - - except URLError as e: - FreeCAD.Console.PrintLog(f"Error loading {url}:\n {e.reason}\n") - return None - except Exception: - return None - else: - return u - - def getserver(url): """returns the server part of an url""" @@ -207,6 +167,19 @@ def get_zip_url(repo): return None +def recognized_git_location(repo) -> bool: + parsed_url = urlparse(repo.url) + if ( + parsed_url.netloc == "github.com" + or parsed_url.netloc == "framagit.com" + or parsed_url.netloc == "gitlab.com" + or parsed_url.netloc == "salsa.debian.org" + ): + return True + else: + return False + + def construct_git_url(repo, filename): "Returns a direct download link to a file in an online Git repo: works with github, gitlab, and framagit" diff --git a/src/Mod/AddonManager/addonmanager_workers.py b/src/Mod/AddonManager/addonmanager_workers.py index 16cc2cd0fc..b0d1a576df 100644 --- a/src/Mod/AddonManager/addonmanager_workers.py +++ b/src/Mod/AddonManager/addonmanager_workers.py @@ -35,10 +35,13 @@ import time import subprocess import sys import platform +import itertools from datetime import datetime -from typing import Union, List +from typing import Union, List, Dict +from enum import Enum, auto -from PySide2 import QtCore, QtNetwork + +from PySide2 import QtCore import FreeCAD @@ -47,12 +50,9 @@ if FreeCAD.GuiUp: import addonmanager_utilities as utils from addonmanager_macro import Macro -from addonmanager_metadata import ( - MetadataDownloadWorker, - MetadataTxtDownloadWorker, - RequirementsTxtDownloadWorker, -) + from AddonManagerRepo import AddonManagerRepo +from NetworkManager import AM_NETWORK_MANAGER translate = FreeCAD.Qt.translate @@ -119,18 +119,7 @@ class ConnectionChecker(QtCore.QThread): translate("AddonsInstaller", "Checking network connection...\n") ) url = "https://api.github.com/zen" - request = utils.urlopen(url) - if QtCore.QThread.currentThread().isInterruptionRequested(): - return - if not request: - self.failure.emit( - translate( - "AddonsInstaller", - "Unable to connect to GitHub: check your internet connection and proxy settings and try again.", - ) - ) - return - result = request.read() + result = AM_NETWORK_MANAGER.blocking_get(url) if QtCore.QThread.currentThread().isInterruptionRequested(): return if not result: @@ -142,7 +131,7 @@ class ConnectionChecker(QtCore.QThread): ) return - result = result.decode("utf8") + result = result.data().decode("utf8") FreeCAD.Console.PrintLog(f"GitHub's zen message response: {result}\n") self.success.emit() @@ -164,12 +153,11 @@ class UpdateWorker(QtCore.QThread): # update info lists global obsolete, macros_reject_list, mod_reject_list, py2only - u = utils.urlopen( + p = AM_NETWORK_MANAGER.blocking_get( "https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/addonflags.json" ) - if u: - p = u.read() - u.close() + if p: + p = p.data().decode("utf8") j = json.loads(p) if "obsolete" in j and "Mod" in j["obsolete"]: obsolete = j["obsolete"]["Mod"] @@ -266,15 +254,12 @@ class UpdateWorker(QtCore.QThread): self.addon_repo.emit(repo) # querying official addons - u = utils.urlopen( + p = AM_NETWORK_MANAGER.blocking_get( "https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/.gitmodules" ) - if not u: + if not p: return - p = u.read() - if isinstance(p, bytes): - p = p.decode("utf-8") - u.close() + p = p.data().decode("utf8") p = re.findall( ( r'(?m)\[submodule\s*"(?P.*)"\]\s*' @@ -642,8 +627,8 @@ class FillMacroListWorker(QtCore.QThread): Reads only the page https://wiki.freecad.org/Macros_recipes """ - u = utils.urlopen("https://wiki.freecad.org/Macros_recipes") - if not u: + p = AM_NETWORK_MANAGER.blocking_get("https://wiki.freecad.org/Macros_recipes") + if not p: FreeCAD.Console.PrintWarning( translate( "AddonsInstaller", @@ -652,10 +637,7 @@ class FillMacroListWorker(QtCore.QThread): + "\n" ) return - p = u.read() - u.close() - if isinstance(p, bytes): - p = p.decode("utf-8") + p = p.data().decode("utf8") macros = re.findall('title="(Macro.*?)"', p) macros = [mac for mac in macros if ("translated" not in mac)] macro_names = [] @@ -835,18 +817,13 @@ class ShowWorker(QtCore.QThread): readmeurl = utils.get_readme_html_url(self.repo) if not readmeurl: FreeCAD.Console.PrintLog(f"README not found for {url}\n") - u = utils.urlopen(readmeurl) - if not u: + p = AM_NETWORK_MANAGER.blocking_get(readmeurl) + if not p: FreeCAD.Console.PrintLog(f"Debug: README not found at {readmeurl}\n") - u = utils.urlopen(readmeurl) - if u: - p = u.read() - if isinstance(p, bytes): - p = p.decode("utf-8") - u.close() - readme = re.findall(regex, p, flags=re.MULTILINE | re.DOTALL) - if readme: - desc = readme[0] + p = p.data().decode("utf8") + readme = re.findall(regex, p, flags=re.MULTILINE | re.DOTALL) + if readme: + desc = readme[0] else: FreeCAD.Console.PrintLog(f"Debug: README not found at {readmeurl}\n") else: @@ -854,12 +831,9 @@ class ShowWorker(QtCore.QThread): readmeurl = utils.get_readme_url(self.repo) if not readmeurl: FreeCAD.Console.PrintLog(f"Debug: README not found for {url}\n") - u = utils.urlopen(readmeurl) - if u: - p = u.read() - if isinstance(p, bytes): - p = p.decode("utf-8") - u.close() + p = AM_NETWORK_MANAGER.blocking_get(readmeurl) + if p: + p = p.data().decode("utf8") desc = utils.fix_relative_links(p, readmeurl.rsplit("/README.md")[0]) if not NOMARKDOWN and have_markdown: desc = markdown.markdown(desc, extensions=["md_in_html"]) @@ -877,25 +851,22 @@ class ShowWorker(QtCore.QThread): desc = message else: FreeCAD.Console.PrintLog("Debug: README not found at {readmeurl}\n") - if desc == "": - # fall back to the description text - u = utils.urlopen(url) - if not u: + if desc == "": + # fall back to the description text + p = AM_NETWORK_MANAGER.blocking_get(url) + if not p: + return + p = p.data().decode("utf8") + descregex = utils.get_desc_regex(self.repo) + if descregex: + desc = re.findall(descregex, p) + if desc: + desc = desc[0] + if not desc: + desc = "Unable to retrieve addon description" + self.repo.description = desc + if QtCore.QThread.currentThread().isInterruptionRequested(): return - p = u.read() - if isinstance(p, bytes): - p = p.decode("utf-8") - u.close() - descregex = utils.get_desc_regex(self.repo) - if descregex: - desc = re.findall(descregex, p) - if desc: - desc = desc[0] - if not desc: - desc = "Unable to retrieve addon description" - self.repo.description = desc - if QtCore.QThread.currentThread().isInterruptionRequested(): - return message = desc if self.repo.update_status == AddonManagerRepo.UpdateStatus.UNCHECKED: # Addon is installed but we haven't checked it yet, so let's check if it has an update @@ -920,7 +891,7 @@ class ShowWorker(QtCore.QThread): AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE ) self.update_status.emit(self.repo) - + if QtCore.QThread.currentThread().isInterruptionRequested(): return self.readme_updated.emit(message) @@ -971,9 +942,9 @@ class ShowWorker(QtCore.QThread): storename = os.path.join(store, wbName + name[-remainChars:]) if not os.path.exists(storename): try: - u = utils.urlopen(path) - imagedata = u.read() - u.close() + imagedata = AM_NETWORK_MANAGER.blocking_get(path) + if not imagedata: + raise Exception except Exception: FreeCAD.Console.PrintLog( "AddonManager: Debug: Error retrieving image from " @@ -987,7 +958,7 @@ class ShowWorker(QtCore.QThread): # lower length limit for path storename = storename[-140:] f = open(storename, "wb") - f.write(imagedata) + f.write(imagedata.data()) f.close() message = message.replace( 'src="' + origpath, @@ -1095,7 +1066,15 @@ class InstallWorkbenchWorker(QtCore.QThread): # Do the git process... self.run_git(target_dir) else: - self.run_zip(target_dir) + + # The zip process uses an event loop, since the download can potentially be quite large + self.launch_zip(target_dir) + self.zip_complete = False + current_thread = QtCore.QThread.currentThread() + while not self.zip_complete: + if current_thread.isInterruptionRequested(): + return + QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents, 50) def update_status(self) -> None: if hasattr(self, "git_progress") and self.isRunning(): @@ -1223,7 +1202,7 @@ class InstallWorkbenchWorker(QtCore.QThread): self.update_metadata() self.success.emit(self.repo, answer) - def run_zip(self, zipdir: str) -> None: + def launch_zip(self, zipdir: str) -> None: "downloads and unzip a zip version from a git repo" bakdir = None @@ -1237,97 +1216,91 @@ class InstallWorkbenchWorker(QtCore.QThread): if not zipurl: self.failure.emit( self.repo, - translate("AddonsInstaller", "Error: Unable to locate zip from") + translate("AddonsInstaller", "Error: Unable to locate ZIP from") + " " + self.repo.name, ) return - try: - u = utils.urlopen(zipurl) - except Exception: + + self.zipdir = zipdir + self.bakdir = bakdir + + AM_NETWORK_MANAGER.progress_made.connect(self.update_zip_status) + AM_NETWORK_MANAGER.progress_complete.connect(self.finish_zip) + self.zip_download_index = AM_NETWORK_MANAGER.submit_monitored_get(zipurl) + + def update_zip_status(self, index: int, bytes_read: int, data_size: int): + if index == self.zip_download_index: + locale = QtCore.QLocale() + if data_size > 10 * 1024 * 1024: # To avoid overflows, show MB instead + MB_read = bytes_read / 1024 / 1024 + MB_total = data_size / 1024 / 1024 + self.progress_made.emit(MB_read, MB_total) + mbytes_str = locale.toString(MB_read) + mbytes_total_str = locale.toString(MB_total) + percent = int(100 * float(MB_read / MB_total)) + self.status_message.emit( + translate( + "AddonsInstaller", + f"Downloading: {mbytes_str}MB of {mbytes_total_str}MB ({percent}%)", + ) + ) + elif data_size > 0: + self.progress_made.emit(bytes_read, data_size) + bytes_str = locale.toString(bytes_read) + bytes_total_str = locale.toString(data_size) + percent = int(100 * float(bytes_read / data_size)) + self.status_message.emit( + translate( + "AddonsInstaller", + f"Downloading: {bytes_str} of {bytes_total_str} bytes ({percent}%)", + ) + ) + else: + MB_read = bytes_read / 1024 / 1024 + bytes_str = locale.toString(MB_read) + self.status_message.emit( + translate( + "AddonsInstaller", + f"Downloading: {bytes_str}MB of unknown total", + ) + ) + + def finish_zip(self, index: int, response_code: int, filename: os.PathLike): + self.zip_complete = True + if response_code != 200: self.failure.emit( self.repo, - translate("AddonsInstaller", "Error: Unable to download") - + " " - + zipurl, - ) - return - if not u: - self.failure.emit( - self.repo, - translate("AddonsInstaller", "Error: Unable to download") - + " " - + zipurl, + translate( + "AddonsInstaller", + f"Error: Error while downloading ZIP file for {self.repo.display_name}", + ), ) return - data_size = 0 - if "content-length" in u.headers: - try: - data_size = int(u.headers["content-length"]) - except Exception: - pass - - current_thread = QtCore.QThread.currentThread() - - # Use an on-disk file and track download progress - locale = QtCore.QLocale() - with tempfile.NamedTemporaryFile(delete=False) as temp: - bytes_to_read = 1024 * 1024 - bytes_read = 0 - while True: - if current_thread.isInterruptionRequested(): - return - chunk = u.read(bytes_to_read) - if not chunk: - break - bytes_read += bytes_to_read - temp.write(chunk) - if data_size: - self.progress_made.emit(bytes_read, data_size) - percent = int(100 * float(bytes_read / data_size)) - MB_read = bytes_read / 1024 / 1024 - MB_total = data_size / 1024 / 1024 - bytes_str = locale.toString(MB_read) - bytes_total_str = locale.toString(MB_total) - self.status_message.emit( - translate( - "AddonsInstaller", - f"Downloading: {bytes_str}MB of {bytes_total_str}MB ({percent}%)", - ) - ) - else: - MB_read = bytes_read / 1024 / 1024 - bytes_str = locale.toString(MB_read) - self.status_message.emit( - translate( - "AddonsInstaller", - f"Downloading: {bytes_str}MB of unknown total", - ) - ) - QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents) - zfile = zipfile.ZipFile(temp) + with zipfile.ZipFile(filename, "r") as zfile: master = zfile.namelist()[0] # github will put everything in a subfolder self.status_message.emit( translate("AddonsInstaller", f"Download complete. Unzipping file...") ) QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents) - zfile.extractall(zipdir) - u.close() - zfile.close() - for filename in os.listdir(zipdir + os.sep + master): - shutil.move( - zipdir + os.sep + master + os.sep + filename, - zipdir + os.sep + filename, - ) - os.rmdir(zipdir + os.sep + master) - if bakdir: - shutil.rmtree(bakdir) - self.update_metadata() - self.success.emit( - self.repo, - translate("AddonsInstaller", "Successfully installed") + " " + zipurl, + zfile.extractall(self.zipdir) + for filename in os.listdir(self.zipdir + os.sep + master): + shutil.move( + self.zipdir + os.sep + master + os.sep + filename, + self.zipdir + os.sep + filename, ) + os.rmdir(self.zipdir + os.sep + master) + if self.bakdir: + shutil.rmtree(self.bakdir) + self.update_metadata() + self.success.emit( + self.repo, + translate( + "AddonsInstaller", + f"Successfully installed {self.repo.display_name} from ZIP file", + ), + ) def update_metadata(self): basedir = FreeCAD.getUserAppDataDir() @@ -1506,150 +1479,216 @@ class UpdateMetadataCacheWorker(QtCore.QThread): progress_made = QtCore.Signal(int, int) package_updated = QtCore.Signal(AddonManagerRepo) - class AtomicCounter(object): - def __init__(self, start=0): - self.lock = threading.Lock() - self.count = start - - def set(self, new_value): - with self.lock: - self.count = new_value - - def get(self): - with self.lock: - return self.count - - def increment(self): - with self.lock: - self.count += 1 - - def decrement(self): - with self.lock: - self.count -= 1 + class RequestType(Enum): + PACKAGE_XML = auto() + METADATA_TXT = auto() + REQUIREMENTS_TXT = auto() + ICON = auto() def __init__(self, repos): QtCore.QThread.__init__(self) self.repos = repos - self.counter = UpdateMetadataCacheWorker.AtomicCounter() + self.requests: Dict[ + int, (AddonManagerRepo, UpdateMetadataCacheWorker.RequestType) + ] = {} + AM_NETWORK_MANAGER.completed.connect(self.download_completed) + self.requests_completed = 0 + self.total_requests = 0 + self.store = os.path.join( + FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata" + ) + self.updated_repos = set() def run(self): current_thread = QtCore.QThread.currentThread() - self.num_downloads_required = len(self.repos) - self.progress_made.emit(0, self.num_downloads_required) - self.status_message.emit( - translate("AddonsInstaller", "Retrieving package metadata...") - ) - store = os.path.join( - FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata" - ) - index_file = os.path.join(store, "index.json") - self.index = {} - if os.path.isfile(index_file): - with open(index_file, "r") as f: - index_string = f.read() - self.index = json.loads(index_string) - download_queue = ( - QtNetwork.QNetworkAccessManager() - ) # Must be created on this thread - download_queue.finished.connect(self.on_finished) - - # Prevent strange internal Qt errors about cache setup by pre-emptively setting - # up a cache. The error this fixes is: - # "caching was enabled after some bytes had been written" - qnam_cache = os.path.join( - FreeCAD.getUserCachePath(), "AddonManager", "QNAM_CACHE" - ) - diskCache = QtNetwork.QNetworkDiskCache() - diskCache.setCacheDirectory(qnam_cache) - download_queue.setCache(diskCache) - - self.downloaders = [] for repo in self.repos: - if repo.metadata_url: + if repo.url and utils.recognized_git_location(repo): # package.xml - downloader = MetadataDownloadWorker(None, repo, self.index) - downloader.start_fetch(download_queue) - downloader.updated.connect(self.on_updated) - self.downloaders.append(downloader) + index = AM_NETWORK_MANAGER.submit_unmonitored_get( + utils.construct_git_url(repo, "package.xml") + ) + self.requests[index] = ( + repo, + UpdateMetadataCacheWorker.RequestType.PACKAGE_XML, + ) + self.total_requests += 1 # metadata.txt - downloader = MetadataTxtDownloadWorker(None, repo) - downloader.start_fetch(download_queue) - downloader.updated.connect(self.on_updated) - self.downloaders.append(downloader) + index = AM_NETWORK_MANAGER.submit_unmonitored_get( + utils.construct_git_url(repo, "metadata.txt") + ) + self.requests[index] = ( + repo, + UpdateMetadataCacheWorker.RequestType.METADATA_TXT, + ) + self.total_requests += 1 # requirements.txt - downloader = RequirementsTxtDownloadWorker(None, repo) - downloader.start_fetch(download_queue) - downloader.updated.connect(self.on_updated) - self.downloaders.append(downloader) + index = AM_NETWORK_MANAGER.submit_unmonitored_get( + utils.construct_git_url(repo, "requirements.txt") + ) + self.requests[index] = ( + repo, + UpdateMetadataCacheWorker.RequestType.REQUIREMENTS_TXT, + ) + self.total_requests += 1 - # Run a local event loop until we've processed all of the downloads: - # this is local to this thread, and does not affect the main event loop - ui_updater = QtCore.QTimer() - ui_updater.timeout.connect(self.send_ui_update) - ui_updater.start(100) # Send an update back to the main thread every 100ms - self.num_downloads_required = len(self.downloaders) - self.num_downloads_completed = UpdateMetadataCacheWorker.AtomicCounter() - aborted = False - while True: + while self.requests: if current_thread.isInterruptionRequested(): - download_queue.finished.disconnect(self.on_finished) - for downloader in self.downloaders: - downloader.updated.disconnect(self.on_updated) - downloader.abort() - aborted = True - if ( - aborted - or self.num_downloads_completed.get() >= self.num_downloads_required - ): - break + AM_NETWORK_MANAGER.completed.disconnect(self.download_completed) + for request in self.requests.keys(): + AM_NETWORK_MANAGER.abort(request) + return + # 50 ms maximum between checks for interruption QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents, 50) - if aborted: - FreeCAD.Console.PrintLog("Metadata update cancelled\n") - return + # This set contains one copy of each of the repos that got some kind of data in + # this process. For those repos, tell the main Addon Manager code that it needs + # to update its copy of the repo, and redraw its information. + for repo in self.updated_repos: + self.package_updated.emit(repo) - # Update and serialize the updated index, overwriting whatever was - # there before - for downloader in self.downloaders: - if hasattr(downloader, "last_sha1"): - self.index[downloader.repo.name] = downloader.last_sha1 - if not os.path.exists(store): - os.makedirs(store) - with open(index_file, "w") as f: - json.dump(self.index, f, indent=" ") + def download_completed( + self, index: int, code: int, data: QtCore.QByteArray + ) -> None: + if index in self.requests: + self.requests_completed += 1 + self.progress_made.emit(self.requests_completed, self.total_requests) + request = self.requests.pop(index) + if code == 200: # HTTP success + self.updated_repos.add(request[0]) # mark this repo as updated + if request[1] == UpdateMetadataCacheWorker.RequestType.PACKAGE_XML: + self.process_package_xml(request[0], data) + elif request[1] == UpdateMetadataCacheWorker.RequestType.METADATA_TXT: + self.process_metadata_txt(request[0], data) + elif ( + request[1] == UpdateMetadataCacheWorker.RequestType.REQUIREMENTS_TXT + ): + self.process_requirements_txt(request[0], data) + elif request[1] == UpdateMetadataCacheWorker.RequestType.ICON: + self.process_icon(request[0], data) - def on_finished(self, reply): - # Called by the QNetworkAccessManager's sub-threads when a fetch - # process completed (in any state) - self.num_downloads_completed.increment() - reply.deleteLater() - - def on_updated(self, repo): - # Called if this repo got new metadata and/or a new icon - self.package_updated.emit(repo) - - def send_ui_update(self): - completed = self.num_downloads_completed.get() - required = self.num_downloads_required - percentage = int(100 * completed / required) - self.progress_made.emit(completed, required) + def process_package_xml(self, repo: AddonManagerRepo, data: QtCore.QByteArray): + repo.repo_type = AddonManagerRepo.RepoType.PACKAGE # By definition + package_cache_directory = os.path.join(self.store, repo.name) + if not os.path.exists(package_cache_directory): + os.makedirs(package_cache_directory) + new_xml_file = os.path.join(package_cache_directory, "package.xml") + with open(new_xml_file, "wb") as f: + f.write(data.data()) + metadata = FreeCAD.Metadata(new_xml_file) + repo.metadata = metadata self.status_message.emit( - translate("AddonsInstaller", "Retrieving package metadata...") - + f" {completed} / {required} ({percentage}%)" + translate("AddonsInstaller", f"Downloaded package.xml for {repo.name}") ) - def terminate_all(self): - got = self.num_downloads_completed.get() - wanted = self.num_downloads_required - if wanted < got: - FreeCAD.Console.PrintWarning( - f"During cache interruption, wanted {wanted}, got {got}, forcibly terminating now...\n" + # Grab a new copy of the icon as well: we couldn't enqueue this earlier because + # we didn't know the path to it, which is stored in the package.xml file. + icon = metadata.Icon + if not icon: + # If there is no icon set for the entire package, see if there are + # any workbenches, which are required to have icons, and grab the first + # one we find: + content = repo.metadata.Content + if "workbench" in content: + wb = content["workbench"][0] + if wb.Icon: + if wb.Subdirectory: + subdir = wb.Subdirectory + else: + subdir = wb.Name + repo.Icon = subdir + wb.Icon + icon = repo.Icon + + icon_url = utils.construct_git_url(repo, icon) + index = AM_NETWORK_MANAGER.submit_unmonitored_get(icon_url) + self.requests[index] = (repo, UpdateMetadataCacheWorker.RequestType.ICON) + self.total_requests += 1 + + def process_metadata_txt(self, repo: AddonManagerRepo, data: QtCore.QByteArray): + self.status_message.emit( + translate( + "AddonsInstaller", f"Downloaded metadata.txt for {repo.display_name}" ) - self.num_downloads_completed.set(wanted) + ) + f = io.StringIO(data.data().decode("utf8")) + while True: + line = f.readline() + if not line: + break + if line.startswith("workbenches="): + depswb = line.split("=")[1].split(",") + for wb in depswb: + wb_name = wb.strip() + if wb_name: + repo.requires.add(wb_name) + FreeCAD.Console.PrintLog( + f"{repo.display_name} requires FreeCAD Addon '{wb_name}'\n" + ) + + elif line.startswith("pylibs="): + depspy = line.split("=")[1].split(",") + for pl in depspy: + dep = pl.strip() + if dep: + repo.python_requires.add(dep) + FreeCAD.Console.PrintLog( + f"{repo.display_name} requires python package '{dep}'\n" + ) + + elif line.startswith("optionalpylibs="): + opspy = line.split("=")[1].split(",") + for pl in opspy: + dep = pl.strip() + if dep: + repo.python_optional.add(dep) + FreeCAD.Console.PrintLog( + f"{repo.display_name} optionally imports python package '{pl.strip()}'\n" + ) + # For review and debugging purposes, store the file locally + package_cache_directory = os.path.join(self.store, repo.name) + if not os.path.exists(package_cache_directory): + os.makedirs(package_cache_directory) + new_xml_file = os.path.join(package_cache_directory, "metadata.txt") + with open(new_xml_file, "wb") as f: + f.write(data.data()) + + def process_requirements_txt(self, repo: AddonManagerRepo, data: QtCore.QByteArray): + self.status_message.emit( + translate( + "AddonsInstaller", + f"Downloaded requirements.txt for {repo.display_name}", + ) + ) + f = io.StringIO(data.data().decode("utf8")) + lines = f.readlines() + for line in lines: + break_chars = " <>=~!+#" + package = line + for n, c in enumerate(line): + if c in break_chars: + package = line[:n].strip() + break + if package: + repo.python_requires.add(package) + # For review and debugging purposes, store the file locally + package_cache_directory = os.path.join(self.store, repo.name) + if not os.path.exists(package_cache_directory): + os.makedirs(package_cache_directory) + new_xml_file = os.path.join(package_cache_directory, "requirements.txt") + with open(new_xml_file, "wb") as f: + f.write(data.data()) + + def process_icon(self, repo: AddonManagerRepo, data: QtCore.QByteArray): + self.status_message.emit( + translate("AddonsInstaller", f"Downloaded icon for {repo.display_name}") + ) + cache_file = repo.get_cached_icon_filename() + with open(cache_file, "wb") as icon_file: + icon_file.write(data.data()) + repo.cached_icon_filename = cache_file if have_git and not NOGIT: diff --git a/src/Mod/AddonManager/proxy_authentication.ui b/src/Mod/AddonManager/proxy_authentication.ui new file mode 100644 index 0000000000..7c804680b5 --- /dev/null +++ b/src/Mod/AddonManager/proxy_authentication.ui @@ -0,0 +1,150 @@ + + + proxy_authentication + + + + 0 + 0 + 536 + 276 + + + + Proxy login required + + + + + + Proxy requires authentication + + + + + + + + + Proxy: + + + + + + + + 0 + 0 + + + + Placeholder for proxy address + + + + + + + Realm: + + + + + + + Placeholder for proxy realm + + + + + + + + + + + Username + + + + + + + + + + Password + + + + + + + QLineEdit::PasswordEchoOnEdit + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + + + + + + + buttonBox + accepted() + proxy_authentication + accept() + + + 248 + 254 + + + 157 + 274 + + + + + buttonBox + rejected() + proxy_authentication + reject() + + + 316 + 260 + + + 286 + 274 + + + + +