diff --git a/src/Mod/AddonManager/AddonManager.py b/src/Mod/AddonManager/AddonManager.py index a87b5a6fc2..f2a27515f6 100644 --- a/src/Mod/AddonManager/AddonManager.py +++ b/src/Mod/AddonManager/AddonManager.py @@ -42,6 +42,8 @@ from package_list import PackageList, PackageListItemModel from package_details import PackageDetails from AddonManagerRepo import AddonManagerRepo +from NetworkManager import HAVE_QTNETWORK + __title__ = "FreeCAD Addon Manager Module" __author__ = "Yorik van Havre", "Jonathan Wiedemann", "Kurt Kremitzki", "Chris Hennes" __url__ = "http://www.freecad.org" @@ -228,9 +230,16 @@ class CommandAddonManager: # This must run on the main GUI thread if hasattr(self, "connection_check_message") and self.connection_check_message: self.connection_check_message.close() - QtWidgets.QMessageBox.critical( - None, translate("AddonsInstaller", "Connection failed"), message - ) + if HAVE_QTNETWORK: + QtWidgets.QMessageBox.critical( + None, translate("AddonsInstaller", "Connection failed"), message + ) + else: + QtWidgets.QMessageBox.critical( + None, + translate("AddonsInstaller", "Missing dependency"), + translate("AddonsInstaller", "Could not import QtNetwork -- see Report View for details. Addon Manager unavailable."), + ) def launch(self) -> None: """Shows the Addon Manager UI""" @@ -821,7 +830,7 @@ class CommandAddonManager: self.packageDetails.show_repo(selected_repo) def show_information(self, message: str) -> None: - """shows generic text in the information pane (which might be collapsed)""" + """shows generic text in the information pane""" self.dialog.labelStatusInfo.setText(message) self.dialog.labelStatusInfo.repaint() diff --git a/src/Mod/AddonManager/CMakeLists.txt b/src/Mod/AddonManager/CMakeLists.txt index 8b77d2846d..56a12e7983 100644 --- a/src/Mod/AddonManager/CMakeLists.txt +++ b/src/Mod/AddonManager/CMakeLists.txt @@ -8,7 +8,6 @@ SET(AddonManager_SRCS AddonManager.py AddonManagerRepo.py addonmanager_macro.py - addonmanager_metadata.py addonmanager_utilities.py addonmanager_workers.py AddonManager.ui @@ -18,6 +17,7 @@ SET(AddonManager_SRCS compact_view.py dependency_resolution_dialog.ui expanded_view.py + NetworkManager.py package_list.py package_details.py ) diff --git a/src/Mod/AddonManager/NetworkManager.py b/src/Mod/AddonManager/NetworkManager.py new file mode 100644 index 0000000000..5c9314bc1c --- /dev/null +++ b/src/Mod/AddonManager/NetworkManager.py @@ -0,0 +1,552 @@ +# -*- coding: utf-8 -*- + +# *************************************************************************** +# * * +# * Copyright (c) 2022 Chris Hennes * +# * * +# * This program is free software; you can redistribute it and/or modify * +# * it under the terms of the GNU Lesser General Public License (LGPL) * +# * as published by the Free Software Foundation; either version 2 of * +# * the License, or (at your option) any later version. * +# * for detail see the LICENSE text file. * +# * * +# * This program is distributed in the hope that it will be useful, * +# * but WITHOUT ANY WARRANTY; without even the implied warranty of * +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * +# * GNU Library General Public License for more details. * +# * * +# * You should have received a copy of the GNU Library General Public * +# * License along with this program; if not, write to the Free Software * +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * +# * USA * +# * * +# *************************************************************************** + + +############################################################################# +# +# ABOUT NETWORK MANAGER +# +# A wrapper around QNetworkAccessManager providing proxy-handling +# capabilities, and simplified access to submitting requests from any +# application thread. +# +# +# USAGE +# +# Once imported, this file provides access to a global object called +# AM_NETWORK_MANAGER. This is a QThread object running on its own thread, but +# designed to be interacted with from any other application thread. It +# provides two principal methods: submit_unmonitored_get() and +# submit_monitored_get(). Use the unmonitored version for small amounts of +# data (suitable for caching in RAM, and without a need to show a progress +# bar during download), and the monitored version for larger amounts of data. +# Both functions take a URL, and return an integer index. That index allows +# tracking of the completed request by attaching to the signals completed(), +# progress_made(), and progress_complete(). All three provide, as the first +# argument to the signal, the index of the request the signal refers to. +# Code attached to those signals should filter them to look for the indices +# of the requests they care about. Requests may complete in any order. +# +# A secondary blocking interface is also provided, for very short network +# accesses: the blocking_get() function blocks until the network transmission +# is complete, directly returning a QByteArray object with the received data. +# Do not run on the main GUI thread! + + +try: + import FreeCAD + + if FreeCAD.GuiUp: + import FreeCADGui + + HAVE_FREECAD = True + translate = FreeCAD.Qt.translate +except Exception: + # For standalone testing support working without the FreeCAD import + HAVE_FREECAD = False + +import threading +from PySide2 import QtCore + +import os +import queue +import itertools +import tempfile +from typing import Dict, List + +# This is the global instance of the NetworkManager that outside code +# should access +AM_NETWORK_MANAGER = None + +HAVE_QTNETWORK = True +try: + from PySide2 import QtNetwork +except Exception: + if HAVE_FREECAD: + FreeCAD.Console.PrintError( + translate( + "AddonsInstaller", + "Could not import QtNetwork -- it does not appear to be installed on your system. Please install the package 'python3-pyside2.qtnetwork' on your system and if possible contact your FreeCAD package maintainer to alert them to the missing dependency. The Addon Manager will not be available.", + ) + + "\n" + ) + else: + print( + "Could not import QtNetwork, unable to test this file. Try installing the python3-pyside2.qtnetwork package." + ) + exit(1) + HAVE_QTNETWORK = False + +if HAVE_QTNETWORK: + + class QueueItem: + def __init__( + self, index: int, request: QtNetwork.QNetworkRequest, track_progress: bool + ): + self.index = index + self.request = request + self.track_progress = track_progress + + class NetworkManager(QtCore.QThread): + """A single global instance of NetworkManager is instantiated and stored as + AM_NETWORK_MANAGER. Outside threads should send GET requests to this class by + calling the submit_unmonitored_request() or submit_monitored_request() function, + as needed. See the documentation of those functions for details.""" + + # Connect to complete for requests with no progress monitoring (e.g. small amounts of data) + completed = QtCore.Signal( + int, int, QtCore.QByteArray + ) # Index, http response code, received data (if any) + + # Connect to progress_made and progress_complete for large amounts of data, which get buffered into a temp file + # That temp file should be deleted when your code is done with it + progress_made = QtCore.Signal( + int, int, int + ) # Index, bytes read, total bytes (may be None) + + progress_complete = QtCore.Signal( + int, int, os.PathLike + ) # Index, http response code, filename + + def __init__(self): + super().__init__() + + # Set up the incoming request queue: requests get put on this queue, and the main event loop + # pulls them off and runs them. + self.counting_iterator = itertools.count() + self.queue = queue.Queue() + self.__last_started_index = 0 + self.__abort_when_found: List[int] = [] + self.replies: Dict[int, QtNetwork.QNetworkReply] = {} + self.file_buffers = {} + + # We support an arbitrary number of threads using synchronous GET calls: + self.synchronous_lock = threading.Lock() + self.synchronous_complete: Dict[int, bool] = {} + self.synchronous_result_data: Dict[int, QtCore.QByteArray] = {} + + def run(self): + """Do not call directly: use start() to begin the event loop on a new thread.""" + + # Create the QNAM on this thread: + self.thread = QtCore.QThread.currentThread() + self.QNAM = QtNetwork.QNetworkAccessManager() + self.QNAM.proxyAuthenticationRequired.connect(self.__authenticate_proxy) + self.QNAM.authenticationRequired.connect(self.__authenticate_resource) + + # A helper connection for our blocking interface + self.completed.connect(self.__synchronous_process_completion) + + # Set up the proxy, if necesssary: + noProxyCheck = True + systemProxyCheck = False + userProxyCheck = False + proxy_string = "" + if HAVE_FREECAD: + pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") + noProxyCheck = pref.GetBool("NoProxyCheck", noProxyCheck) + systemProxyCheck = pref.GetBool("SystemProxyCheck", systemProxyCheck) + userProxyCheck = pref.GetBool("UserProxyCheck", userProxyCheck) + proxy_string = pref.GetString("ProxyUrl", "") + else: + print("Please select a proxy type:") + print("1) No proxy") + print("2) Use system proxy settings") + print("3) Custom proxy settings") + result = input("Choice: ") + if result == "1": + pass + elif result == "2": + noProxyCheck = False + systemProxyCheck = True + elif result == "3": + noProxyCheck = False + userProxyCheck = True + proxy_string = input("Enter your proxy server (host:port): ") + else: + print(f"Got {result}, expected 1, 2, or 3.") + app.quit() + + if noProxyCheck: + pass + elif systemProxyCheck: + query = QtNetwork.QNetworkProxyQuery( + QtCore.QUrl("https://github.com/FreeCAD/FreeCAD") + ) + proxy = QtNetwork.QNetworkProxyFactory.systemProxyForQuery(query) + if proxy and proxy[0]: + self.QNAM.setProxy( + proxy[0] + ) # This may still be QNetworkProxy.NoProxy + elif userProxyCheck: + host, _, port_string = proxy_string.rpartition(":") + port = 0 if not port_string else int(port_string) + # For now assume an HttpProxy, but eventually this should be a parameter + proxy = QtNetwork.QNetworkProxy( + QtNetwork.QNetworkProxy.HttpProxy, host, port + ) + self.QNAM.setProxy(proxy) + + qnam_cache = QtCore.QStandardPaths.writableLocation(QtCore.QStandardPaths.CacheLocation); + os.makedirs(qnam_cache,exist_ok=True) + diskCache = QtNetwork.QNetworkDiskCache() + diskCache.setCacheDirectory(qnam_cache) + self.QNAM.setCache(diskCache) + + # Start an event loop + while True: + if QtCore.QThread.currentThread().isInterruptionRequested(): + # Support shutting down the entire thread, but this should be very rarely used. + # Callers should generally just call abort() on each network call they want to + # terminate, using requestInterruption() will terminate ALL network requests. + if not HAVE_FREECAD: + print( + "Shutting down all active network requests...", flush=True + ) + self.abort_all() + self.queue.join() + if not HAVE_FREECAD: + print("All requests terminated.", flush=True) + return + try: + item = self.queue.get_nowait() + if item: + if item.index in self.__abort_when_found: + self.__abort_when_found.remove(item.index) + continue # Do not do anything with this item, it's been aborted... + reply = self.QNAM.get(item.request) + + self.__last_started_index = item.index + reply.finished.connect(lambda i=item: self.__reply_finished(i)) + reply.redirected.connect( + lambda url, r=reply: self.__on_redirect(r, url) + ) + reply.sslErrors.connect(self.__on_ssl_error) + if item.track_progress: + reply.readyRead.connect( + lambda i=item.index: self.__data_incoming(i) + ) + reply.downloadProgress.connect( + lambda a, b, i=item.index: self.progress_made.emit( + i, a, b + ) + ) + self.replies[item.index] = reply + except queue.Empty: + pass + QtCore.QCoreApplication.processEvents() + + def submit_unmonitored_get(self, url: str) -> int: + """Adds this request to the queue, and returns an index that can be used by calling code + in conjunction with the completed() signal to handle the results of the call. All data is + kept in memory, and the completed() call includes a direct handle to the bytes returned. It + is not called until the data transfer has finished and the connection is closed.""" + + current_index = next(self.counting_iterator) # A thread-safe counter + # Use a queue because we can only put things on the QNAM from the main event loop thread + self.queue.put( + QueueItem( + current_index, self.__create_get_request(url), track_progress=False + ) + ) + return current_index + + def submit_monitored_get(self, url: str) -> int: + """Adds this request to the queue, and returns an index that can be used by calling code + in conjunction with the progress_made() and progress_completed() signals to handle the + results of the call. All data is cached to disk, and progress is reported periodically + as the underlying QNetworkReply reports its progress. The progress_completed() signal + contains a path to a temporary file with the stored data. Calling code should delete this + file when done with it (or move it into its final place, etc.).""" + + current_index = next(self.counting_iterator) # A thread-safe counter + # Use a queue because we can only put things on the QNAM from the main event loop thread + self.queue.put( + QueueItem( + current_index, self.__create_get_request(url), track_progress=True + ) + ) + return current_index + + def blocking_get(self, url: str) -> QtCore.QByteArray: + """Submits a GET request to the QNetworkAccessManager and block until it is complete""" + + current_index = next(self.counting_iterator) # A thread-safe counter + with self.synchronous_lock: + self.synchronous_complete[current_index] = False + + self.queue.put( + QueueItem( + current_index, self.__create_get_request(url), track_progress=False + ) + ) + while not self.synchronous_complete[current_index]: + if QtCore.QThread.currentThread().isInterruptionRequested(): + return None + QtCore.QCoreApplication.processEvents() + + with self.synchronous_lock: + self.synchronous_complete.pop(current_index) + return self.synchronous_result_data.pop(current_index) + + def __synchronous_process_completion( + self, index: int, code: int, data: QtCore.QByteArray + ) -> None: + with self.synchronous_lock: + if index in self.synchronous_complete: + if code == 200: + self.synchronous_result_data[index] = data + self.synchronous_complete[index] = True + + def __create_get_request(self, url: str) -> QtNetwork.QNetworkRequest: + request = QtNetwork.QNetworkRequest(QtCore.QUrl(url)) + request.setAttribute( + QtNetwork.QNetworkRequest.RedirectPolicyAttribute, + QtNetwork.QNetworkRequest.UserVerifiedRedirectPolicy, + ) + request.setAttribute( + QtNetwork.QNetworkRequest.CacheSaveControlAttribute, True + ) + request.setAttribute( + QtNetwork.QNetworkRequest.CacheLoadControlAttribute, + QtNetwork.QNetworkRequest.PreferCache, + ) + request.setAttribute( + QtNetwork.QNetworkRequest.BackgroundRequestAttribute, True + ) + return request + + def abort_all(self): + """Abort ALL network calls in progress, including clearing the queue""" + for reply in self.replies: + if reply.isRunning(): + reply.abort() + while True: + try: + self.queue.get() + self.queue.task_done() + except queue.Empty: + break + + def abort(self, index: int): + if index in self.replies and self.replies[index].isRunning(): + self.replies[index].abort() + elif index < self.__last_started_index: + # It's still in the queue. Mark it for later destruction. + self.__abort_when_found.append(index) + + def __authenticate_proxy( + self, + reply: QtNetwork.QNetworkProxy, + authenticator: QtNetwork.QAuthenticator, + ): + if HAVE_FREECAD and FreeCAD.GuiUp: + proxy_authentication = FreeCADGui.PySideUic.loadUi( + os.path.join(os.path.dirname(__file__), "proxy_authentication.ui") + ) + # Show the right labels, etc. + proxy_authentication.labelProxyAddress.setText( + f"{reply.hostName()}:{reply.port()}" + ) + if authenticator.realm(): + proxy_authentication.labelProxyRealm.setText(authenticator.realm()) + else: + proxy_authentication.labelProxyRealm.hide() + proxy_authentication.labelRealmCaption.hide() + result = proxy_authentication.exec() + if result == QtWidgets.QDialogButtonBox.Ok: + authenticator.setUser(proxy_authentication.lineEditUsername.text()) + authenticator.setPassword( + proxy_authentication.lineEditPassword.text() + ) + else: + username = input("Proxy username: ") + import getpass + + password = getpass.getpass() + authenticator.setUser(username) + authenticator.setPassword(password) + + def __authenticate_resource( + self, + _reply: QtNetwork.QNetworkReply, + _authenticator: QtNetwork.QAuthenticator, + ): + pass + + def __on_redirect(self, reply, _): + # For now just blindly follow all redirects + reply.redirectAllowed.emit() + + def __on_ssl_error(self, reply: str, errors: List[str]): + if HAVE_FREECAD: + FreeCAD.Console.PrintWarning( + translate("AddonsInstaller", "Error with encrypted connection") + + "\n:" + ) + FreeCAD.Console.PrintWarning(reply) + for error in errors: + FreeCAD.Console.PrintWarning(error) + else: + print("Error with encrypted connection") + for error in errors: + print(error) + + def __data_incoming(self, index: int): + reply = self.replies[index] + chunk_size = reply.bytesAvailable() + buffer = reply.read(chunk_size) + if not index in self.file_buffers: + f = tempfile.NamedTemporaryFile("wb", delete=False) + self.file_buffers[index] = f + else: + f = self.file_buffers[index] + f.write(buffer) + + def __reply_finished(self, item: QueueItem) -> None: + reply = self.replies.pop(item.index) + response_code = reply.attribute( + QtNetwork.QNetworkRequest.HttpStatusCodeAttribute + ) + self.queue.task_done() + if reply.error() == QtNetwork.QNetworkReply.NetworkError.NoError: + if item.track_progress: + f = self.file_buffers[item.index] + f.close() + self.progress_complete.emit(item.index, response_code, f.name) + else: + data = reply.readAll() + self.completed.emit(item.index, response_code, data) + else: + if item.track_progress: + self.progress_complete.emit(item.index, response_code, "") + else: + self.completed.emit(item.index, response_code, None) + +else: # HAVE_QTNETWORK is false: + + class NetworkManager(QtCore.QThread): + """A dummy class to enable an offline mode when the QtNetwork package is not yet installed""" + + completed = QtCore.Signal( + int, int, bytes + ) # Emitted as soon as the request is made, with a connection failed error + progress_made = QtCore.Signal( + int, int, int + ) # Never emitted, no progress is made here + progress_complete = QtCore.Signal( + int, int, os.PathLike + ) # Emitted as soon as the request is made, with a connection failed error + + def __init__(self): + super().__init__() + self.monitored_queue = queue.Queue() + self.unmonitored_queue = queue.Queue() + + def run(self): + while True: + try: + index = self.monitored_queue.get_nowait() + self.progress_complete.emit( + index, 418, "--ERR418--" + ) # Refuse to provide data + self.monitored_queue.task_done() + except queue.Empty: + pass + try: + index = self.unmonitored_queue.get_nowait() + self.completed.emit(index, 418, None) + self.unmonitored_queue.task_done() + except queue.Empty: + pass + if QtCore.QThread.currentThread().isInterruptionRequested(): + return + QtCore.QCoreApplication.processEvents() + + def submit_unmonitored_request(self, _) -> int: + current_index = next(itertools.count()) + self.unmonitored_queue.put(current_index) + return current_index + + def submit_monitored_request(self, _) -> int: + current_index = next(itertools.count()) + self.monitored_queue.put(current_index) + return current_index + + def blocking_get(self, _: str) -> QtCore.QByteArray: + return None + + def abort_all( + self, + ): + pass # Nothing to do + + def abort(self, _): + pass # Nothing to do + + +AM_NETWORK_MANAGER = NetworkManager() +AM_NETWORK_MANAGER.start() + + +if __name__ == "__main__": + + app = QtCore.QCoreApplication() + + count = 0 + + # For testing, create several network requests and send them off in quick succession: + # (Choose small downloads, no need for significant data) + urls = [ + "https://api.github.com/zen", + "http://climate.ok.gov/index.php/climate/rainfall_table/local_data", + "https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/AIANNHA/MapServer", + ] + + def handle_completion(index: int, code: int, data): + global count + if code == 200: + print( + f"For request {index+1}, response was {data.size()} bytes.", flush=True + ) + else: + print( + f"For request {index+1}, request failed with HTTP result code {code}", + flush=True, + ) + + count += 1 + if count >= len(urls): + print(f"Shutting down...", flush=True) + AM_NETWORK_MANAGER.requestInterruption() + AM_NETWORK_MANAGER.wait(5000) + app.quit() + + AM_NETWORK_MANAGER.completed.connect(handle_completion) + for url in urls: + AM_NETWORK_MANAGER.submit_unmonitored_get(url) + + app.exec_() + + print("Done with all requests.") diff --git a/src/Mod/AddonManager/addonmanager_macro.py b/src/Mod/AddonManager/addonmanager_macro.py index 73797ffaeb..4dfd1f8674 100644 --- a/src/Mod/AddonManager/addonmanager_macro.py +++ b/src/Mod/AddonManager/addonmanager_macro.py @@ -30,9 +30,10 @@ import time from typing import Dict, Tuple, List, Union import FreeCAD +from NetworkManager import AM_NETWORK_MANAGER + +translate = FreeCAD.Qt.translate -from addonmanager_utilities import translate -from addonmanager_utilities import urlopen from addonmanager_utilities import remove_directory_if_empty try: @@ -165,28 +166,25 @@ class Macro(object): def fill_details_from_wiki(self, url): code = "" - u = urlopen(url) - if u is None: + p = AM_NETWORK_MANAGER.blocking_get(url) + if not p: FreeCAD.Console.PrintWarning( translate( "AddonsInstaller", - f"Could not connect to {url} - check connection and proxy settings", + f"Unable to open macro wiki page at {url}", ) + "\n" ) return - p = u.read() - if isinstance(p, bytes): - p = p.decode("utf-8") - u.close() + p = p.data().decode("utf8") # check if the macro page has its code hosted elsewhere, download if # needed if "rawcodeurl" in p: rawcodeurl = re.findall('rawcodeurl.*?href="(http.*?)">', p) if rawcodeurl: rawcodeurl = rawcodeurl[0] - u2 = urlopen(rawcodeurl) - if u2 is None: + u2 = AM_NETWORK_MANAGER.blocking_get(rawcodeurl) + if not u2: FreeCAD.Console.PrintWarning( translate( "AddonsInstaller", @@ -195,18 +193,7 @@ class Macro(object): + "\n" ) return - response = "" - block = 8192 - while True: - data = u2.read(block) - if not data: - break - if isinstance(data, bytes): - data = data.decode("utf-8") - response += data - if response: - code = response - u2.close() + code = u2.data().decode("utf8") if not code: code = re.findall(r"
(.*?)
", p.replace("\n", "--endl--")) if code: diff --git a/src/Mod/AddonManager/addonmanager_metadata.py b/src/Mod/AddonManager/addonmanager_metadata.py deleted file mode 100644 index b6e8eebc46..0000000000 --- a/src/Mod/AddonManager/addonmanager_metadata.py +++ /dev/null @@ -1,315 +0,0 @@ -# *************************************************************************** -# * * -# * Copyright (c) 2021 Chris Hennes * -# * * -# * This program is free software; you can redistribute it and/or modify * -# * it under the terms of the GNU Lesser General Public License (LGPL) * -# * as published by the Free Software Foundation; either version 2 of * -# * the License, or (at your option) any later version. * -# * for detail see the LICENCE text file. * -# * * -# * This program is distributed in the hope that it will be useful, * -# * but WITHOUT ANY WARRANTY; without even the implied warranty of * -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * -# * GNU Library General Public License for more details. * -# * * -# * You should have received a copy of the GNU Library General Public * -# * License along with this program; if not, write to the Free Software * -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * -# * USA * -# * * -# *************************************************************************** - -import FreeCAD - -import os -import io -import hashlib -from typing import Dict, List, Set - -from PySide2 import QtCore, QtNetwork -from PySide2.QtCore import QObject - -import addonmanager_utilities as utils -from AddonManagerRepo import AddonManagerRepo - -translate = FreeCAD.Qt.translate - - -class DownloadWorker(QObject): - updated = QtCore.Signal(AddonManagerRepo) - - def __init__(self, parent, url: str): - """repo is an AddonManagerRepo object, and index is a dictionary of SHA1 hashes of the package.xml files in the cache""" - - super().__init__(parent) - self.url = url - - def start_fetch(self, network_manager: QtNetwork.QNetworkAccessManager): - """Asynchronously begin the network access. Intended as a set-and-forget black box for downloading files.""" - - self.request = QtNetwork.QNetworkRequest(QtCore.QUrl(self.url)) - self.request.setAttribute( - QtNetwork.QNetworkRequest.RedirectPolicyAttribute, - QtNetwork.QNetworkRequest.UserVerifiedRedirectPolicy, - ) - self.request.setAttribute( - QtNetwork.QNetworkRequest.CacheSaveControlAttribute, True - ) - self.request.setAttribute( - QtNetwork.QNetworkRequest.CacheLoadControlAttribute, - QtNetwork.QNetworkRequest.PreferCache, - ) - self.request.setAttribute( - QtNetwork.QNetworkRequest.BackgroundRequestAttribute, True - ) - - self.fetch_task = network_manager.get(self.request) - self.fetch_task.finished.connect(self.resolve_fetch) - self.fetch_task.redirected.connect(self.on_redirect) - self.fetch_task.sslErrors.connect(self.on_ssl_error) - - def abort(self): - if not self.fetch_task.isFinished(): - self.fetch_task.abort() - - def on_redirect(self, _): - # For now just blindly follow all redirects - self.fetch_task.redirectAllowed.emit() - - def on_ssl_error(self, reply: str, errors: List[str]): - FreeCAD.Console.PrintWarning( - translate("AddonsInstaller", "Error with encrypted connection") + "\n:" - ) - FreeCAD.Console.PrintWarning(reply) - for error in errors: - FreeCAD.Console.PrintWarning(error) - - -class MetadataDownloadWorker(DownloadWorker): - """A worker for downloading package.xml and associated icon(s) - - To use, instantiate an object of this class and call the start_fetch() function - with a QNetworkAccessManager. It is expected that many of these objects will all - be created and associated with the same QNAM, which will then handle the actual - asynchronous downloads in some Qt-defined number of threads. To monitor progress - you should connect to the QNAM's "finished" signal, and ensure it is called the - number of times you expect based on how many workers you have enqueued. - - """ - - def __init__(self, parent, repo: AddonManagerRepo, index: Dict[str, str]): - """repo is an AddonManagerRepo object, and index is a dictionary of SHA1 hashes of the package.xml files in the cache""" - - super().__init__(parent, repo.metadata_url) - self.repo = repo - self.index = index - self.store = os.path.join( - FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata" - ) - self.last_sha1 = "" - - def resolve_fetch(self): - """Called when the data fetch completed, either with an error, or if it found the metadata file""" - - if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError: - FreeCAD.Console.PrintLog(f"Found a package.xml file for {self.repo.name}\n") - self.repo.repo_type = AddonManagerRepo.RepoType.PACKAGE - new_xml = self.fetch_task.readAll() - hasher = hashlib.sha1() - hasher.update(new_xml.data()) - new_sha1 = hasher.hexdigest() - self.last_sha1 = new_sha1 - # Determine if we need to download the icon: only do that if the - # package.xml file changed (since - # a change in the version number will show up as a change in the - # SHA1, without having to actually - # read the metadata) - if self.repo.name in self.index: - cached_sha1 = self.index[self.repo.name] - if cached_sha1 != new_sha1: - self.update_local_copy(new_xml) - else: - # Assume that if the package.xml file didn't change, - # neither did the icon, so don't waste - # resources downloading it - xml_file = os.path.join(self.store, self.repo.name, "package.xml") - self.repo.metadata = FreeCAD.Metadata(xml_file) - else: - # There is no local copy yet, so we definitely have to update - # the cache - self.update_local_copy(new_xml) - elif ( - self.fetch_task.error() - == QtNetwork.QNetworkReply.NetworkError.ContentNotFoundError - ): - pass - elif ( - self.fetch_task.error() - == QtNetwork.QNetworkReply.NetworkError.OperationCanceledError - ): - pass - else: - FreeCAD.Console.PrintWarning( - translate("AddonsInstaller", "Failed to connect to URL") - + f":\n{self.url}\n {self.fetch_task.error()}\n" - ) - - def update_local_copy(self, new_xml): - # We have to update the local copy of the metadata file and re-download - # the icon file - - name = self.repo.name - repo_url = self.repo.url - package_cache_directory = os.path.join(self.store, name) - if not os.path.exists(package_cache_directory): - os.makedirs(package_cache_directory) - new_xml_file = os.path.join(package_cache_directory, "package.xml") - with open(new_xml_file, "wb") as f: - f.write(new_xml.data()) - metadata = FreeCAD.Metadata(new_xml_file) - self.repo.metadata = metadata - self.repo.repo_type = AddonManagerRepo.RepoType.PACKAGE - icon = metadata.Icon - - if not icon: - # If there is no icon set for the entire package, see if there are - # any workbenches, which are required to have icons, and grab the first - # one we find: - content = self.repo.metadata.Content - if "workbench" in content: - wb = content["workbench"][0] - if wb.Icon: - if wb.Subdirectory: - subdir = wb.Subdirectory - else: - subdir = wb.Name - self.repo.Icon = subdir + wb.Icon - icon = self.repo.Icon - - icon_url = utils.construct_git_url(self.repo, icon) - icon_stream = utils.urlopen(icon_url) - if icon and icon_stream and icon_url: - icon_data = icon_stream.read() - cache_file = self.repo.get_cached_icon_filename() - with open(cache_file, "wb") as icon_file: - icon_file.write(icon_data) - self.repo.cached_icon_filename = cache_file - self.updated.emit(self.repo) - - -class MetadataTxtDownloadWorker(DownloadWorker): - """A worker for downloading metadata.txt""" - - def __init__(self, parent, repo: AddonManagerRepo): - super().__init__(parent, utils.construct_git_url(repo, "metadata.txt")) - self.repo = repo - - def resolve_fetch(self): - """Called when the data fetch completed, either with an error, or if it found the metadata file""" - - if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError: - FreeCAD.Console.PrintLog( - f"Found a metadata.txt file for {self.repo.name}\n" - ) - new_deps = self.fetch_task.readAll() - self.parse_file(new_deps.data().decode("utf8")) - elif ( - self.fetch_task.error() - == QtNetwork.QNetworkReply.NetworkError.ContentNotFoundError - ): - pass - elif ( - self.fetch_task.error() - == QtNetwork.QNetworkReply.NetworkError.OperationCanceledError - ): - pass - else: - FreeCAD.Console.PrintWarning( - translate("AddonsInstaller", "Failed to connect to URL") - + f":\n{self.url}\n {self.fetch_task.error()}\n" - ) - - def parse_file(self, data: str) -> None: - f = io.StringIO(data) - while True: - line = f.readline() - if not line: - break - if line.startswith("workbenches="): - depswb = line.split("=")[1].split(",") - for wb in depswb: - wb_name = wb.strip() - if wb_name: - self.repo.requires.add(wb_name) - FreeCAD.Console.PrintLog( - f"{self.repo.display_name} requires FreeCAD Addon '{wb_name}'\n" - ) - - elif line.startswith("pylibs="): - depspy = line.split("=")[1].split(",") - for pl in depspy: - dep = pl.strip() - if dep: - self.repo.python_requires.add(dep) - FreeCAD.Console.PrintLog( - f"{self.repo.display_name} requires python package '{dep}'\n" - ) - - elif line.startswith("optionalpylibs="): - opspy = line.split("=")[1].split(",") - for pl in opspy: - dep = pl.strip() - if dep: - self.repo.python_optional.add(dep) - FreeCAD.Console.PrintLog( - f"{self.repo.display_name} optionally imports python package '{pl.strip()}'\n" - ) - self.updated.emit(self.repo) - - -class RequirementsTxtDownloadWorker(DownloadWorker): - """A worker for downloading requirements.txt""" - - def __init__(self, parent, repo: AddonManagerRepo): - super().__init__(parent, utils.construct_git_url(repo, "requirements.txt")) - self.repo = repo - - def resolve_fetch(self): - """Called when the data fetch completed, either with an error, or if it found the metadata file""" - - if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError: - FreeCAD.Console.PrintLog( - f"Found a requirements.txt file for {self.repo.name}\n" - ) - new_deps = self.fetch_task.readAll() - self.parse_file(new_deps.data().decode("utf8")) - elif ( - self.fetch_task.error() - == QtNetwork.QNetworkReply.NetworkError.ContentNotFoundError - ): - pass - elif ( - self.fetch_task.error() - == QtNetwork.QNetworkReply.NetworkError.OperationCanceledError - ): - pass - else: - FreeCAD.Console.PrintWarning( - translate("AddonsInstaller", "Failed to connect to URL") - + f":\n{self.url}\n {self.fetch_task.error()}\n" - ) - - def parse_file(self, data: str) -> None: - f = io.StringIO(data) - lines = f.readlines() - for line in lines: - break_chars = " <>=~!+#" - package = line - for n, c in enumerate(line): - if c in break_chars: - package = line[:n].strip() - break - if package: - self.repo.python_requires.add(package) - self.updated.emit(self.repo) diff --git a/src/Mod/AddonManager/addonmanager_utilities.py b/src/Mod/AddonManager/addonmanager_utilities.py index 43c09dfc6b..50b441d4f6 100644 --- a/src/Mod/AddonManager/addonmanager_utilities.py +++ b/src/Mod/AddonManager/addonmanager_utilities.py @@ -25,7 +25,7 @@ import os import re import ctypes import ssl -from typing import Union +from typing import Union, Optional import urllib from urllib.request import Request @@ -53,7 +53,6 @@ else: except AttributeError: pass - # @package AddonManager_utilities # \ingroup ADDONMANAGER # \brief Utilities to work across different platforms, providers and python versions @@ -72,7 +71,7 @@ def translate(context, text, disambig=None): def symlink(source, link_name): - "creates a symlink of a file, if possible" + """Creates a symlink of a file, if possible. Note that it fails on most modern Windows installations""" if os.path.exists(link_name) or os.path.lexists(link_name): pass @@ -81,6 +80,8 @@ def symlink(source, link_name): if callable(os_symlink): os_symlink(source, link_name) else: + # NOTE: This does not work on most normal Windows 10 and later installations, unless developer + # mode is turned on. Make sure to catch any exception thrown and have a fallback plan. csl = ctypes.windll.kernel32.CreateSymbolicLinkW csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32) csl.restype = ctypes.c_ubyte @@ -92,47 +93,6 @@ def symlink(source, link_name): raise ctypes.WinError() -def urlopen(url: str) -> Union[None, HTTPResponse]: - """Opens an url with urllib and streams it to a temp file""" - - timeout = 5 - - # Proxy an ssl configuration - pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons") - if pref.GetBool("NoProxyCheck", True): - proxies = {} - else: - if pref.GetBool("SystemProxyCheck", False): - proxy = urllib.request.getproxies() - proxies = {"http": proxy.get("http"), "https": proxy.get("http")} - elif pref.GetBool("UserProxyCheck", False): - proxy = pref.GetString("ProxyUrl", "") - proxies = {"http": proxy, "https": proxy} - - if ssl_ctx: - handler = urllib.request.HTTPSHandler(context=ssl_ctx) - else: - handler = {} - proxy_support = urllib.request.ProxyHandler(proxies) - opener = urllib.request.build_opener(proxy_support, handler) - urllib.request.install_opener(opener) - - # Url opening - req = urllib.request.Request( - url, headers={"User-Agent": "Mozilla/5.0 Magic Browser"} - ) - try: - u = urllib.request.urlopen(req, timeout=timeout) - - except URLError as e: - FreeCAD.Console.PrintLog(f"Error loading {url}:\n {e.reason}\n") - return None - except Exception: - return None - else: - return u - - def getserver(url): """returns the server part of an url""" @@ -207,6 +167,19 @@ def get_zip_url(repo): return None +def recognized_git_location(repo) -> bool: + parsed_url = urlparse(repo.url) + if ( + parsed_url.netloc == "github.com" + or parsed_url.netloc == "framagit.com" + or parsed_url.netloc == "gitlab.com" + or parsed_url.netloc == "salsa.debian.org" + ): + return True + else: + return False + + def construct_git_url(repo, filename): "Returns a direct download link to a file in an online Git repo: works with github, gitlab, and framagit" diff --git a/src/Mod/AddonManager/addonmanager_workers.py b/src/Mod/AddonManager/addonmanager_workers.py index 16cc2cd0fc..b0d1a576df 100644 --- a/src/Mod/AddonManager/addonmanager_workers.py +++ b/src/Mod/AddonManager/addonmanager_workers.py @@ -35,10 +35,13 @@ import time import subprocess import sys import platform +import itertools from datetime import datetime -from typing import Union, List +from typing import Union, List, Dict +from enum import Enum, auto -from PySide2 import QtCore, QtNetwork + +from PySide2 import QtCore import FreeCAD @@ -47,12 +50,9 @@ if FreeCAD.GuiUp: import addonmanager_utilities as utils from addonmanager_macro import Macro -from addonmanager_metadata import ( - MetadataDownloadWorker, - MetadataTxtDownloadWorker, - RequirementsTxtDownloadWorker, -) + from AddonManagerRepo import AddonManagerRepo +from NetworkManager import AM_NETWORK_MANAGER translate = FreeCAD.Qt.translate @@ -119,18 +119,7 @@ class ConnectionChecker(QtCore.QThread): translate("AddonsInstaller", "Checking network connection...\n") ) url = "https://api.github.com/zen" - request = utils.urlopen(url) - if QtCore.QThread.currentThread().isInterruptionRequested(): - return - if not request: - self.failure.emit( - translate( - "AddonsInstaller", - "Unable to connect to GitHub: check your internet connection and proxy settings and try again.", - ) - ) - return - result = request.read() + result = AM_NETWORK_MANAGER.blocking_get(url) if QtCore.QThread.currentThread().isInterruptionRequested(): return if not result: @@ -142,7 +131,7 @@ class ConnectionChecker(QtCore.QThread): ) return - result = result.decode("utf8") + result = result.data().decode("utf8") FreeCAD.Console.PrintLog(f"GitHub's zen message response: {result}\n") self.success.emit() @@ -164,12 +153,11 @@ class UpdateWorker(QtCore.QThread): # update info lists global obsolete, macros_reject_list, mod_reject_list, py2only - u = utils.urlopen( + p = AM_NETWORK_MANAGER.blocking_get( "https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/addonflags.json" ) - if u: - p = u.read() - u.close() + if p: + p = p.data().decode("utf8") j = json.loads(p) if "obsolete" in j and "Mod" in j["obsolete"]: obsolete = j["obsolete"]["Mod"] @@ -266,15 +254,12 @@ class UpdateWorker(QtCore.QThread): self.addon_repo.emit(repo) # querying official addons - u = utils.urlopen( + p = AM_NETWORK_MANAGER.blocking_get( "https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/.gitmodules" ) - if not u: + if not p: return - p = u.read() - if isinstance(p, bytes): - p = p.decode("utf-8") - u.close() + p = p.data().decode("utf8") p = re.findall( ( r'(?m)\[submodule\s*"(?P.*)"\]\s*' @@ -642,8 +627,8 @@ class FillMacroListWorker(QtCore.QThread): Reads only the page https://wiki.freecad.org/Macros_recipes """ - u = utils.urlopen("https://wiki.freecad.org/Macros_recipes") - if not u: + p = AM_NETWORK_MANAGER.blocking_get("https://wiki.freecad.org/Macros_recipes") + if not p: FreeCAD.Console.PrintWarning( translate( "AddonsInstaller", @@ -652,10 +637,7 @@ class FillMacroListWorker(QtCore.QThread): + "\n" ) return - p = u.read() - u.close() - if isinstance(p, bytes): - p = p.decode("utf-8") + p = p.data().decode("utf8") macros = re.findall('title="(Macro.*?)"', p) macros = [mac for mac in macros if ("translated" not in mac)] macro_names = [] @@ -835,18 +817,13 @@ class ShowWorker(QtCore.QThread): readmeurl = utils.get_readme_html_url(self.repo) if not readmeurl: FreeCAD.Console.PrintLog(f"README not found for {url}\n") - u = utils.urlopen(readmeurl) - if not u: + p = AM_NETWORK_MANAGER.blocking_get(readmeurl) + if not p: FreeCAD.Console.PrintLog(f"Debug: README not found at {readmeurl}\n") - u = utils.urlopen(readmeurl) - if u: - p = u.read() - if isinstance(p, bytes): - p = p.decode("utf-8") - u.close() - readme = re.findall(regex, p, flags=re.MULTILINE | re.DOTALL) - if readme: - desc = readme[0] + p = p.data().decode("utf8") + readme = re.findall(regex, p, flags=re.MULTILINE | re.DOTALL) + if readme: + desc = readme[0] else: FreeCAD.Console.PrintLog(f"Debug: README not found at {readmeurl}\n") else: @@ -854,12 +831,9 @@ class ShowWorker(QtCore.QThread): readmeurl = utils.get_readme_url(self.repo) if not readmeurl: FreeCAD.Console.PrintLog(f"Debug: README not found for {url}\n") - u = utils.urlopen(readmeurl) - if u: - p = u.read() - if isinstance(p, bytes): - p = p.decode("utf-8") - u.close() + p = AM_NETWORK_MANAGER.blocking_get(readmeurl) + if p: + p = p.data().decode("utf8") desc = utils.fix_relative_links(p, readmeurl.rsplit("/README.md")[0]) if not NOMARKDOWN and have_markdown: desc = markdown.markdown(desc, extensions=["md_in_html"]) @@ -877,25 +851,22 @@ class ShowWorker(QtCore.QThread): desc = message else: FreeCAD.Console.PrintLog("Debug: README not found at {readmeurl}\n") - if desc == "": - # fall back to the description text - u = utils.urlopen(url) - if not u: + if desc == "": + # fall back to the description text + p = AM_NETWORK_MANAGER.blocking_get(url) + if not p: + return + p = p.data().decode("utf8") + descregex = utils.get_desc_regex(self.repo) + if descregex: + desc = re.findall(descregex, p) + if desc: + desc = desc[0] + if not desc: + desc = "Unable to retrieve addon description" + self.repo.description = desc + if QtCore.QThread.currentThread().isInterruptionRequested(): return - p = u.read() - if isinstance(p, bytes): - p = p.decode("utf-8") - u.close() - descregex = utils.get_desc_regex(self.repo) - if descregex: - desc = re.findall(descregex, p) - if desc: - desc = desc[0] - if not desc: - desc = "Unable to retrieve addon description" - self.repo.description = desc - if QtCore.QThread.currentThread().isInterruptionRequested(): - return message = desc if self.repo.update_status == AddonManagerRepo.UpdateStatus.UNCHECKED: # Addon is installed but we haven't checked it yet, so let's check if it has an update @@ -920,7 +891,7 @@ class ShowWorker(QtCore.QThread): AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE ) self.update_status.emit(self.repo) - + if QtCore.QThread.currentThread().isInterruptionRequested(): return self.readme_updated.emit(message) @@ -971,9 +942,9 @@ class ShowWorker(QtCore.QThread): storename = os.path.join(store, wbName + name[-remainChars:]) if not os.path.exists(storename): try: - u = utils.urlopen(path) - imagedata = u.read() - u.close() + imagedata = AM_NETWORK_MANAGER.blocking_get(path) + if not imagedata: + raise Exception except Exception: FreeCAD.Console.PrintLog( "AddonManager: Debug: Error retrieving image from " @@ -987,7 +958,7 @@ class ShowWorker(QtCore.QThread): # lower length limit for path storename = storename[-140:] f = open(storename, "wb") - f.write(imagedata) + f.write(imagedata.data()) f.close() message = message.replace( 'src="' + origpath, @@ -1095,7 +1066,15 @@ class InstallWorkbenchWorker(QtCore.QThread): # Do the git process... self.run_git(target_dir) else: - self.run_zip(target_dir) + + # The zip process uses an event loop, since the download can potentially be quite large + self.launch_zip(target_dir) + self.zip_complete = False + current_thread = QtCore.QThread.currentThread() + while not self.zip_complete: + if current_thread.isInterruptionRequested(): + return + QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents, 50) def update_status(self) -> None: if hasattr(self, "git_progress") and self.isRunning(): @@ -1223,7 +1202,7 @@ class InstallWorkbenchWorker(QtCore.QThread): self.update_metadata() self.success.emit(self.repo, answer) - def run_zip(self, zipdir: str) -> None: + def launch_zip(self, zipdir: str) -> None: "downloads and unzip a zip version from a git repo" bakdir = None @@ -1237,97 +1216,91 @@ class InstallWorkbenchWorker(QtCore.QThread): if not zipurl: self.failure.emit( self.repo, - translate("AddonsInstaller", "Error: Unable to locate zip from") + translate("AddonsInstaller", "Error: Unable to locate ZIP from") + " " + self.repo.name, ) return - try: - u = utils.urlopen(zipurl) - except Exception: + + self.zipdir = zipdir + self.bakdir = bakdir + + AM_NETWORK_MANAGER.progress_made.connect(self.update_zip_status) + AM_NETWORK_MANAGER.progress_complete.connect(self.finish_zip) + self.zip_download_index = AM_NETWORK_MANAGER.submit_monitored_get(zipurl) + + def update_zip_status(self, index: int, bytes_read: int, data_size: int): + if index == self.zip_download_index: + locale = QtCore.QLocale() + if data_size > 10 * 1024 * 1024: # To avoid overflows, show MB instead + MB_read = bytes_read / 1024 / 1024 + MB_total = data_size / 1024 / 1024 + self.progress_made.emit(MB_read, MB_total) + mbytes_str = locale.toString(MB_read) + mbytes_total_str = locale.toString(MB_total) + percent = int(100 * float(MB_read / MB_total)) + self.status_message.emit( + translate( + "AddonsInstaller", + f"Downloading: {mbytes_str}MB of {mbytes_total_str}MB ({percent}%)", + ) + ) + elif data_size > 0: + self.progress_made.emit(bytes_read, data_size) + bytes_str = locale.toString(bytes_read) + bytes_total_str = locale.toString(data_size) + percent = int(100 * float(bytes_read / data_size)) + self.status_message.emit( + translate( + "AddonsInstaller", + f"Downloading: {bytes_str} of {bytes_total_str} bytes ({percent}%)", + ) + ) + else: + MB_read = bytes_read / 1024 / 1024 + bytes_str = locale.toString(MB_read) + self.status_message.emit( + translate( + "AddonsInstaller", + f"Downloading: {bytes_str}MB of unknown total", + ) + ) + + def finish_zip(self, index: int, response_code: int, filename: os.PathLike): + self.zip_complete = True + if response_code != 200: self.failure.emit( self.repo, - translate("AddonsInstaller", "Error: Unable to download") - + " " - + zipurl, - ) - return - if not u: - self.failure.emit( - self.repo, - translate("AddonsInstaller", "Error: Unable to download") - + " " - + zipurl, + translate( + "AddonsInstaller", + f"Error: Error while downloading ZIP file for {self.repo.display_name}", + ), ) return - data_size = 0 - if "content-length" in u.headers: - try: - data_size = int(u.headers["content-length"]) - except Exception: - pass - - current_thread = QtCore.QThread.currentThread() - - # Use an on-disk file and track download progress - locale = QtCore.QLocale() - with tempfile.NamedTemporaryFile(delete=False) as temp: - bytes_to_read = 1024 * 1024 - bytes_read = 0 - while True: - if current_thread.isInterruptionRequested(): - return - chunk = u.read(bytes_to_read) - if not chunk: - break - bytes_read += bytes_to_read - temp.write(chunk) - if data_size: - self.progress_made.emit(bytes_read, data_size) - percent = int(100 * float(bytes_read / data_size)) - MB_read = bytes_read / 1024 / 1024 - MB_total = data_size / 1024 / 1024 - bytes_str = locale.toString(MB_read) - bytes_total_str = locale.toString(MB_total) - self.status_message.emit( - translate( - "AddonsInstaller", - f"Downloading: {bytes_str}MB of {bytes_total_str}MB ({percent}%)", - ) - ) - else: - MB_read = bytes_read / 1024 / 1024 - bytes_str = locale.toString(MB_read) - self.status_message.emit( - translate( - "AddonsInstaller", - f"Downloading: {bytes_str}MB of unknown total", - ) - ) - QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents) - zfile = zipfile.ZipFile(temp) + with zipfile.ZipFile(filename, "r") as zfile: master = zfile.namelist()[0] # github will put everything in a subfolder self.status_message.emit( translate("AddonsInstaller", f"Download complete. Unzipping file...") ) QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents) - zfile.extractall(zipdir) - u.close() - zfile.close() - for filename in os.listdir(zipdir + os.sep + master): - shutil.move( - zipdir + os.sep + master + os.sep + filename, - zipdir + os.sep + filename, - ) - os.rmdir(zipdir + os.sep + master) - if bakdir: - shutil.rmtree(bakdir) - self.update_metadata() - self.success.emit( - self.repo, - translate("AddonsInstaller", "Successfully installed") + " " + zipurl, + zfile.extractall(self.zipdir) + for filename in os.listdir(self.zipdir + os.sep + master): + shutil.move( + self.zipdir + os.sep + master + os.sep + filename, + self.zipdir + os.sep + filename, ) + os.rmdir(self.zipdir + os.sep + master) + if self.bakdir: + shutil.rmtree(self.bakdir) + self.update_metadata() + self.success.emit( + self.repo, + translate( + "AddonsInstaller", + f"Successfully installed {self.repo.display_name} from ZIP file", + ), + ) def update_metadata(self): basedir = FreeCAD.getUserAppDataDir() @@ -1506,150 +1479,216 @@ class UpdateMetadataCacheWorker(QtCore.QThread): progress_made = QtCore.Signal(int, int) package_updated = QtCore.Signal(AddonManagerRepo) - class AtomicCounter(object): - def __init__(self, start=0): - self.lock = threading.Lock() - self.count = start - - def set(self, new_value): - with self.lock: - self.count = new_value - - def get(self): - with self.lock: - return self.count - - def increment(self): - with self.lock: - self.count += 1 - - def decrement(self): - with self.lock: - self.count -= 1 + class RequestType(Enum): + PACKAGE_XML = auto() + METADATA_TXT = auto() + REQUIREMENTS_TXT = auto() + ICON = auto() def __init__(self, repos): QtCore.QThread.__init__(self) self.repos = repos - self.counter = UpdateMetadataCacheWorker.AtomicCounter() + self.requests: Dict[ + int, (AddonManagerRepo, UpdateMetadataCacheWorker.RequestType) + ] = {} + AM_NETWORK_MANAGER.completed.connect(self.download_completed) + self.requests_completed = 0 + self.total_requests = 0 + self.store = os.path.join( + FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata" + ) + self.updated_repos = set() def run(self): current_thread = QtCore.QThread.currentThread() - self.num_downloads_required = len(self.repos) - self.progress_made.emit(0, self.num_downloads_required) - self.status_message.emit( - translate("AddonsInstaller", "Retrieving package metadata...") - ) - store = os.path.join( - FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata" - ) - index_file = os.path.join(store, "index.json") - self.index = {} - if os.path.isfile(index_file): - with open(index_file, "r") as f: - index_string = f.read() - self.index = json.loads(index_string) - download_queue = ( - QtNetwork.QNetworkAccessManager() - ) # Must be created on this thread - download_queue.finished.connect(self.on_finished) - - # Prevent strange internal Qt errors about cache setup by pre-emptively setting - # up a cache. The error this fixes is: - # "caching was enabled after some bytes had been written" - qnam_cache = os.path.join( - FreeCAD.getUserCachePath(), "AddonManager", "QNAM_CACHE" - ) - diskCache = QtNetwork.QNetworkDiskCache() - diskCache.setCacheDirectory(qnam_cache) - download_queue.setCache(diskCache) - - self.downloaders = [] for repo in self.repos: - if repo.metadata_url: + if repo.url and utils.recognized_git_location(repo): # package.xml - downloader = MetadataDownloadWorker(None, repo, self.index) - downloader.start_fetch(download_queue) - downloader.updated.connect(self.on_updated) - self.downloaders.append(downloader) + index = AM_NETWORK_MANAGER.submit_unmonitored_get( + utils.construct_git_url(repo, "package.xml") + ) + self.requests[index] = ( + repo, + UpdateMetadataCacheWorker.RequestType.PACKAGE_XML, + ) + self.total_requests += 1 # metadata.txt - downloader = MetadataTxtDownloadWorker(None, repo) - downloader.start_fetch(download_queue) - downloader.updated.connect(self.on_updated) - self.downloaders.append(downloader) + index = AM_NETWORK_MANAGER.submit_unmonitored_get( + utils.construct_git_url(repo, "metadata.txt") + ) + self.requests[index] = ( + repo, + UpdateMetadataCacheWorker.RequestType.METADATA_TXT, + ) + self.total_requests += 1 # requirements.txt - downloader = RequirementsTxtDownloadWorker(None, repo) - downloader.start_fetch(download_queue) - downloader.updated.connect(self.on_updated) - self.downloaders.append(downloader) + index = AM_NETWORK_MANAGER.submit_unmonitored_get( + utils.construct_git_url(repo, "requirements.txt") + ) + self.requests[index] = ( + repo, + UpdateMetadataCacheWorker.RequestType.REQUIREMENTS_TXT, + ) + self.total_requests += 1 - # Run a local event loop until we've processed all of the downloads: - # this is local to this thread, and does not affect the main event loop - ui_updater = QtCore.QTimer() - ui_updater.timeout.connect(self.send_ui_update) - ui_updater.start(100) # Send an update back to the main thread every 100ms - self.num_downloads_required = len(self.downloaders) - self.num_downloads_completed = UpdateMetadataCacheWorker.AtomicCounter() - aborted = False - while True: + while self.requests: if current_thread.isInterruptionRequested(): - download_queue.finished.disconnect(self.on_finished) - for downloader in self.downloaders: - downloader.updated.disconnect(self.on_updated) - downloader.abort() - aborted = True - if ( - aborted - or self.num_downloads_completed.get() >= self.num_downloads_required - ): - break + AM_NETWORK_MANAGER.completed.disconnect(self.download_completed) + for request in self.requests.keys(): + AM_NETWORK_MANAGER.abort(request) + return + # 50 ms maximum between checks for interruption QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents, 50) - if aborted: - FreeCAD.Console.PrintLog("Metadata update cancelled\n") - return + # This set contains one copy of each of the repos that got some kind of data in + # this process. For those repos, tell the main Addon Manager code that it needs + # to update its copy of the repo, and redraw its information. + for repo in self.updated_repos: + self.package_updated.emit(repo) - # Update and serialize the updated index, overwriting whatever was - # there before - for downloader in self.downloaders: - if hasattr(downloader, "last_sha1"): - self.index[downloader.repo.name] = downloader.last_sha1 - if not os.path.exists(store): - os.makedirs(store) - with open(index_file, "w") as f: - json.dump(self.index, f, indent=" ") + def download_completed( + self, index: int, code: int, data: QtCore.QByteArray + ) -> None: + if index in self.requests: + self.requests_completed += 1 + self.progress_made.emit(self.requests_completed, self.total_requests) + request = self.requests.pop(index) + if code == 200: # HTTP success + self.updated_repos.add(request[0]) # mark this repo as updated + if request[1] == UpdateMetadataCacheWorker.RequestType.PACKAGE_XML: + self.process_package_xml(request[0], data) + elif request[1] == UpdateMetadataCacheWorker.RequestType.METADATA_TXT: + self.process_metadata_txt(request[0], data) + elif ( + request[1] == UpdateMetadataCacheWorker.RequestType.REQUIREMENTS_TXT + ): + self.process_requirements_txt(request[0], data) + elif request[1] == UpdateMetadataCacheWorker.RequestType.ICON: + self.process_icon(request[0], data) - def on_finished(self, reply): - # Called by the QNetworkAccessManager's sub-threads when a fetch - # process completed (in any state) - self.num_downloads_completed.increment() - reply.deleteLater() - - def on_updated(self, repo): - # Called if this repo got new metadata and/or a new icon - self.package_updated.emit(repo) - - def send_ui_update(self): - completed = self.num_downloads_completed.get() - required = self.num_downloads_required - percentage = int(100 * completed / required) - self.progress_made.emit(completed, required) + def process_package_xml(self, repo: AddonManagerRepo, data: QtCore.QByteArray): + repo.repo_type = AddonManagerRepo.RepoType.PACKAGE # By definition + package_cache_directory = os.path.join(self.store, repo.name) + if not os.path.exists(package_cache_directory): + os.makedirs(package_cache_directory) + new_xml_file = os.path.join(package_cache_directory, "package.xml") + with open(new_xml_file, "wb") as f: + f.write(data.data()) + metadata = FreeCAD.Metadata(new_xml_file) + repo.metadata = metadata self.status_message.emit( - translate("AddonsInstaller", "Retrieving package metadata...") - + f" {completed} / {required} ({percentage}%)" + translate("AddonsInstaller", f"Downloaded package.xml for {repo.name}") ) - def terminate_all(self): - got = self.num_downloads_completed.get() - wanted = self.num_downloads_required - if wanted < got: - FreeCAD.Console.PrintWarning( - f"During cache interruption, wanted {wanted}, got {got}, forcibly terminating now...\n" + # Grab a new copy of the icon as well: we couldn't enqueue this earlier because + # we didn't know the path to it, which is stored in the package.xml file. + icon = metadata.Icon + if not icon: + # If there is no icon set for the entire package, see if there are + # any workbenches, which are required to have icons, and grab the first + # one we find: + content = repo.metadata.Content + if "workbench" in content: + wb = content["workbench"][0] + if wb.Icon: + if wb.Subdirectory: + subdir = wb.Subdirectory + else: + subdir = wb.Name + repo.Icon = subdir + wb.Icon + icon = repo.Icon + + icon_url = utils.construct_git_url(repo, icon) + index = AM_NETWORK_MANAGER.submit_unmonitored_get(icon_url) + self.requests[index] = (repo, UpdateMetadataCacheWorker.RequestType.ICON) + self.total_requests += 1 + + def process_metadata_txt(self, repo: AddonManagerRepo, data: QtCore.QByteArray): + self.status_message.emit( + translate( + "AddonsInstaller", f"Downloaded metadata.txt for {repo.display_name}" ) - self.num_downloads_completed.set(wanted) + ) + f = io.StringIO(data.data().decode("utf8")) + while True: + line = f.readline() + if not line: + break + if line.startswith("workbenches="): + depswb = line.split("=")[1].split(",") + for wb in depswb: + wb_name = wb.strip() + if wb_name: + repo.requires.add(wb_name) + FreeCAD.Console.PrintLog( + f"{repo.display_name} requires FreeCAD Addon '{wb_name}'\n" + ) + + elif line.startswith("pylibs="): + depspy = line.split("=")[1].split(",") + for pl in depspy: + dep = pl.strip() + if dep: + repo.python_requires.add(dep) + FreeCAD.Console.PrintLog( + f"{repo.display_name} requires python package '{dep}'\n" + ) + + elif line.startswith("optionalpylibs="): + opspy = line.split("=")[1].split(",") + for pl in opspy: + dep = pl.strip() + if dep: + repo.python_optional.add(dep) + FreeCAD.Console.PrintLog( + f"{repo.display_name} optionally imports python package '{pl.strip()}'\n" + ) + # For review and debugging purposes, store the file locally + package_cache_directory = os.path.join(self.store, repo.name) + if not os.path.exists(package_cache_directory): + os.makedirs(package_cache_directory) + new_xml_file = os.path.join(package_cache_directory, "metadata.txt") + with open(new_xml_file, "wb") as f: + f.write(data.data()) + + def process_requirements_txt(self, repo: AddonManagerRepo, data: QtCore.QByteArray): + self.status_message.emit( + translate( + "AddonsInstaller", + f"Downloaded requirements.txt for {repo.display_name}", + ) + ) + f = io.StringIO(data.data().decode("utf8")) + lines = f.readlines() + for line in lines: + break_chars = " <>=~!+#" + package = line + for n, c in enumerate(line): + if c in break_chars: + package = line[:n].strip() + break + if package: + repo.python_requires.add(package) + # For review and debugging purposes, store the file locally + package_cache_directory = os.path.join(self.store, repo.name) + if not os.path.exists(package_cache_directory): + os.makedirs(package_cache_directory) + new_xml_file = os.path.join(package_cache_directory, "requirements.txt") + with open(new_xml_file, "wb") as f: + f.write(data.data()) + + def process_icon(self, repo: AddonManagerRepo, data: QtCore.QByteArray): + self.status_message.emit( + translate("AddonsInstaller", f"Downloaded icon for {repo.display_name}") + ) + cache_file = repo.get_cached_icon_filename() + with open(cache_file, "wb") as icon_file: + icon_file.write(data.data()) + repo.cached_icon_filename = cache_file if have_git and not NOGIT: diff --git a/src/Mod/AddonManager/proxy_authentication.ui b/src/Mod/AddonManager/proxy_authentication.ui new file mode 100644 index 0000000000..7c804680b5 --- /dev/null +++ b/src/Mod/AddonManager/proxy_authentication.ui @@ -0,0 +1,150 @@ + + + proxy_authentication + + + + 0 + 0 + 536 + 276 + + + + Proxy login required + + + + + + Proxy requires authentication + + + + + + + + + Proxy: + + + + + + + + 0 + 0 + + + + Placeholder for proxy address + + + + + + + Realm: + + + + + + + Placeholder for proxy realm + + + + + + + + + + + Username + + + + + + + + + + Password + + + + + + + QLineEdit::PasswordEchoOnEdit + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + + + + + + + buttonBox + accepted() + proxy_authentication + accept() + + + 248 + 254 + + + 157 + 274 + + + + + buttonBox + rejected() + proxy_authentication + reject() + + + 316 + 260 + + + 286 + 274 + + + + +