kindred/create
1
1
Fork 0
Code Issues 53 Pull Requests Actions Packages Projects 9 Releases 2 Wiki Activity

Addon Manager: Create NetworkManager class

Browse Source 
To enable single-login authenticated proxy use, and simplified multi-threaded
network accesses, this commit adds a new wrapper around a QNetworkAccessManager
and includes a global instantiation of the class intended to exist for the
lifetime of the program. This instance can be used to enqueue any number of
network requests, which the manager will send out to the networking subsystem
in an appropriate manner.
This commit is contained in:
Chris Hennes
2022-01-14 11:52:31 -06:00
parent cad0d01883
commit e9d6a2c4a4
8 changed files with 1055 additions and 660 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/Mod/AddonManager/AddonManager.py
View File

@@ -42,6 +42,8 @@ from package_list import PackageList, PackageListItemModel
from package_details import PackageDetails
from AddonManagerRepo import AddonManagerRepo
from NetworkManager import HAVE_QTNETWORK
__title__ = "FreeCAD Addon Manager Module"
__author__ = "Yorik van Havre", "Jonathan Wiedemann", "Kurt Kremitzki", "Chris Hennes"
__url__ = "http://www.freecad.org"
@@ -228,9 +230,16 @@ class CommandAddonManager:
# This must run on the main GUI thread
if hasattr(self, "connection_check_message") and self.connection_check_message:
self.connection_check_message.close()
QtWidgets.QMessageBox.critical(
None, translate("AddonsInstaller", "Connection failed"), message
)
if HAVE_QTNETWORK:
QtWidgets.QMessageBox.critical(
None, translate("AddonsInstaller", "Connection failed"), message
)
else:
QtWidgets.QMessageBox.critical(
None,
translate("AddonsInstaller", "Missing dependency"),
translate("AddonsInstaller", "Could not import QtNetwork -- see Report View for details. Addon Manager unavailable."),
)
def launch(self) -> None:
"""Shows the Addon Manager UI"""
@@ -821,7 +830,7 @@ class CommandAddonManager:
self.packageDetails.show_repo(selected_repo)
def show_information(self, message: str) -> None:
"""shows generic text in the information pane (which might be collapsed)"""
"""shows generic text in the information pane"""
self.dialog.labelStatusInfo.setText(message)
self.dialog.labelStatusInfo.repaint()

2
src/Mod/AddonManager/CMakeLists.txt
View File

@@ -8,7 +8,6 @@ SET(AddonManager_SRCS
AddonManager.py
AddonManagerRepo.py
addonmanager_macro.py
addonmanager_metadata.py
addonmanager_utilities.py
addonmanager_workers.py
AddonManager.ui
@@ -18,6 +17,7 @@ SET(AddonManager_SRCS
compact_view.py
dependency_resolution_dialog.ui
expanded_view.py
NetworkManager.py
package_list.py
package_details.py
)

552
src/Mod/AddonManager/NetworkManager.py Normal file
View File

@@ -0,0 +1,552 @@
# -*- coding: utf-8 -*-
# ***************************************************************************
# * *
# * Copyright (c) 2022 Chris Hennes <chennes@pioneerlibrarysystem.org> *
# * *
# * This program is free software; you can redistribute it and/or modify *
# * it under the terms of the GNU Lesser General Public License (LGPL) *
# * as published by the Free Software Foundation; either version 2 of *
# * the License, or (at your option) any later version. *
# * for detail see the LICENSE text file. *
# * *
# * This program is distributed in the hope that it will be useful, *
# * but WITHOUT ANY WARRANTY; without even the implied warranty of *
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
# * GNU Library General Public License for more details. *
# * *
# * You should have received a copy of the GNU Library General Public *
# * License along with this program; if not, write to the Free Software *
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 *
# * USA *
# * *
# ***************************************************************************
#############################################################################
#
# ABOUT NETWORK MANAGER
#
# A wrapper around QNetworkAccessManager providing proxy-handling
# capabilities, and simplified access to submitting requests from any
# application thread.
#
#
# USAGE
#
# Once imported, this file provides access to a global object called
# AM_NETWORK_MANAGER. This is a QThread object running on its own thread, but
# designed to be interacted with from any other application thread. It
# provides two principal methods: submit_unmonitored_get() and
# submit_monitored_get(). Use the unmonitored version for small amounts of
# data (suitable for caching in RAM, and without a need to show a progress
# bar during download), and the monitored version for larger amounts of data.
# Both functions take a URL, and return an integer index. That index allows
# tracking of the completed request by attaching to the signals completed(),
# progress_made(), and progress_complete(). All three provide, as the first
# argument to the signal, the index of the request the signal refers to.
# Code attached to those signals should filter them to look for the indices
# of the requests they care about. Requests may complete in any order.
#
# A secondary blocking interface is also provided, for very short network
# accesses: the blocking_get() function blocks until the network transmission
# is complete, directly returning a QByteArray object with the received data.
# Do not run on the main GUI thread!
try:
import FreeCAD
if FreeCAD.GuiUp:
import FreeCADGui
HAVE_FREECAD = True
translate = FreeCAD.Qt.translate
except Exception:
# For standalone testing support working without the FreeCAD import
HAVE_FREECAD = False
import threading
from PySide2 import QtCore
import os
import queue
import itertools
import tempfile
from typing import Dict, List
# This is the global instance of the NetworkManager that outside code
# should access
AM_NETWORK_MANAGER = None
HAVE_QTNETWORK = True
try:
from PySide2 import QtNetwork
except Exception:
if HAVE_FREECAD:
FreeCAD.Console.PrintError(
translate(
"AddonsInstaller",
"Could not import QtNetwork -- it does not appear to be installed on your system. Please install the package 'python3-pyside2.qtnetwork' on your system and if possible contact your FreeCAD package maintainer to alert them to the missing dependency. The Addon Manager will not be available.",
)
+ "\n"
)
else:
print(
"Could not import QtNetwork, unable to test this file. Try installing the python3-pyside2.qtnetwork package."
)
exit(1)
HAVE_QTNETWORK = False
if HAVE_QTNETWORK:
class QueueItem:
def __init__(
self, index: int, request: QtNetwork.QNetworkRequest, track_progress: bool
):
self.index = index
self.request = request
self.track_progress = track_progress
class NetworkManager(QtCore.QThread):
"""A single global instance of NetworkManager is instantiated and stored as
AM_NETWORK_MANAGER. Outside threads should send GET requests to this class by
calling the submit_unmonitored_request() or submit_monitored_request() function,
as needed. See the documentation of those functions for details."""
# Connect to complete for requests with no progress monitoring (e.g. small amounts of data)
completed = QtCore.Signal(
int, int, QtCore.QByteArray
) # Index, http response code, received data (if any)
# Connect to progress_made and progress_complete for large amounts of data, which get buffered into a temp file
# That temp file should be deleted when your code is done with it
progress_made = QtCore.Signal(
int, int, int
) # Index, bytes read, total bytes (may be None)
progress_complete = QtCore.Signal(
int, int, os.PathLike
) # Index, http response code, filename
def __init__(self):
super().__init__()
# Set up the incoming request queue: requests get put on this queue, and the main event loop
# pulls them off and runs them.
self.counting_iterator = itertools.count()
self.queue = queue.Queue()
self.__last_started_index = 0
self.__abort_when_found: List[int] = []
self.replies: Dict[int, QtNetwork.QNetworkReply] = {}
self.file_buffers = {}
# We support an arbitrary number of threads using synchronous GET calls:
self.synchronous_lock = threading.Lock()
self.synchronous_complete: Dict[int, bool] = {}
self.synchronous_result_data: Dict[int, QtCore.QByteArray] = {}
def run(self):
"""Do not call directly: use start() to begin the event loop on a new thread."""
# Create the QNAM on this thread:
self.thread = QtCore.QThread.currentThread()
self.QNAM = QtNetwork.QNetworkAccessManager()
self.QNAM.proxyAuthenticationRequired.connect(self.__authenticate_proxy)
self.QNAM.authenticationRequired.connect(self.__authenticate_resource)
# A helper connection for our blocking interface
self.completed.connect(self.__synchronous_process_completion)
# Set up the proxy, if necesssary:
noProxyCheck = True
systemProxyCheck = False
userProxyCheck = False
proxy_string = ""
if HAVE_FREECAD:
pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons")
noProxyCheck = pref.GetBool("NoProxyCheck", noProxyCheck)
systemProxyCheck = pref.GetBool("SystemProxyCheck", systemProxyCheck)
userProxyCheck = pref.GetBool("UserProxyCheck", userProxyCheck)
proxy_string = pref.GetString("ProxyUrl", "")
else:
print("Please select a proxy type:")
print("1) No proxy")
print("2) Use system proxy settings")
print("3) Custom proxy settings")
result = input("Choice: ")
if result == "1":
pass
elif result == "2":
noProxyCheck = False
systemProxyCheck = True
elif result == "3":
noProxyCheck = False
userProxyCheck = True
proxy_string = input("Enter your proxy server (host:port): ")
else:
print(f"Got {result}, expected 1, 2, or 3.")
app.quit()
if noProxyCheck:
pass
elif systemProxyCheck:
query = QtNetwork.QNetworkProxyQuery(
QtCore.QUrl("https://github.com/FreeCAD/FreeCAD")
)
proxy = QtNetwork.QNetworkProxyFactory.systemProxyForQuery(query)
if proxy and proxy[0]:
self.QNAM.setProxy(
proxy[0]
) # This may still be QNetworkProxy.NoProxy
elif userProxyCheck:
host, _, port_string = proxy_string.rpartition(":")
port = 0 if not port_string else int(port_string)
# For now assume an HttpProxy, but eventually this should be a parameter
proxy = QtNetwork.QNetworkProxy(
QtNetwork.QNetworkProxy.HttpProxy, host, port
)
self.QNAM.setProxy(proxy)
qnam_cache = QtCore.QStandardPaths.writableLocation(QtCore.QStandardPaths.CacheLocation);
os.makedirs(qnam_cache,exist_ok=True)
diskCache = QtNetwork.QNetworkDiskCache()
diskCache.setCacheDirectory(qnam_cache)
self.QNAM.setCache(diskCache)
# Start an event loop
while True:
if QtCore.QThread.currentThread().isInterruptionRequested():
# Support shutting down the entire thread, but this should be very rarely used.
# Callers should generally just call abort() on each network call they want to
# terminate, using requestInterruption() will terminate ALL network requests.
if not HAVE_FREECAD:
print(
"Shutting down all active network requests...", flush=True
)
self.abort_all()
self.queue.join()
if not HAVE_FREECAD:
print("All requests terminated.", flush=True)
return
try:
item = self.queue.get_nowait()
if item:
if item.index in self.__abort_when_found:
self.__abort_when_found.remove(item.index)
continue # Do not do anything with this item, it's been aborted...
reply = self.QNAM.get(item.request)
self.__last_started_index = item.index
reply.finished.connect(lambda i=item: self.__reply_finished(i))
reply.redirected.connect(
lambda url, r=reply: self.__on_redirect(r, url)
)
reply.sslErrors.connect(self.__on_ssl_error)
if item.track_progress:
reply.readyRead.connect(
lambda i=item.index: self.__data_incoming(i)
)
reply.downloadProgress.connect(
lambda a, b, i=item.index: self.progress_made.emit(
i, a, b
)
)
self.replies[item.index] = reply
except queue.Empty:
pass
QtCore.QCoreApplication.processEvents()
def submit_unmonitored_get(self, url: str) -> int:
"""Adds this request to the queue, and returns an index that can be used by calling code
in conjunction with the completed() signal to handle the results of the call. All data is
kept in memory, and the completed() call includes a direct handle to the bytes returned. It
is not called until the data transfer has finished and the connection is closed."""
current_index = next(self.counting_iterator) # A thread-safe counter
# Use a queue because we can only put things on the QNAM from the main event loop thread
self.queue.put(
QueueItem(
current_index, self.__create_get_request(url), track_progress=False
)
)
return current_index
def submit_monitored_get(self, url: str) -> int:
"""Adds this request to the queue, and returns an index that can be used by calling code
in conjunction with the progress_made() and progress_completed() signals to handle the
results of the call. All data is cached to disk, and progress is reported periodically
as the underlying QNetworkReply reports its progress. The progress_completed() signal
contains a path to a temporary file with the stored data. Calling code should delete this
file when done with it (or move it into its final place, etc.)."""
current_index = next(self.counting_iterator) # A thread-safe counter
# Use a queue because we can only put things on the QNAM from the main event loop thread
self.queue.put(
QueueItem(
current_index, self.__create_get_request(url), track_progress=True
)
)
return current_index
def blocking_get(self, url: str) -> QtCore.QByteArray:
"""Submits a GET request to the QNetworkAccessManager and block until it is complete"""
current_index = next(self.counting_iterator) # A thread-safe counter
with self.synchronous_lock:
self.synchronous_complete[current_index] = False
self.queue.put(
QueueItem(
current_index, self.__create_get_request(url), track_progress=False
)
)
while not self.synchronous_complete[current_index]:
if QtCore.QThread.currentThread().isInterruptionRequested():
return None
QtCore.QCoreApplication.processEvents()
with self.synchronous_lock:
self.synchronous_complete.pop(current_index)
return self.synchronous_result_data.pop(current_index)
def __synchronous_process_completion(
self, index: int, code: int, data: QtCore.QByteArray
) -> None:
with self.synchronous_lock:
if index in self.synchronous_complete:
if code == 200:
self.synchronous_result_data[index] = data
self.synchronous_complete[index] = True
def __create_get_request(self, url: str) -> QtNetwork.QNetworkRequest:
request = QtNetwork.QNetworkRequest(QtCore.QUrl(url))
request.setAttribute(
QtNetwork.QNetworkRequest.RedirectPolicyAttribute,
QtNetwork.QNetworkRequest.UserVerifiedRedirectPolicy,
)
request.setAttribute(
QtNetwork.QNetworkRequest.CacheSaveControlAttribute, True
)
request.setAttribute(
QtNetwork.QNetworkRequest.CacheLoadControlAttribute,
QtNetwork.QNetworkRequest.PreferCache,
)
request.setAttribute(
QtNetwork.QNetworkRequest.BackgroundRequestAttribute, True
)
return request
def abort_all(self):
"""Abort ALL network calls in progress, including clearing the queue"""
for reply in self.replies:
if reply.isRunning():
reply.abort()
while True:
try:
self.queue.get()
self.queue.task_done()
except queue.Empty:
break
def abort(self, index: int):
if index in self.replies and self.replies[index].isRunning():
self.replies[index].abort()
elif index < self.__last_started_index:
# It's still in the queue. Mark it for later destruction.
self.__abort_when_found.append(index)
def __authenticate_proxy(
self,
reply: QtNetwork.QNetworkProxy,
authenticator: QtNetwork.QAuthenticator,
):
if HAVE_FREECAD and FreeCAD.GuiUp:
proxy_authentication = FreeCADGui.PySideUic.loadUi(
os.path.join(os.path.dirname(__file__), "proxy_authentication.ui")
)
# Show the right labels, etc.
proxy_authentication.labelProxyAddress.setText(
f"{reply.hostName()}:{reply.port()}"
)
if authenticator.realm():
proxy_authentication.labelProxyRealm.setText(authenticator.realm())
else:
proxy_authentication.labelProxyRealm.hide()
proxy_authentication.labelRealmCaption.hide()
result = proxy_authentication.exec()
if result == QtWidgets.QDialogButtonBox.Ok:
authenticator.setUser(proxy_authentication.lineEditUsername.text())
authenticator.setPassword(
proxy_authentication.lineEditPassword.text()
)
else:
username = input("Proxy username: ")
import getpass
password = getpass.getpass()
authenticator.setUser(username)
authenticator.setPassword(password)
def __authenticate_resource(
self,
_reply: QtNetwork.QNetworkReply,
_authenticator: QtNetwork.QAuthenticator,
):
pass
def __on_redirect(self, reply, _):
# For now just blindly follow all redirects
reply.redirectAllowed.emit()
def __on_ssl_error(self, reply: str, errors: List[str]):
if HAVE_FREECAD:
FreeCAD.Console.PrintWarning(
translate("AddonsInstaller", "Error with encrypted connection")
+ "\n:"
)
FreeCAD.Console.PrintWarning(reply)
for error in errors:
FreeCAD.Console.PrintWarning(error)
else:
print("Error with encrypted connection")
for error in errors:
print(error)
def __data_incoming(self, index: int):
reply = self.replies[index]
chunk_size = reply.bytesAvailable()
buffer = reply.read(chunk_size)
if not index in self.file_buffers:
f = tempfile.NamedTemporaryFile("wb", delete=False)
self.file_buffers[index] = f
else:
f = self.file_buffers[index]
f.write(buffer)
def __reply_finished(self, item: QueueItem) -> None:
reply = self.replies.pop(item.index)
response_code = reply.attribute(
QtNetwork.QNetworkRequest.HttpStatusCodeAttribute
)
self.queue.task_done()
if reply.error() == QtNetwork.QNetworkReply.NetworkError.NoError:
if item.track_progress:
f = self.file_buffers[item.index]
f.close()
self.progress_complete.emit(item.index, response_code, f.name)
else:
data = reply.readAll()
self.completed.emit(item.index, response_code, data)
else:
if item.track_progress:
self.progress_complete.emit(item.index, response_code, "")
else:
self.completed.emit(item.index, response_code, None)
else: # HAVE_QTNETWORK is false:
class NetworkManager(QtCore.QThread):
"""A dummy class to enable an offline mode when the QtNetwork package is not yet installed"""
completed = QtCore.Signal(
int, int, bytes
) # Emitted as soon as the request is made, with a connection failed error
progress_made = QtCore.Signal(
int, int, int
) # Never emitted, no progress is made here
progress_complete = QtCore.Signal(
int, int, os.PathLike
) # Emitted as soon as the request is made, with a connection failed error
def __init__(self):
super().__init__()
self.monitored_queue = queue.Queue()
self.unmonitored_queue = queue.Queue()
def run(self):
while True:
try:
index = self.monitored_queue.get_nowait()
self.progress_complete.emit(
index, 418, "--ERR418--"
) # Refuse to provide data
self.monitored_queue.task_done()
except queue.Empty:
pass
try:
index = self.unmonitored_queue.get_nowait()
self.completed.emit(index, 418, None)
self.unmonitored_queue.task_done()
except queue.Empty:
pass
if QtCore.QThread.currentThread().isInterruptionRequested():
return
QtCore.QCoreApplication.processEvents()
def submit_unmonitored_request(self, _) -> int:
current_index = next(itertools.count())
self.unmonitored_queue.put(current_index)
return current_index
def submit_monitored_request(self, _) -> int:
current_index = next(itertools.count())
self.monitored_queue.put(current_index)
return current_index
def blocking_get(self, _: str) -> QtCore.QByteArray:
return None
def abort_all(
self,
):
pass # Nothing to do
def abort(self, _):
pass # Nothing to do
AM_NETWORK_MANAGER = NetworkManager()
AM_NETWORK_MANAGER.start()
if __name__ == "__main__":
app = QtCore.QCoreApplication()
count = 0
# For testing, create several network requests and send them off in quick succession:
# (Choose small downloads, no need for significant data)
urls = [
"https://api.github.com/zen",
"http://climate.ok.gov/index.php/climate/rainfall_table/local_data",
"https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/AIANNHA/MapServer",
]
def handle_completion(index: int, code: int, data):
global count
if code == 200:
print(
f"For request {index+1}, response was {data.size()} bytes.", flush=True
)
else:
print(
f"For request {index+1}, request failed with HTTP result code {code}",
flush=True,
)
count += 1
if count >= len(urls):
print(f"Shutting down...", flush=True)
AM_NETWORK_MANAGER.requestInterruption()
AM_NETWORK_MANAGER.wait(5000)
app.quit()
AM_NETWORK_MANAGER.completed.connect(handle_completion)
for url in urls:
AM_NETWORK_MANAGER.submit_unmonitored_get(url)
app.exec_()
print("Done with all requests.")

33
src/Mod/AddonManager/addonmanager_macro.py
View File

@@ -30,9 +30,10 @@ import time
from typing import Dict, Tuple, List, Union
import FreeCAD
from NetworkManager import AM_NETWORK_MANAGER
translate = FreeCAD.Qt.translate
from addonmanager_utilities import translate
from addonmanager_utilities import urlopen
from addonmanager_utilities import remove_directory_if_empty
try:
@@ -165,28 +166,25 @@ class Macro(object):
def fill_details_from_wiki(self, url):
code = ""
u = urlopen(url)
if u is None:
p = AM_NETWORK_MANAGER.blocking_get(url)
if not p:
FreeCAD.Console.PrintWarning(
translate(
"AddonsInstaller",
f"Could not connect to {url} - check connection and proxy settings",
f"Unable to open macro wiki page at {url}",
)
+ "\n"
)
return
p = u.read()
if isinstance(p, bytes):
p = p.decode("utf-8")
u.close()
p = p.data().decode("utf8")
# check if the macro page has its code hosted elsewhere, download if
# needed
if "rawcodeurl" in p:
rawcodeurl = re.findall('rawcodeurl.*?href="(http.*?)">', p)
if rawcodeurl:
rawcodeurl = rawcodeurl[0]
u2 = urlopen(rawcodeurl)
if u2 is None:
u2 = AM_NETWORK_MANAGER.blocking_get(rawcodeurl)
if not u2:
FreeCAD.Console.PrintWarning(
translate(
"AddonsInstaller",
@@ -195,18 +193,7 @@ class Macro(object):
+ "\n"
)
return
response = ""
block = 8192
while True:
data = u2.read(block)
if not data:
break
if isinstance(data, bytes):
data = data.decode("utf-8")
response += data
if response:
code = response
u2.close()
code = u2.data().decode("utf8")
if not code:
code = re.findall(r"<pre>(.*?)</pre>", p.replace("\n", "--endl--"))
if code:

315
src/Mod/AddonManager/addonmanager_metadata.py
View File

@@ -1,315 +0,0 @@
# ***************************************************************************
# * *
# * Copyright (c) 2021 Chris Hennes <chennes@pioneerlibrarysystem.org> *
# * *
# * This program is free software; you can redistribute it and/or modify *
# * it under the terms of the GNU Lesser General Public License (LGPL) *
# * as published by the Free Software Foundation; either version 2 of *
# * the License, or (at your option) any later version. *
# * for detail see the LICENCE text file. *
# * *
# * This program is distributed in the hope that it will be useful, *
# * but WITHOUT ANY WARRANTY; without even the implied warranty of *
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
# * GNU Library General Public License for more details. *
# * *
# * You should have received a copy of the GNU Library General Public *
# * License along with this program; if not, write to the Free Software *
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 *
# * USA *
# * *
# ***************************************************************************
import FreeCAD
import os
import io
import hashlib
from typing import Dict, List, Set
from PySide2 import QtCore, QtNetwork
from PySide2.QtCore import QObject
import addonmanager_utilities as utils
from AddonManagerRepo import AddonManagerRepo
translate = FreeCAD.Qt.translate
class DownloadWorker(QObject):
updated = QtCore.Signal(AddonManagerRepo)
def __init__(self, parent, url: str):
"""repo is an AddonManagerRepo object, and index is a dictionary of SHA1 hashes of the package.xml files in the cache"""
super().__init__(parent)
self.url = url
def start_fetch(self, network_manager: QtNetwork.QNetworkAccessManager):
"""Asynchronously begin the network access. Intended as a set-and-forget black box for downloading files."""
self.request = QtNetwork.QNetworkRequest(QtCore.QUrl(self.url))
self.request.setAttribute(
QtNetwork.QNetworkRequest.RedirectPolicyAttribute,
QtNetwork.QNetworkRequest.UserVerifiedRedirectPolicy,
)
self.request.setAttribute(
QtNetwork.QNetworkRequest.CacheSaveControlAttribute, True
)
self.request.setAttribute(
QtNetwork.QNetworkRequest.CacheLoadControlAttribute,
QtNetwork.QNetworkRequest.PreferCache,
)
self.request.setAttribute(
QtNetwork.QNetworkRequest.BackgroundRequestAttribute, True
)
self.fetch_task = network_manager.get(self.request)
self.fetch_task.finished.connect(self.resolve_fetch)
self.fetch_task.redirected.connect(self.on_redirect)
self.fetch_task.sslErrors.connect(self.on_ssl_error)
def abort(self):
if not self.fetch_task.isFinished():
self.fetch_task.abort()
def on_redirect(self, _):
# For now just blindly follow all redirects
self.fetch_task.redirectAllowed.emit()
def on_ssl_error(self, reply: str, errors: List[str]):
FreeCAD.Console.PrintWarning(
translate("AddonsInstaller", "Error with encrypted connection") + "\n:"
)
FreeCAD.Console.PrintWarning(reply)
for error in errors:
FreeCAD.Console.PrintWarning(error)
class MetadataDownloadWorker(DownloadWorker):
"""A worker for downloading package.xml and associated icon(s)
To use, instantiate an object of this class and call the start_fetch() function
with a QNetworkAccessManager. It is expected that many of these objects will all
be created and associated with the same QNAM, which will then handle the actual
asynchronous downloads in some Qt-defined number of threads. To monitor progress
you should connect to the QNAM's "finished" signal, and ensure it is called the
number of times you expect based on how many workers you have enqueued.
"""
def __init__(self, parent, repo: AddonManagerRepo, index: Dict[str, str]):
"""repo is an AddonManagerRepo object, and index is a dictionary of SHA1 hashes of the package.xml files in the cache"""
super().__init__(parent, repo.metadata_url)
self.repo = repo
self.index = index
self.store = os.path.join(
FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata"
)
self.last_sha1 = ""
def resolve_fetch(self):
"""Called when the data fetch completed, either with an error, or if it found the metadata file"""
if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError:
FreeCAD.Console.PrintLog(f"Found a package.xml file for {self.repo.name}\n")
self.repo.repo_type = AddonManagerRepo.RepoType.PACKAGE
new_xml = self.fetch_task.readAll()
hasher = hashlib.sha1()
hasher.update(new_xml.data())
new_sha1 = hasher.hexdigest()
self.last_sha1 = new_sha1
# Determine if we need to download the icon: only do that if the
# package.xml file changed (since
# a change in the version number will show up as a change in the
# SHA1, without having to actually
# read the metadata)
if self.repo.name in self.index:
cached_sha1 = self.index[self.repo.name]
if cached_sha1 != new_sha1:
self.update_local_copy(new_xml)
else:
# Assume that if the package.xml file didn't change,
# neither did the icon, so don't waste
# resources downloading it
xml_file = os.path.join(self.store, self.repo.name, "package.xml")
self.repo.metadata = FreeCAD.Metadata(xml_file)
else:
# There is no local copy yet, so we definitely have to update
# the cache
self.update_local_copy(new_xml)
elif (
self.fetch_task.error()
== QtNetwork.QNetworkReply.NetworkError.ContentNotFoundError
):
pass
elif (
self.fetch_task.error()
== QtNetwork.QNetworkReply.NetworkError.OperationCanceledError
):
pass
else:
FreeCAD.Console.PrintWarning(
translate("AddonsInstaller", "Failed to connect to URL")
+ f":\n{self.url}\n {self.fetch_task.error()}\n"
)
def update_local_copy(self, new_xml):
# We have to update the local copy of the metadata file and re-download
# the icon file
name = self.repo.name
repo_url = self.repo.url
package_cache_directory = os.path.join(self.store, name)
if not os.path.exists(package_cache_directory):
os.makedirs(package_cache_directory)
new_xml_file = os.path.join(package_cache_directory, "package.xml")
with open(new_xml_file, "wb") as f:
f.write(new_xml.data())
metadata = FreeCAD.Metadata(new_xml_file)
self.repo.metadata = metadata
self.repo.repo_type = AddonManagerRepo.RepoType.PACKAGE
icon = metadata.Icon
if not icon:
# If there is no icon set for the entire package, see if there are
# any workbenches, which are required to have icons, and grab the first
# one we find:
content = self.repo.metadata.Content
if "workbench" in content:
wb = content["workbench"][0]
if wb.Icon:
if wb.Subdirectory:
subdir = wb.Subdirectory
else:
subdir = wb.Name
self.repo.Icon = subdir + wb.Icon
icon = self.repo.Icon
icon_url = utils.construct_git_url(self.repo, icon)
icon_stream = utils.urlopen(icon_url)
if icon and icon_stream and icon_url:
icon_data = icon_stream.read()
cache_file = self.repo.get_cached_icon_filename()
with open(cache_file, "wb") as icon_file:
icon_file.write(icon_data)
self.repo.cached_icon_filename = cache_file
self.updated.emit(self.repo)
class MetadataTxtDownloadWorker(DownloadWorker):
"""A worker for downloading metadata.txt"""
def __init__(self, parent, repo: AddonManagerRepo):
super().__init__(parent, utils.construct_git_url(repo, "metadata.txt"))
self.repo = repo
def resolve_fetch(self):
"""Called when the data fetch completed, either with an error, or if it found the metadata file"""
if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError:
FreeCAD.Console.PrintLog(
f"Found a metadata.txt file for {self.repo.name}\n"
)
new_deps = self.fetch_task.readAll()
self.parse_file(new_deps.data().decode("utf8"))
elif (
self.fetch_task.error()
== QtNetwork.QNetworkReply.NetworkError.ContentNotFoundError
):
pass
elif (
self.fetch_task.error()
== QtNetwork.QNetworkReply.NetworkError.OperationCanceledError
):
pass
else:
FreeCAD.Console.PrintWarning(
translate("AddonsInstaller", "Failed to connect to URL")
+ f":\n{self.url}\n {self.fetch_task.error()}\n"
)
def parse_file(self, data: str) -> None:
f = io.StringIO(data)
while True:
line = f.readline()
if not line:
break
if line.startswith("workbenches="):
depswb = line.split("=")[1].split(",")
for wb in depswb:
wb_name = wb.strip()
if wb_name:
self.repo.requires.add(wb_name)
FreeCAD.Console.PrintLog(
f"{self.repo.display_name} requires FreeCAD Addon '{wb_name}'\n"
)
elif line.startswith("pylibs="):
depspy = line.split("=")[1].split(",")
for pl in depspy:
dep = pl.strip()
if dep:
self.repo.python_requires.add(dep)
FreeCAD.Console.PrintLog(
f"{self.repo.display_name} requires python package '{dep}'\n"
)
elif line.startswith("optionalpylibs="):
opspy = line.split("=")[1].split(",")
for pl in opspy:
dep = pl.strip()
if dep:
self.repo.python_optional.add(dep)
FreeCAD.Console.PrintLog(
f"{self.repo.display_name} optionally imports python package '{pl.strip()}'\n"
)
self.updated.emit(self.repo)
class RequirementsTxtDownloadWorker(DownloadWorker):
"""A worker for downloading requirements.txt"""
def __init__(self, parent, repo: AddonManagerRepo):
super().__init__(parent, utils.construct_git_url(repo, "requirements.txt"))
self.repo = repo
def resolve_fetch(self):
"""Called when the data fetch completed, either with an error, or if it found the metadata file"""
if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError:
FreeCAD.Console.PrintLog(
f"Found a requirements.txt file for {self.repo.name}\n"
)
new_deps = self.fetch_task.readAll()
self.parse_file(new_deps.data().decode("utf8"))
elif (
self.fetch_task.error()
== QtNetwork.QNetworkReply.NetworkError.ContentNotFoundError
):
pass
elif (
self.fetch_task.error()
== QtNetwork.QNetworkReply.NetworkError.OperationCanceledError
):
pass
else:
FreeCAD.Console.PrintWarning(
translate("AddonsInstaller", "Failed to connect to URL")
+ f":\n{self.url}\n {self.fetch_task.error()}\n"
)
def parse_file(self, data: str) -> None:
f = io.StringIO(data)
lines = f.readlines()
for line in lines:
break_chars = " <>=~!+#"
package = line
for n, c in enumerate(line):
if c in break_chars:
package = line[:n].strip()
break
if package:
self.repo.python_requires.add(package)
self.updated.emit(self.repo)

61
src/Mod/AddonManager/addonmanager_utilities.py
View File

@@ -25,7 +25,7 @@ import os
import re
import ctypes
import ssl
from typing import Union
from typing import Union, Optional
import urllib
from urllib.request import Request
@@ -53,7 +53,6 @@ else:
except AttributeError:
pass
# @package AddonManager_utilities
# \ingroup ADDONMANAGER
# \brief Utilities to work across different platforms, providers and python versions
@@ -72,7 +71,7 @@ def translate(context, text, disambig=None):
def symlink(source, link_name):
"creates a symlink of a file, if possible"
"""Creates a symlink of a file, if possible. Note that it fails on most modern Windows installations"""
if os.path.exists(link_name) or os.path.lexists(link_name):
pass
@@ -81,6 +80,8 @@ def symlink(source, link_name):
if callable(os_symlink):
os_symlink(source, link_name)
else:
# NOTE: This does not work on most normal Windows 10 and later installations, unless developer
# mode is turned on. Make sure to catch any exception thrown and have a fallback plan.
csl = ctypes.windll.kernel32.CreateSymbolicLinkW
csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
csl.restype = ctypes.c_ubyte
@@ -92,47 +93,6 @@ def symlink(source, link_name):
raise ctypes.WinError()
def urlopen(url: str) -> Union[None, HTTPResponse]:
"""Opens an url with urllib and streams it to a temp file"""
timeout = 5
# Proxy an ssl configuration
pref = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons")
if pref.GetBool("NoProxyCheck", True):
proxies = {}
else:
if pref.GetBool("SystemProxyCheck", False):
proxy = urllib.request.getproxies()
proxies = {"http": proxy.get("http"), "https": proxy.get("http")}
elif pref.GetBool("UserProxyCheck", False):
proxy = pref.GetString("ProxyUrl", "")
proxies = {"http": proxy, "https": proxy}
if ssl_ctx:
handler = urllib.request.HTTPSHandler(context=ssl_ctx)
else:
handler = {}
proxy_support = urllib.request.ProxyHandler(proxies)
opener = urllib.request.build_opener(proxy_support, handler)
urllib.request.install_opener(opener)
# Url opening
req = urllib.request.Request(
url, headers={"User-Agent": "Mozilla/5.0 Magic Browser"}
)
try:
u = urllib.request.urlopen(req, timeout=timeout)
except URLError as e:
FreeCAD.Console.PrintLog(f"Error loading {url}:\n {e.reason}\n")
return None
except Exception:
return None
else:
return u
def getserver(url):
"""returns the server part of an url"""
@@ -207,6 +167,19 @@ def get_zip_url(repo):
return None
def recognized_git_location(repo) -> bool:
parsed_url = urlparse(repo.url)
if (
parsed_url.netloc == "github.com"
or parsed_url.netloc == "framagit.com"
or parsed_url.netloc == "gitlab.com"
or parsed_url.netloc == "salsa.debian.org"
):
return True
else:
return False
def construct_git_url(repo, filename):
"Returns a direct download link to a file in an online Git repo: works with github, gitlab, and framagit"

585
src/Mod/AddonManager/addonmanager_workers.py
View File

@@ -35,10 +35,13 @@ import time
import subprocess
import sys
import platform
import itertools
from datetime import datetime
from typing import Union, List
from typing import Union, List, Dict
from enum import Enum, auto
from PySide2 import QtCore, QtNetwork
from PySide2 import QtCore
import FreeCAD
@@ -47,12 +50,9 @@ if FreeCAD.GuiUp:
import addonmanager_utilities as utils
from addonmanager_macro import Macro
from addonmanager_metadata import (
MetadataDownloadWorker,
MetadataTxtDownloadWorker,
RequirementsTxtDownloadWorker,
)
from AddonManagerRepo import AddonManagerRepo
from NetworkManager import AM_NETWORK_MANAGER
translate = FreeCAD.Qt.translate
@@ -119,18 +119,7 @@ class ConnectionChecker(QtCore.QThread):
translate("AddonsInstaller", "Checking network connection...\n")
)
url = "https://api.github.com/zen"
request = utils.urlopen(url)
if QtCore.QThread.currentThread().isInterruptionRequested():
return
if not request:
self.failure.emit(
translate(
"AddonsInstaller",
"Unable to connect to GitHub: check your internet connection and proxy settings and try again.",
)
)
return
result = request.read()
result = AM_NETWORK_MANAGER.blocking_get(url)
if QtCore.QThread.currentThread().isInterruptionRequested():
return
if not result:
@@ -142,7 +131,7 @@ class ConnectionChecker(QtCore.QThread):
)
return
result = result.decode("utf8")
result = result.data().decode("utf8")
FreeCAD.Console.PrintLog(f"GitHub's zen message response: {result}\n")
self.success.emit()
@@ -164,12 +153,11 @@ class UpdateWorker(QtCore.QThread):
# update info lists
global obsolete, macros_reject_list, mod_reject_list, py2only
u = utils.urlopen(
p = AM_NETWORK_MANAGER.blocking_get(
"https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/addonflags.json"
)
if u:
p = u.read()
u.close()
if p:
p = p.data().decode("utf8")
j = json.loads(p)
if "obsolete" in j and "Mod" in j["obsolete"]:
obsolete = j["obsolete"]["Mod"]
@@ -266,15 +254,12 @@ class UpdateWorker(QtCore.QThread):
self.addon_repo.emit(repo)
# querying official addons
u = utils.urlopen(
p = AM_NETWORK_MANAGER.blocking_get(
"https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/.gitmodules"
)
if not u:
if not p:
return
p = u.read()
if isinstance(p, bytes):
p = p.decode("utf-8")
u.close()
p = p.data().decode("utf8")
p = re.findall(
(
r'(?m)\[submodule\s*"(?P<name>.*)"\]\s*'
@@ -642,8 +627,8 @@ class FillMacroListWorker(QtCore.QThread):
Reads only the page https://wiki.freecad.org/Macros_recipes
"""
u = utils.urlopen("https://wiki.freecad.org/Macros_recipes")
if not u:
p = AM_NETWORK_MANAGER.blocking_get("https://wiki.freecad.org/Macros_recipes")
if not p:
FreeCAD.Console.PrintWarning(
translate(
"AddonsInstaller",
@@ -652,10 +637,7 @@ class FillMacroListWorker(QtCore.QThread):
+ "\n"
)
return
p = u.read()
u.close()
if isinstance(p, bytes):
p = p.decode("utf-8")
p = p.data().decode("utf8")
macros = re.findall('title="(Macro.*?)"', p)
macros = [mac for mac in macros if ("translated" not in mac)]
macro_names = []
@@ -835,18 +817,13 @@ class ShowWorker(QtCore.QThread):
readmeurl = utils.get_readme_html_url(self.repo)
if not readmeurl:
FreeCAD.Console.PrintLog(f"README not found for {url}\n")
u = utils.urlopen(readmeurl)
if not u:
p = AM_NETWORK_MANAGER.blocking_get(readmeurl)
if not p:
FreeCAD.Console.PrintLog(f"Debug: README not found at {readmeurl}\n")
u = utils.urlopen(readmeurl)
if u:
p = u.read()
if isinstance(p, bytes):
p = p.decode("utf-8")
u.close()
readme = re.findall(regex, p, flags=re.MULTILINE | re.DOTALL)
if readme:
desc = readme[0]
p = p.data().decode("utf8")
readme = re.findall(regex, p, flags=re.MULTILINE | re.DOTALL)
if readme:
desc = readme[0]
else:
FreeCAD.Console.PrintLog(f"Debug: README not found at {readmeurl}\n")
else:
@@ -854,12 +831,9 @@ class ShowWorker(QtCore.QThread):
readmeurl = utils.get_readme_url(self.repo)
if not readmeurl:
FreeCAD.Console.PrintLog(f"Debug: README not found for {url}\n")
u = utils.urlopen(readmeurl)
if u:
p = u.read()
if isinstance(p, bytes):
p = p.decode("utf-8")
u.close()
p = AM_NETWORK_MANAGER.blocking_get(readmeurl)
if p:
p = p.data().decode("utf8")
desc = utils.fix_relative_links(p, readmeurl.rsplit("/README.md")[0])
if not NOMARKDOWN and have_markdown:
desc = markdown.markdown(desc, extensions=["md_in_html"])
@@ -877,25 +851,22 @@ class ShowWorker(QtCore.QThread):
desc = message
else:
FreeCAD.Console.PrintLog("Debug: README not found at {readmeurl}\n")
if desc == "":
# fall back to the description text
u = utils.urlopen(url)
if not u:
if desc == "":
# fall back to the description text
p = AM_NETWORK_MANAGER.blocking_get(url)
if not p:
return
p = p.data().decode("utf8")
descregex = utils.get_desc_regex(self.repo)
if descregex:
desc = re.findall(descregex, p)
if desc:
desc = desc[0]
if not desc:
desc = "Unable to retrieve addon description"
self.repo.description = desc
if QtCore.QThread.currentThread().isInterruptionRequested():
return
p = u.read()
if isinstance(p, bytes):
p = p.decode("utf-8")
u.close()
descregex = utils.get_desc_regex(self.repo)
if descregex:
desc = re.findall(descregex, p)
if desc:
desc = desc[0]
if not desc:
desc = "Unable to retrieve addon description"
self.repo.description = desc
if QtCore.QThread.currentThread().isInterruptionRequested():
return
message = desc
if self.repo.update_status == AddonManagerRepo.UpdateStatus.UNCHECKED:
# Addon is installed but we haven't checked it yet, so let's check if it has an update
@@ -920,7 +891,7 @@ class ShowWorker(QtCore.QThread):
AddonManagerRepo.UpdateStatus.NO_UPDATE_AVAILABLE
)
self.update_status.emit(self.repo)
if QtCore.QThread.currentThread().isInterruptionRequested():
return
self.readme_updated.emit(message)
@@ -971,9 +942,9 @@ class ShowWorker(QtCore.QThread):
storename = os.path.join(store, wbName + name[-remainChars:])
if not os.path.exists(storename):
try:
u = utils.urlopen(path)
imagedata = u.read()
u.close()
imagedata = AM_NETWORK_MANAGER.blocking_get(path)
if not imagedata:
raise Exception
except Exception:
FreeCAD.Console.PrintLog(
"AddonManager: Debug: Error retrieving image from "
@@ -987,7 +958,7 @@ class ShowWorker(QtCore.QThread):
# lower length limit for path
storename = storename[-140:]
f = open(storename, "wb")
f.write(imagedata)
f.write(imagedata.data())
f.close()
message = message.replace(
'src="' + origpath,
@@ -1095,7 +1066,15 @@ class InstallWorkbenchWorker(QtCore.QThread):
# Do the git process...
self.run_git(target_dir)
else:
self.run_zip(target_dir)
# The zip process uses an event loop, since the download can potentially be quite large
self.launch_zip(target_dir)
self.zip_complete = False
current_thread = QtCore.QThread.currentThread()
while not self.zip_complete:
if current_thread.isInterruptionRequested():
return
QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents, 50)
def update_status(self) -> None:
if hasattr(self, "git_progress") and self.isRunning():
@@ -1223,7 +1202,7 @@ class InstallWorkbenchWorker(QtCore.QThread):
self.update_metadata()
self.success.emit(self.repo, answer)
def run_zip(self, zipdir: str) -> None:
def launch_zip(self, zipdir: str) -> None:
"downloads and unzip a zip version from a git repo"
bakdir = None
@@ -1237,97 +1216,91 @@ class InstallWorkbenchWorker(QtCore.QThread):
if not zipurl:
self.failure.emit(
self.repo,
translate("AddonsInstaller", "Error: Unable to locate zip from")
translate("AddonsInstaller", "Error: Unable to locate ZIP from")
+ " "
+ self.repo.name,
)
return
try:
u = utils.urlopen(zipurl)
except Exception:
self.zipdir = zipdir
self.bakdir = bakdir
AM_NETWORK_MANAGER.progress_made.connect(self.update_zip_status)
AM_NETWORK_MANAGER.progress_complete.connect(self.finish_zip)
self.zip_download_index = AM_NETWORK_MANAGER.submit_monitored_get(zipurl)
def update_zip_status(self, index: int, bytes_read: int, data_size: int):
if index == self.zip_download_index:
locale = QtCore.QLocale()
if data_size > 10 * 1024 * 1024: # To avoid overflows, show MB instead
MB_read = bytes_read / 1024 / 1024
MB_total = data_size / 1024 / 1024
self.progress_made.emit(MB_read, MB_total)
mbytes_str = locale.toString(MB_read)
mbytes_total_str = locale.toString(MB_total)
percent = int(100 * float(MB_read / MB_total))
self.status_message.emit(
translate(
"AddonsInstaller",
f"Downloading: {mbytes_str}MB of {mbytes_total_str}MB ({percent}%)",
)
)
elif data_size > 0:
self.progress_made.emit(bytes_read, data_size)
bytes_str = locale.toString(bytes_read)
bytes_total_str = locale.toString(data_size)
percent = int(100 * float(bytes_read / data_size))
self.status_message.emit(
translate(
"AddonsInstaller",
f"Downloading: {bytes_str} of {bytes_total_str} bytes ({percent}%)",
)
)
else:
MB_read = bytes_read / 1024 / 1024
bytes_str = locale.toString(MB_read)
self.status_message.emit(
translate(
"AddonsInstaller",
f"Downloading: {bytes_str}MB of unknown total",
)
)
def finish_zip(self, index: int, response_code: int, filename: os.PathLike):
self.zip_complete = True
if response_code != 200:
self.failure.emit(
self.repo,
translate("AddonsInstaller", "Error: Unable to download")
+ " "
+ zipurl,
)
return
if not u:
self.failure.emit(
self.repo,
translate("AddonsInstaller", "Error: Unable to download")
+ " "
+ zipurl,
translate(
"AddonsInstaller",
f"Error: Error while downloading ZIP file for {self.repo.display_name}",
),
)
return
data_size = 0
if "content-length" in u.headers:
try:
data_size = int(u.headers["content-length"])
except Exception:
pass
current_thread = QtCore.QThread.currentThread()
# Use an on-disk file and track download progress
locale = QtCore.QLocale()
with tempfile.NamedTemporaryFile(delete=False) as temp:
bytes_to_read = 1024 * 1024
bytes_read = 0
while True:
if current_thread.isInterruptionRequested():
return
chunk = u.read(bytes_to_read)
if not chunk:
break
bytes_read += bytes_to_read
temp.write(chunk)
if data_size:
self.progress_made.emit(bytes_read, data_size)
percent = int(100 * float(bytes_read / data_size))
MB_read = bytes_read / 1024 / 1024
MB_total = data_size / 1024 / 1024
bytes_str = locale.toString(MB_read)
bytes_total_str = locale.toString(MB_total)
self.status_message.emit(
translate(
"AddonsInstaller",
f"Downloading: {bytes_str}MB of {bytes_total_str}MB ({percent}%)",
)
)
else:
MB_read = bytes_read / 1024 / 1024
bytes_str = locale.toString(MB_read)
self.status_message.emit(
translate(
"AddonsInstaller",
f"Downloading: {bytes_str}MB of unknown total",
)
)
QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents)
zfile = zipfile.ZipFile(temp)
with zipfile.ZipFile(filename, "r") as zfile:
master = zfile.namelist()[0] # github will put everything in a subfolder
self.status_message.emit(
translate("AddonsInstaller", f"Download complete. Unzipping file...")
)
QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents)
zfile.extractall(zipdir)
u.close()
zfile.close()
for filename in os.listdir(zipdir + os.sep + master):
shutil.move(
zipdir + os.sep + master + os.sep + filename,
zipdir + os.sep + filename,
)
os.rmdir(zipdir + os.sep + master)
if bakdir:
shutil.rmtree(bakdir)
self.update_metadata()
self.success.emit(
self.repo,
translate("AddonsInstaller", "Successfully installed") + " " + zipurl,
zfile.extractall(self.zipdir)
for filename in os.listdir(self.zipdir + os.sep + master):
shutil.move(
self.zipdir + os.sep + master + os.sep + filename,
self.zipdir + os.sep + filename,
)
os.rmdir(self.zipdir + os.sep + master)
if self.bakdir:
shutil.rmtree(self.bakdir)
self.update_metadata()
self.success.emit(
self.repo,
translate(
"AddonsInstaller",
f"Successfully installed {self.repo.display_name} from ZIP file",
),
)
def update_metadata(self):
basedir = FreeCAD.getUserAppDataDir()
@@ -1506,150 +1479,216 @@ class UpdateMetadataCacheWorker(QtCore.QThread):
progress_made = QtCore.Signal(int, int)
package_updated = QtCore.Signal(AddonManagerRepo)
class AtomicCounter(object):
def __init__(self, start=0):
self.lock = threading.Lock()
self.count = start
def set(self, new_value):
with self.lock:
self.count = new_value
def get(self):
with self.lock:
return self.count
def increment(self):
with self.lock:
self.count += 1
def decrement(self):
with self.lock:
self.count -= 1
class RequestType(Enum):
PACKAGE_XML = auto()
METADATA_TXT = auto()
REQUIREMENTS_TXT = auto()
ICON = auto()
def __init__(self, repos):
QtCore.QThread.__init__(self)
self.repos = repos
self.counter = UpdateMetadataCacheWorker.AtomicCounter()
self.requests: Dict[
int, (AddonManagerRepo, UpdateMetadataCacheWorker.RequestType)
] = {}
AM_NETWORK_MANAGER.completed.connect(self.download_completed)
self.requests_completed = 0
self.total_requests = 0
self.store = os.path.join(
FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata"
)
self.updated_repos = set()
def run(self):
current_thread = QtCore.QThread.currentThread()
self.num_downloads_required = len(self.repos)
self.progress_made.emit(0, self.num_downloads_required)
self.status_message.emit(
translate("AddonsInstaller", "Retrieving package metadata...")
)
store = os.path.join(
FreeCAD.getUserCachePath(), "AddonManager", "PackageMetadata"
)
index_file = os.path.join(store, "index.json")
self.index = {}
if os.path.isfile(index_file):
with open(index_file, "r") as f:
index_string = f.read()
self.index = json.loads(index_string)
download_queue = (
QtNetwork.QNetworkAccessManager()
) # Must be created on this thread
download_queue.finished.connect(self.on_finished)
# Prevent strange internal Qt errors about cache setup by pre-emptively setting
# up a cache. The error this fixes is:
# "caching was enabled after some bytes had been written"
qnam_cache = os.path.join(
FreeCAD.getUserCachePath(), "AddonManager", "QNAM_CACHE"
)
diskCache = QtNetwork.QNetworkDiskCache()
diskCache.setCacheDirectory(qnam_cache)
download_queue.setCache(diskCache)
self.downloaders = []
for repo in self.repos:
if repo.metadata_url:
if repo.url and utils.recognized_git_location(repo):
# package.xml
downloader = MetadataDownloadWorker(None, repo, self.index)
downloader.start_fetch(download_queue)
downloader.updated.connect(self.on_updated)
self.downloaders.append(downloader)
index = AM_NETWORK_MANAGER.submit_unmonitored_get(
utils.construct_git_url(repo, "package.xml")
)
self.requests[index] = (
repo,
UpdateMetadataCacheWorker.RequestType.PACKAGE_XML,
)
self.total_requests += 1
# metadata.txt
downloader = MetadataTxtDownloadWorker(None, repo)
downloader.start_fetch(download_queue)
downloader.updated.connect(self.on_updated)
self.downloaders.append(downloader)
index = AM_NETWORK_MANAGER.submit_unmonitored_get(
utils.construct_git_url(repo, "metadata.txt")
)
self.requests[index] = (
repo,
UpdateMetadataCacheWorker.RequestType.METADATA_TXT,
)
self.total_requests += 1
# requirements.txt
downloader = RequirementsTxtDownloadWorker(None, repo)
downloader.start_fetch(download_queue)
downloader.updated.connect(self.on_updated)
self.downloaders.append(downloader)
index = AM_NETWORK_MANAGER.submit_unmonitored_get(
utils.construct_git_url(repo, "requirements.txt")
)
self.requests[index] = (
repo,
UpdateMetadataCacheWorker.RequestType.REQUIREMENTS_TXT,
)
self.total_requests += 1
# Run a local event loop until we've processed all of the downloads:
# this is local to this thread, and does not affect the main event loop
ui_updater = QtCore.QTimer()
ui_updater.timeout.connect(self.send_ui_update)
ui_updater.start(100) # Send an update back to the main thread every 100ms
self.num_downloads_required = len(self.downloaders)
self.num_downloads_completed = UpdateMetadataCacheWorker.AtomicCounter()
aborted = False
while True:
while self.requests:
if current_thread.isInterruptionRequested():
download_queue.finished.disconnect(self.on_finished)
for downloader in self.downloaders:
downloader.updated.disconnect(self.on_updated)
downloader.abort()
aborted = True
if (
aborted
or self.num_downloads_completed.get() >= self.num_downloads_required
):
break
AM_NETWORK_MANAGER.completed.disconnect(self.download_completed)
for request in self.requests.keys():
AM_NETWORK_MANAGER.abort(request)
return
# 50 ms maximum between checks for interruption
QtCore.QCoreApplication.processEvents(QtCore.QEventLoop.AllEvents, 50)
if aborted:
FreeCAD.Console.PrintLog("Metadata update cancelled\n")
return
# This set contains one copy of each of the repos that got some kind of data in
# this process. For those repos, tell the main Addon Manager code that it needs
# to update its copy of the repo, and redraw its information.
for repo in self.updated_repos:
self.package_updated.emit(repo)
# Update and serialize the updated index, overwriting whatever was
# there before
for downloader in self.downloaders:
if hasattr(downloader, "last_sha1"):
self.index[downloader.repo.name] = downloader.last_sha1
if not os.path.exists(store):
os.makedirs(store)
with open(index_file, "w") as f:
json.dump(self.index, f, indent=" ")
def download_completed(
self, index: int, code: int, data: QtCore.QByteArray
) -> None:
if index in self.requests:
self.requests_completed += 1
self.progress_made.emit(self.requests_completed, self.total_requests)
request = self.requests.pop(index)
if code == 200: # HTTP success
self.updated_repos.add(request[0]) # mark this repo as updated
if request[1] == UpdateMetadataCacheWorker.RequestType.PACKAGE_XML:
self.process_package_xml(request[0], data)
elif request[1] == UpdateMetadataCacheWorker.RequestType.METADATA_TXT:
self.process_metadata_txt(request[0], data)
elif (
request[1] == UpdateMetadataCacheWorker.RequestType.REQUIREMENTS_TXT
):
self.process_requirements_txt(request[0], data)
elif request[1] == UpdateMetadataCacheWorker.RequestType.ICON:
self.process_icon(request[0], data)
def on_finished(self, reply):
# Called by the QNetworkAccessManager's sub-threads when a fetch
# process completed (in any state)
self.num_downloads_completed.increment()
reply.deleteLater()
def on_updated(self, repo):
# Called if this repo got new metadata and/or a new icon
self.package_updated.emit(repo)
def send_ui_update(self):
completed = self.num_downloads_completed.get()
required = self.num_downloads_required
percentage = int(100 * completed / required)
self.progress_made.emit(completed, required)
def process_package_xml(self, repo: AddonManagerRepo, data: QtCore.QByteArray):
repo.repo_type = AddonManagerRepo.RepoType.PACKAGE # By definition
package_cache_directory = os.path.join(self.store, repo.name)
if not os.path.exists(package_cache_directory):
os.makedirs(package_cache_directory)
new_xml_file = os.path.join(package_cache_directory, "package.xml")
with open(new_xml_file, "wb") as f:
f.write(data.data())
metadata = FreeCAD.Metadata(new_xml_file)
repo.metadata = metadata
self.status_message.emit(
translate("AddonsInstaller", "Retrieving package metadata...")
+ f" {completed} / {required} ({percentage}%)"
translate("AddonsInstaller", f"Downloaded package.xml for {repo.name}")
)
def terminate_all(self):
got = self.num_downloads_completed.get()
wanted = self.num_downloads_required
if wanted < got:
FreeCAD.Console.PrintWarning(
f"During cache interruption, wanted {wanted}, got {got}, forcibly terminating now...\n"
# Grab a new copy of the icon as well: we couldn't enqueue this earlier because
# we didn't know the path to it, which is stored in the package.xml file.
icon = metadata.Icon
if not icon:
# If there is no icon set for the entire package, see if there are
# any workbenches, which are required to have icons, and grab the first
# one we find:
content = repo.metadata.Content
if "workbench" in content:
wb = content["workbench"][0]
if wb.Icon:
if wb.Subdirectory:
subdir = wb.Subdirectory
else:
subdir = wb.Name
repo.Icon = subdir + wb.Icon
icon = repo.Icon
icon_url = utils.construct_git_url(repo, icon)
index = AM_NETWORK_MANAGER.submit_unmonitored_get(icon_url)
self.requests[index] = (repo, UpdateMetadataCacheWorker.RequestType.ICON)
self.total_requests += 1
def process_metadata_txt(self, repo: AddonManagerRepo, data: QtCore.QByteArray):
self.status_message.emit(
translate(
"AddonsInstaller", f"Downloaded metadata.txt for {repo.display_name}"
)
self.num_downloads_completed.set(wanted)
)
f = io.StringIO(data.data().decode("utf8"))
while True:
line = f.readline()
if not line:
break
if line.startswith("workbenches="):
depswb = line.split("=")[1].split(",")
for wb in depswb:
wb_name = wb.strip()
if wb_name:
repo.requires.add(wb_name)
FreeCAD.Console.PrintLog(
f"{repo.display_name} requires FreeCAD Addon '{wb_name}'\n"
)
elif line.startswith("pylibs="):
depspy = line.split("=")[1].split(",")
for pl in depspy:
dep = pl.strip()
if dep:
repo.python_requires.add(dep)
FreeCAD.Console.PrintLog(
f"{repo.display_name} requires python package '{dep}'\n"
)
elif line.startswith("optionalpylibs="):
opspy = line.split("=")[1].split(",")
for pl in opspy:
dep = pl.strip()
if dep:
repo.python_optional.add(dep)
FreeCAD.Console.PrintLog(
f"{repo.display_name} optionally imports python package '{pl.strip()}'\n"
)
# For review and debugging purposes, store the file locally
package_cache_directory = os.path.join(self.store, repo.name)
if not os.path.exists(package_cache_directory):
os.makedirs(package_cache_directory)
new_xml_file = os.path.join(package_cache_directory, "metadata.txt")
with open(new_xml_file, "wb") as f:
f.write(data.data())
def process_requirements_txt(self, repo: AddonManagerRepo, data: QtCore.QByteArray):
self.status_message.emit(
translate(
"AddonsInstaller",
f"Downloaded requirements.txt for {repo.display_name}",
)
)
f = io.StringIO(data.data().decode("utf8"))
lines = f.readlines()
for line in lines:
break_chars = " <>=~!+#"
package = line
for n, c in enumerate(line):
if c in break_chars:
package = line[:n].strip()
break
if package:
repo.python_requires.add(package)
# For review and debugging purposes, store the file locally
package_cache_directory = os.path.join(self.store, repo.name)
if not os.path.exists(package_cache_directory):
os.makedirs(package_cache_directory)
new_xml_file = os.path.join(package_cache_directory, "requirements.txt")
with open(new_xml_file, "wb") as f:
f.write(data.data())
def process_icon(self, repo: AddonManagerRepo, data: QtCore.QByteArray):
self.status_message.emit(
translate("AddonsInstaller", f"Downloaded icon for {repo.display_name}")
)
cache_file = repo.get_cached_icon_filename()
with open(cache_file, "wb") as icon_file:
icon_file.write(data.data())
repo.cached_icon_filename = cache_file
if have_git and not NOGIT:

150
src/Mod/AddonManager/proxy_authentication.ui Normal file
View File

@@ -0,0 +1,150 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>proxy_authentication</class>
<widget class="QDialog" name="proxy_authentication">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>536</width>
<height>276</height>
</rect>
</property>
<property name="windowTitle">
<string>Proxy login required</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout">
<item>
<widget class="QLabel" name="label">
<property name="text">
<string>Proxy requires authentication</string>
</property>
</widget>
</item>
<item>
<layout class="QGridLayout" name="gridLayout_2">
<item row="0" column="0">
<widget class="QLabel" name="label_4">
<property name="text">
<string>Proxy:</string>
</property>
</widget>
</item>
<item row="0" column="1">
<widget class="QLabel" name="labelProxyAddress">
<property name="sizePolicy">
<sizepolicy hsizetype="Expanding" vsizetype="Preferred">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="text">
<string>Placeholder for proxy address</string>
</property>
</widget>
</item>
<item row="1" column="0">
<widget class="QLabel" name="labelRealmCaption">
<property name="text">
<string>Realm:</string>
</property>
</widget>
</item>
<item row="1" column="1">
<widget class="QLabel" name="labelProxyRealm">
<property name="text">
<string>Placeholder for proxy realm</string>
</property>
</widget>
</item>
</layout>
</item>
<item>
<layout class="QGridLayout" name="gridLayout">
<item row="0" column="0">
<widget class="QLabel" name="label_2">
<property name="text">
<string>Username</string>
</property>
</widget>
</item>
<item row="0" column="1">
<widget class="QLineEdit" name="lineEditUsername"/>
</item>
<item row="1" column="0">
<widget class="QLabel" name="label_3">
<property name="text">
<string>Password</string>
</property>
</widget>
</item>
<item row="1" column="1">
<widget class="QLineEdit" name="lineEditPassword">
<property name="echoMode">
<enum>QLineEdit::PasswordEchoOnEdit</enum>
</property>
</widget>
</item>
</layout>
</item>
<item>
<spacer name="verticalSpacer">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QDialogButtonBox" name="buttonBox">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="standardButtons">
<set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
</property>
</widget>
</item>
</layout>
</widget>
<resources/>
<connections>
<connection>
<sender>buttonBox</sender>
<signal>accepted()</signal>
<receiver>proxy_authentication</receiver>
<slot>accept()</slot>
<hints>
<hint type="sourcelabel">
<x>248</x>
<y>254</y>
</hint>
<hint type="destinationlabel">
<x>157</x>
<y>274</y>
</hint>
</hints>
</connection>
<connection>
<sender>buttonBox</sender>
<signal>rejected()</signal>
<receiver>proxy_authentication</receiver>
<slot>reject()</slot>
<hints>
<hint type="sourcelabel">
<x>316</x>
<y>260</y>
</hint>
<hint type="destinationlabel">
<x>286</x>
<y>274</y>
</hint>
</hints>
</connection>
</connections>
</ui>