Addon Manager: Rework backend to use package.xml

This shifts to use the model-view-controller pattern for the list of addons,
and moves to using a full model class rather than an indexed array for the
data storage and management. This enables much more information to be stored
as part of the new AddonManagerRepo data type. It now wraps the Macro class
for macros, supports Preference Packs, and provides access to the Metadata
object.

src/Mod/AddonManager/AddonManager.ui

@@ -28,40 +28,105 @@
      <widget class="QWidget" name="verticalLayoutWidget">
       <layout class="QVBoxLayout" name="verticalLayout_2">
        <item>
-        <widget class="QTabWidget" name="tabWidget">
-         <property name="currentIndex">
-          <number>0</number>
+        <layout class="QHBoxLayout" name="horizontalLayout_6">
+         <item>
+          <widget class="QLabel" name="label">
+           <property name="text">
+            <string>Show packages containing:</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QComboBox" name="comboPackageType">
+           <item>
+            <property name="text">
+             <string>All</string>
+            </property>
+           </item>
+           <item>
+            <property name="text">
+             <string>Workbenches</string>
+            </property>
+           </item>
+           <item>
+            <property name="text">
+             <string>Macros</string>
+            </property>
+           </item>
+           <item>
+            <property name="text">
+             <string>Preference Packs</string>
+            </property>
+           </item>
+          </widget>
+         </item>
+        </layout>
+       </item>
+       <item>
+        <layout class="QHBoxLayout" name="horizontalLayout_5">
+         <item>
+          <widget class="QLineEdit" name="lineEditFilter">
+           <property name="placeholderText">
+            <string>Filter</string>
+           </property>
+           <property name="clearButtonEnabled">
+            <bool>true</bool>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QLabel" name="labelFilterValidity">
+           <property name="text">
+            <string>OK</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+       <item>
+        <widget class="QTableView" name="tablePackages">
+         <property name="editTriggers">
+          <set>QAbstractItemView::NoEditTriggers</set>
          </property>
-         <widget class="QWidget" name="tab">
-          <attribute name="title">
-           <string>Workbenches</string>
-          </attribute>
-          <layout class="QVBoxLayout" name="verticalLayout_5">
-           <item>
-            <widget class="QListWidget" name="listWorkbenches"/>
-           </item>
-          </layout>
-         </widget>
-         <widget class="QWidget" name="tab_2">
-          <attribute name="title">
-           <string>Macros</string>
-          </attribute>
-          <layout class="QVBoxLayout" name="verticalLayout_6">
-           <item>
-            <widget class="QListWidget" name="listMacros"/>
-           </item>
-           <item>
-            <widget class="QPushButton" name="buttonExecute">
-             <property name="toolTip">
-              <string>Executes the selected macro, if installed</string>
-             </property>
-             <property name="text">
-              <string>Execute</string>
-             </property>
-            </widget>
-           </item>
-          </layout>
-         </widget>
+         <property name="showDropIndicator" stdset="0">
+          <bool>false</bool>
+         </property>
+         <property name="selectionMode">
+          <enum>QAbstractItemView::SingleSelection</enum>
+         </property>
+         <property name="selectionBehavior">
+          <enum>QAbstractItemView::SelectRows</enum>
+         </property>
+         <property name="iconSize">
+          <size>
+           <width>16</width>
+           <height>16</height>
+          </size>
+         </property>
+         <property name="showGrid">
+          <bool>false</bool>
+         </property>
+         <property name="sortingEnabled">
+          <bool>true</bool>
+         </property>
+         <property name="wordWrap">
+          <bool>false</bool>
+         </property>
+         <attribute name="horizontalHeaderMinimumSectionSize">
+          <number>16</number>
+         </attribute>
+         <attribute name="verticalHeaderVisible">
+          <bool>false</bool>
+         </attribute>
+         <attribute name="verticalHeaderMinimumSectionSize">
+          <number>12</number>
+         </attribute>
+         <attribute name="verticalHeaderDefaultSectionSize">
+          <number>16</number>
+         </attribute>
+         <attribute name="verticalHeaderStretchLastSection">
+          <bool>false</bool>
+         </attribute>
         </widget>
        </item>
       </layout>
@@ -95,13 +160,125 @@
          </item>
         </layout>
        </item>
+       <item>
+        <layout class="QHBoxLayout" name="horizontalLayout_4">
+         <item>
+          <widget class="QLabel" name="labelIcon">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="minimumSize">
+            <size>
+             <width>64</width>
+             <height>64</height>
+            </size>
+           </property>
+           <property name="baseSize">
+            <size>
+             <width>64</width>
+             <height>64</height>
+            </size>
+           </property>
+           <property name="text">
+            <string>Icon</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <layout class="QVBoxLayout" name="verticalLayout">
+           <property name="spacing">
+            <number>0</number>
+           </property>
+           <property name="sizeConstraint">
+            <enum>QLayout::SetDefaultConstraint</enum>
+           </property>
+           <item>
+            <widget class="QLabel" name="labelPackageName">
+             <property name="text">
+              <string>&lt;h1&gt;Package Name&lt;/h1&gt;</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QLabel" name="labelVersion">
+             <property name="text">
+              <string>&lt;em&gt;Version&lt;/em&gt;</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QLabel" name="labelMaintainer">
+             <property name="text">
+              <string>Maintainer</string>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </item>
+        </layout>
+       </item>
+       <item>
+        <widget class="QLabel" name="labelDescription">
+         <property name="text">
+          <string>Description</string>
+         </property>
+         <property name="textFormat">
+          <enum>Qt::PlainText</enum>
+         </property>
+         <property name="wordWrap">
+          <bool>true</bool>
+         </property>
+        </widget>
+       </item>
+       <item>
+        <layout class="QGridLayout" name="urlGrid">
+         <item row="0" column="0">
+          <widget class="QLabel" name="labelUrlType">
+           <property name="text">
+            <string>URL Type</string>
+           </property>
+          </widget>
+         </item>
+         <item row="0" column="1">
+          <widget class="QLabel" name="labelUrl">
+           <property name="text">
+            <string>Url</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+       <item>
+        <widget class="QLabel" name="labelContents">
+         <property name="text">
+          <string>Package contents:</string>
+         </property>
+        </widget>
+       </item>
        <item>
         <widget class="QTextBrowser" name="description"/>
        </item>
+       <item>
+        <widget class="QPushButton" name="buttonExecute">
+         <property name="text">
+          <string>Run Macro</string>
+         </property>
+        </widget>
+       </item>
       </layout>
      </widget>
     </widget>
    </item>
+   <item>
+    <widget class="QLabel" name="labelStatusInfo">
+     <property name="text">
+      <string>labelStatusInfo</string>
+     </property>
+    </widget>
+   </item>
    <item>
     <layout class="QHBoxLayout" name="horizontalLayout_2">
      <item>
@@ -109,6 +286,9 @@
        <property name="value">
         <number>24</number>
        </property>
+       <property name="textVisible">
+        <bool>false</bool>
+       </property>
        <property name="format">
         <string>Downloading info...</string>
        </property>

src/Mod/AddonManager/AddonManagerRepo.py

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+#***************************************************************************
+#*                                                                         *
+#*   Copyright (c) 2021 Chris Hennes <chennes@pioneerlibrarysystem.org>    *
+#*                                                                         *
+#*   This program is free software; you can redistribute it and/or modify  *
+#*   it under the terms of the GNU Lesser General Public License (LGPL)    *
+#*   as published by the Free Software Foundation; either version 2 of     *
+#*   the License, or (at your option) any later version.                   *
+#*   for detail see the LICENCE text file.                                 *
+#*                                                                         *
+#*   This program is distributed in the hope that it will be useful,       *
+#*   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+#*   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
+#*   GNU Library General Public License for more details.                  *
+#*                                                                         *
+#*   You should have received a copy of the GNU Library General Public     *
+#*   License along with this program; if not, write to the Free Software   *
+#*   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  *
+#*   USA                                                                   *
+#*                                                                         *
+#***************************************************************************
+
+import FreeCAD
+
+import os
+
+from addonmanager_macro import Macro
+
+class AddonManagerRepo:
+    "Encapsulate information about a FreeCAD addon"
+
+    from enum import Enum
+
+    class RepoType(Enum):
+        WORKBENCH = 1
+        MACRO = 2
+        PACKAGE = 3
+
+        def __str__(self) ->str :
+            if self.value == 1:
+                return "Workbench"
+            elif self.value == 2:
+                return "Macro"
+            elif self.value == 3:
+                return "Package"
+
+    class UpdateStatus(Enum):
+        NOT_INSTALLED = 0
+        UNCHECKED = 1
+        NO_UPDATE_AVAILABLE = 2
+        UPDATE_AVAILABLE = 3
+        PENDING_RESTART = 4
+
+        def __lt__(self, other):
+            if self.__class__ is other.__class__:
+              return self.value < other.value
+            return NotImplemented
+
+        def __str__(self) ->str :
+            if self.value == 0:
+                return "Not installed"
+            elif self.value == 1:
+                return "Unchecked"
+            elif self.value == 2:
+                return "No update available"
+            elif self.value == 3:
+                return "Update available"
+            elif self.value == 4:
+                return "Restart required"
+
+    def __init__ (self, name:str, url:str, status:UpdateStatus, branch:str):
+        self.name = name
+        self.url = url
+        self.branch = branch
+        self.update_status = status
+        self.repo_type = AddonManagerRepo.RepoType.WORKBENCH
+        self.description = None
+        from addonmanager_utilities import construct_git_url
+        self.metadata_url = "" if not self.url else construct_git_url(self, "package.xml")
+        self.metadata = None
+        self.icon = None
+        self.cached_icon_filename = ""
+        self.macro = None # Bridge to Gaël Écorchard's macro management class
+
+    def __str__ (self) -> str:
+        result  = f"FreeCAD {self.repo_type}\n"
+        result += f"Name: {self.name}\n"
+        result += f"URL: {self.url}\n"
+        result += "Has metadata\n" if self.metadata is not None else "No metadata found\n"
+        if self.macro is not None:
+            result += "Has linked Macro object\n"
+        return result
+
+    @classmethod
+    def from_macro (self, macro:Macro):
+        if macro.is_installed():
+            status = AddonManagerRepo.UpdateStatus.UNCHECKED
+        else: 
+            status = AddonManagerRepo.UpdateStatus.NOT_INSTALLED
+        instance = AddonManagerRepo(macro.name, macro.url, status, "master")
+        instance.macro = macro
+        instance.repo_type = AddonManagerRepo.RepoType.MACRO
+        instance.description = macro.desc
+        return instance
+        
+    def contains_workbench(self) -> bool:
+        """ Determine if this package contains (or is) a workbench """
+
+        if self.repo_type == AddonManagerRepo.RepoType.WORKBENCH:
+            return True
+        elif self.repo_type == AddonManagerRepo.RepoType.PACKAGE:
+            content = self.metadata.Content
+            return "workbench" in content
+        else:
+            return False
+
+    def contains_macro(self) -> bool:
+        """ Determine if this package contains (or is) a macro """
+
+        if self.repo_type == AddonManagerRepo.RepoType.MACRO:
+            return True
+        elif self.repo_type == AddonManagerRepo.RepoType.PACKAGE:
+            content = self.metadata.Content
+            return "macro" in content
+        else:
+            return False
+
+    def contains_preference_pack(self) -> bool:
+        """ Determine if this package contains a preference pack """
+
+        if self.repo_type == AddonManagerRepo.RepoType.PACKAGE:
+            content = self.metadata.Content
+            return "preferencepack" in content
+        else:
+            return False
+
+    def get_cached_icon_filename(self) ->str:
+        """ Get the filename for the locally-cached copy of the icon """
+
+        if self.cached_icon_filename:
+            return self.cached_icon_filename
+
+        real_icon = self.metadata.Icon
+        if not real_icon:
+            # If there is no icon set for the entire package, see if there are any workbenches, which
+            # are required to have icons, and grab the first one we find:
+            content = self.metadata.Content
+            if "workbench" in content:
+                wb = content["workbench"][0]
+                if wb.Icon:
+                    if wb.Subdirectory:
+                        subdir = wb.Subdirectory
+                    else:
+                        subdir = wb.Name
+                    real_icon = subdir + wb.Icon
+
+        real_icon = real_icon.replace("/", os.path.sep) # Required path separator in the metadata.xml file to local separator
+
+        _, file_extension = os.path.splitext(real_icon)
+        store = os.path.join(FreeCAD.getUserAppDataDir(), "AddonManager", "PackageMetadata")
+        self.cached_icon_filename = os.path.join(store, self.name, "cached_icon"+file_extension)
+
+        return self.cached_icon_filename

src/Mod/AddonManager/CMakeLists.txt

@@ -6,7 +6,9 @@ SET(AddonManager_SRCS
     Init.py
     InitGui.py
     AddonManager.py
+    AddonManagerRepo.py
     addonmanager_macro.py
+    addonmanager_metadata.py
     addonmanager_utilities.py
     addonmanager_workers.py
     AddonManager.ui

src/Mod/AddonManager/addonmanager_macro.py

@@ -24,12 +24,14 @@
 import os
 import re
 import sys
-
+import codecs
+import shutil
 
 import FreeCAD
 
 from addonmanager_utilities import translate
 from addonmanager_utilities import urlopen
+from addonmanager_utilities import remove_directory_if_empty
 
 try:
     from HTMLParser import HTMLParser
@@ -39,10 +41,10 @@ except ImportError:
 
 #  @package AddonManager_macro
 #  \ingroup ADDONMANAGER
-#  \brief Unified handler for FreeCAD macros that can be obtained from different sources
+#  \brief Unified handler for FreeCAD macros that can be obtained from
+#  different sources
 #  @{
 
-
 class Macro(object):
     """This class provides a unified way to handle macros coming from different sources"""
 
@@ -70,12 +72,11 @@ class Macro(object):
     def is_installed(self):
         if self.on_git and not self.src_filename:
             return False
-        return (os.path.exists(os.path.join(FreeCAD.getUserMacroDir(True), self.filename))
-                or os.path.exists(os.path.join(FreeCAD.getUserMacroDir(True), "Macro_" + self.filename)))
+        return (os.path.exists(os.path.join(FreeCAD.getUserMacroDir(True), self.filename)) or os.path.exists(os.path.join(FreeCAD.getUserMacroDir(True), "Macro_" + self.filename)))
 
     def fill_details_from_file(self, filename):
         with open(filename) as f:
-            # Number of parsed fields of metadata. For now, __Comment__,
+            # Number of parsed fields of metadata.  For now, __Comment__,
             # __Web__, __Version__, __Files__.
             number_of_required_fields = 4
             re_desc = re.compile(r"^__Comment__\s*=\s*(['\"])(.*)\1")
@@ -109,23 +110,24 @@ class Macro(object):
         code = ""
         u = urlopen(url)
         if u is None:
-            print("AddonManager: Debug: connection is lost (proxy setting changed?)", url)
+            FreeCAD.Console.PrintWarning("AddonManager: Debug: connection is lost (proxy setting changed?)", url, "\n")
             return
         p = u.read()
-        if sys.version_info.major >= 3 and isinstance(p, bytes):
+        if isinstance(p, bytes):
             p = p.decode("utf-8")
         u.close()
-        # check if the macro page has its code hosted elsewhere, download if needed
+        # check if the macro page has its code hosted elsewhere, download if
+        # needed
         if "rawcodeurl" in p:
             rawcodeurl = re.findall("rawcodeurl.*?href=\"(http.*?)\">", p)
             if rawcodeurl:
                 rawcodeurl = rawcodeurl[0]
                 u2 = urlopen(rawcodeurl)
                 if u2 is None:
-                    print("AddonManager: Debug: unable to open URL", rawcodeurl)
+                    FreeCAD.Console.PrintWarning("AddonManager: Debug: unable to open URL", rawcodeurl, "\n")
                     return
                 # code = u2.read()
-                # github is slow to respond... We need to use this trick below
+                # github is slow to respond...  We need to use this trick below
                 response = ""
                 block = 8192
                 # expected = int(u2.headers["content-length"])
@@ -134,7 +136,7 @@ class Macro(object):
                     data = u2.read(block)
                     if not data:
                         break
-                    if sys.version_info.major >= 3 and isinstance(data, bytes):
+                    if isinstance(data, bytes):
                         data = data.decode("utf-8")
                     response += data
                 if response:
@@ -147,25 +149,101 @@ class Macro(object):
                 code = sorted(code, key=len)[-1]
                 code = code.replace("--endl--", "\n")
                 # Clean HTML escape codes.
-                if sys.version_info.major < 3:
-                    code = code.decode("utf8")
                 code = unescape(code)
                 code = code.replace(b"\xc2\xa0".decode("utf-8"), " ")
-                if sys.version_info.major < 3:
-                    code = code.encode("utf8")
             else:
-                FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "Unable to fetch the code of this macro."))
+                FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "Unable to fetch the code of this macro.") + "\n")
 
         desc = re.findall(r"<td class=\"ctEven left macro-description\">(.*?)</td>", p.replace("\n", " "))
         if desc:
             desc = desc[0]
         else:
             FreeCAD.Console.PrintWarning(translate("AddonsInstaller",
-                                                   "Unable to retrieve a description for this macro."))
+                                                   "Unable to retrieve a description for this macro.") + "\n")
             desc = "No description available"
         self.desc = desc
         self.url = url
+        if isinstance(code, list):
+            flat_code = ""
+            for chunk in code:
+                flat_code += chunk
+            code = flat_code
         self.code = code
         self.parsed = True
 
+    def install(self, macro_dir:str) -> bool:
+        """Install a macro and all its related files
+
+        Returns True if the macro was installed correctly.
+
+        Parameters
+        ----------
+        - macro_dir: the directory to install into
+        """
+
+        if not self.code:
+            return False
+        if not os.path.isdir(macro_dir):
+            try:
+                os.makedirs(macro_dir)
+            except OSError:
+                FreeCAD.Console.PrintError(f"Failed to create {macro_dir}\n")
+                return False
+        macro_path = os.path.join(macro_dir, self.filename)
+        try:
+            with codecs.open(macro_path, 'w', 'utf-8') as macrofile:
+                macrofile.write(self.code)
+        except IOError:
+            FreeCAD.Console.PrintError(f"Failed to write {macro_path}\n")
+            return False
+        # Copy related files, which are supposed to be given relative to
+        # self.src_filename.
+        base_dir = os.path.dirname(self.src_filename)
+        for other_file in self.other_files:
+            dst_dir = os.path.join(macro_dir, os.path.dirname(other_file))
+            if not os.path.isdir(dst_dir):
+                try:
+                    os.makedirs(dst_dir)
+                except OSError:
+                    FreeCAD.Console.PrintError(f"Failed to create {dst_dir}\n")
+                    return False
+            src_file = os.path.join(base_dir, other_file)
+            dst_file = os.path.join(macro_dir, other_file)
+            try:
+                shutil.copy(src_file, dst_file)
+            except IOError:
+                FreeCAD.Console.PrintError(f"Failed to copy {src_file} to {dst_file}\n")
+                return False
+    
+        FreeCAD.Console.PrintMessage(f"Macro {self.name} was installed successfully.\n")
+        return True
+
+
+    def remove(self) -> bool:
+        """Remove a macro and all its related files
+
+        Returns True if the macro was removed correctly.
+        """
+
+        if not self.is_installed():
+            # Macro not installed, nothing to do.
+            return True
+        macro_dir = FreeCAD.getUserMacroDir(True)
+        macro_path = os.path.join(macro_dir, self.filename)
+        macro_path_with_macro_prefix = os.path.join(macro_dir, 'Macro_' + self.filename)
+        if os.path.exists(macro_path):
+            os.remove(macro_path)
+        elif os.path.exists(macro_path_with_macro_prefix):
+            os.remove(macro_path_with_macro_prefix)
+        # Remove related files, which are supposed to be given relative to
+        # self.src_filename.
+        for other_file in self.other_files:
+            dst_file = os.path.join(macro_dir, other_file)
+            try:
+                os.remove(dst_file)
+                remove_directory_if_empty(os.path.dirname(dst_file))
+            except Exception:
+                FreeCAD.Console.PrintWarning(f"Failed to remove macro file '{dst_file}': it might not exist, or its permissions changed\n")
+        return True
+
 #  @}

src/Mod/AddonManager/addonmanager_metadata.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+#***************************************************************************
+#*                                                                         *
+#* Copyright (c) 2021 Chris Hennes <chennes@pioneerlibrarysystem.org>      *
+#*                                                                         *
+#* This program is free software; you can redistribute it and/or modify    *
+#* it under the terms of the GNU Lesser General Public License (LGPL)      *
+#* as published by the Free Software Foundation; either version 2 of       *
+#* the License, or (at your option) any later version.                     *
+#* for detail see the LICENCE text file.                                   *
+#*                                                                         *
+#* This program is distributed in the hope that it will be useful,         *
+#* but WITHOUT ANY WARRANTY; without even the implied warranty of          *
+#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
+#* GNU Library General Public License for more details.                    *
+#*                                                                         *
+#* You should have received a copy of the GNU Library General Public       *
+#* License along with this program; if not, write to the Free Software     *
+#* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307     *
+#* USA                                                                     *
+#*                                                                         *
+#***************************************************************************
+
+import FreeCAD
+
+import git
+import tempfile
+import os
+import hashlib
+
+from PySide2 import QtCore, QtNetwork
+from PySide2.QtCore import QObject
+
+import addonmanager_utilities as utils
+from AddonManagerRepo import AddonManagerRepo
+
+class MetadataDownloadWorker(QObject):
+    """A worker for downloading package.xml and associated icon(s)
+    
+    To use, instantiate an object of this class and call the start_fetch() function
+    with a QNetworkAccessManager. It is expected that many of these objects will all
+    be created and associated with the same QNAM, which will then handle the actual
+    asynchronous downloads in some Qt-defined number of threads. To monitor progress
+    you should connect to the QNAM's "finished" signal, and ensure it is called the
+    number of times you expect based on how many workers you have enqueued.
+    
+    """
+
+    updated = QtCore.Signal(AddonManagerRepo) 
+
+    def __init__(self, parent, repo, index):
+        "repo is an AddonManagerRepo object, and index is a dictionary of SHA1 hashes of the package.xml files in the cache"
+
+        super().__init__(parent)
+        self.repo = repo
+        self.index = index
+        self.store = os.path.join(FreeCAD.getUserAppDataDir(), "AddonManager", "PackageMetadata")
+        self.last_sha1 = ""
+        self.url = self.repo.metadata_url
+
+    def start_fetch(self, network_manager):
+        "Asynchronously begin the network access. Intended as a set-and-forget black box for downloading metadata."        
+        self.request = QtNetwork.QNetworkRequest(QtCore.QUrl(self.url))
+        self.request.setAttribute(QtNetwork.QNetworkRequest.RedirectPolicyAttribute,
+                                  QtNetwork.QNetworkRequest.UserVerifiedRedirectPolicy)
+
+        self.fetch_task = network_manager.get(self.request)
+        self.fetch_task.finished.connect(self.resolve_fetch)
+        self.fetch_task.redirected.connect(self.on_redirect)
+
+    def abort(self):
+        self.fetch_task.abort()
+
+    def on_redirect(self, url):
+        # For now just blindly follow all redirects
+        self.fetch_task.redirectAllowed.emit()
+
+    def resolve_fetch(self):
+        "Called when the data fetch completed, either with an error, or if it found the metadata file"
+        if self.fetch_task.error() == QtNetwork.QNetworkReply.NetworkError.NoError:
+            FreeCAD.Console.PrintMessage(f"Found a metadata file for {self.repo.name}\n")
+            self.repo.repo_type = AddonManagerRepo.RepoType.PACKAGE
+            new_xml = self.fetch_task.readAll()
+            hasher = hashlib.sha1()
+            hasher.update(new_xml)
+            new_sha1 = hasher.hexdigest()
+            self.last_sha1 = new_sha1
+            # Determine if we need to download the icon: only do that if the
+            # package.xml file changed (since
+            # a change in the version number will show up as a change in the
+            # SHA1, without having to actually
+            # read the metadata)
+            if self.repo.name in self.index:
+                cached_sha1 = self.index[self.repo.name]
+                if cached_sha1 != new_sha1:
+                    self.update_local_copy(new_xml)
+                else:
+                    # Assume that if the package.xml file didn't change,
+                    # neither did the icon, so don't waste
+                    # resources downloading it
+                    xml_file = os.path.join(self.store, self.repo.name, "package.xml")
+                    self.repo.metadata = FreeCAD.Metadata(xml_file)
+            else:
+                # There is no local copy yet, so we definitely have to update
+                # the cache
+                self.update_local_copy(new_xml)
+
+    def update_local_copy(self, new_xml):
+        # We have to update the local copy of the metadata file and re-download
+        # the icon file
+
+        name = self.repo.name
+        repo_url = self.repo.url
+        package_cache_directory = os.path.join(self.store, name)
+        if not os.path.exists(package_cache_directory):
+            os.makedirs(package_cache_directory)
+        new_xml_file = os.path.join(package_cache_directory, "package.xml")
+        with open(new_xml_file, "wb") as f:
+            f.write(new_xml)
+        metadata = FreeCAD.Metadata(new_xml_file)
+        self.repo.metadata = metadata
+        self.repo.repo_type = AddonManagerRepo.RepoType.PACKAGE
+        icon = metadata.Icon
+
+        if not icon:
+            # If there is no icon set for the entire package, see if there are
+            # any workbenches, which
+            # are required to have icons, and grab the first one we find:
+            content = self.repo.metadata.Content
+            if "workbench" in content:
+                wb = content["workbench"][0]
+                if wb.Icon:
+                    if wb.Subdirectory:
+                        subdir = wb.Subdirectory
+                    else:
+                        subdir = wb.Name
+                    self.repo.Icon = subdir + wb.Icon
+                    icon = self.repo.Icon
+
+        icon_url = utils.construct_git_url(self.repo, icon)
+        icon_stream = utils.urlopen(icon_url)
+        if icon and icon_stream and icon_url:
+            icon_data = icon_stream.read()
+            cache_file = self.repo.get_cached_icon_filename()
+            with open(cache_file, "wb") as icon_file:
+                icon_file.write(icon_data)
+                self.repo.cached_icon_filename = cache_file
+        self.updated.emit(self.repo)

src/Mod/AddonManager/addonmanager_utilities.py

@@ -27,12 +27,15 @@ import re
 import shutil
 import sys
 import ctypes
+import tempfile
+import ssl
 
-import urllib.request as urllib2
+import urllib
+from urllib.request import Request
 from urllib.error import URLError
 from urllib.parse import urlparse
 
-from PySide import QtGui, QtCore
+from PySide2 import QtGui, QtCore, QtWidgets
 
 import FreeCAD
 import FreeCADGui
@@ -46,7 +49,9 @@ except ImportError:
     pass
 else:
     try:
-        ssl_ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
+        #ssl_ctx = ssl.create_default_context(cafile=certifi.where())
+        ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+        #ssl_ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
     except AttributeError:
         pass
 
@@ -61,18 +66,17 @@ def translate(context, text, disambig=None):
     "Main translation function"
 
     try:
-        _encoding = QtGui.QApplication.UnicodeUTF8
+        _encoding = QtWidgets.QApplication.UnicodeUTF8
     except AttributeError:
-        return QtGui.QApplication.translate(context, text, disambig)
+        return QtWidgets.QApplication.translate(context, text, disambig)
     else:
-        return QtGui.QApplication.translate(context, text, disambig, _encoding)
+        return QtWidgets.QApplication.translate(context, text, disambig, _encoding)
 
 
 def symlink(source, link_name):
     "creates a symlink of a file, if possible"
 
     if os.path.exists(link_name) or os.path.lexists(link_name):
-        # print("macro already exists")
         pass
     else:
         os_symlink = getattr(os, "symlink", None)
@@ -90,8 +94,8 @@ def symlink(source, link_name):
                 raise ctypes.WinError()
 
 
-def urlopen(url):
-    """Opens an url with urllib2"""
+def urlopen(url:str):
+    """Opens an url with urllib and streams it to a temp file"""
 
     timeout = 5
 
@@ -101,25 +105,29 @@ def urlopen(url):
         proxies = {}
     else:
         if pref.GetBool("SystemProxyCheck", False):
-            proxy = urllib2.getproxies()
+            proxy = urllib.request.getproxies()
             proxies = {"http": proxy.get('http'), "https": proxy.get('http')}
         elif pref.GetBool("UserProxyCheck", False):
             proxy = pref.GetString("ProxyUrl", "")
             proxies = {"http": proxy, "https": proxy}
 
     if ssl_ctx:
-        handler = urllib2.HTTPSHandler(context=ssl_ctx)
+        handler = urllib.request.HTTPSHandler(context=ssl_ctx)
     else:
         handler = {}
-    proxy_support = urllib2.ProxyHandler(proxies)
-    opener = urllib2.build_opener(proxy_support, handler)
-    urllib2.install_opener(opener)
+    proxy_support = urllib.request.ProxyHandler(proxies)
+    opener = urllib.request.build_opener(proxy_support, handler)
+    urllib.request.install_opener(opener)
 
     # Url opening
-    req = urllib2.Request(url,
+    req = urllib.request.Request(url,
                           headers={'User-Agent': "Magic Browser"})
     try:
-        u = urllib2.urlopen(req, timeout=timeout)
+        u = urllib.request.urlopen(req, timeout=timeout)
+
+    except URLError as e:
+        FreeCAD.Console.PrintError(f"Error loading {url}:\n {e.reason}\n")
+        return None
     except Exception:
         return None
     else:
@@ -151,76 +159,7 @@ def update_macro_details(old_macro, new_macro):
             setattr(old_macro, attr, getattr(new_macro, attr))
 
 
-def install_macro(macro, macro_repo_dir):
-    """Install a macro and all its related files
 
-    Returns True if the macro was installed correctly.
-
-    Parameters
-    ----------
-    - macro: an addonmanager_macro.Macro instance
-    """
-
-    if not macro.code:
-        return False
-    macro_dir = FreeCAD.getUserMacroDir(True)
-    if not os.path.isdir(macro_dir):
-        try:
-            os.makedirs(macro_dir)
-        except OSError:
-            return False
-    macro_path = os.path.join(macro_dir, macro.filename)
-    try:
-        with codecs.open(macro_path, 'w', 'utf-8') as macrofile:
-            macrofile.write(macro.code)
-    except IOError:
-        return False
-    # Copy related files, which are supposed to be given relative to
-    # macro.src_filename.
-    base_dir = os.path.dirname(macro.src_filename)
-    for other_file in macro.other_files:
-        dst_dir = os.path.join(macro_dir, os.path.dirname(other_file))
-        if not os.path.isdir(dst_dir):
-            try:
-                os.makedirs(dst_dir)
-            except OSError:
-                return False
-        src_file = os.path.join(base_dir, other_file)
-        dst_file = os.path.join(macro_dir, other_file)
-        try:
-            shutil.copy(src_file, dst_file)
-        except IOError:
-            return False
-    return True
-
-
-def remove_macro(macro):
-    """Remove a macro and all its related files
-
-    Returns True if the macro was removed correctly.
-
-    Parameters
-    ----------
-    - macro: an addonmanager_macro.Macro instance
-    """
-
-    if not macro.is_installed():
-        # Macro not installed, nothing to do.
-        return True
-    macro_dir = FreeCAD.getUserMacroDir(True)
-    macro_path = os.path.join(macro_dir, macro.filename)
-    macro_path_with_macro_prefix = os.path.join(macro_dir, 'Macro_' + macro.filename)
-    if os.path.exists(macro_path):
-        os.remove(macro_path)
-    elif os.path.exists(macro_path_with_macro_prefix):
-        os.remove(macro_path_with_macro_prefix)
-    # Remove related files, which are supposed to be given relative to
-    # macro.src_filename.
-    for other_file in macro.other_files:
-        dst_file = os.path.join(macro_dir, other_file)
-        remove_directory_if_empty(os.path.dirname(dst_file))
-        os.remove(dst_file)
-    return True
 
 
 def remove_directory_if_empty(dir):
@@ -238,72 +177,81 @@ def remove_directory_if_empty(dir):
 def restart_freecad():
     "Shuts down and restarts FreeCAD"
 
-    args = QtGui.QApplication.arguments()[1:]
+    args = QtWidgets.QApplication.arguments()[1:]
     if FreeCADGui.getMainWindow().close():
-        QtCore.QProcess.startDetached(QtGui.QApplication.applicationFilePath(), args)
+        QtCore.QProcess.startDetached(QtWidgets.QApplication.applicationFilePath(), args)
 
 
-def get_zip_url(baseurl):
+def get_zip_url(repo):
     "Returns the location of a zip file from a repo, if available"
 
-    parsedUrl = urlparse(baseurl)
+    parsedUrl = urlparse(repo.url)
     if parsedUrl.netloc == "github.com":
-        return baseurl+"/archive/master.zip"
+        return f"{repo.url}/archive/{repo.branch}.zip"
     elif parsedUrl.netloc == "framagit.org" or parsedUrl.netloc == "gitlab.com":
         # https://framagit.org/freecad-france/mooc-workbench/-/archive/master/mooc-workbench-master.zip
         reponame = baseurl.strip("/").split("/")[-1]
-        return baseurl+"/-/archive/master/"+reponame+"-master.zip"
+        return f"{repo.url}/-/archive/{repo.branch}/{repo.name}-{repo.branch}.zip"
     else:
-        print("Debug: addonmanager_utilities.get_zip_url: Unknown git host:", parsedUrl.netloc)
+        FreeCAD.Console.PrintWarning("Debug: addonmanager_utilities.get_zip_url: Unknown git host:", parsedUrl.netloc)
         return None
 
+def construct_git_url(repo, filename):
+    "Returns a direct download link to a file in an online Git repo: works with github, gitlab, and framagit"
 
-def get_readme_url(url):
-    "Returns the location of a readme file"
-
-    parsedUrl = urlparse(url)
-    if parsedUrl.netloc == "github.com" or parsedUrl.netloc == "framagit.com":
-        return url+"/raw/master/README.md"
-    elif parsedUrl.netloc == "gitlab.com":
-        return url+"/-/raw/master/README.md"
+    parsed_url = urlparse(repo.url)
+    if parsed_url.netloc == "github.com" or parsed_url.netloc == "framagit.com":
+        return f"{repo.url}/raw/{repo.branch}/{filename}"
+    elif parsed_url.netloc == "gitlab.com":
+        return f"{repo.url}/-/raw/{repo.branch}/{filename}"
     else:
-        print("Debug: addonmanager_utilities.get_readme_url: Unknown git host:", url)
+        FreeCAD.Console.PrintLog("Debug: addonmanager_utilities.construct_git_url: Unknown git host:", parsed_url.netloc)
     return None
 
+def get_readme_url(repo):
+    "Returns the location of a readme file"
 
-def get_desc_regex(url):
+    return construct_git_url(repo, "README.md")
+
+def get_metadata_url(url):
+    "Returns the location of a package.xml metadata file"
+
+    return construct_git_url(repo, "package.xml")
+
+
+def get_desc_regex(repo):
     """Returns a regex string that extracts a WB description to be displayed in the description
     panel of the Addon manager, if the README could not be found"""
 
-    parsedUrl = urlparse(url)
+    parsedUrl = urlparse(repo.url)
     if parsedUrl.netloc == "github.com":
         return r'<meta property="og:description" content="(.*?)"'
     elif parsedUrl.netloc == "framagit.org" or parsedUrl.netloc == "gitlab.com":
         return r'<meta.*?content="(.*?)".*?og:description.*?>'
-    print("Debug: addonmanager_utilities.get_desc_regex: Unknown git host:", url)
+    FreeCAD.Console.PrintWarning("Debug: addonmanager_utilities.get_desc_regex: Unknown git host:", repo.url)
     return None
 
 
-def get_readme_html_url(url):
+def get_readme_html_url(repo):
     """Returns the location of a html file containing readme"""
 
-    parsedUrl = urlparse(url)
+    parsedUrl = urlparse(repo.url)
     if parsedUrl.netloc == "github.com":
-        return url + "/blob/master/README.md"
+        return f"{repo.url}/blob/{repo.branch}/README.md"
     else:
-        print("Debug: addonmanager_utilities.get_readme_html_url: Unknown git host:", url)
+        FreeCAD.Console.PrintWarning("Debug: addonmanager_utilities.get_readme_html_url: Unknown git host:", repo.url)
         return None
 
 
-def get_readme_regex(url):
+def get_readme_regex(repo):
     """Return a regex string that extracts the contents to be displayed in the description
     panel of the Addon manager, from raw HTML data (the readme's html rendering usually)"""
 
-    parsedUrl = urlparse(url)
+    parsedUrl = urlparse(repo.url)
     if parsedUrl.netloc == "github.com":
         return "<article.*?>(.*?)</article>"
     else:
-        print("Debug: addonmanager_utilities.get_readme_regex: Unknown git host:", url)
+        FreeCAD.Console.PrintWarning("Debug: addonmanager_utilities.get_readme_regex: Unknown git host:", repo.url)
         return None
 
 
@@ -319,7 +267,7 @@ def fix_relative_links(text, base_url):
             if len(parts) < 2 or not re.match(r"^http|^www|^.+\.|^/", parts[0]):
                 newlink = os.path.join(base_url, link.lstrip('./'))
                 line = line.replace(link, newlink)
-                print("Debug: replaced " + link + " with " + newlink)
+                FreeCAD.Console.PrintLog("Debug: replaced " + link + " with " + newlink)
         new_text = new_text + '\n' + line
     return new_text
 

src/Mod/Start/StartPage/LoadNew.py

@@ -27,4 +27,4 @@ FreeCAD.newDocument()
 FreeCADGui.activeDocument().activeView().viewDefaultOrientation()
 
 from StartPage import StartPage
-StartPage.postStart()
+StartPage.postStart()