From 07db27d0dd9ca2d734d30368ee3624a3382980df Mon Sep 17 00:00:00 2001 From: "Matsievskiy S.V" Date: Thu, 17 Sep 2020 20:17:03 +0300 Subject: [PATCH] Addon Manager: change lookup mechanism Switch addon lookup mechanism from parsing html page to extracting info from .gitmodules file. This simplifies logic and allows using non-Github repos. Readme for Github repos are extracted from HTML pages using regex. Gitlab pages are converted to HTML using Python Markdown lib if present, falling back to displaying raw markdown. In this case image links are converted from relative to absolute paths. --- requirements.txt | 3 +- .../AddonManager/addonmanager_utilities.py | 79 ++++++---- src/Mod/AddonManager/addonmanager_workers.py | 149 +++++++++++------- 3 files changed, 141 insertions(+), 90 deletions(-) diff --git a/requirements.txt b/requirements.txt index bc837bc7fb..ca6a0a7f3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ matplotlib==3.0.2; python_version >= '3.0' PySide==1.2.4 # PySide2==5.12.0 Shiboken==1.2.2 -six==1.12.0 +six==1.12.0 +Markdown==3.2.2 diff --git a/src/Mod/AddonManager/addonmanager_utilities.py b/src/Mod/AddonManager/addonmanager_utilities.py index 1aaa93ac4c..c1d6d081da 100644 --- a/src/Mod/AddonManager/addonmanager_utilities.py +++ b/src/Mod/AddonManager/addonmanager_utilities.py @@ -116,8 +116,10 @@ def urlopen(url): urllib2.install_opener(opener) # Url opening + req = urllib2.Request(url, + headers={'User-Agent' : "Magic Browser"}) try: - u = urllib2.urlopen(url, timeout=timeout) + u = urllib2.urlopen(req, timeout=timeout) except: return None else: @@ -259,7 +261,7 @@ def getZipUrl(baseurl): url = getserver(baseurl).strip("/") if url.endswith("github.com"): return baseurl+"/archive/master.zip" - elif url.endswith("framagit.org"): + elif url.endswith("framagit.org") or url.endswith("gitlab.com"): # https://framagit.org/freecad-france/mooc-workbench/-/archive/master/mooc-workbench-master.zip reponame = baseurl.strip("/").split("/")[-1] return baseurl+"/-/archive/master/"+reponame+"-master.zip" @@ -272,12 +274,37 @@ def getReadmeUrl(url): "Returns the location of a readme file" - if ("github" in url) or ("framagit" in url): - return url+"/blob/master/README.md" - print("Debug: addonmanager_utilities.getReadmeUrl: Unknown git host:",url) + if "github" in url or "framagit" in url or "gitlab" in url: + return url+"/raw/master/README.md" + else: + print("Debug: addonmanager_utilities.getReadmeUrl: Unknown git host:",url) return None +def getDescRegex(url): + + """Returns a regex string that extracts a WB description to be displayed in the description + panel of the Addon manager, if the README could not be found""" + + if "github" in url: + return "" + print("Debug: addonmanager_utilities.getDescRegex: Unknown git host:",url) + return None + + +def getReadmeHTMLUrl(url): + + "Returns the location of a html file containing readme" + + if ("github" in url): + return url+"/blob/master/README.md" + else: + print("Debug: addonmanager_utilities.getReadmeUrl: Unknown git host:",url) + return None + + def getReadmeRegex(url): """Return a regex string that extracts the contents to be displayed in the description @@ -285,32 +312,24 @@ def getReadmeRegex(url): if ("github" in url): return "(.*?)" - elif ("framagit" in url): - return None # the readme content on framagit is generated by javascript so unretrievable by urlopen - print("Debug: addonmanager_utilities.getReadmeRegex: Unknown git host:",url) - return None + else: + print("Debug: addonmanager_utilities.getReadmeRegex: Unknown git host:",url) + return None -def getDescRegex(url): - - """Returns a regex string that extracts a WB description to be displayed in the description - panel of the Addon manager, if the README could not be found""" +def fixRelativeLinks(text, base_url): - if ("github" in url): - return "" - print("Debug: addonmanager_utilities.getDescRegex: Unknown git host:",url) - return None + """Replace markdown image relative links with + absolute ones using the base URL""" -def getRepoUrl(text): - - "finds an URL in a given piece of text extracted from github's HTML" - - if ("href" in text): - return "https://github.com/" + re.findall("href=\"\/(.*?)\/tree",text)[0] - elif ("MOOC" in text): - # Bad hack for now... We need to do better - return "https://framagit.org/freecad-france/mooc-workbench" - print("Debug: addonmanager_utilities.getRepoUrl: Unable to find repo:",text) - return None + new_text = "" + for line in text.splitlines(): + for link in (re.findall(r"!\[.*?\]\((.*?)\)", line) + + re.findall(r"src\s*=\s*[\"'](.+?)[\"']", line)): + parts = link.split('/') + if len(parts) < 2 or not re.match(r"^http|^www|^.+\.|^/", parts[0]): + newlink = os.path.join(base_url, link.lstrip('./')) + line = line.replace(link, newlink) + print("Debug: replaced " + link + " with " + newlink) + new_text = new_text + '\n' + line + return new_text diff --git a/src/Mod/AddonManager/addonmanager_workers.py b/src/Mod/AddonManager/addonmanager_workers.py index 4113fbdafe..3afaf2e08e 100644 --- a/src/Mod/AddonManager/addonmanager_workers.py +++ b/src/Mod/AddonManager/addonmanager_workers.py @@ -40,27 +40,17 @@ from addonmanager_macro import Macro # \brief Multithread workers for the addon manager # Blacklisted addons -MACROS_BLACKLIST = ["BOLTS", - "WorkFeatures", - "how to install", - "PartsLibrary", - "FCGear"] +macros_blacklist = [] # These addons will print an additional message informing the user -OBSOLETE = ["assembly2", - "drawing_dimensioning", - "cura_engine"] +obsolete = [] # These addons will print an additional message informing the user Python2 only -PY2ONLY = ["geodata", - "GDT", - "timber", - "flamingo", - "reconstruction", - "animation"] +py2only = [] NOGIT = False # for debugging purposes, set this to True to always use http downloads +NOMARKDOWN = False # for debugging purposes, set this to True to disable Markdown lib """Multithread workers for the Addon Manager""" @@ -82,7 +72,31 @@ class UpdateWorker(QtCore.QThread): "populates the list of addons" self.progressbar_show.emit(True) - u = utils.urlopen("https://github.com/FreeCAD/FreeCAD-addons") + + # update info lists + global obsolete, macros_blacklist, py2only + u = utils.urlopen("https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/addonflags.json") + if u: + p = u.read() + if sys.version_info.major >= 3 and isinstance(p, bytes): + p = p.decode("utf-8") + u.close() + hit = re.findall(r'"obsolete"[^\{]*?{[^\{]*?"Mod":\[(?P[^\[\]]+?)\]}', + p.replace('\n', '').replace(' ', '')) + if hit: + obsolete = hit[0].replace('"', '').split(',') + hit = re.findall(r'"blacklisted"[^\{]*?{[^\{]*?"Macro":\[(?P[^\[\]]+?)\]}', + p.replace('\n', '').replace(' ', '')) + if hit: + macros_blacklist = hit[0].replace('"', '').split(',') + hit = re.findall(r'"py2only"[^\{]*?{[^\{]*?"Mod":\[(?P[^\[\]]+?)\]}', + p.replace('\n', '').replace(' ', '')) + if hit: + py2only = hit[0].replace('"', '').split(',') + else: + print("Debug: addon_flags.json not found") + + u = utils.urlopen("https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/.gitmodules") if not u: self.progressbar_show.emit(False) self.done.emit() @@ -92,32 +106,23 @@ class UpdateWorker(QtCore.QThread): if sys.version_info.major >= 3 and isinstance(p, bytes): p = p.decode("utf-8") u.close() - p = p.replace("\n"," ") - p = re.findall("octicon-file-submodule(.*?)message",p) + p = re.findall((r"(?m)\[submodule\s*\"(?P.*)\"\]\s*" + r"path\s*=\s*(?P.+)\s*" + r"url\s*=\s*(?Phttps?://.*)"), p) basedir = FreeCAD.getUserAppDataDir() moddir = basedir + os.sep + "Mod" repos = [] # querying official addons - for l in p: - #name = re.findall("data-skip-pjax=\"true\">(.*?)<",l)[0] - res = re.findall("title=\"(.*?) @",l) - if res: - name = res[0] - else: - print("AddonMananger: Debug: couldn't find title in",l) - continue + for name, path, url in p: self.info_label.emit(name) - #url = re.findall("title=\"(.*?) @",l)[0] - url = utils.getRepoUrl(l) - if url: - addondir = moddir + os.sep + name - #print ("found:",name," at ",url) - if os.path.exists(addondir) and os.listdir(addondir): - # make sure the folder exists and it contains files! - state = 1 - else: - state = 0 - repos.append([name,url,state]) + url = url.split(".git")[0] + addondir = moddir + os.sep + name + if os.path.exists(addondir) and os.listdir(addondir): + # make sure the folder exists and it contains files! + state = 1 + else: + state = 0 + repos.append([name, url, state]) # querying custom addons customaddons = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons").GetString("CustomRepositories","").split("\n") for url in customaddons: @@ -316,7 +321,7 @@ class FillMacroListWorker(QtCore.QThread): for mac in macros: macname = mac[6:] # Remove "Macro ". macname = macname.replace("&","&") - if (macname not in MACROS_BLACKLIST) and ('recipes' not in macname.lower()): + if (macname not in macros_blacklist) and ('recipes' not in macname.lower()): macro = Macro(macname) macro.on_wiki = True self.macros.append(macro) @@ -355,24 +360,50 @@ class ShowWorker(QtCore.QThread): url = self.repos[self.idx][1] self.info_label.emit(translate("AddonsInstaller", "Retrieving info from") + ' ' + str(url)) desc = "" - # get the README if possible - readmeurl = utils.getReadmeUrl(url) - if not readmeurl: - print("Debug: README not found for",url) - u = utils.urlopen(readmeurl) - if not u: - print("Debug: README not found at",readmeurl) - if u: - p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): - p = p.decode("utf-8") - u.close() - readmeregex = utils.getReadmeRegex(url) - if readmeregex: - readme = re.findall(readmeregex,p,flags=re.MULTILINE|re.DOTALL) + regex = utils.getReadmeRegex(url) + if regex: + # extract readme from html via regex + readmeurl = utils.getReadmeHTMLUrl(url) + if not readmeurl: + print("Debug: README not found for",url) + u = utils.urlopen(readmeurl) + if not u: + print("Debug: README not found at",readmeurl) + u = utils.urlopen(readmeurl) + if u: + p = u.read() + if sys.version_info.major >= 3 and isinstance(p, bytes): + p = p.decode("utf-8") + u.close() + readme = re.findall(regex,p,flags=re.MULTILINE|re.DOTALL) if readme: - desc += readme[0] - if not desc: + desc = readme[0] + else: + print("Debug: README not found at",readmeurl) + else: + # convert raw markdown using lib + readmeurl = utils.getReadmeUrl(url) + if not readmeurl: + print("Debug: README not found for",url) + u = utils.urlopen(readmeurl) + if u: + p = u.read() + if sys.version_info.major >= 3 and isinstance(p, bytes): + p = p.decode("utf-8") + u.close() + desc = utils.fixRelativeLinks(p,readmeurl.rsplit("/README.md")[0]) + try: + if NOMARKDOWN: + raise ImportError + import markdown # try to use system Markdown lib + desc = markdown.markdown(desc,extensions=['md_in_html']) + except ImportError: + message = "
"+translate("AddonsInstaller","Raw markdown displayed")+"

" + message += translate("AddonsInstaller","Python Markdown library is missing.")+"
" + desc + "
" + desc = message + else: + print("Debug: README not found at",readmeurl) + if desc == "": # fall back to the description text u = utils.urlopen(url) if not u: @@ -387,7 +418,7 @@ class ShowWorker(QtCore.QThread): if descregex: desc = re.findall(descregex,p) if desc: - desc = "
"+desc[0] + desc = desc[0] if not desc: desc = "Unable to retrieve addon description" self.repos[self.idx].append(desc) @@ -447,12 +478,12 @@ class ShowWorker(QtCore.QThread): message = desc + '

Addon repository: ' + self.repos[self.idx][1] + '' # If the Addon is obsolete, let the user know through the Addon UI - if self.repos[self.idx][0] in OBSOLETE: + if self.repos[self.idx][0] in obsolete: message = "
"+translate("AddonsInstaller","This addon is marked as obsolete")+"

" message += translate("AddonsInstaller","This usually means it is no longer maintained, and some more advanced addon in this list provides the same functionality.")+"
" + desc # If the Addon is Python 2 only, let the user know through the Addon UI - if self.repos[self.idx][0] in PY2ONLY: + if self.repos[self.idx][0] in py2only: message = "
"+translate("AddonsInstaller","This addon is marked as Python 2 Only")+"

" message += translate("AddonsInstaller","This workbench may no longer be maintained and installing it on a Python 3 system will more than likely result in errors at startup or while in use.")+"
" + desc @@ -622,7 +653,7 @@ class InstallWorker(QtCore.QThread): self.progressbar_show.emit(True) if os.path.exists(clonedir): self.info_label.emit("Updating module...") - if sys.version_info.major > 2 and str(self.repos[idx][0]) in PY2ONLY: + if sys.version_info.major > 2 and str(self.repos[idx][0]) in py2only: FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "User requested updating a Python 2 workbench on a system running Python 3 - ")+str(self.repos[idx][0])+"\n") if git: if not os.path.exists(clonedir + os.sep + '.git'): @@ -656,7 +687,7 @@ class InstallWorker(QtCore.QThread): self.info_label.emit("Checking module dependencies...") depsok,answer = self.checkDependencies(self.repos[idx][1]) if depsok: - if sys.version_info.major > 2 and str(self.repos[idx][0]) in PY2ONLY: + if sys.version_info.major > 2 and str(self.repos[idx][0]) in py2only: FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "User requested installing a Python 2 workbench on a system running Python 3 - ")+str(self.repos[idx][0])+"\n") if git: self.info_label.emit("Cloning module...")