diff --git a/requirements.txt b/requirements.txt index bc837bc7fb..ca6a0a7f3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ matplotlib==3.0.2; python_version >= '3.0' PySide==1.2.4 # PySide2==5.12.0 Shiboken==1.2.2 -six==1.12.0 +six==1.12.0 +Markdown==3.2.2 diff --git a/src/Mod/AddonManager/addonmanager_utilities.py b/src/Mod/AddonManager/addonmanager_utilities.py index 1aaa93ac4c..c1d6d081da 100644 --- a/src/Mod/AddonManager/addonmanager_utilities.py +++ b/src/Mod/AddonManager/addonmanager_utilities.py @@ -116,8 +116,10 @@ def urlopen(url): urllib2.install_opener(opener) # Url opening + req = urllib2.Request(url, + headers={'User-Agent' : "Magic Browser"}) try: - u = urllib2.urlopen(url, timeout=timeout) + u = urllib2.urlopen(req, timeout=timeout) except: return None else: @@ -259,7 +261,7 @@ def getZipUrl(baseurl): url = getserver(baseurl).strip("/") if url.endswith("github.com"): return baseurl+"/archive/master.zip" - elif url.endswith("framagit.org"): + elif url.endswith("framagit.org") or url.endswith("gitlab.com"): # https://framagit.org/freecad-france/mooc-workbench/-/archive/master/mooc-workbench-master.zip reponame = baseurl.strip("/").split("/")[-1] return baseurl+"/-/archive/master/"+reponame+"-master.zip" @@ -272,12 +274,37 @@ def getReadmeUrl(url): "Returns the location of a readme file" - if ("github" in url) or ("framagit" in url): - return url+"/blob/master/README.md" - print("Debug: addonmanager_utilities.getReadmeUrl: Unknown git host:",url) + if "github" in url or "framagit" in url or "gitlab" in url: + return url+"/raw/master/README.md" + else: + print("Debug: addonmanager_utilities.getReadmeUrl: Unknown git host:",url) return None +def getDescRegex(url): + + """Returns a regex string that extracts a WB description to be displayed in the description + panel of the Addon manager, if the README could not be found""" + + if "github" in url: + return "" + print("Debug: addonmanager_utilities.getDescRegex: Unknown git host:",url) + return None + + +def getReadmeHTMLUrl(url): + + "Returns the location of a html file containing readme" + + if ("github" in url): + return url+"/blob/master/README.md" + else: + print("Debug: addonmanager_utilities.getReadmeUrl: Unknown git host:",url) + return None + + def getReadmeRegex(url): """Return a regex string that extracts the contents to be displayed in the description @@ -285,32 +312,24 @@ def getReadmeRegex(url): if ("github" in url): return "(.*?)" - elif ("framagit" in url): - return None # the readme content on framagit is generated by javascript so unretrievable by urlopen - print("Debug: addonmanager_utilities.getReadmeRegex: Unknown git host:",url) - return None + else: + print("Debug: addonmanager_utilities.getReadmeRegex: Unknown git host:",url) + return None -def getDescRegex(url): - - """Returns a regex string that extracts a WB description to be displayed in the description - panel of the Addon manager, if the README could not be found""" +def fixRelativeLinks(text, base_url): - if ("github" in url): - return "" - print("Debug: addonmanager_utilities.getDescRegex: Unknown git host:",url) - return None + """Replace markdown image relative links with + absolute ones using the base URL""" -def getRepoUrl(text): - - "finds an URL in a given piece of text extracted from github's HTML" - - if ("href" in text): - return "https://github.com/" + re.findall("href=\"\/(.*?)\/tree",text)[0] - elif ("MOOC" in text): - # Bad hack for now... We need to do better - return "https://framagit.org/freecad-france/mooc-workbench" - print("Debug: addonmanager_utilities.getRepoUrl: Unable to find repo:",text) - return None + new_text = "" + for line in text.splitlines(): + for link in (re.findall(r"!\[.*?\]\((.*?)\)", line) + + re.findall(r"src\s*=\s*[\"'](.+?)[\"']", line)): + parts = link.split('/') + if len(parts) < 2 or not re.match(r"^http|^www|^.+\.|^/", parts[0]): + newlink = os.path.join(base_url, link.lstrip('./')) + line = line.replace(link, newlink) + print("Debug: replaced " + link + " with " + newlink) + new_text = new_text + '\n' + line + return new_text diff --git a/src/Mod/AddonManager/addonmanager_workers.py b/src/Mod/AddonManager/addonmanager_workers.py index 4113fbdafe..3afaf2e08e 100644 --- a/src/Mod/AddonManager/addonmanager_workers.py +++ b/src/Mod/AddonManager/addonmanager_workers.py @@ -40,27 +40,17 @@ from addonmanager_macro import Macro # \brief Multithread workers for the addon manager # Blacklisted addons -MACROS_BLACKLIST = ["BOLTS", - "WorkFeatures", - "how to install", - "PartsLibrary", - "FCGear"] +macros_blacklist = [] # These addons will print an additional message informing the user -OBSOLETE = ["assembly2", - "drawing_dimensioning", - "cura_engine"] +obsolete = [] # These addons will print an additional message informing the user Python2 only -PY2ONLY = ["geodata", - "GDT", - "timber", - "flamingo", - "reconstruction", - "animation"] +py2only = [] NOGIT = False # for debugging purposes, set this to True to always use http downloads +NOMARKDOWN = False # for debugging purposes, set this to True to disable Markdown lib """Multithread workers for the Addon Manager""" @@ -82,7 +72,31 @@ class UpdateWorker(QtCore.QThread): "populates the list of addons" self.progressbar_show.emit(True) - u = utils.urlopen("https://github.com/FreeCAD/FreeCAD-addons") + + # update info lists + global obsolete, macros_blacklist, py2only + u = utils.urlopen("https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/addonflags.json") + if u: + p = u.read() + if sys.version_info.major >= 3 and isinstance(p, bytes): + p = p.decode("utf-8") + u.close() + hit = re.findall(r'"obsolete"[^\{]*?{[^\{]*?"Mod":\[(?P[^\[\]]+?)\]}', + p.replace('\n', '').replace(' ', '')) + if hit: + obsolete = hit[0].replace('"', '').split(',') + hit = re.findall(r'"blacklisted"[^\{]*?{[^\{]*?"Macro":\[(?P[^\[\]]+?)\]}', + p.replace('\n', '').replace(' ', '')) + if hit: + macros_blacklist = hit[0].replace('"', '').split(',') + hit = re.findall(r'"py2only"[^\{]*?{[^\{]*?"Mod":\[(?P[^\[\]]+?)\]}', + p.replace('\n', '').replace(' ', '')) + if hit: + py2only = hit[0].replace('"', '').split(',') + else: + print("Debug: addon_flags.json not found") + + u = utils.urlopen("https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/.gitmodules") if not u: self.progressbar_show.emit(False) self.done.emit() @@ -92,32 +106,23 @@ class UpdateWorker(QtCore.QThread): if sys.version_info.major >= 3 and isinstance(p, bytes): p = p.decode("utf-8") u.close() - p = p.replace("\n"," ") - p = re.findall("octicon-file-submodule(.*?)message",p) + p = re.findall((r"(?m)\[submodule\s*\"(?P.*)\"\]\s*" + r"path\s*=\s*(?P.+)\s*" + r"url\s*=\s*(?Phttps?://.*)"), p) basedir = FreeCAD.getUserAppDataDir() moddir = basedir + os.sep + "Mod" repos = [] # querying official addons - for l in p: - #name = re.findall("data-skip-pjax=\"true\">(.*?)<",l)[0] - res = re.findall("title=\"(.*?) @",l) - if res: - name = res[0] - else: - print("AddonMananger: Debug: couldn't find title in",l) - continue + for name, path, url in p: self.info_label.emit(name) - #url = re.findall("title=\"(.*?) @",l)[0] - url = utils.getRepoUrl(l) - if url: - addondir = moddir + os.sep + name - #print ("found:",name," at ",url) - if os.path.exists(addondir) and os.listdir(addondir): - # make sure the folder exists and it contains files! - state = 1 - else: - state = 0 - repos.append([name,url,state]) + url = url.split(".git")[0] + addondir = moddir + os.sep + name + if os.path.exists(addondir) and os.listdir(addondir): + # make sure the folder exists and it contains files! + state = 1 + else: + state = 0 + repos.append([name, url, state]) # querying custom addons customaddons = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons").GetString("CustomRepositories","").split("\n") for url in customaddons: @@ -316,7 +321,7 @@ class FillMacroListWorker(QtCore.QThread): for mac in macros: macname = mac[6:] # Remove "Macro ". macname = macname.replace("&","&") - if (macname not in MACROS_BLACKLIST) and ('recipes' not in macname.lower()): + if (macname not in macros_blacklist) and ('recipes' not in macname.lower()): macro = Macro(macname) macro.on_wiki = True self.macros.append(macro) @@ -355,24 +360,50 @@ class ShowWorker(QtCore.QThread): url = self.repos[self.idx][1] self.info_label.emit(translate("AddonsInstaller", "Retrieving info from") + ' ' + str(url)) desc = "" - # get the README if possible - readmeurl = utils.getReadmeUrl(url) - if not readmeurl: - print("Debug: README not found for",url) - u = utils.urlopen(readmeurl) - if not u: - print("Debug: README not found at",readmeurl) - if u: - p = u.read() - if sys.version_info.major >= 3 and isinstance(p, bytes): - p = p.decode("utf-8") - u.close() - readmeregex = utils.getReadmeRegex(url) - if readmeregex: - readme = re.findall(readmeregex,p,flags=re.MULTILINE|re.DOTALL) + regex = utils.getReadmeRegex(url) + if regex: + # extract readme from html via regex + readmeurl = utils.getReadmeHTMLUrl(url) + if not readmeurl: + print("Debug: README not found for",url) + u = utils.urlopen(readmeurl) + if not u: + print("Debug: README not found at",readmeurl) + u = utils.urlopen(readmeurl) + if u: + p = u.read() + if sys.version_info.major >= 3 and isinstance(p, bytes): + p = p.decode("utf-8") + u.close() + readme = re.findall(regex,p,flags=re.MULTILINE|re.DOTALL) if readme: - desc += readme[0] - if not desc: + desc = readme[0] + else: + print("Debug: README not found at",readmeurl) + else: + # convert raw markdown using lib + readmeurl = utils.getReadmeUrl(url) + if not readmeurl: + print("Debug: README not found for",url) + u = utils.urlopen(readmeurl) + if u: + p = u.read() + if sys.version_info.major >= 3 and isinstance(p, bytes): + p = p.decode("utf-8") + u.close() + desc = utils.fixRelativeLinks(p,readmeurl.rsplit("/README.md")[0]) + try: + if NOMARKDOWN: + raise ImportError + import markdown # try to use system Markdown lib + desc = markdown.markdown(desc,extensions=['md_in_html']) + except ImportError: + message = "
"+translate("AddonsInstaller","Raw markdown displayed")+"

" + message += translate("AddonsInstaller","Python Markdown library is missing.")+"
" + desc + "
" + desc = message + else: + print("Debug: README not found at",readmeurl) + if desc == "": # fall back to the description text u = utils.urlopen(url) if not u: @@ -387,7 +418,7 @@ class ShowWorker(QtCore.QThread): if descregex: desc = re.findall(descregex,p) if desc: - desc = "
"+desc[0] + desc = desc[0] if not desc: desc = "Unable to retrieve addon description" self.repos[self.idx].append(desc) @@ -447,12 +478,12 @@ class ShowWorker(QtCore.QThread): message = desc + '

Addon repository: ' + self.repos[self.idx][1] + '' # If the Addon is obsolete, let the user know through the Addon UI - if self.repos[self.idx][0] in OBSOLETE: + if self.repos[self.idx][0] in obsolete: message = "
"+translate("AddonsInstaller","This addon is marked as obsolete")+"

" message += translate("AddonsInstaller","This usually means it is no longer maintained, and some more advanced addon in this list provides the same functionality.")+"
" + desc # If the Addon is Python 2 only, let the user know through the Addon UI - if self.repos[self.idx][0] in PY2ONLY: + if self.repos[self.idx][0] in py2only: message = "
"+translate("AddonsInstaller","This addon is marked as Python 2 Only")+"

" message += translate("AddonsInstaller","This workbench may no longer be maintained and installing it on a Python 3 system will more than likely result in errors at startup or while in use.")+"
" + desc @@ -622,7 +653,7 @@ class InstallWorker(QtCore.QThread): self.progressbar_show.emit(True) if os.path.exists(clonedir): self.info_label.emit("Updating module...") - if sys.version_info.major > 2 and str(self.repos[idx][0]) in PY2ONLY: + if sys.version_info.major > 2 and str(self.repos[idx][0]) in py2only: FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "User requested updating a Python 2 workbench on a system running Python 3 - ")+str(self.repos[idx][0])+"\n") if git: if not os.path.exists(clonedir + os.sep + '.git'): @@ -656,7 +687,7 @@ class InstallWorker(QtCore.QThread): self.info_label.emit("Checking module dependencies...") depsok,answer = self.checkDependencies(self.repos[idx][1]) if depsok: - if sys.version_info.major > 2 and str(self.repos[idx][0]) in PY2ONLY: + if sys.version_info.major > 2 and str(self.repos[idx][0]) in py2only: FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "User requested installing a Python 2 workbench on a system running Python 3 - ")+str(self.repos[idx][0])+"\n") if git: self.info_label.emit("Cloning module...")