Addon Manager: change lookup mechanism
Switch addon lookup mechanism from parsing html page to extracting info from .gitmodules file. This simplifies logic and allows using non-Github repos. Readme for Github repos are extracted from HTML pages using regex. Gitlab pages are converted to HTML using Python Markdown lib if present, falling back to displaying raw markdown. In this case image links are converted from relative to absolute paths.
This commit is contained in:
Matsievskiy S.V
committed by
Yorik van Havre
Yorik van Havre
parent
2b49d60ac5
commit
07db27d0dd
@@ -3,4 +3,5 @@ matplotlib==3.0.2; python_version >= '3.0'
|
||||
PySide==1.2.4
|
||||
# PySide2==5.12.0
|
||||
Shiboken==1.2.2
|
||||
six==1.12.0
|
||||
six==1.12.0
|
||||
Markdown==3.2.2
|
||||
|
||||
@@ -116,8 +116,10 @@ def urlopen(url):
|
||||
urllib2.install_opener(opener)
|
||||
|
||||
# Url opening
|
||||
req = urllib2.Request(url,
|
||||
headers={'User-Agent' : "Magic Browser"})
|
||||
try:
|
||||
u = urllib2.urlopen(url, timeout=timeout)
|
||||
u = urllib2.urlopen(req, timeout=timeout)
|
||||
except:
|
||||
return None
|
||||
else:
|
||||
@@ -259,7 +261,7 @@ def getZipUrl(baseurl):
|
||||
url = getserver(baseurl).strip("/")
|
||||
if url.endswith("github.com"):
|
||||
return baseurl+"/archive/master.zip"
|
||||
elif url.endswith("framagit.org"):
|
||||
elif url.endswith("framagit.org") or url.endswith("gitlab.com"):
|
||||
# https://framagit.org/freecad-france/mooc-workbench/-/archive/master/mooc-workbench-master.zip
|
||||
reponame = baseurl.strip("/").split("/")[-1]
|
||||
return baseurl+"/-/archive/master/"+reponame+"-master.zip"
|
||||
@@ -272,12 +274,37 @@ def getReadmeUrl(url):
|
||||
|
||||
"Returns the location of a readme file"
|
||||
|
||||
if ("github" in url) or ("framagit" in url):
|
||||
return url+"/blob/master/README.md"
|
||||
print("Debug: addonmanager_utilities.getReadmeUrl: Unknown git host:",url)
|
||||
if "github" in url or "framagit" in url or "gitlab" in url:
|
||||
return url+"/raw/master/README.md"
|
||||
else:
|
||||
print("Debug: addonmanager_utilities.getReadmeUrl: Unknown git host:",url)
|
||||
return None
|
||||
|
||||
|
||||
def getDescRegex(url):
|
||||
|
||||
"""Returns a regex string that extracts a WB description to be displayed in the description
|
||||
panel of the Addon manager, if the README could not be found"""
|
||||
|
||||
if "github" in url:
|
||||
return "<meta property=\"og:description\" content=\"(.*?)\""
|
||||
elif "framagit" in url or "gitlab" in url:
|
||||
return "<meta.*?content=\"(.*?)\".*?og\:description.*?>"
|
||||
print("Debug: addonmanager_utilities.getDescRegex: Unknown git host:",url)
|
||||
return None
|
||||
|
||||
|
||||
def getReadmeHTMLUrl(url):
|
||||
|
||||
"Returns the location of a html file containing readme"
|
||||
|
||||
if ("github" in url):
|
||||
return url+"/blob/master/README.md"
|
||||
else:
|
||||
print("Debug: addonmanager_utilities.getReadmeUrl: Unknown git host:",url)
|
||||
return None
|
||||
|
||||
|
||||
def getReadmeRegex(url):
|
||||
|
||||
"""Return a regex string that extracts the contents to be displayed in the description
|
||||
@@ -285,32 +312,24 @@ def getReadmeRegex(url):
|
||||
|
||||
if ("github" in url):
|
||||
return "<article.*?>(.*?)</article>"
|
||||
elif ("framagit" in url):
|
||||
return None # the readme content on framagit is generated by javascript so unretrievable by urlopen
|
||||
print("Debug: addonmanager_utilities.getReadmeRegex: Unknown git host:",url)
|
||||
return None
|
||||
else:
|
||||
print("Debug: addonmanager_utilities.getReadmeRegex: Unknown git host:",url)
|
||||
return None
|
||||
|
||||
|
||||
def getDescRegex(url):
|
||||
|
||||
"""Returns a regex string that extracts a WB description to be displayed in the description
|
||||
panel of the Addon manager, if the README could not be found"""
|
||||
def fixRelativeLinks(text, base_url):
|
||||
|
||||
if ("github" in url):
|
||||
return "<meta property=\"og:description\" content=\"(.*?)\""
|
||||
elif ("framagit" in url):
|
||||
return "<meta.*?content=\"(.*?)\".*?og\:description.*?>"
|
||||
print("Debug: addonmanager_utilities.getDescRegex: Unknown git host:",url)
|
||||
return None
|
||||
"""Replace markdown image relative links with
|
||||
absolute ones using the base URL"""
|
||||
|
||||
def getRepoUrl(text):
|
||||
|
||||
"finds an URL in a given piece of text extracted from github's HTML"
|
||||
|
||||
if ("href" in text):
|
||||
return "https://github.com/" + re.findall("href=\"\/(.*?)\/tree",text)[0]
|
||||
elif ("MOOC" in text):
|
||||
# Bad hack for now... We need to do better
|
||||
return "https://framagit.org/freecad-france/mooc-workbench"
|
||||
print("Debug: addonmanager_utilities.getRepoUrl: Unable to find repo:",text)
|
||||
return None
|
||||
new_text = ""
|
||||
for line in text.splitlines():
|
||||
for link in (re.findall(r"!\[.*?\]\((.*?)\)", line) +
|
||||
re.findall(r"src\s*=\s*[\"'](.+?)[\"']", line)):
|
||||
parts = link.split('/')
|
||||
if len(parts) < 2 or not re.match(r"^http|^www|^.+\.|^/", parts[0]):
|
||||
newlink = os.path.join(base_url, link.lstrip('./'))
|
||||
line = line.replace(link, newlink)
|
||||
print("Debug: replaced " + link + " with " + newlink)
|
||||
new_text = new_text + '\n' + line
|
||||
return new_text
|
||||
|
||||
@@ -40,27 +40,17 @@ from addonmanager_macro import Macro
|
||||
# \brief Multithread workers for the addon manager
|
||||
|
||||
# Blacklisted addons
|
||||
MACROS_BLACKLIST = ["BOLTS",
|
||||
"WorkFeatures",
|
||||
"how to install",
|
||||
"PartsLibrary",
|
||||
"FCGear"]
|
||||
macros_blacklist = []
|
||||
|
||||
# These addons will print an additional message informing the user
|
||||
OBSOLETE = ["assembly2",
|
||||
"drawing_dimensioning",
|
||||
"cura_engine"]
|
||||
obsolete = []
|
||||
|
||||
# These addons will print an additional message informing the user Python2 only
|
||||
PY2ONLY = ["geodata",
|
||||
"GDT",
|
||||
"timber",
|
||||
"flamingo",
|
||||
"reconstruction",
|
||||
"animation"]
|
||||
py2only = []
|
||||
|
||||
NOGIT = False # for debugging purposes, set this to True to always use http downloads
|
||||
|
||||
NOMARKDOWN = False # for debugging purposes, set this to True to disable Markdown lib
|
||||
|
||||
"""Multithread workers for the Addon Manager"""
|
||||
|
||||
@@ -82,7 +72,31 @@ class UpdateWorker(QtCore.QThread):
|
||||
"populates the list of addons"
|
||||
|
||||
self.progressbar_show.emit(True)
|
||||
u = utils.urlopen("https://github.com/FreeCAD/FreeCAD-addons")
|
||||
|
||||
# update info lists
|
||||
global obsolete, macros_blacklist, py2only
|
||||
u = utils.urlopen("https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/addonflags.json")
|
||||
if u:
|
||||
p = u.read()
|
||||
if sys.version_info.major >= 3 and isinstance(p, bytes):
|
||||
p = p.decode("utf-8")
|
||||
u.close()
|
||||
hit = re.findall(r'"obsolete"[^\{]*?{[^\{]*?"Mod":\[(?P<obsolete>[^\[\]]+?)\]}',
|
||||
p.replace('\n', '').replace(' ', ''))
|
||||
if hit:
|
||||
obsolete = hit[0].replace('"', '').split(',')
|
||||
hit = re.findall(r'"blacklisted"[^\{]*?{[^\{]*?"Macro":\[(?P<blacklisted>[^\[\]]+?)\]}',
|
||||
p.replace('\n', '').replace(' ', ''))
|
||||
if hit:
|
||||
macros_blacklist = hit[0].replace('"', '').split(',')
|
||||
hit = re.findall(r'"py2only"[^\{]*?{[^\{]*?"Mod":\[(?P<py2only>[^\[\]]+?)\]}',
|
||||
p.replace('\n', '').replace(' ', ''))
|
||||
if hit:
|
||||
py2only = hit[0].replace('"', '').split(',')
|
||||
else:
|
||||
print("Debug: addon_flags.json not found")
|
||||
|
||||
u = utils.urlopen("https://raw.githubusercontent.com/FreeCAD/FreeCAD-addons/master/.gitmodules")
|
||||
if not u:
|
||||
self.progressbar_show.emit(False)
|
||||
self.done.emit()
|
||||
@@ -92,32 +106,23 @@ class UpdateWorker(QtCore.QThread):
|
||||
if sys.version_info.major >= 3 and isinstance(p, bytes):
|
||||
p = p.decode("utf-8")
|
||||
u.close()
|
||||
p = p.replace("\n"," ")
|
||||
p = re.findall("octicon-file-submodule(.*?)message",p)
|
||||
p = re.findall((r"(?m)\[submodule\s*\"(?P<name>.*)\"\]\s*"
|
||||
r"path\s*=\s*(?P<path>.+)\s*"
|
||||
r"url\s*=\s*(?P<url>https?://.*)"), p)
|
||||
basedir = FreeCAD.getUserAppDataDir()
|
||||
moddir = basedir + os.sep + "Mod"
|
||||
repos = []
|
||||
# querying official addons
|
||||
for l in p:
|
||||
#name = re.findall("data-skip-pjax=\"true\">(.*?)<",l)[0]
|
||||
res = re.findall("title=\"(.*?) @",l)
|
||||
if res:
|
||||
name = res[0]
|
||||
else:
|
||||
print("AddonMananger: Debug: couldn't find title in",l)
|
||||
continue
|
||||
for name, path, url in p:
|
||||
self.info_label.emit(name)
|
||||
#url = re.findall("title=\"(.*?) @",l)[0]
|
||||
url = utils.getRepoUrl(l)
|
||||
if url:
|
||||
addondir = moddir + os.sep + name
|
||||
#print ("found:",name," at ",url)
|
||||
if os.path.exists(addondir) and os.listdir(addondir):
|
||||
# make sure the folder exists and it contains files!
|
||||
state = 1
|
||||
else:
|
||||
state = 0
|
||||
repos.append([name,url,state])
|
||||
url = url.split(".git")[0]
|
||||
addondir = moddir + os.sep + name
|
||||
if os.path.exists(addondir) and os.listdir(addondir):
|
||||
# make sure the folder exists and it contains files!
|
||||
state = 1
|
||||
else:
|
||||
state = 0
|
||||
repos.append([name, url, state])
|
||||
# querying custom addons
|
||||
customaddons = FreeCAD.ParamGet("User parameter:BaseApp/Preferences/Addons").GetString("CustomRepositories","").split("\n")
|
||||
for url in customaddons:
|
||||
@@ -316,7 +321,7 @@ class FillMacroListWorker(QtCore.QThread):
|
||||
for mac in macros:
|
||||
macname = mac[6:] # Remove "Macro ".
|
||||
macname = macname.replace("&","&")
|
||||
if (macname not in MACROS_BLACKLIST) and ('recipes' not in macname.lower()):
|
||||
if (macname not in macros_blacklist) and ('recipes' not in macname.lower()):
|
||||
macro = Macro(macname)
|
||||
macro.on_wiki = True
|
||||
self.macros.append(macro)
|
||||
@@ -355,24 +360,50 @@ class ShowWorker(QtCore.QThread):
|
||||
url = self.repos[self.idx][1]
|
||||
self.info_label.emit(translate("AddonsInstaller", "Retrieving info from") + ' ' + str(url))
|
||||
desc = ""
|
||||
# get the README if possible
|
||||
readmeurl = utils.getReadmeUrl(url)
|
||||
if not readmeurl:
|
||||
print("Debug: README not found for",url)
|
||||
u = utils.urlopen(readmeurl)
|
||||
if not u:
|
||||
print("Debug: README not found at",readmeurl)
|
||||
if u:
|
||||
p = u.read()
|
||||
if sys.version_info.major >= 3 and isinstance(p, bytes):
|
||||
p = p.decode("utf-8")
|
||||
u.close()
|
||||
readmeregex = utils.getReadmeRegex(url)
|
||||
if readmeregex:
|
||||
readme = re.findall(readmeregex,p,flags=re.MULTILINE|re.DOTALL)
|
||||
regex = utils.getReadmeRegex(url)
|
||||
if regex:
|
||||
# extract readme from html via regex
|
||||
readmeurl = utils.getReadmeHTMLUrl(url)
|
||||
if not readmeurl:
|
||||
print("Debug: README not found for",url)
|
||||
u = utils.urlopen(readmeurl)
|
||||
if not u:
|
||||
print("Debug: README not found at",readmeurl)
|
||||
u = utils.urlopen(readmeurl)
|
||||
if u:
|
||||
p = u.read()
|
||||
if sys.version_info.major >= 3 and isinstance(p, bytes):
|
||||
p = p.decode("utf-8")
|
||||
u.close()
|
||||
readme = re.findall(regex,p,flags=re.MULTILINE|re.DOTALL)
|
||||
if readme:
|
||||
desc += readme[0]
|
||||
if not desc:
|
||||
desc = readme[0]
|
||||
else:
|
||||
print("Debug: README not found at",readmeurl)
|
||||
else:
|
||||
# convert raw markdown using lib
|
||||
readmeurl = utils.getReadmeUrl(url)
|
||||
if not readmeurl:
|
||||
print("Debug: README not found for",url)
|
||||
u = utils.urlopen(readmeurl)
|
||||
if u:
|
||||
p = u.read()
|
||||
if sys.version_info.major >= 3 and isinstance(p, bytes):
|
||||
p = p.decode("utf-8")
|
||||
u.close()
|
||||
desc = utils.fixRelativeLinks(p,readmeurl.rsplit("/README.md")[0])
|
||||
try:
|
||||
if NOMARKDOWN:
|
||||
raise ImportError
|
||||
import markdown # try to use system Markdown lib
|
||||
desc = markdown.markdown(desc,extensions=['md_in_html'])
|
||||
except ImportError:
|
||||
message = " <div style=\"width: 100%; text-align:center; background: #91bbe0;\"><strong style=\"color: #FFFFFF;\">"+translate("AddonsInstaller","Raw markdown displayed")+"</strong><br/><br/>"
|
||||
message += translate("AddonsInstaller","Python Markdown library is missing.")+"<br/></div><hr/><pre>" + desc + "</pre>"
|
||||
desc = message
|
||||
else:
|
||||
print("Debug: README not found at",readmeurl)
|
||||
if desc == "":
|
||||
# fall back to the description text
|
||||
u = utils.urlopen(url)
|
||||
if not u:
|
||||
@@ -387,7 +418,7 @@ class ShowWorker(QtCore.QThread):
|
||||
if descregex:
|
||||
desc = re.findall(descregex,p)
|
||||
if desc:
|
||||
desc = "<br/>"+desc[0]
|
||||
desc = desc[0]
|
||||
if not desc:
|
||||
desc = "Unable to retrieve addon description"
|
||||
self.repos[self.idx].append(desc)
|
||||
@@ -447,12 +478,12 @@ class ShowWorker(QtCore.QThread):
|
||||
message = desc + '<br/><br/>Addon repository: <a href="' + self.repos[self.idx][1] + '">' + self.repos[self.idx][1] + '</a>'
|
||||
|
||||
# If the Addon is obsolete, let the user know through the Addon UI
|
||||
if self.repos[self.idx][0] in OBSOLETE:
|
||||
if self.repos[self.idx][0] in obsolete:
|
||||
message = " <div style=\"width: 100%; text-align:center; background: #FFB3B3;\"><strong style=\"color: #FFFFFF; background: #FF0000;\">"+translate("AddonsInstaller","This addon is marked as obsolete")+"</strong><br/><br/>"
|
||||
message += translate("AddonsInstaller","This usually means it is no longer maintained, and some more advanced addon in this list provides the same functionality.")+"<br/></div><hr/>" + desc
|
||||
|
||||
# If the Addon is Python 2 only, let the user know through the Addon UI
|
||||
if self.repos[self.idx][0] in PY2ONLY:
|
||||
if self.repos[self.idx][0] in py2only:
|
||||
message = " <div style=\"width: 100%; text-align:center; background: #ffe9b3;\"><strong style=\"color: #FFFFFF; background: #ff8000;\">"+translate("AddonsInstaller","This addon is marked as Python 2 Only")+"</strong><br/><br/>"
|
||||
message += translate("AddonsInstaller","This workbench may no longer be maintained and installing it on a Python 3 system will more than likely result in errors at startup or while in use.")+"<br/></div><hr/>" + desc
|
||||
|
||||
@@ -622,7 +653,7 @@ class InstallWorker(QtCore.QThread):
|
||||
self.progressbar_show.emit(True)
|
||||
if os.path.exists(clonedir):
|
||||
self.info_label.emit("Updating module...")
|
||||
if sys.version_info.major > 2 and str(self.repos[idx][0]) in PY2ONLY:
|
||||
if sys.version_info.major > 2 and str(self.repos[idx][0]) in py2only:
|
||||
FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "User requested updating a Python 2 workbench on a system running Python 3 - ")+str(self.repos[idx][0])+"\n")
|
||||
if git:
|
||||
if not os.path.exists(clonedir + os.sep + '.git'):
|
||||
@@ -656,7 +687,7 @@ class InstallWorker(QtCore.QThread):
|
||||
self.info_label.emit("Checking module dependencies...")
|
||||
depsok,answer = self.checkDependencies(self.repos[idx][1])
|
||||
if depsok:
|
||||
if sys.version_info.major > 2 and str(self.repos[idx][0]) in PY2ONLY:
|
||||
if sys.version_info.major > 2 and str(self.repos[idx][0]) in py2only:
|
||||
FreeCAD.Console.PrintWarning(translate("AddonsInstaller", "User requested installing a Python 2 workbench on a system running Python 3 - ")+str(self.repos[idx][0])+"\n")
|
||||
if git:
|
||||
self.info_label.emit("Cloning module...")
|
||||
|
||||
Reference in New Issue
Block a user