diff --git a/src/Tools/offlinedoc/README b/src/Tools/offlinedoc/README deleted file mode 100644 index 621fe07ef4..0000000000 --- a/src/Tools/offlinedoc/README +++ /dev/null @@ -1,41 +0,0 @@ -this suite of tools can be used to retrieve a local copy -from the FreeCAD wiki and then use it to generate qhelp -and pdf files. The downloading of the entire wiki is now -a huge operation, prone to network errors, so it has been -cut into 2 parts, one to retrieve a list of files to -download and another to actually download the files. - -1) run "buildwikiindex.py" to build an index file containing - a list of all the files to download - -2) run "downloadwiki.py". If connection drops, run it again, - the already downloaded files will be skipped. - -2b) Dirty hack: run "fixlinks.py" to fix wrong html links - (downloadwiki.py should be fixed in the future) - -3) run "buildqhelp.py" to generate freecad.qhc and freecad.qch - files - -4) run "buildpdf.py" to generate freecad.pdf (wkhtmltopdf must be installed) - -5) the qhelp files can be tested with "assistant -collectionFile freecad.qhc" - -6) If you have already downloaded the whole wiki, run "update.py" immediately - after, to create a list of revision IDs for each page. - -7) Once the initial revisions list has been created, the "update.py" script - can be ran anytime in the future, to check for pages that have changed - since the stored revision ID. The script is meant to run twice, one to get - a list of pages that have changed, and another one to download the changed - pages (and all their dependencies) again. - -8) To split the generated freecad.qch into parts that are smaller than 50Mb - (github limit): split -d --byte=49M localwiki/freecad.qch localwiki/freecad.qch.part - -9) To join the parts again (for testing): cat localwiki/freecad.qch.part* >> test.qch - Then check that test.qch has the same md5 number than localwiki/freecad.qch - -10) To test: assistant -collectionFile localwiki/freecad.qhc - - diff --git a/src/Tools/offlinedoc/buildpdf.py b/src/Tools/offlinedoc/buildpdf.py deleted file mode 100755 index 8469886268..0000000000 --- a/src/Tools/offlinedoc/buildpdf.py +++ /dev/null @@ -1,322 +0,0 @@ -#!/usr/bin/env python - -#*************************************************************************** -#* * -#* Copyright (c) 2009 Yorik van Havre * -#* * -#* This program is free software; you can redistribute it and/or modify * -#* it under the terms of the GNU Lesser General Public License (LGPL) * -#* as published by the Free Software Foundation; either version 2 of * -#* the License, or (at your option) any later version. * -#* for detail see the LICENCE text file. * -#* * -#* This program is distributed in the hope that it will be useful, * -#* but WITHOUT ANY WARRANTY; without even the implied warranty of * -#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * -#* GNU Library General Public License for more details. * -#* * -#* You should have received a copy of the GNU Library General Public * -#* License along with this program; if not, write to the Free Software * -#* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * -#* USA * -#* * -#*************************************************************************** - -__title__="buildpdf" -__author__ = "Yorik van Havre " -__url__ = "http://www.freecadweb.org" - -""" -This script builds a pdf file from a local copy of the wiki -""" - -Workbenches=["Part","PartDesign","Sketcher","Constraints","Draft","Path","Fem","Arch","TechDraw","Raytracing","OpenSCAD","Robot","Mesh"] - -TOC="""Online_Help_Startpage -About_FreeCAD -Feature_list -Installing -Getting_started -Mouse_Model -Document_structure -Property_editor -Import_Export -Workbenches - -WorkbenchesList - -Interface_Customization -Preferences_Editor -Macros -Introduction_to_Python -Python_scripting_tutorial -Topological_data_scripting -Mesh_Scripting -Mesh_to_Part -Scenegraph -Pivy - -begin - -PySide -PySide_Beginner_Examples -PySide_Medium_Examples -PySide_Advanced_Examples - -end - -Scripted_objects -Embedding_FreeCAD -Embedding_FreeCADGui -Code_snippets""" - -import sys, os, shutil, time -from urllib.request import urlopen - -# CONFIGURATION ################################################# - -INDEX = "Online_Help_Toc" # the start page from where to crawl the wiki -PDFCONVERTOR = 'wkhtmltopdf' # can be 'pisa', 'htmldoc', 'wkhtmltopdf' or 'firefox' -VERBOSE = True # set true to get output messages -INCLUDECOMMANDS = True # if true, the command pages of each workbench are included after each WB page -OVERWRITE = False # if true, pdf files are recreated even if already existing -FIREFOXPDFFOLDER = os.path.expanduser("~")+os.sep+"PDF" # if firefox is used, set this to where it places its pdf files by default -COVER = "http://www.freecadweb.org/wiki/images/7/79/Freecad-pdf-cover.svg" - -# END CONFIGURATION ############################################## - - -FOLDER = "./localwiki" - -fcount = dcount = 0 - -def crawl(): - "creates a pdf file from the localwiki folder" - - # tests ############################################### - - if PDFCONVERTOR == 'pisa': - try: - import ho.pisa as pisa - except Exception: - "Error: Python-pisa not installed, exiting." - return 1 - elif PDFCONVERTOR == 'htmldoc': - if os.system('htmldoc --version'): - print("Error: Htmldoc not found, exiting.") - return 1 - try: - from PyPDF2 import PdfFileReader,PdfFileWriter - except Exception: - print("Error: Python-pypdf2 not installed, exiting.") - - # run ######################################################## - - buildpdffiles() - joinpdf() - - if VERBOSE: print("All done!") - return 0 - - -def buildpdffiles(): - "scans a folder for html files and converts them all to pdf" - templist = os.listdir(FOLDER) - if PDFCONVERTOR == 'wkhtmltopdf': - makeStyleSheet() - global fileslist - fileslist = [] - for i in templist: - if i[-5:] == '.html': - fileslist.append(i) - print("converting ",len(fileslist)," pages") - i = 1 - for f in fileslist: - print(i," : ",f) - if PDFCONVERTOR == 'pisa': - createpdf_pisa(f[:-5]) - elif PDFCONVERTOR == 'wkhtmltopdf': - createpdf_wkhtmltopdf(f[:-5]) - elif PDFCONVERTOR == 'firefox': - createpdf_firefox(f[:-5]) - else: - createpdf_htmldoc(f[:-5]) - i += 1 - - -def fetch_resources(uri, rel): - """ - Callback to allow pisa/reportlab to retrieve Images,Stylesheets, etc. - 'uri' is the href attribute from the html link element. - 'rel' gives a relative path, but it's not used here. - - Note from Yorik: Not working!! - """ - path = os.path.join(FOLDER,uri.replace("./", "")) - return path - -def createpdf_pisa(pagename): - "creates a pdf file from a saved page using pisa (python module)" - import ho.pisa as pisa - if (not exists(pagename+".pdf",image=True)) or OVERWRITE: - infile = open(FOLDER + os.sep + pagename+'.html','ro') - outfile = open(FOLDER + os.sep + pagename+'.pdf','wb') - if VERBOSE: print("Converting " + pagename + " to pdf...") - pdf = pisa.CreatePDF(infile,outfile,FOLDER,link_callback=fetch_resources) - outfile.close() - if pdf.err: - return pdf.err - return 0 - - -def createpdf_firefox(pagename): - "creates a pdf file from a saved page using firefox (needs command line printing extension)" - # the default printer will be used, so make sure it is set to pdf - # command line printing extension http://forums.mozillazine.org/viewtopic.php?f=38&t=2729795 - if (not exists(pagename+".pdf",image=True)) or OVERWRITE: - infile = FOLDER + os.sep + pagename+'.html' - outfile = FOLDER + os.sep + pagename+'.pdf' - os.system('firefox -print ' + infile) - time.sleep(6) - if os.path.exists(FIREFOXPDFFOLDER + os.sep + pagename + ".pdf"): - shutil.move(FIREFOXPDFFOLDER+os.sep+pagename+".pdf",outfile) - else: - print("-----------------------------------------> Couldn't find print output!") - - -def createpdf_htmldoc(pagename): - "creates a pdf file from a saved page using htmldoc (external app, but supports images)" - if (not exists(pagename+".pdf",image=True)) or OVERWRITE: - infile = FOLDER + os.sep + pagename+'.html' - outfile = FOLDER + os.sep + pagename+'.pdf' - return os.system('htmldoc --webpage --textfont sans --browserwidth 840 -f '+outfile+' '+infile) - - -def createpdf_wkhtmltopdf(pagename): - "creates a pdf file from a saved page using htmldoc (external app, but supports images)" - if (not exists(pagename+".pdf",image=True)) or OVERWRITE: - infile = FOLDER + os.sep + pagename+'.html' - outfile = FOLDER + os.sep + pagename+'.pdf' - cmd = 'wkhtmltopdf -L 5mm --user-style-sheet '+FOLDER+os.sep+'wkhtmltopdf.css '+infile+' '+outfile - print(cmd) - #return os.system(cmd) - else: - print("skipping") - - -def joinpdf(): - "creates one pdf file from several others, following order from the cover" - from PyPDF2 import PdfFileReader,PdfFileWriter - if VERBOSE: print("Building table of contents...") - - result = PdfFileWriter() - createCover() - inputfile = PdfFileReader(open(FOLDER+os.sep+'Cover.pdf','rb')) - result.addPage(inputfile.getPage(0)) - count = 1 - - tocfile = createTOC() - parent = False - for page in tocfile: - page = page.strip() - if page: - if page[0] == "#": - continue - if page == "begin": - parent = True - continue - if page == "end": - parent = False - continue - if VERBOSE: print('Appending ',page, " at position ",count) - title = page.replace("_"," ") - pdffile = page + ".pdf" - if exists(pdffile,True): - inputfile = PdfFileReader(open(FOLDER + os.sep + pdffile,'rb')) - numpages = inputfile.getNumPages() - for i in range(numpages): - result.addPage(inputfile.getPage(i)) - if parent == True: - parent = result.addBookmark(title,count) - elif parent == False: - result.addBookmark(title,count) - else: - result.addBookmark(title,count,parent) - count += numpages - else: - print("page ",pdffile," not found, aborting.") - sys.exit() - - if VERBOSE: print("Writing...") - outputfile = open(FOLDER+os.sep+"freecad.pdf",'wb') - result.write(outputfile) - outputfile.close() - if VERBOSE: - print(' ') - print('Successfully created '+FOLDER+os.sep+'freecad.pdf') - - -def createTOC(): - "populates the TOC" - tocfile = TOC.split("\n") - files = [f for f in os.listdir(FOLDER) if f.endswith(".pdf")] - wbpages = [] - for wb in Workbenches: - wbpage += "begin" - if wb+"_Workbench" in files: - wbpages.append(wb+"_Workbench") - for f in files: - if f.lower().startswith(wb.lower()+"_"): - if (not f.lower().endswith("_workbench")) and (not f.lower().endswith("tutorial")): - wb.append(f) - if wb+"_tutorial" in files: - wbpages.append(wb+"_tutorial") - wbpages.append("end") - toc = [] - for i in tocfile: - if i == "WorkbenchesList": - toc.extend(wbpages) - else: - toc.append(i) - return toc - - -def local(page,image=False): - "returns a local path for a given page/image" - if image: - return FOLDER + os.sep + page - else: - return FOLDER + os.sep + page + '.html' - - -def exists(page,image=False): - "checks if given page/image already exists" - path = local(page,image) - if os.path.exists(path): return True - return False - - -def makeStyleSheet(): - "Creates a stylesheet for wkhtmltopdf" - outputfile = open(FOLDER+os.sep+"wkhtmltopdf.css",'wb') - outputfile.write(""" -.printfooter { - display:none !important; -} -""") - outputfile.close() - - -def createCover(): - "downloads and creates a cover page" - if VERBOSE: print("fetching " + COVER) - data = (urlopen(COVER).read()) - path = FOLDER + os.sep + "Cover.svg" - fil = open(path,'wb') - fil.write(data) - fil.close() - os.system('inkscape --export-pdf='+FOLDER+os.sep+'Cover.pdf'+' '+FOLDER+os.sep+'Cover.svg') - - -if __name__ == "__main__": - crawl() diff --git a/src/Tools/offlinedoc/buildqhelp.py b/src/Tools/offlinedoc/buildqhelp.py deleted file mode 100755 index 5608154d7b..0000000000 --- a/src/Tools/offlinedoc/buildqhelp.py +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env python3 - -#*************************************************************************** -#* * -#* Copyright (c) 2009 Yorik van Havre * -#* * -#* This program is free software; you can redistribute it and/or modify * -#* it under the terms of the GNU Lesser General Public License (LGPL) * -#* as published by the Free Software Foundation; either version 2 of * -#* the License, or (at your option) any later version. * -#* for detail see the LICENCE text file. * -#* * -#* This program is distributed in the hope that it will be useful, * -#* but WITHOUT ANY WARRANTY; without even the implied warranty of * -#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * -#* GNU Library General Public License for more details. * -#* * -#* You should have received a copy of the GNU Library General Public * -#* License along with this program; if not, write to the Free Software * -#* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * -#* USA * -#* * -#*************************************************************************** - -__title__="wiki2qhelp" -__author__ = "Yorik van Havre " -__url__ = "http://www.freecadweb.org" - -""" -This script builds qhrlp files from a local copy of the wiki -""" - -import os, re, shutil - -# CONFIGURATION ################################################# - -FOLDER = "./localwiki" -INDEX = "Online_Help_Toc" # the start page from where to crawl the wiki -VERBOSE = True # to display what's going on. Otherwise, runs totally silent. -QHELPCOMPILER = 'qhelpgenerator' -RELEASE = '0.19' - -# END CONFIGURATION ############################################## - -fcount = dcount = 0 - -def crawl(): - "downloads an entire wiki site" - - # tests ############################################### - - if os.system(QHELPCOMPILER +' -v'): - print ("Error: QAssistant not fully installed, exiting.") - return 1 - - # run ######################################################## - - qhp = buildtoc() - qhcp = createCollProjectFile() - shutil.copy("../../Gui/Icons/freecad-icon-64.png","localwiki/freecad-icon-64.png") - if generate(qhcp): - print ("Error while generating") - return 1 - if compile(qhp): - print ("Error while compiling") - return 1 - if VERBOSE: print ("All done!") - #i=raw_input("Copy the files to their correct location in the source tree? y/n (default=no) ") - #if i.upper() in ["Y","YES"]: - # shutil.copy("localwiki/freecad.qch","../../Doc/freecad.qch") - # shutil.copy("localwiki/freecad.qhc","../../Doc/freecad.qhc") - #else: - print ('Files freecad.qch and freecad.qhc are in localwiki. Test with "assistant -collectionFile localwiki/freecad.qhc"') - return 0 - -def compile(qhpfile): - "compiles the whole html doc with qassistant" - qchfile = FOLDER + os.sep + "freecad.qch" - if not os.system(QHELPCOMPILER + ' '+qhpfile+' -o '+qchfile): - if VERBOSE: print ("Successfully created",qchfile) - return 0 - return 1 - -def generate(qhcpfile): - "generates qassistant-specific settings like icon, title, ..." - txt=""" -
FreeCAD """+RELEASE+""" help files
-http://www.freecadweb.org
- """ - about=open(FOLDER + os.sep + "about.txt","w") - about.write(txt) - about.close() - qhcfile = FOLDER + os.sep + "freecad.qhc" - if not os.system(QHELPCOMPILER+' '+qhcpfile+' -o '+qhcfile): - if VERBOSE: print ("Successfully created ",qhcfile) - return 0 - return 1 - -def createCollProjectFile(): - qprojectfile = ''' - - - FreeCAD User Manual - freecad-icon-64.png - freecad/freecad - qthelp://org.freecad.usermanual/doc/Online_Help_Startpage.html - - About FreeCAD - - - about.txt - freecad-icon-64.png - - true - true - true - - - - - freecad.qhp - freecad.qch - - - - freecad.qch - - - -''' - if VERBOSE: print ("Building project file...") - qfilename = FOLDER + os.sep + "freecad.qhcp" - f = open(qfilename,'w') - f.write(qprojectfile) - f.close() - if VERBOSE: print ("Done writing qhcp file:",qfilename) - return qfilename - -def buildtoc(): - ''' - gets the table of contents page and parses its - contents into a clean lists structure - ''' - - qhelpfile = ''' - - org.freecad.usermanual - doc - - - - - - - - - - - - -''' - - def getname(line): - line = re.compile('
  • ').sub('',line) - line = re.compile('
    • ').sub('',line) - title = line.strip() - link = '' - if "]*>(.*?)',line)[0].strip() - link = re.findall('href="(.*?)"',line)[0].strip() - if link: - if not link.endswith(".html"): - link = link + ".html" - if link.startswith("/"): - link = link[1:] - if not link: link = 'default.html' - if title.startswith(" <","><") - html = re.findall("",html)[0] - items = re.findall(']*>.*?|',html) - inserttoc = '
    \n' - insertkeywords = '' - for item in items: - if not ("
      " in item): - if ("
    " in item): - inserttoc += '
    \n' - else: - link = '' - title,link=getname(item) - if link: - link='" ref="'+link - insertkeywords += ('\n') - if link and title: - inserttoc += ('
    \n') - else: - subitems = item.split("
      ") - for i in range(len(subitems)): - link = '' - title,link=getname(subitems[i]) - if link: - link='" ref="'+link - insertkeywords += ('\n') - trail = '' - if i == len(subitems)-1: trail = '' - if link and title: - inserttoc += ('
      '+trail+'\n') - inserttoc += '
      \n' - - insertfiles = "\n" - for fil in os.listdir(FOLDER): - insertfiles += (""+fil+"\n") - insertfiles += "\n" - - qhelpfile = re.compile('').sub(insertkeywords,qhelpfile) - qhelpfile = re.compile('').sub(inserttoc,qhelpfile) - qhelpfile = re.compile('').sub(insertfiles,qhelpfile) - qfilename = FOLDER + os.sep + "freecad.qhp" - f = open(qfilename,'w') - f.write(qhelpfile) - f.close() - if VERBOSE: print ("Done writing qhp file:",qfilename) - return qfilename - -if __name__ == "__main__": - crawl() - diff --git a/src/Tools/offlinedoc/buildwikiindex.py b/src/Tools/offlinedoc/buildwikiindex.py deleted file mode 100755 index 34f7730110..0000000000 --- a/src/Tools/offlinedoc/buildwikiindex.py +++ /dev/null @@ -1,222 +0,0 @@ -#!/usr/bin/env python - -#*************************************************************************** -#* * -#* Copyright (c) 2009 Yorik van Havre * -#* * -#* This program is free software; you can redistribute it and/or modify * -#* it under the terms of the GNU Lesser General Public License (LGPL) * -#* as published by the Free Software Foundation; either version 2 of * -#* the License, or (at your option) any later version. * -#* for detail see the LICENCE text file. * -#* * -#* This program is distributed in the hope that it will be useful, * -#* but WITHOUT ANY WARRANTY; without even the implied warranty of * -#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * -#* GNU Library General Public License for more details. * -#* * -#* You should have received a copy of the GNU Library General Public * -#* License along with this program; if not, write to the Free Software * -#* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * -#* USA * -#* * -#*************************************************************************** - -from __future__ import print_function - -__title__="buildwikiindex.py" -__author__ = "Yorik van Havre " -__url__ = "http://www.freecadweb.org" - -""" -This script parses the contents of a wiki site and saves a file containing -names of pages and images to be downloaded. -""" - -import sys, os, re -from urllib2 import urlopen, HTTPError - -# CONFIGURATION ################################################# - -URL = "https://www.freecadweb.org/wiki" #default URL if no URL is passed -INDEX = "Online_Help_Toc" # the start page from where to crawl the wiki -NORETRIEVE = ['Manual','Developer_hub','Power_users_hub','Users_hub','Source_documentation', - 'User_hub','Main_Page','About_this_site','Interesting_links','Syndication_feeds', - 'FreeCAD:General_disclaimer','FreeCAD:About','FreeCAD:Privacy_policy','WikiPages'] # pages that won't be fetched (kept online) -NORETRIEVE += ['Constraint_Concentric','Constraint_EqualLength','Constraint_ExternalAngle', - 'Constraint_Horizontal','Constraint_HorizontalDistance','Constraint_Internal_Alignment', - 'Constraint_InternalAngle','Constraint_Length','Constraint_Lock','Constraint_Parallel', - 'Constraint_Perpendicular','Constraint_PointOnEnd','Constraint_PointOnMidPoint', - 'Constraint_PointOnObject','Constraint_PointOnPoint','Constraint_PointOnStart', - 'Constraint_PointToObject','Constraint_Radius','Constraint_SnellsLaw', - 'Constraint_Symmetric','Constraint_Tangent','Constraint_TangentToEnd', - 'Constraint_TangentToStart','Constraint_Vertical', - 'Join_Cutout','Join_Embed','Part_BooleanFragment','Part_Sections','Curves_HelicalSweep', - 'CurvedShapes_FlyingWingS800','CurvedShapes_HortenHIX','CurvedShapes_SurfaceCut', - 'CurvedShapes_InterpolatedMiddle','CurvedShapes_CurvedSegment','Arch_Cell', - 'Std_ClippingPlane','Std_AboutQt'] # pages that have been renamed but still dangle around... -GETTRANSLATIONS = False # Set true if you want to get the translations too. -MAXFAIL = 3 # max number of retries if download fails -VERBOSE = True # to display what's going on. Otherwise, runs totally silent. -WRITETHROUGH = True # if true, fetched files are constantly written to disk, in case of failure. - -# END CONFIGURATION ############################################## - -wikiindex = "/index.php?title=" - -def crawl(pagename=[]): - "downloads an entire wiki site" - todolist = [] - processed = [] - count = 1 - if pagename: - if not isinstance(pagename,list): - pagename = [pagename] - todolist = pagename - else: - if os.path.exists("wikifiles.txt"): - f = open("wikifiles.txt","r") - if VERBOSE: print ("Reading existing list...") - for l in f.readlines(): - if l.strip() != "": - if VERBOSE: print ("Adding ",l) - processed.append(l.strip()) - f.close() - if os.path.exists("todolist.txt"): - f = open("todolist.txt","r") - if VERBOSE: print ("Reading existing todo list...") - for l in f.readlines(): - if l.strip() != "": - todolist.append(l.strip()) - f.close() - else: - indexpages,imgs = get(INDEX) - todolist.extend(indexpages) - while todolist: - targetpage = todolist.pop() - if (not targetpage in NORETRIEVE): - if VERBOSE: print (count, ": Scanning ", targetpage) - pages,images = get(targetpage) - count += 1 - processed.append(targetpage) - processed.extend(images) - if VERBOSE: print ("got",len(pages),"links") - for p in pages: - if (not (p in todolist)) and (not (p in processed)): - todolist.append(p) - if WRITETHROUGH: - writeList(processed) - writeList(todolist,"todolist.txt") - if VERBOSE: print ("Fetched ", count, " pages") - if not WRITETHROUGH: - writeList(processed) - if pagename: - return processed - return 0 - -def get(page): - "downloads a single page, returns the other pages it links to" - html = fetchpage(page) - html = cleanhtml(html) - pages = getlinks(html) - images = getimagelinks(html) - return pages,images - -def cleanhtml(html): - "cleans given html code from dirty script stuff" - html = html.replace('\n','Wlinebreak') # removing linebreaks for regex processing - html = re.compile('(.*)]+column-content+[^>]+>').sub('',html) # stripping before content - html = re.compile(']+column-one+[^>]+>.*').sub('',html) # stripping after content - html = re.compile('').sub('',html) # removing comment tags - html = re.compile(']*>.*?').sub('',html) # removing script tags - html = re.compile('').sub('',html) # removing IE tags - html = re.compile('
      ]*>.*?
      ').sub('',html) # removing nav div - html = re.compile('

      ]*>.*?

      ').sub('',html) # removing print subtitle - html = re.compile('Retrieved from').sub('Online version:',html) # changing online title - html = re.compile('
      ').sub('',html) # removing nav stuff - html = re.compile('
      ').sub('',html) # removing nav stuff - html = re.compile('
      ').sub('',html) # removing translations links - html = re.compile('
      ]*>.*?',html) - pages = [] - for l in links: - # rg = re.findall('php\?title=(.*)\" title',l) - rg = re.findall('href=.*?php\?title=(.*?)"',l) - if not rg: - rg = re.findall('href="\/wiki\/(.*?)"',l) - if not rg: - rg = re.findall('href=".*?wiki\\.freecadweb\\.org\/(.*?)"',l) - if not rg: - rg = re.findall('href="\/(.*?)"',l) - if "images" in rg: - rg = None - elif "mediawiki" in rg: - rg = None - if rg: - rg = rg[0] - if not "Command_Reference" in rg: - if "#" in rg: - rg = rg.split('#')[0] - if ":" in rg: - NORETRIEVE.append(rg) - if "&" in rg: - NORETRIEVE.append(rg) - if ";" in rg: - NORETRIEVE.append(rg) - if "/" in rg: - if not GETTRANSLATIONS: - NORETRIEVE.append(rg) - if not rg in NORETRIEVE: - pages.append(rg) - print ("got link: ",rg) - return pages - -def getimagelinks(html): - "returns a list of image links found in an html file" - imlinks = re.findall(' * -#* * -#* This program is free software; you can redistribute it and/or modify * -#* it under the terms of the GNU Lesser General Public License (LGPL) * -#* as published by the Free Software Foundation; either version 2 of * -#* the License, or (at your option) any later version. * -#* for detail see the LICENCE text file. * -#* * -#* This program is distributed in the hope that it will be useful, * -#* but WITHOUT ANY WARRANTY; without even the implied warranty of * -#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * -#* GNU Library General Public License for more details. * -#* * -#* You should have received a copy of the GNU Library General Public * -#* License along with this program; if not, write to the Free Software * -#* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * -#* USA * -#* * -#*************************************************************************** - -from __future__ import print_function - -__title__="downloadwiki" -__author__ = "Yorik van Havre " -__url__ = "http://www.freecadweb.org" - -""" -This script retrieves the contents of a wiki site from a pages list -""" - -import os, re -from urllib2 import urlopen, HTTPError - -# CONFIGURATION ################################################# - -DEFAULTURL = "https://www.freecadweb.org" #default URL if no URL is passed -INDEX = "Online_Help_Toc" # the start page from where to crawl the wiki -NORETRIEVE = ['Manual','Developer_hub','Power_users_hub','Users_hub','Source_documentation', - 'User_hub','Main_Page','About_this_site','Interesting_links','Syndication_feeds', - 'FreeCAD:General_disclaimer','FreeCAD:About','FreeCAD:Privacy_policy','WikiPages'] # pages that won't be fetched (kept online) -NORETRIEVE += ['Constraint_Concentric','Constraint_EqualLength','Constraint_ExternalAngle', - 'Constraint_Horizontal','Constraint_HorizontalDistance','Constraint_Internal_Alignment', - 'Constraint_InternalAngle','Constraint_Length','Constraint_Lock','Constraint_Parallel', - 'Constraint_Perpendicular','Constraint_PointOnEnd','Constraint_PointOnMidPoint', - 'Constraint_PointOnObject','Constraint_PointOnPoint','Constraint_PointOnStart', - 'Constraint_PointToObject','Constraint_Radius','Constraint_SnellsLaw', - 'Constraint_Symmetric','Constraint_Tangent','Constraint_TangentToEnd', - 'Constraint_TangentToStart','Constraint_Vertical'] # pages that have been renamed but still dangle around...GETTRANSLATIONS = False # Set true if you want to get the translations too. -MAXFAIL = 3 # max number of retries if download fails -VERBOSE = True # to display what's going on. Otherwise, runs totally silent. -GETTRANSLATIONS = False # Set true if you want to get the translations too. - -# END CONFIGURATION ############################################## - -FOLDER = "./localwiki" -LISTFILE = "wikifiles.txt" -URL = DEFAULTURL -wikiindex = "/wiki/index.php?title=" -imageprefix = "/wiki/" -defaultfile = " " -css = """/* Basic CSS for offline wiki rendering */ - -body { - font-family: Fira Sans,Arial,Helvetica,sans-serif; - text-align: justify; - max-width: 800px; - } - -h1 { - font-size: 2.4em; - font-weight: bold; - padding: 5px; - border-radius: 5px; - } - -h2 { - font-weight: normal; - font-size: 1.6em; - border-bottom: 1px solid #ddd; - } - -h3 { - padding-left: 20px; - } - -img { - max-width: 100%; - } - -li { - margin-top: 10px; - } - -pre, .mw-code { - text-align: left; - padding: 5px 5px 5px 20px; - font-family: mono; - border-radius: 2px; - } - -a:link, a:visited { - font-weight: bold; - text-decoration: none; - color: #2969C4; - } - -a:hover { - text-decoration: underline; - } - -.printfooter { - font-size: 0.8em; - color: #333333; - border-top: 1px solid #333; - margin-top: 20px; - } - -.wikitable #toc { - font-size: 0.8em; - } - -.ct, .ctTitle, .ctOdd, .ctEven th { - font-size: 1em; - text-align: left; - width: 190px; - float: right; - margin-top: 10px; - border-radius: 2px; - } - -.ct { - margin-left: 15px; - padding: 10px; - } -#mw-navigation, .mw-jump-link, .docnav, .NavFrame { - display:none; /*TODO remove on next build (included below)*/ - } -""" - -def crawl(): - "downloads an entire wiki site" - global processed - processed = [] - if VERBOSE: print ("crawling ", URL, ", saving in ", FOLDER) - if not os.path.isdir(FOLDER): os.mkdir(FOLDER) - file = open(FOLDER + os.sep + "wiki.css",'wb') - file.write(css) - file.close() - dfile = open(FOLDER + os.sep + "default.html",'wb') - dfile.write(defaultfile) - dfile.close() - lfile = open(LISTFILE) - global locallist - locallist = [] - for l in lfile: locallist.append(l.replace("\n","")) - lfile.close() - todolist = locallist[:] - print ("getting ",len(todolist)," files...") - count = 1 - get(INDEX) - while todolist: - targetpage = todolist.pop() - if VERBOSE: print (count,(3-len(str(count)))*" ", ": Fetching ", targetpage) - get(targetpage) - count += 1 - if VERBOSE: print ("Fetched ", count, " pages") - if VERBOSE: print ("All done!") - return 0 - -def get(page): - "downloads a single page" - localpage = page - if "Command_Reference" in localpage: - localpage = localpage.replace("Category:","") - localpage = localpage.replace("&pagefrom=","+") - localpage = localpage.replace("#mw-pages","") - if page[-4:] in [".png",".jpg",".svg",".gif","jpeg",".PNG",".JPG"]: - fetchimage(page) - elif not exists(localpage): - html = fetchpage(page) - html = cleanhtml(html) - pages = getlinks(html) - html = cleanlinks(html,pages) - html = cleanimagelinks(html) - output(html,page) - else: - if VERBOSE: print (" skipping ",page) - -def getlinks(html): - "returns a list of wikipage links in html file" - links = re.findall(']*>.*?',html) - pages = [] - for l in links: - # rg = re.findall('php\?title=(.*)\" title',l) - rg = re.findall('href=.*?php\?title=(.*?)"',l) - if not rg: - rg = re.findall('href="\/wiki\/(.*?)"',l) - if rg: - rg = rg[0] - if not "Command_Reference" in rg: - if "#" in rg: - rg = rg.split('#')[0] - if ":" in rg: - NORETRIEVE.append(rg) - if ";" in rg: - NORETRIEVE.append(rg) - if "&" in rg: - NORETRIEVE.append(rg) - if "/" in rg: - if not GETTRANSLATIONS: - NORETRIEVE.append(rg) - pages.append(rg) - return pages - -def getimagelinks(html): - "returns a list of image links found in an html file" - return re.findall(']+>').sub('',html) # stripping before content - html = re.compile('
      ]*>.*?
      ').sub('',html) # removing nav div - html = re.compile('

      ]*>.*?

      ').sub('',html) # removing print subtitle - html = re.compile('Retrieved from').sub('Online version:',html) # changing online title - html = re.compile('
      ').sub('',html) # removing nav stuff - html = re.compile('
      ').sub('',html) # removing nav stuff - html = re.compile('
      ').sub('',html) # removing toc - html = re.compile('width=\"100%\" style=\"float: right; width: 230px; margin-left: 1em\"').sub('',html) # removing command box styling - #html = re.compile('
      ').sub('',html) # removing translations links - if not GETTRANSLATIONS: - html = re.compile('
      ').sub('',html) # removing translations links - html = re.compile('Wlinebreak').sub('\n',html) # restoring original linebreaks - return html - - -def cleanlinks(html, pages=None): - "cleans page links found in html" - if not pages: pages = getlinks(html) - for page in pages: - if page in NORETRIEVE: - output = 'href="' + URL + wikiindex + page + '"' - else: - output = 'href="' + page.replace("/","-") + '.html"' - html = re.compile('href="[^"]+' + page + '"').sub(output,html) - if "Command_Reference" in output: - html = html.replace("Category:","") - html = html.replace("&pagefrom=","+") - html = html.replace("#mw-pages",".html") - html = html.replace("/wiki/index.php?title=Command_Reference","Command_Reference") - return html - -def cleanimagelinks(html,links=None): - "cleans image links in given html" - if not links: links = getimagelinks(html) - if links: - for l in links: - nl = re.findall('.*/(.*)',l) - if nl: html = html.replace(l,nl[0]) - # fetchimage(l) - return html - -def fetchpage(page): - "retrieves given page from the wiki" - print (" downloading: ",URL + wikiindex + page) - failcount = 0 - while failcount < MAXFAIL: - try: - html = (urlopen(URL + wikiindex + page).read()) - return html - except HTTPError: - failcount += 1 - print ('Error: unable to fetch page ' + page) - -def fetchimage(imagelink): - "retrieves given image from the wiki and saves it" - if imagelink[0:5] == "File:": - print ("Skipping file page link") - return - filename = re.findall('.*/(.*)',imagelink)[0] - if not exists(filename,image=True): - failcount = 0 - while failcount < MAXFAIL: - try: - if VERBOSE: print (" downloading " + URL + imageprefix + imagelink) - data = (urlopen(URL + imageprefix + imagelink)).read() - path = local(filename,image=True) - file = open(path,'wb') - file.write(data) - file.close() - except Exception: - failcount += 1 - else: - processed.append(filename) - if VERBOSE: print (" saving ",local(filename,image=True)) - return - print ('Error: unable to fetch file ' + filename) - else: - if VERBOSE: print (" skipping ",filename) - -def local(page,image=False): - "returns a local path for a given page/image" - if image: - return FOLDER + os.sep + page - else: - return FOLDER + os.sep + page + '.html' - -def exists(page,image=False): - "checks if given page/image already exists" - path = local(page.replace("/","-"),image) - if os.path.exists(path): return True - return False - -def webroot(url): - return re.findall('(http://.*?)/',url)[0] - -def output(html,page): - "encapsulates raw html code into nice html body" - title = page.replace("_"," ") - header = "" - header += "" + title + "" - header += '' - header += "" - header += "" - header += "

      " + title + "

      " - footer = "" - html = header+html+footer - filename = local(page.replace("/","-")) - if "Command_Reference" in filename: - filename = filename.replace("Category:","") - filename = filename.replace("&pagefrom=","+") - filename = filename.replace("#mw-pages","") - filename = filename.replace(".html.html",".html") - print (" saving ",filename) - file = open(filename,'wb') - file.write(html) - file.close() - -if __name__ == "__main__": - crawl() - diff --git a/src/Tools/offlinedoc/fixlinks.py b/src/Tools/offlinedoc/fixlinks.py deleted file mode 100755 index 68bd70caf4..0000000000 --- a/src/Tools/offlinedoc/fixlinks.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 - -#*************************************************************************** -#* * -#* Copyright (c) 2021 Yorik van Havre * -#* * -#* This program is free software; you can redistribute it and/or modify * -#* it under the terms of the GNU Lesser General Public License (LGPL) * -#* as published by the Free Software Foundation; either version 2 of * -#* the License, or (at your option) any later version. * -#* for detail see the LICENCE text file. * -#* * -#* This program is distributed in the hope that it will be useful, * -#* but WITHOUT ANY WARRANTY; without even the implied warranty of * -#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * -#* GNU Library General Public License for more details. * -#* * -#* You should have received a copy of the GNU Library General Public * -#* License along with this program; if not, write to the Free Software * -#* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * -#* USA * -#* * -#*************************************************************************** - -"""This script fixes links like href="/Arch_Wall" into href="/Arch_Wall.html" where needed. Dirty hack, downloadwiki.py should be fixed instead""" - -import os,re -files = [f for f in os.listdir("localwiki") if f.endswith(".html")] -for fn in files: - f = open(os.path.join("localwiki",fn)) - b = f.read() - f.close() - b = b.replace("\n","--endl--") - for href in re.findall("href=\".*?\"",b): - if (not "." in href) and (not "#" in href): - repl = href[:-1]+".html\"" - if "href=\"/" in repl: - repl = repl.replace("href=\"/","href=\"") - print(fn," : replacing",href,"with",repl) - b = b.replace(href,repl) - b = b.replace("--endl--","\n") - f = open(os.path.join("localwiki",fn),"w") - f.write(b) - f.close() - diff --git a/src/Tools/offlinedoc/update.py b/src/Tools/offlinedoc/update.py deleted file mode 100755 index 0ff3eae205..0000000000 --- a/src/Tools/offlinedoc/update.py +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env python - -#*************************************************************************** -#* * -#* Copyright (c) 2017 Yorik van Havre * -#* * -#* This program is free software; you can redistribute it and/or modify * -#* it under the terms of the GNU Lesser General Public License (LGPL) * -#* as published by the Free Software Foundation; either version 2 of * -#* the License, or (at your option) any later version. * -#* for detail see the LICENCE text file. * -#* * -#* This program is distributed in the hope that it will be useful, * -#* but WITHOUT ANY WARRANTY; without even the implied warranty of * -#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * -#* GNU Library General Public License for more details. * -#* * -#* You should have received a copy of the GNU Library General Public * -#* License along with this program; if not, write to the Free Software * -#* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * -#* USA * -#* * -#*************************************************************************** - -__title__="update.py" -__author__ = "Yorik van Havre " -__url__ = "http://www.freecadweb.org" - -""" -This script needs to be run after the wiki has been fully downloaded. It has three usages: - -1) If no revisions.txt file is found, it parses the contents of the wikifiles.txt file - and, for each entry, it retrieves a corresponding revision ID, and creates a revisions.txt file - -2) If a revisions.txt file exists but no update.txt file exists, it crawls through all entries of - wikifiles.txt, and for each one, compares the current revision with the one stored in revisions.txt. - An update.txt file is created with all pages that have different revision IDs - -3) If update.txt exists, each entry of it will be scanned again for new links and all the needed - files downloaded. Revision.txt and wikifiles.txt get also updated. -""" - -import sys, os, re -from urllib.request import urlopen -from urllib.error import HTTPError - -# CONFIGURATION ################################################# - -URL = "https://www.freecadweb.org/wiki" #default URL if no URL is passed -GETTRANSLATIONS = False # Set true if you want to get the translations too. -MAXFAIL = 3 # max number of retries if download fails -VERBOSE = True # to display what's going on. Otherwise, runs totally silent. - -# END CONFIGURATION ############################################## - -wikiindex = "/index.php?title=" - -def update(pagename=None): - - if not os.path.exists("revisions.txt"): # case 1) - if not os.path.exists("wikifiles.txt"): - print("No wikifiles.txt found. Aborting") - sys.exit() - pages = [] - f = open("wikifiles.txt","r") - if VERBOSE: print("Reading existing list...") - for l in f.readlines(): - if l.strip() != "": - if not "/wiki/" in l: - if VERBOSE: print("Adding ",l.strip()) - pages.append(l.strip()) - f.close() - if VERBOSE: print("Added ",str(len(pages))," entries") - i = 1 - revs = [] - for page in pages: - rev = getRevision(page) - if VERBOSE: print(str(i)," revision: ",rev) - revs.append(page+":"+rev) - i += 1 - writeList(revs,"revisions.txt") - print("All done. Successfully written revisions.txt with ",len(revs)," entries.") - - elif os.path.exists("revisions.txt") and (not os.path.exists("updates.txt")): # case 2) - f = open("revisions.txt","r") - if VERBOSE: print("Reading revisions list...") - revisions = {} - for l in f.readlines(): - if l.strip() != "": - r = l.strip().split(":") - p = ":".join(r[:-1]) - if VERBOSE: print("Adding ",p) - revisions[p] = r[1] - f.close() - if VERBOSE: print("Added ",str(len(list(revisions.keys())))," entries") - updates = [] - i = 1 - for page in list(revisions.keys()): - rev = getRevision(page) - if rev != revisions[page]: - if VERBOSE: print(str(i),page," has a new revision: ",rev) - updates.append(page) - else: - if VERBOSE: print(str(i),page," is up to date ") - i += 1 - if updates: - writeList(updates,"updates.txt") - print("All done. Successfully written updates.txt with ",len(updates)," entries.") - else: - print("Everything up to date. Nothing to be done.") - - elif os.path.exists("revisions.txt") and os.path.exists("updates.txt"): # case 3) - if not os.path.exists("wikifiles.txt"): - print("No wikifiles.txt found. Aborting") - sys.exit() - wikifiles = [] - f = open("wikifiles.txt","r") - if VERBOSE: print("Reading wikifiles list...") - for l in f.readlines(): - if l.strip() != "": - wikifiles.append(l.strip()) - f.close() - if VERBOSE: print("Read ",str(len(wikifiles))," entries") - f = open("revisions.txt","r") - if VERBOSE: print("Reading revisions list...") - revisions = {} - for l in f.readlines(): - if l.strip() != "": - r = l.strip().split(":") - p = ":".join(r[:-1]) - revisions[p] = r[1] - f.close() - todo = [] - f = open("updates.txt","r") - if VERBOSE: print("Reading updates list...") - for l in f.readlines(): - if l.strip() != "": - todo.append(l.strip()) - f.close() - if VERBOSE: print(str(len(todo))," pages to scan...") - import buildwikiindex - buildwikiindex.WRITETHROUGH = False - buildwikiindex.VERBOSE = VERBOSE - updates = [] - for t in todo: - if VERBOSE: print("Scanning ",t) - updates.extend(buildwikiindex.crawl(t)) - updates = [u for u in updates if not u in wikifiles] - if VERBOSE: print(str(len(updates))," files to download...") - import downloadwiki - i = 1 - for u in updates: - if VERBOSE: print(i, ": Fetching ", u) - downloadwiki.get(u) - if not "/wiki/" in u: - rev = getRevision(u) - revisions[u] = rev - if not u in wikifiles: - wikifiles.append(u) - i += 1 - if VERBOSE: print("Updating wikifiles and revisions...") - writeList(wikifiles,"wikifiles.txt") - updatedrevs = [] - for k in list(revisions.keys()): - updatedrevs.append(k+":"+revisions[k]) - writeList(updatedrevs,"revisions.txt") - os.remove("updates.txt") - if VERBOSE: print("All done!") - -def getRevision(page): - html = fetchPage(page) - revs = re.findall("wgCurRevisionId\"\:(.*?),",html) - if len(revs) == 1: - return revs[0] - print('Error: unable to get revision ID of ' + page) - sys.exit() - -def fetchPage(page): - "retrieves given page from the wiki" - print("fetching: ",page) - failcount = 0 - while failcount < MAXFAIL: - try: - html = (urlopen(URL + wikiindex + page).read()) - return html - except HTTPError: - failcount += 1 - print('Error: unable to fetch page ' + page) - sys.exit() - -def writeList(pages,filename): - f = open(filename,"wb") - for p in pages: - f.write(p+"\n") - f.close() - if VERBOSE: print("written ",filename) - -if __name__ == "__main__": - update(sys.argv[1:])