From f5b6c20699f224995b02d8f54db252de9c12fde1 Mon Sep 17 00:00:00 2001 From: Benjamin Nauck Date: Sat, 27 Feb 2021 12:45:48 +0100 Subject: [PATCH] Tools: updatecrowdin.py rewrite to support new api Crowdin has depreciated their old api and will shut it down completely in december 2021. What's new: * rewritten in Python 3 (drops support for discontinued Python 2) * uses Crowdin's api v2 * runs on vanilla Python 3, (no external modules) * automatically adds new translation from the src directory * supports api token to be specified in an environment variable (Makes GitHub integration easier in the future) * requires project id to be explicitly set with an environment variable (Makes testing safer) The api token can be set either in ~/.crowdin-freecad-token similar to how the old api key was set, or by specifying it in an environment variable. The later has been added to make it easier with GitHub-Crowdin integration in the future. The requirement to explicitly set CROWDIN_PROJECT_ID has been introduced to avoid accidentally using the main Crowdin project while testing the script. The script has been tested on Python versions 3.7, 3.8 and 3.9. Co-authored-by: Mattias Pierre --- src/Tools/updatecrowdin.py | 363 +++++++++++++++++++++---------------- 1 file changed, 210 insertions(+), 153 deletions(-) diff --git a/src/Tools/updatecrowdin.py b/src/Tools/updatecrowdin.py index 2ce9c30de2..a5a1b707e2 100755 --- a/src/Tools/updatecrowdin.py +++ b/src/Tools/updatecrowdin.py @@ -1,187 +1,244 @@ -#!/usr/bin/python +#!/usr/bin/env python3 +# *************************************************************************** +# * * +# * Copyright (c) 2015 Yorik van Havre * +# * Copyright (c) 2021 Benjamin Nauck * +# * Copyright (c) 2021 Mattias Pierre * +# * * +# * This program is free software; you can redistribute it and/or modify * +# * it under the terms of the GNU Library General Public License (LGPL) * +# * as published by the Free Software Foundation; either version 2 of * +# * the License, or (at your option) any later version. * +# * for detail see the LICENCE text file. * +# * * +# * This program is distributed in the hope that it will be useful, * +# * but WITHOUT ANY WARRANTY; without even the implied warranty of * +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * +# * GNU Library General Public License for more details. * +# * * +# * You should have received a copy of the GNU Library General Public * +# * License along with this program; if not, write to the Free Software * +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * +# * USA * +# * * +# *************************************************************************** -#*************************************************************************** -#* * -#* Copyright (c) 2015 Yorik van Havre * -#* * -#* This program is free software; you can redistribute it and/or modify * -#* it under the terms of the GNU Library General Public License (LGPL) * -#* as published by the Free Software Foundation; either version 2 of * -#* the License, or (at your option) any later version. * -#* for detail see the LICENCE text file. * -#* * -#* This program is distributed in the hope that it will be useful, * -#* but WITHOUT ANY WARRANTY; without even the implied warranty of * -#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * -#* GNU Library General Public License for more details. * -#* * -#* You should have received a copy of the GNU Library General Public * -#* License along with this program; if not, write to the Free Software * -#* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * -#* USA * -#* * -#*************************************************************************** - -from __future__ import print_function - -''' +""" This utility offers several commands to interact with the FreeCAD project on crowdin. -For it to work, you need a .crowdin-freecad file in your user's folder, that contains -the API key that gives access to the crowdin FreeCAD project. +For it to work, you need a ~/.crowdin-freecad-token file in your user's folder, that contains +the API access token that gives access to the crowdin FreeCAD project. +The API token can also be specified in the CROWDIN_TOKEN environment variable. + +The CROWDIN_PROJECT_ID environment variable must be set. Usage: - updatecrowdin.py command + updatecrowdin.py [] Available commands: - status: prints a status of the translations - update: updates crowdin the current version of .ts files found in the source code - build: builds a new downloadable package on crowdin with all translated strings - download: downloads the latest build + status: prints a status of the translations + update: updates crowdin the current version of .ts files found in the source code + build: builds a new downloadable package on crowdin with all translated strings + download [build_id]: downloads build specified by 'build_id' or latest if build_id is left blank Example: ./updatecrowdin.py update -''' +Setting the project name adhoc: + + CROWDIN_PROJECT_ID=some_project ./updatecrowdin.py update +""" # See crowdin API docs at https://crowdin.com/page/api +import concurrent.futures +import glob +import json +import os +import sys +from collections import namedtuple +from functools import lru_cache +from os.path import basename, splitext +from urllib.parse import quote_plus +from urllib.request import Request +from urllib.request import urlopen +from urllib.request import urlretrieve + +TsFile = namedtuple('TsFile', ['filename', 'src_path']) + +LEGACY_NAMING_MAP = {'Draft.ts': 'draft.ts'} -import sys,os,xml.sax,pycurl,StringIO +class CrowdinUpdater: + + BASE_URL = 'https://api.crowdin.com/api/v2' + + def __init__(self, token, project_identifier, multithread=True): + self.token = token + self.project_identifier = project_identifier + self.multithread = multithread + + @lru_cache() + def _get_project_id(self): + url = f'{self.BASE_URL}/projects/' + response = self._make_api_req(url) + + for project in [p['data'] for p in response]: + if project['identifier'] == project_identifier: + return project['id'] + + raise Exception('No project identifier found!') + + def _make_project_api_req(self, project_path, *args, **kwargs): + url = f'{self.BASE_URL}/projects/{self._get_project_id()}{project_path}' + return self._make_api_req(url=url, *args, **kwargs) + + def _make_api_req(self, url, extra_headers={}, method='GET', data=None): + headers = {'Authorization': 'Bearer ' + load_token(), **extra_headers} + + if type(data) is dict: + headers['Content-Type'] = 'application/json' + data = json.dumps(data).encode('utf-8') + + request = Request(url, headers=headers, method=method, data=data) + return json.loads(urlopen(request).read())['data'] + + def _get_files_info(self): + files = self._make_project_api_req('/files?limit=250') + return {f['data']['path'].strip('/'): str(f['data']['id']) for f in files} + + def _add_storage(self, filename, fp): + response = self._make_api_req(f'{self.BASE_URL}/storages', data=fp, method='POST', extra_headers={ + 'Crowdin-API-FileName': filename, + 'Content-Type': 'application/octet-stream' + }) + return response['id'] + + def _update_file(self, project_id, ts_file, files_info): + filename = quote_plus(ts_file.filename) + + with open(ts_file.src_path, 'rb') as fp: + storage_id = self._add_storage(filename, fp) + + if filename in files_info: + file_id = files_info[filename] + self._make_project_api_req(f'/files/{file_id}', method='PUT', data={ + 'storageId': storage_id, + 'updateOption': 'keep_translations_and_approvals' + }) + print(f'{filename} updated') + else: + self._make_project_api_req('/files', data={ + 'storageId': storage_id, + 'name': filename + }) + print(f'{filename} uploaded') + + def status(self): + response = self._make_project_api_req('/languages/progress') + return [item['data'] for item in response] + + def download(self, build_id): + filename = f'{self.project_identifier}.zip' + response = self._make_project_api_req(f'/translations/builds/{build_id}/download') + urlretrieve(response['url'], filename) + print('download complete') + + def build(self): + self._make_project_api_req('/translations/builds', data={}, method='POST') + + def build_status(self): + response = self._make_project_api_req('/translations/builds') + return [item['data'] for item in response] + + def update(self, ts_files): + files_info = self._get_files_info() + futures = [] + + with concurrent.futures.ThreadPoolExecutor() as executor: + for ts_file in ts_files: + if self.multithread: + future = executor.submit(self._update_file, self.project_identifier, ts_file, files_info) + futures.append(future) + else: + self._update_file(self.project_identifier, ts_file, files_info) + + # This blocks until all futures are complete and will also throw any exception + for future in futures: + future.result() -files = [ ["AddonManager.ts", "/Mod/AddonManager/Resources/translations/AddonManager.ts"], - ["Arch.ts", "/Mod/Arch/Resources/translations/Arch.ts"], - ["Assembly.ts", "/Mod/Assembly/Gui/Resources/translations/Assembly.ts"], - ["draft.ts", "/Mod/Draft/Resources/translations/Draft.ts"], - ["Drawing.ts", "/Mod/Drawing/Gui/Resources/translations/Drawing.ts"], - ["Fem.ts", "/Mod/Fem/Gui/Resources/translations/Fem.ts"], - ["FreeCAD.ts", "/Gui/Language/FreeCAD.ts"], - ["Image.ts", "/Mod/Image/Gui/Resources/translations/Image.ts"], - ["Mesh.ts", "/Mod/Mesh/Gui/Resources/translations/Mesh.ts"], - ["MeshPart.ts", "/Mod/MeshPart/Gui/Resources/translations/MeshPart.ts"], - ["OpenSCAD.ts", "/Mod/OpenSCAD/Resources/translations/OpenSCAD.ts"], - ["Part.ts", "/Mod/Part/Gui/Resources/translations/Part.ts"], - ["PartDesign.ts", "/Mod/PartDesign/Gui/Resources/translations/PartDesign.ts"], - ["Plot.ts", "/Mod/Plot/resources/translations/Plot.ts"], - ["Points.ts", "/Mod/Points/Gui/Resources/translations/Points.ts"], - ["Raytracing.ts", "/Mod/Raytracing/Gui/Resources/translations/Raytracing.ts"], - ["ReverseEngineering.ts","/Mod/ReverseEngineering/Gui/Resources/translations/ReverseEngineering.ts"], - ["Robot.ts", "/Mod/Robot/Gui/Resources/translations/Robot.ts"], - ["Ship.ts", "/Mod/Ship/resources/translations/Ship.ts"], - ["Sketcher.ts", "/Mod/Sketcher/Gui/Resources/translations/Sketcher.ts"], - ["StartPage.ts", "/Mod/Start/Gui/Resources/translations/StartPage.ts"], - ["Test.ts", "/Mod/Test/Gui/Resources/translations/Test.ts"], - ["Web.ts", "/Mod/Web/Gui/Resources/translations/Web.ts"], - ["Spreadsheet.ts", "/Mod/Spreadsheet/Gui/Resources/translations/Spreadsheet.ts"], - ["Path.ts", "/Mod/Path/Gui/Resources/translations/Path.ts"], - ["Tux.ts", "/Mod/Tux/Resources/translations/Tux.ts"], - ["TechDraw.ts", "/Mod/TechDraw/Gui/Resources/translations/TechDraw.ts"], - ] - - -# handler for the command responses -class ResponseHandler( xml.sax.ContentHandler ): - - def __init__(self): - self.current = "" - self.data = "" - self.translated = 1 - self.total = 1 - - def startElement(self, tag, attributes): - self.current = tag - if tag == "file": - self.data += attributes["status"] - elif tag == "error": - self.data == "Error: " - - def endElement(self, tag): - if self.current in ["language","success","error"]: - self.data = "" - self.translated = 1 - self.total = 1 - self.current = "" - - def characters(self, content): - if self.current == "name": - self.data += content - elif self.current == "phrases": - self.total = int(content) - elif self.current == "translated": - self.translated = int(content) - pc = int((float(self.translated)/self.total)*100) - self.data += " : " + str(pc) + "%\n" - elif self.current == "message": - self.data += content - +def load_token(): + # load API token stored in ~/.crowdin-freecad-token + config_file = os.path.expanduser('~')+os.sep+".crowdin-freecad-token" + if os.path.exists(config_file): + with open(config_file) as file: + return file.read().strip() + return None if __name__ == "__main__": + command = None - # only one argument allowed - arg = sys.argv[1:] - if len(arg) != 1: - print(__doc__) + args = sys.argv[1:] + if args: + command = args[0] + + token = os.environ.get('CROWDIN_TOKEN', load_token()) + if command and not token: + print('Token not found') sys.exit() - arg = arg[0] - # getting API key stored in ~/.crowdin-freecad - configfile = os.path.expanduser("~")+os.sep+".crowdin-freecad" - if not os.path.exists(configfile): - print("Config file not found!") + project_identifier = os.environ.get('CROWDIN_PROJECT_ID') + if not project_identifier: + print('CROWDIN_PROJECT_ID env var must be set') sys.exit() - f = open(configfile) - url = "https://api.crowdin.com/api/project/freecad/" - key = "?key="+f.read().strip() - f.close() - if arg == "status": - c = pycurl.Curl() - c.setopt(pycurl.URL, url+"status"+key+"&xml") - b = StringIO.StringIO() - c.setopt(pycurl.WRITEFUNCTION, b.write) - c.perform() - c.close() - handler = ResponseHandler() - xml.sax.parseString(b.getvalue(),handler) - print(handler.data) + updater = CrowdinUpdater(token, project_identifier) - elif arg == "build": - print("Building (warning, this can be invoked only once per 30 minutes)...") - c = pycurl.Curl() - c.setopt(pycurl.URL, url+"export"+key) - b = StringIO.StringIO() - c.setopt(pycurl.WRITEFUNCTION, b.write) - c.perform() - c.close() - handler = ResponseHandler() - xml.sax.parseString(b.getvalue(),handler) - print(handler.data) + if command == "status": + status = updater.status() + for item in status: + print(f"language: {item['languageId']}") + print(f" translation progress: {item['translationProgress']}%") + print(f" approval progress: {item['approvalProgress']}%") - elif arg == "download": - print("Downloading all.zip in current directory...") - cmd = "wget -O freecad.zip "+url+"download/all.zip"+key - os.system(cmd) + elif command == "build-status": + for item in updater.build_status(): + print(f" id: {item['id']} progress: {item['progress']}% status: {item['status']}") - elif arg == "update": - basepath = os.path.dirname(os.path.abspath(".")) - for f in files: - print("Sending ",f[0],"...") - c = pycurl.Curl() - fields = [('files['+f[0]+']', (c.FORM_FILE, basepath+f[1]))] - c.setopt(pycurl.URL, url+"update-file"+key) - c.setopt(pycurl.HTTPPOST, fields) - b = StringIO.StringIO() - c.setopt(pycurl.WRITEFUNCTION, b.write) - c.perform() - c.close() - handler = ResponseHandler() - xml.sax.parseString(b.getvalue(),handler) - print(handler.data) + elif command == "build": + updater.build() + elif command == "download": + if len(args) == 2: + updater.download(args[1]) + else: + stat = updater.build_status() + if not stat: + print('no builds found') + elif len(stat) == 1: + updater.download(stat[0]['id']) + else: + print('available builds:') + for item in stat: + print(f" id: {item['id']} progress: {item['progress']}% status: {item['status']}") + print('please specify a build id') + + elif command == "update": + # Find all ts files. However, this contains the lang-specific files too. Let's drop those + all_ts_files = glob.glob('../**/*.ts', recursive=True) + # Remove the file extensions + ts_files_wo_ext = [splitext(f)[0] for f in all_ts_files] + # Filter out any file that has another file as a substring. E.g. Draft is a substring of Draft_en + main_ts_files = list(filter(lambda f: not [a for a in ts_files_wo_ext if a in f and f != a], ts_files_wo_ext)) + # Create tuples to map Crowdin name with local path name + names_and_path = [(f'{basename(f)}.ts', f'{f}.ts') for f in main_ts_files] + # Accomodate for legacy naming + ts_files = [TsFile(LEGACY_NAMING_MAP[a] if a in LEGACY_NAMING_MAP else a, b) for (a, b) in names_and_path] + + updater.update(ts_files) else: print(__doc__)