| |
@@ -0,0 +1,241 @@
|
| |
+ #!/usr/bin/env python3
|
| |
+ """Calls `./src-to-pot.sh` and `./po-to-src.sh` for all source repos"""
|
| |
+
|
| |
+ import argparse
|
| |
+ import os
|
| |
+ import urllib.request
|
| |
+ import subprocess
|
| |
+
|
| |
+ from urllib.parse import urlparse
|
| |
+
|
| |
+ import yaml
|
| |
+
|
| |
+ SELECTED_COMPONENT = None
|
| |
+
|
| |
+ def main():
|
| |
+ """Handle params"""
|
| |
+ global SELECTED_COMPONENT
|
| |
+
|
| |
+ parser = argparse.ArgumentParser(
|
| |
+ description="Calls `./src-to-pot.sh` and `./po-to-src.sh` for all source repos")
|
| |
+ parser.add_argument("--clone_sources", choices=['true', 'false'], required=True,
|
| |
+ help="Should we clone/update sources or use local content?")
|
| |
+ parser.add_argument("--commit_l10n", choices=['true', 'false'], required=True,
|
| |
+ help="Should we commit changes on l10n repos?")
|
| |
+ parser.add_argument("--commit_tsources", choices=['true', 'false'], required=True,
|
| |
+ help="Should we commit translated sources?")
|
| |
+ parser.add_argument("--component", required=False,
|
| |
+ help="Antora component to convert, all if unset")
|
| |
+ args = parser.parse_args()
|
| |
+
|
| |
+ SELECTED_COMPONENT = args.component
|
| |
+
|
| |
+ source_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/"
|
| |
+ trans_source_dir = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/"
|
| |
+ l10n_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/"
|
| |
+
|
| |
+ os.makedirs(source_dir, exist_ok=True)
|
| |
+ os.makedirs(trans_source_dir, exist_ok=True)
|
| |
+ os.makedirs(l10n_dir, exist_ok=True)
|
| |
+
|
| |
+ if args.clone_sources == "true":
|
| |
+ clone_sources()
|
| |
+
|
| |
+ clone_translated_source()
|
| |
+ make_component_lists()
|
| |
+
|
| |
+ if args.commit_l10n == "true":
|
| |
+ commit_l10n_repos()
|
| |
+
|
| |
+ if args.commit_tsources == "true":
|
| |
+ commit_translated_sources()
|
| |
+
|
| |
+ def call_src_to_pot(url, branch='master', basedir='/', module="ROOT"):
|
| |
+ """Call src-to-ot.sh to convert english source (adoc) content to pot files"""
|
| |
+ print("call_src_to_pot: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module)
|
| |
+ output = os.path.dirname(os.path.abspath(__file__)) + "/l10n/"
|
| |
+
|
| |
+ subprocess.run(['./src-to-pot.sh', "--src-branch", branch,
|
| |
+ "--src-basedir", basedir, "--module", module, url, output], check=True)
|
| |
+
|
| |
+ def call_po_to_src(url, branch='master', basedir='/', module="ROOT"):
|
| |
+ """Call po-to-src.sh to convert po files to translated sources (adoc)"""
|
| |
+ print("call_po_to_src: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module)
|
| |
+ output = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/"
|
| |
+
|
| |
+ subprocess.run(['./po-to-src.sh', "--src-branch", branch,
|
| |
+ "--src-basedir", basedir, "--module", module, url,
|
| |
+ output], check=True)
|
| |
+
|
| |
+ def clone_source(doc_url, branch="master"):
|
| |
+ """Clone english sources in sources folder (update content if existing)"""
|
| |
+ repo_name = urlparse(doc_url).hostname+urlparse(doc_url).path[1:].replace('.git', '')
|
| |
+ repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name
|
| |
+
|
| |
+ if os.path.exists(repo_dir):
|
| |
+ subprocess.run(['git', 'fetch'], check=True, cwd=repo_dir)
|
| |
+ subprocess.run(['git', 'checkout', branch], check=True, cwd=repo_dir)
|
| |
+ subprocess.run(['git', 'pull'], check=True, cwd=repo_dir)
|
| |
+ else:
|
| |
+ subprocess.run(["git", "clone", "-b", branch, doc_url, repo_dir], check=True)
|
| |
+
|
| |
+ def clone_l10n(repo_name):
|
| |
+ """Clone localization repos in l10n folder (update content if existing)"""
|
| |
+ print("* clone_l10n: " + repo_name)
|
| |
+
|
| |
+ url = "https://pagure.io/fedora-l10n/" + repo_name
|
| |
+ repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name
|
| |
+
|
| |
+ if os.path.exists(repo_dir):
|
| |
+ subprocess.run(['git', 'pull'], check=True, cwd=repo_dir)
|
| |
+ else:
|
| |
+ subprocess.run(["git", "clone", url, repo_dir, "--config",
|
| |
+ "remote.origin.pushurl=ssh://git@pagure.io/fedora-l10n/{r}.git".format(r=repo_name)],
|
| |
+ check=True)
|
| |
+
|
| |
+ def clone_translated_source():
|
| |
+ """Clone translated sources (update content if existing)"""
|
| |
+ print("* clone_translated_source")
|
| |
+ repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/"
|
| |
+
|
| |
+ url = "ssh://git@pagure.io/fedora-docs/translated-sources.git"
|
| |
+
|
| |
+ if os.path.exists(repo_dir):
|
| |
+ subprocess.run(['git', 'pull'], check=True, cwd=repo_dir)
|
| |
+ else:
|
| |
+ subprocess.run(["git", "clone", url, repo_dir], check=True)
|
| |
+
|
| |
+ def commit_translated_source():
|
| |
+ """Commit translated sources"""
|
| |
+ print("* commit_translated_source")
|
| |
+ repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/"
|
| |
+
|
| |
+ subprocess.run(['git', 'add', '.'], check=True, cwd=repo_dir)
|
| |
+
|
| |
+ subprocess.run(["git", "commit", "-m", "automatic update of translated content"],
|
| |
+ check=True, cwd=repo_dir)
|
| |
+
|
| |
+ subprocess.run(['git', 'push'], check=True, cwd=repo_dir)
|
| |
+
|
| |
+
|
| |
+ def commit_l10n_repos():
|
| |
+ """Commit and push changes of localization repositories"""
|
| |
+
|
| |
+ print("* commit_l10n_repos")
|
| |
+ repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/"
|
| |
+
|
| |
+ # commit each folders of the l10n path
|
| |
+ [commit_l10n(r) for r in next(os.walk(repo_dir))[1]]
|
| |
+
|
| |
+
|
| |
+ def commit_l10n(repo_name):
|
| |
+ """Commit and push changes of a localization repo"""
|
| |
+
|
| |
+ print("* commit_l10n: " + repo_name)
|
| |
+
|
| |
+ url = "https://pagure.io/fedora-l10n/" + repo_name
|
| |
+ repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name
|
| |
+
|
| |
+ subprocess.run(['git', 'add', '.'], check=True, cwd=repo_dir)
|
| |
+
|
| |
+ subprocess.run(["git", "commit", "-m", "automatic update of pot/po files"],
|
| |
+ check=True, cwd=repo_dir)
|
| |
+
|
| |
+ subprocess.run(['git', 'push'], check=True, cwd=repo_dir)
|
| |
+
|
| |
+ def clone_sources():
|
| |
+ """List sources repositories from antora yaml file"""
|
| |
+ print("* clone_sources:")
|
| |
+
|
| |
+ # download site.yml
|
| |
+ urllib.request.urlretrieve(
|
| |
+ "https://pagure.io/fedora-docs/docs-fp-o/raw/master/f/site.yml", "site.yml"
|
| |
+ )
|
| |
+
|
| |
+ # Read site.yml
|
| |
+ with open("site.yml", 'r') as stream:
|
| |
+ data_loaded = yaml.load(stream, Loader=yaml.SafeLoader)
|
| |
+
|
| |
+ # Parse site.yml
|
| |
+ for source in data_loaded['content']['sources']:
|
| |
+ doc_url = source['url']
|
| |
+
|
| |
+ if 'start_path' in source:
|
| |
+ clone_source(doc_url)
|
| |
+
|
| |
+ elif 'branches' in source:
|
| |
+ for branch in source['branches']:
|
| |
+ clone_source(doc_url, branch)
|
| |
+ else:
|
| |
+ clone_source(doc_url)
|
| |
+
|
| |
+
|
| |
+ # Remove site.yml
|
| |
+ os.remove("site.yml")
|
| |
+
|
| |
+ def get_component(comp_dir, basedir='/', branch='master'):
|
| |
+ """Run actions on the component (clone repos, run scripts, commit changes)"""
|
| |
+ print("\n\nget_component: "+ comp_dir + "/" + basedir + " " + branch)
|
| |
+ subprocess.run(['git', 'fetch'], check=True, cwd=comp_dir)
|
| |
+ subprocess.run(['git', 'checkout', branch], check=True, cwd=comp_dir)
|
| |
+ subprocess.run(['git', 'pull'], check=True, cwd=comp_dir)
|
| |
+
|
| |
+ with open(comp_dir +"/"+ basedir + "/antora.yml", 'r') as stream:
|
| |
+ data_loaded = yaml.load(stream, Loader=yaml.SafeLoader)
|
| |
+
|
| |
+ component = data_loaded['name']
|
| |
+ version = data_loaded['version']
|
| |
+
|
| |
+ if SELECTED_COMPONENT not in (None, component):
|
| |
+ print("--component was set to {s}, ignoring {c}".format(s=SELECTED_COMPONENT, c=component))
|
| |
+ return
|
| |
+
|
| |
+ modules = []
|
| |
+ for (dirpath, dirnames, filenames) in os.walk(comp_dir + "/" + basedir + "/modules"):
|
| |
+ modules.extend(dirnames)
|
| |
+ break
|
| |
+
|
| |
+ if not modules:
|
| |
+ raise ValueError('There should be at least one module!')
|
| |
+
|
| |
+ for module in modules:
|
| |
+ if module == "ROOT":
|
| |
+ target_repo_name = component
|
| |
+ else:
|
| |
+ target_repo_name = component + "-" + module
|
| |
+ clone_l10n(target_repo_name)
|
| |
+ call_src_to_pot(url=comp_dir, basedir=basedir, branch=branch, module=module)
|
| |
+ call_po_to_src(url=comp_dir, basedir=basedir, branch=branch, module=module)
|
| |
+
|
| |
+ def make_component_lists():
|
| |
+ """Deduct the component name from site.yml (l10n repos are per component)"""
|
| |
+
|
| |
+ # download site.yml
|
| |
+ urllib.request.urlretrieve(
|
| |
+ "https://pagure.io/fedora-docs/docs-fp-o/raw/master/f/site.yml", "site.yml"
|
| |
+ )
|
| |
+
|
| |
+ # Read site.yml
|
| |
+ with open("site.yml", 'r') as stream:
|
| |
+ data_loaded = yaml.load(stream, Loader=yaml.SafeLoader)
|
| |
+
|
| |
+ # Parse site.yml
|
| |
+ for source in data_loaded['content']['sources']:
|
| |
+ doc_url = source['url']
|
| |
+ repo_name = urlparse(doc_url).hostname+urlparse(doc_url).path[1:].replace('.git', '')
|
| |
+ repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name
|
| |
+
|
| |
+ if 'start_path' in source:
|
| |
+ get_component(repo_dir, basedir=source['start_path'])
|
| |
+
|
| |
+ elif 'branches' in source:
|
| |
+ for branch in source['branches']:
|
| |
+ get_component(repo_dir, branch=branch)
|
| |
+ else:
|
| |
+ get_component(repo_dir)
|
| |
+
|
| |
+ # Remove site.yml
|
| |
+ os.remove("site.yml")
|
| |
+
|
| |
+ if __name__ == '__main__':
|
| |
+ main()
|
| |
jibecfed
commented 5 years ago
here is the work we did together.
we rewrote all-po-to-src.sh and all-src-to-pot.sh in a build.py script