From 59036c2eb15cf5afe22afaa2c7626592b7629694 Mon Sep 17 00:00:00 2001 From: Adam Samalik Date: Oct 02 2019 18:29:34 +0000 Subject: rewrite language building --- diff --git a/build-scripts/Dockerfile b/build-scripts/Dockerfile index 2f7cd80..0f41de8 100644 --- a/build-scripts/Dockerfile +++ b/build-scripts/Dockerfile @@ -1,13 +1,13 @@ FROM fedora:29 RUN dnf -y module install nodejs:10/default && \ - dnf -y install git && \ + dnf -y install git python3-yaml && \ dnf clean all RUN npm i -g @antora/cli@2.0 @antora/site-generator-default@2.0 -ADD rebuild-site.sh /antora/rebuild-site.sh +ADD build-translated.py /antora/build-translated.py WORKDIR /antora -CMD /antora/rebuild-site.sh +CMD /antora/build-translated.py diff --git a/build-scripts/build-translated.py b/build-scripts/build-translated.py new file mode 100755 index 0000000..fad81b0 --- /dev/null +++ b/build-scripts/build-translated.py @@ -0,0 +1,199 @@ +#!/usr/bin/python3 + +import tempfile, yaml, os, errno, subprocess, copy, datetime, shutil, sys + +GLOBAL_CONFIG = { + "docs_repo_url": "https://pagure.io/fedora-docs/docs-fp-o.git", + "docs_repo_branch": "stg", + "translated_sources_repo_url": "https://pagure.io/fedora-docs/translated-sources.git" +} + + +def log(msg): + print(msg) + + +def get_languages(config): + languages_dict = {} + + with tempfile.TemporaryDirectory() as workdir: + translated_sources_repo = os.path.join(workdir, "translated_sources_repo") + subprocess.run(["git", "clone", config["translated_sources_repo_url"], translated_sources_repo]) + + languages = [] + filename_blacklist = [".git"] + for filename in os.listdir(translated_sources_repo): + filepath = os.path.join(translated_sources_repo, filename) + if os.path.isdir(filepath) and filename not in filename_blacklist: + languages.append(filename) + + languages.sort() + + for lang in languages: + languages_dict[lang] = [] + + lang_dir = os.path.join(translated_sources_repo, lang) + for component in os.listdir(lang_dir): + version_dir = os.path.join(lang_dir, component) + for version in os.listdir(version_dir): + start_path = "{lang}/{component}/{version}".format( + lang=lang, component=component, version=version) + + # This is a workaround for cases when a component doesn't have + # the ROOT module that should contain the antora.yml. + # Without the ROOT module the translation scripts won't + # pick up the antora.yml causing the docs build to fail. + # So we're only including directories with the antora.yml file. + if "antora.yml" in os.listdir(os.path.join(version_dir, version)): + languages_dict[lang].append(start_path) + + return languages_dict + + +def generate_lang_switch_ui(languages): + template_start = """ +
+ +
+ """ + template_end = """ +
+
+ """ + + template_list = [] + template_list.append(template_start) + for language in languages: + link = '{language}'.format( + language=language) + template_list.append(link) + template_list.append(template_end) + + return "\n".join(template_list) + + +def main(): + config = GLOBAL_CONFIG + + log("") + log("===== Getting a list of languages =====") + languages = get_languages(config) + log(" Languages: {}".format(" ".join(languages))) + + + log("") + log("===== Generating the language switch UI =====") + lang_switch_ui = generate_lang_switch_ui(["en-US"] + list(languages.keys())) + + + log("") + log("===== Getting the site definition =====") + with tempfile.TemporaryDirectory() as workdir: + docs_repo = os.path.join(workdir, "docs_repo") + subprocess.run(["git", "clone", "--branch", config["docs_repo_branch"], config["docs_repo_url"], docs_repo]) + + with open(os.path.join(docs_repo, "site.yml"), "r") as file: + original_site_yml = yaml.safe_load(file) + + + log("") + log("===== Creating the language switch UI files =====") + ui_dir = os.path.join(docs_repo, "supplemental-ui", "partials") + try: + os.makedirs(ui_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + ui_file = os.path.join(ui_dir, "page-languages.hbs") + with open(ui_file, "w") as file: + file.write(lang_switch_ui) + + log("") + log("===== Generating site.lang.yml files =====") + for lang, start_paths in languages.items(): + lang_site_yml = copy.deepcopy(original_site_yml) + + lang_site_yml["output"]["dir"] = "./public/{lang}".format(lang=lang) + + lang_site_yml["content"] = {} + lang_site_yml["content"]["branches"] = "master" + lang_site_yml["content"]["sources"] = [] + + for start_path in start_paths: + source = {} + source["url"] = config["translated_sources_repo_url"] + source["start_path"] = start_path + + lang_site_yml["content"]["sources"].append(source) + + filename = os.path.join(docs_repo, "site-{lang}.yml".format(lang=lang)) + with open(filename, "w") as file: + file.write(yaml.dump(lang_site_yml)) + + log(" {lang} done".format(lang=lang)) + + + # Timestamp to be included in the footer of the docs + timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S UTC') + antora_env = {"ANTORA_DATE": timestamp} + + # First, building and copying the en-US site + log("") + log("===== Building the en-US site =====") + result = subprocess.run(["antora", "--html-url-extension-style=indexify", os.path.join(docs_repo, "site.yml")], env=antora_env) + + if result.returncode != 0: + log("ERROR building the en-US site") + sys.exit(1) + + + log("") + log("===== Copying the en-US site =====") + source_dir = os.path.join(docs_repo, "public") + target_dir_copying = "/antora/output/en-US.building" + target_dir_copied = "/antora/output/en-US.building/en-US" + target_dir_final = "/antora/output/en-US" + shutil.rmtree(target_dir_copying, ignore_errors=True) + subprocess.run(["cp", "-r", source_dir, target_dir_copying]) + shutil.rmtree(target_dir_final, ignore_errors=True) + shutil.move(target_dir_copied, target_dir_final) + shutil.rmtree(target_dir_copying, ignore_errors=True) + + log("") + log("===== Copying the index.html =====") + source_index_html = os.path.join(docs_repo, "static", "index.html") + target_index_html = "/antora/output/index.html" + shutil.copy(source_index_html, target_index_html) + + + # Second, building and copying all the translated sites + for lang in languages: + log("") + log("===== Building the {lang} site =====".format(lang=lang)) + filename = "site-{lang}.yml".format(lang=lang) + result = subprocess.run(["antora", "--html-url-extension-style=indexify", os.path.join(docs_repo, filename)], env=antora_env) + + if result.returncode != 0: + log("ERROR building the {lang} site".format(lang=lang)) + sys.exit(1) + + + for lang in languages: + log("") + log("===== Copying the {lang} site =====".format(lang=lang)) + source_dir = os.path.join(docs_repo, "public") + target_dir_copying = "/antora/output/{lang}.building".format(lang=lang) + target_dir_copied = "/antora/output/{lang}.building/{lang}".format(lang=lang) + target_dir_final = "/antora/output/{lang}".format(lang=lang) + shutil.rmtree(target_dir_copying, ignore_errors=True) + subprocess.run(["cp", "-r", source_dir, target_dir_copying]) + shutil.rmtree(target_dir_final, ignore_errors=True) + shutil.move(target_dir_copied, target_dir_final) + shutil.rmtree(target_dir_copying, ignore_errors=True) + + + + +if __name__ == "__main__": + main() diff --git a/build-scripts/rebuild-site.sh b/build-scripts/rebuild-site.sh deleted file mode 100755 index 52b6f34..0000000 --- a/build-scripts/rebuild-site.sh +++ /dev/null @@ -1,97 +0,0 @@ -#/bin/bash - -echo "" -echo "$(date +%G-%m-%d-%H%M%S): Building Fedora Docs..." -echo "" - -SOURCE_REPO="https://pagure.io/fedora-docs/docs-fp-o.git" -BUILD_CMD="antora --html-url-extension-style=indexify site.yml" - -target="/antora" - -if [ -z ${BUILD_ENV+x} ]; then - BUILD_ENV="master" -fi - - -if ! workdir=$(mktemp -d) ; then - echo "Error creating a tempdir. Exiting." - exit 1 -fi - -if ! git clone --single-branch --branch $BUILD_ENV $SOURCE_REPO "$workdir/source" ; then - echo "" - echo "Error cloning the source repo." - rm -rf "$workdir" - exit 1 -else - echo "OK cloned source repo" -fi - -if [ ! -f "$workdir/source/site.yml" ]; then - echo "" - echo "Error! There is no site.yml in the source." - rm -rf "$workdir" - exit 1 -fi - -pushd "$workdir/source/" > /dev/null - - -export ANTORA_DATE=$(date -u) - -if ! $BUILD_CMD ; then - echo "" - echo "Error building the site." - rm -rf "$workdir" - exit 1 -fi - -# FIXME: Building languages, this code needs some love - -languages="cs fr jp" - -create_language_menu() { - { - echo '
' - echo " " - echo '
' - echo ' en-US' - for lang in $languages; do - echo " $lang" - done - echo '
' - echo '
' - } > "$workdir/source/supplemental-ui/partials/page-languages.hbs" -} - -for current_lang in $languages; do - create_language_menu "$current_lang" - if ! antora --html-url-extension-style=indexify site-$current_lang.yml ; then - echo "" - echo "Error building translated site for $current_lang." - rm -rf "$workdir" - exit 1 - fi -done - -create_language_menu "en-US" - -# End building languages - -popd > /dev/null - -cp -r "$workdir/source/public/en-US" "$target/output/en-US.building" && rm -rf "$target/output/en-US" && mv "$target/output/en-US.building" "$target/output/en-US" || exit 1 -cp "$workdir/source/static/index.html" "$target/output/index.html" || exit 1 - -# copy languages -for current_lang in $languages; do - cp -r "$workdir/source/public/$current_lang" "$target/output/$current_lang.building" && rm -rf "$target/output/$current_lang" && mv "$target/output/$current_lang.building" "$target/output/$current_lang" || exit 1 -done -# end languages - -rm -rf "$workdir" - -echo "" -echo "$(date +%G-%m-%d-%H%M%S): Building of Fedora Docs has finished successfully!" -echo "" diff --git a/build-translated.sh b/build-translated.sh deleted file mode 100755 index 11b657c..0000000 --- a/build-translated.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -languages="" - -for file in site-*; -do - # we have site-???.yml, we want ??? - suffix=${file:5} # remove "site-" - lang=${suffix%.yml} # remove ".yml" - languages="$languages $lang" -done - -create_language_menu() { - { - echo '
' - echo " " - echo '
' - echo ' en-US' - for lang in $languages; do - echo " $lang" - done - echo '
' - echo '
' - } > ./supplemental-ui/partials/page-languages.hbs -} - -create_language_menu "en-US" -exit - -if [ "$(uname)" == "Darwin" ]; then - for current_lang in $languages; do - create_language_menu "$current_lang" - docker run -e "LANGUAGE=$current_lang" -e "ANTORA_DATE=$(date -u)" --rm -it -v $(pwd):/antora antora/antora --html-url-extension-style=indexify site-$current_lang.yml - done - -elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then - for current_lang in $languages; do - create_language_menu "$current_lang" - podman run -e "LANGUAGE=$current_lang" -e "ANTORA_DATE=$(date -u)" --rm -it -v $(pwd):/antora:z antora/antora --html-url-extension-style=indexify site-$current_lang.yml - done -fi - -create_language_menu "en-US" diff --git a/prepare-translated.py b/prepare-translated.py deleted file mode 100755 index d81f9bb..0000000 --- a/prepare-translated.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 -"""Prepare site.yml for each translated sites""" - -import os -import subprocess -import tempfile -import yaml - -def main(): - """ Read site.yml content, replace `sources` content with localized content""" - lang_sources = get_language_sources() - - with open("site.yml", 'r') as stream: - data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) - - for lang in lang_sources: - print(lang) - - with open('site-'+lang+'.yml', 'w') as outfile: - data_loaded['content']['sources'] = lang_sources[lang] - yaml.dump(data_loaded, outfile, default_flow_style=False) - - -def get_language_sources(): - """ Get the list of languages + modules + branches from translated-sources""" - sources = {} - with tempfile.TemporaryDirectory() as tmp: - t_sources_url = "https://pagure.io/fedora-docs/translated-sources.git" - subprocess.run(["git", "clone", t_sources_url], check=True, cwd=tmp) - - path = tmp+"/translated-sources" - - # we don't want ".git" folder - langs = [d for d in next(os.walk(path))[1] if not d[0] == '.'] - for lang in langs: - sources[lang] = [] - for module in next(os.walk(path+"/"+lang))[1]: - for branch in next(os.walk(path+"/"+lang + "/" + module))[1]: - hop = {} - hop["url"] = t_sources_url - hop["start_path"] = lang+"/"+module+"/"+branch - sources[lang].append(hop) - - return sources - - -if __name__ == '__main__': - main() diff --git a/site-cs.yml b/site-cs.yml deleted file mode 100644 index 4d407cb..0000000 --- a/site-cs.yml +++ /dev/null @@ -1,80 +0,0 @@ -asciidoc: - extensions: - - ./lib/extensions/package-inline-macro.js -content: - branches: master - sources: - - start_path: cs/teleirc-sig/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/remix-building/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/quick-docs/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/project/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/packaging-guidelines/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/neurofedora/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/modularity/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/mindshare/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/mindshare-committee/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/mentored-projects/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/java-packaging-howto/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/iot/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/flatpak/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/fesco/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/fedora/rawhide - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/fedora/f29 - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/fedora/f28 - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/fedora/f27 - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/fedora/f26 - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/fedora-silverblue/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/fedora-docs/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/engineering/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/docs/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/diversity-inclusion/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/council/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/containers/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/commops/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: cs/badges/master - url: https://pagure.io/fedora-docs/translated-sources.git -output: - clean: true - destinations: - - provider: archive - dir: ./public/en-US -runtime: - cache_dir: ./cache - pull: true -site: - start_page: docs::index.adoc - title: Fedora Docs Site - url: https://docs.fedoraproject.org/en-US/ -ui: - bundle: - snapshot: true - url: https://asamalik.fedorapeople.org/docs-translations/ui-bundle.zip - default_layout: with_menu - supplemental_files: ./supplemental-ui diff --git a/site-fr.yml b/site-fr.yml deleted file mode 100644 index 0f86a5c..0000000 --- a/site-fr.yml +++ /dev/null @@ -1,84 +0,0 @@ -asciidoc: - extensions: - - ./lib/extensions/package-inline-macro.js -content: - branches: master - sources: - - start_path: fr/teleirc-sig/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/taiga-docs/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/remix-building/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/quick-docs/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/project/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/packaging-guidelines/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/neurofedora/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/modularity/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/mindshare/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/mindshare-committee/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/mentored-projects/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/java-packaging-howto/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/iot/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/flatpak/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/fesco/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/fedora/rawhide - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/fedora/f29 - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/fedora/f28 - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/fedora/f27 - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/fedora/f26 - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/fedora-silverblue/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/fedora-docs/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/engineering/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/docs/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/diversity-inclusion/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/council/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/containers/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/commops/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/ci/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: fr/badges/master - url: https://pagure.io/fedora-docs/translated-sources.git -output: - clean: true - destinations: - - provider: archive - dir: ./public/en-US -runtime: - cache_dir: ./cache - pull: true -site: - start_page: docs::index.adoc - title: Fedora Docs Site - url: https://docs.fedoraproject.org/en-US/ -ui: - bundle: - snapshot: true - url: https://asamalik.fedorapeople.org/docs-translations/ui-bundle.zip - default_layout: with_menu - supplemental_files: ./supplemental-ui diff --git a/site-ja.yml b/site-ja.yml deleted file mode 100644 index 3633ba2..0000000 --- a/site-ja.yml +++ /dev/null @@ -1,80 +0,0 @@ -asciidoc: - extensions: - - ./lib/extensions/package-inline-macro.js -content: - branches: master - sources: - - start_path: ja/teleirc-sig/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/remix-building/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/quick-docs/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/project/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/packaging-guidelines/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/neurofedora/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/modularity/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/mindshare/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/mindshare-committee/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/mentored-projects/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/java-packaging-howto/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/iot/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/flatpak/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/fesco/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/fedora/rawhide - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/fedora/f29 - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/fedora/f28 - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/fedora/f27 - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/fedora/f26 - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/fedora-silverblue/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/fedora-docs/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/engineering/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/docs/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/diversity-inclusion/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/council/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/containers/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/commops/master - url: https://pagure.io/fedora-docs/translated-sources.git - - start_path: ja/badges/master - url: https://pagure.io/fedora-docs/translated-sources.git -output: - clean: true - destinations: - - provider: archive - dir: ./public/en-US -runtime: - cache_dir: ./cache - pull: true -site: - start_page: docs::index.adoc - title: Fedora Docs Site - url: https://docs.fedoraproject.org/en-US/ -ui: - bundle: - snapshot: true - url: https://asamalik.fedorapeople.org/docs-translations/ui-bundle.zip - default_layout: with_menu - supplemental_files: ./supplemental-ui diff --git a/supplemental-ui/partials/page-languages.hbs b/supplemental-ui/partials/page-languages.hbs index b65342e..b5858f3 100644 --- a/supplemental-ui/partials/page-languages.hbs +++ b/supplemental-ui/partials/page-languages.hbs @@ -2,8 +2,5 @@
en-US - cs - fr - jp