| |
@@ -0,0 +1,468 @@
|
| |
+ #!/usr/bin/python3
|
| |
+
|
| |
+
|
| |
+ # The script requires the following environment variables to be set:
|
| |
+ #
|
| |
+ # Variable Values Effect
|
| |
+ # -----------------|-----------------------------------------------------------
|
| |
+ # BUILD_ENV prod Build the production version of the site
|
| |
+ # using the "prod" branch. DEFAULT
|
| |
+ # stg Build the staging version of the site
|
| |
+ # using the "stg" branch.
|
| |
+ # -----------------|-----------------------------------------------------------
|
| |
+ # BUILD_LANGS english Build the "en-US" version only. DEFAULT
|
| |
+ # translated Build only the translated versions.
|
| |
+ # all Build the "en-US" and the translated versions.
|
| |
+ #
|
| |
+ #
|
| |
+
|
| |
+
|
| |
+ import tempfile, yaml, os, errno, subprocess, copy, datetime, shutil, sys, glob
|
| |
+
|
| |
+
|
| |
+ def get_config():
|
| |
+ config = {}
|
| |
+
|
| |
+ config["docs_repo_branch"] = os.getenv(
|
| |
+ "BUILD_ENV",
|
| |
+ "prod")
|
| |
+
|
| |
+ config["build_langs"] = os.getenv(
|
| |
+ "BUILD_LANGS",
|
| |
+ "english")
|
| |
+
|
| |
+ config["docs_repo_url"] = os.getenv(
|
| |
+ "DOCS_REPO_URL",
|
| |
+ "https://pagure.io/fedora-docs/docs-fp-o.git")
|
| |
+
|
| |
+ config["translated_sources_repo_url"] = os.getenv(
|
| |
+ "TRANSLATED_SOURCES_REPO_URL",
|
| |
+ "https://pagure.io/fedora-docs/translated-sources.git"
|
| |
+ )
|
| |
+
|
| |
+ config["translated_adockeywords_repo_url"] = os.getenv(
|
| |
+ "TRANSLATED_ADOCKEYWORDS_REPO_URL",
|
| |
+ "https://pagure.io/fedora-docs-l10n/asciidoc-keywords.git")
|
| |
+
|
| |
+ return config
|
| |
+
|
| |
+
|
| |
+ def log(msg):
|
| |
+ print(msg, flush=True)
|
| |
+
|
| |
+
|
| |
+ def get_languages(config):
|
| |
+ languages_dict = {}
|
| |
+
|
| |
+ with tempfile.TemporaryDirectory() as workdir:
|
| |
+ translated_sources_repo = os.path.join(workdir, "translated_sources_repo")
|
| |
+ subprocess.run(["git", "clone", "--depth=1", config["translated_sources_repo_url"], translated_sources_repo])
|
| |
+
|
| |
+ languages = []
|
| |
+ filename_blacklist = [".git"]
|
| |
+ for filename in os.listdir(translated_sources_repo):
|
| |
+ filepath = os.path.join(translated_sources_repo, filename)
|
| |
+ if os.path.isdir(filepath) and filename not in filename_blacklist:
|
| |
+ languages.append(filename)
|
| |
+
|
| |
+ languages.sort()
|
| |
+
|
| |
+ for lang in languages:
|
| |
+ languages_dict[lang] = []
|
| |
+
|
| |
+ lang_dir = os.path.join(translated_sources_repo, lang)
|
| |
+ for component in os.listdir(lang_dir):
|
| |
+ version_dir = os.path.join(lang_dir, component)
|
| |
+ for version in os.listdir(version_dir):
|
| |
+ start_path = "{lang}/{component}/{version}".format(
|
| |
+ lang=lang, component=component, version=version)
|
| |
+
|
| |
+ # This is a workaround for cases when a component doesn't have
|
| |
+ # the ROOT module that should contain the antora.yml.
|
| |
+ # Without the ROOT module the translation scripts won't
|
| |
+ # pick up the antora.yml causing the docs build to fail.
|
| |
+ # So we're only including directories with the antora.yml file.
|
| |
+ if "antora.yml" in os.listdir(os.path.join(version_dir, version)):
|
| |
+ languages_dict[lang].append(start_path)
|
| |
+
|
| |
+ return languages_dict
|
| |
+
|
| |
+
|
| |
+ def generate_lang_switch_ui(languages):
|
| |
+ template_start = """
|
| |
+ <div class="page-languages">
|
| |
+ <button class="languages-menu-toggle" title="Show other languages of the site">
|
| |
+ {{{env.ANTORA_LANGUAGE}}}
|
| |
+ </button>
|
| |
+ <div class="languages-menu">
|
| |
+ """
|
| |
+ template_end = """
|
| |
+ </div>
|
| |
+ </div>
|
| |
+ """
|
| |
+
|
| |
+ template_list = []
|
| |
+ template_list.append(template_start)
|
| |
+ for language in languages:
|
| |
+ link = '<a class="language" href="{{{{siteRootPath}}}}/../{language}{{{{page.url}}}}">{language}</a>'.format(
|
| |
+ language=language)
|
| |
+ template_list.append(link)
|
| |
+ template_list.append(template_end)
|
| |
+
|
| |
+ return "\n".join(template_list)
|
| |
+
|
| |
+ def prepare_translated_sources(translated_sources, site_yml, languages, config):
|
| |
+ with tempfile.TemporaryDirectory() as workdir:
|
| |
+ # Location of the translated-sources repo
|
| |
+ # (This repo only holds content that has some translations done,
|
| |
+ # so it might be incomplete.)
|
| |
+ translated_sources_original = os.path.join(workdir, "translated_sources_original")
|
| |
+
|
| |
+ # Location of the English sources in the same structure
|
| |
+ # as translated-sources, so:
|
| |
+ # COMPONENT/VERSION/antora.yml
|
| |
+ # COMPONENT/VERSION/modules/MODULE1
|
| |
+ # COMPONENT/VERSION/modules/MODULE2
|
| |
+ # COMPONENT/VERSION/modules/...
|
| |
+ en_sources = os.path.join(workdir, "en_sources")
|
| |
+
|
| |
+ # Clone the original translated-sources
|
| |
+ subprocess.run(["git", "clone", "--depth=1", config["translated_sources_repo_url"], translated_sources_original])
|
| |
+
|
| |
+ # Get a list of the original English repos
|
| |
+ repos = []
|
| |
+ default_branch = site_yml["content"].get("branches", ["master"])
|
| |
+ if type(default_branch) is str:
|
| |
+ default_branch = [default_branch]
|
| |
+
|
| |
+ for repo_data in site_yml["content"]["sources"]:
|
| |
+
|
| |
+ branches = repo_data.get("branches", default_branch)
|
| |
+ if type(branches) is str:
|
| |
+ branches = [branches]
|
| |
+
|
| |
+ start_path = repo_data.get("start_path", "")
|
| |
+
|
| |
+ for branch in branches:
|
| |
+ repo = {}
|
| |
+ repo["url"] = repo_data["url"]
|
| |
+ repo["branch"] = branch
|
| |
+ repo["start_path"] = start_path
|
| |
+
|
| |
+ repos.append(repo)
|
| |
+
|
| |
+ # Clome the original English sources and put them into the
|
| |
+ # desired structure in en_sources (described above)
|
| |
+ components = {}
|
| |
+ for repo in repos:
|
| |
+ with tempfile.TemporaryDirectory() as tmp_repo_root:
|
| |
+ log("")
|
| |
+ log("Cloning {url} {branch}".format(url=repo["url"], branch=repo["branch"]))
|
| |
+ subprocess.run(["git", "clone", "--branch", repo["branch"], "--depth=1", repo["url"], tmp_repo_root])
|
| |
+
|
| |
+ repo_dir = os.path.join(tmp_repo_root, repo["start_path"])
|
| |
+ antora_yml_file = os.path.join(repo_dir, "antora.yml")
|
| |
+ with open(antora_yml_file, "r") as file:
|
| |
+ antora_yml = yaml.safe_load(file)
|
| |
+
|
| |
+ component = antora_yml["name"]
|
| |
+ version = antora_yml["version"]
|
| |
+ # if component version is null (~), fallback to "master"
|
| |
+ if not version:
|
| |
+ version = "master"
|
| |
+
|
| |
+ # Saving components and all their versions
|
| |
+ if component not in components:
|
| |
+ components[component] = set()
|
| |
+ components[component].add(version)
|
| |
+
|
| |
+ for module in os.listdir(os.path.join(repo_dir, "modules")):
|
| |
+ # Now this is looping over modules accross all components,
|
| |
+ # so component, module, and version variables are available
|
| |
+
|
| |
+ try:
|
| |
+ os.makedirs(os.path.join(en_sources, component, version, "modules"))
|
| |
+ except OSError as e:
|
| |
+ if e.errno != errno.EEXIST:
|
| |
+ raise
|
| |
+
|
| |
+ original_module_dir = os.path.join(repo_dir, "modules", module)
|
| |
+ module_dir = os.path.join(en_sources, component, version, "modules", module)
|
| |
+
|
| |
+ subprocess.run(["cp", "-a", original_module_dir, module_dir])
|
| |
+
|
| |
+ if module == "ROOT" or "nav" in antora_yml:
|
| |
+ # if this is the main antora.yml file
|
| |
+ subprocess.run(["cp", "-a", antora_yml_file, os.path.join(en_sources, component, version, "antora.yml")])
|
| |
+ log("----- copying antora.yml for {component} {module} {version}".format(
|
| |
+ component=component, module=module, version=version))
|
| |
+ else:
|
| |
+ log("----- skipping antora.yml for {component} {module} {version}".format(
|
| |
+ component=component, module=module, version=version))
|
| |
+
|
| |
+ # Set up the language structure in translated_sources
|
| |
+ for language in languages:
|
| |
+ lang_dir = os.path.join(translated_sources, language)
|
| |
+ try:
|
| |
+ os.makedirs(lang_dir)
|
| |
+ except OSError as e:
|
| |
+ if e.errno != errno.EEXIST:
|
| |
+ raise
|
| |
+
|
| |
+ # Copy the English sources in the translated_sources
|
| |
+ for language in languages:
|
| |
+ lang_dir = os.path.join(translated_sources, language)
|
| |
+ for component in components:
|
| |
+ en_component_dir = os.path.join(en_sources, component)
|
| |
+ subprocess.run(["cp", "-a", en_component_dir, lang_dir])
|
| |
+
|
| |
+ # And finally copy the original translated sources
|
| |
+ # into translated_sources
|
| |
+ for language in languages:
|
| |
+ src = os.path.join(translated_sources_original, language)
|
| |
+ subprocess.run(["cp", "-a", src, translated_sources + "/"])
|
| |
+
|
| |
+ return components
|
| |
+
|
| |
+
|
| |
+ def prepare_localized_admonitions(languages, config):
|
| |
+ """ Asciidoc use keywords for admonitions and others items """
|
| |
+ keywords = {}
|
| |
+
|
| |
+ with tempfile.TemporaryDirectory() as workdir:
|
| |
+ translated_keywords_repo = os.path.join(workdir, "asciidoc-keywords")
|
| |
+ subprocess.run(["git", "clone", "--depth=1", config["translated_adockeywords_repo_url"], translated_keywords_repo])
|
| |
+
|
| |
+ languages = []
|
| |
+ log(translated_keywords_repo + "/langs/*/asciidoc-attributes.yml")
|
| |
+ for filename in glob.glob(translated_keywords_repo + "/langs/*/asciidoc-attributes.yml"):
|
| |
+ languages.append(filename.rsplit("/")[::-1][1])
|
| |
+
|
| |
+ for lang in languages:
|
| |
+ file = translated_keywords_repo + "/langs/" + lang + "/asciidoc-attributes.yml"
|
| |
+ with open(file, 'r') as stream:
|
| |
+ try:
|
| |
+ keywords[lang] = yaml.load(stream)
|
| |
+ except yaml.YAMLError as exc:
|
| |
+ print(exc)
|
| |
+
|
| |
+ return keywords
|
| |
+
|
| |
+
|
| |
+ def init_git_repo(path):
|
| |
+ try:
|
| |
+ os.makedirs(path)
|
| |
+ except OSError as e:
|
| |
+ if e.errno != errno.EEXIST:
|
| |
+ raise
|
| |
+ subprocess.run(["git", "init"], cwd=path)
|
| |
+ subprocess.run(["git", "config", "user.name", "Your Name"], cwd=path)
|
| |
+ subprocess.run(["git", "config", "user.email", "you@example.com"], cwd=path)
|
| |
+ subprocess.run(["git", "commit", "--allow-empty", "-m", "init"], cwd=path)
|
| |
+
|
| |
+
|
| |
+ def main():
|
| |
+ config = get_config()
|
| |
+
|
| |
+ with tempfile.TemporaryDirectory() as workdir:
|
| |
+
|
| |
+ #####--------------------------------------#####
|
| |
+ ##### Preparation #####
|
| |
+ #####--------------------------------------#####
|
| |
+
|
| |
+ # Location of the docs-fp-o repo
|
| |
+ docs_repo = os.path.join(workdir, "docs_repo")
|
| |
+
|
| |
+ # Location of the translated sources used for the build
|
| |
+ # (That's the translated-sources repo with the missing files added
|
| |
+ # from the English sources.)
|
| |
+ translated_sources = os.path.join(workdir, "translated_sources")
|
| |
+ init_git_repo(translated_sources)
|
| |
+
|
| |
+ log("")
|
| |
+ log("===== Getting the site definition (site.yml) =====")
|
| |
+ subprocess.run(["git", "clone", "--branch", config["docs_repo_branch"], "--depth=1", config["docs_repo_url"], docs_repo])
|
| |
+
|
| |
+ with open(os.path.join(docs_repo, "site.yml"), "r") as file:
|
| |
+ original_site_yml = yaml.safe_load(file)
|
| |
+
|
| |
+ log("")
|
| |
+ log("===== Getting a list of languages =====")
|
| |
+ languages = get_languages(config)
|
| |
+ log(" Languages: {}".format(" ".join(languages)))
|
| |
+
|
| |
+ log("")
|
| |
+ log("===== Generating the language switch UI =====")
|
| |
+ lang_switch_ui = generate_lang_switch_ui(["en-US"] + list(languages.keys()))
|
| |
+
|
| |
+ ui_dir = os.path.join(docs_repo, "supplemental-ui", "partials")
|
| |
+ try:
|
| |
+ os.makedirs(ui_dir)
|
| |
+ except OSError as e:
|
| |
+ if e.errno != errno.EEXIST:
|
| |
+ raise
|
| |
+
|
| |
+ ui_file = os.path.join(ui_dir, "page-languages.hbs")
|
| |
+ with open(ui_file, "w") as file:
|
| |
+ file.write(lang_switch_ui)
|
| |
+
|
| |
+ # Timestamp to be included in the footer of the docs
|
| |
+ timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S UTC')
|
| |
+ antora_env = copy.deepcopy(os.environ)
|
| |
+ antora_env["ANTORA_DATE"] = timestamp
|
| |
+
|
| |
+
|
| |
+
|
| |
+ #####--------------------------------------#####
|
| |
+ ##### English site build #####
|
| |
+ #####--------------------------------------#####
|
| |
+
|
| |
+ if config["build_langs"] == "english" or config["build_langs"] == "all":
|
| |
+
|
| |
+ log("")
|
| |
+ log("===== Building the en-US site =====")
|
| |
+ antora_env["ANTORA_LANGUAGE"] = "en-US"
|
| |
+ result = subprocess.run(["antora", "--html-url-extension-style=indexify", os.path.join(docs_repo, "site.yml")], env=antora_env)
|
| |
+
|
| |
+ if result.returncode != 0:
|
| |
+ log("ERROR building the en-US site")
|
| |
+ sys.exit(1)
|
| |
+
|
| |
+ log("")
|
| |
+ log("===== Copying the en-US site =====")
|
| |
+ source_dir = os.path.join(docs_repo, "public")
|
| |
+ target_dir_copying = "/antora/output/en-US.building"
|
| |
+ target_dir_copied = "/antora/output/en-US.building/en-US"
|
| |
+ target_dir_final = "/antora/output/en-US"
|
| |
+ shutil.rmtree(target_dir_copying, ignore_errors=True)
|
| |
+ subprocess.run(["cp", "-a", source_dir, target_dir_copying])
|
| |
+ shutil.rmtree(target_dir_final, ignore_errors=True)
|
| |
+ shutil.move(target_dir_copied, target_dir_final)
|
| |
+ shutil.rmtree(target_dir_copying, ignore_errors=True)
|
| |
+
|
| |
+ log("")
|
| |
+ log("===== Copying the index.html =====")
|
| |
+ source_index_html = os.path.join(docs_repo, "static", "index.html")
|
| |
+ target_index_html = "/antora/output/index.html"
|
| |
+ shutil.copy(source_index_html, target_index_html)
|
| |
+
|
| |
+
|
| |
+
|
| |
+ #####--------------------------------------#####
|
| |
+ ##### Translated site build #####
|
| |
+ #####--------------------------------------#####
|
| |
+
|
| |
+
|
| |
+ if config["build_langs"] == "translated" or config["build_langs"] == "all":
|
| |
+
|
| |
+ log("")
|
| |
+ log("===== Preparing the translated sources =====")
|
| |
+ components = prepare_translated_sources(translated_sources, original_site_yml, languages, config)
|
| |
+
|
| |
+ log("")
|
| |
+ log("===== Preparing the translated ascidoc keywords =====")
|
| |
+ keywords = prepare_localized_admonitions(languages, config)
|
| |
+
|
| |
+ log("")
|
| |
+ log("===== Generating site.lang.yml files =====")
|
| |
+ for lang in languages:
|
| |
+ lang_site_yml = copy.deepcopy(original_site_yml)
|
| |
+
|
| |
+ lang_site_yml["output"]["dir"] = "./public/{lang}".format(lang=lang)
|
| |
+
|
| |
+ lang_site_yml["content"] = {}
|
| |
+ # Branches are set to HEAD because the script uses a locally-generated
|
| |
+ # content on top of the original translated-sources repo
|
| |
+ lang_site_yml["content"]["branches"] = "HEAD"
|
| |
+ # disable the "Edit this Page"
|
| |
+ lang_site_yml["content"]["edit_url"] = False
|
| |
+ lang_site_yml["content"]["sources"] = []
|
| |
+
|
| |
+ for component, versions in components.items():
|
| |
+ for version in versions:
|
| |
+ source = {}
|
| |
+ source["url"] = translated_sources
|
| |
+ source["start_path"] = "{lang}/{component}/{version}".format(
|
| |
+ lang=lang, component=component, version=version)
|
| |
+
|
| |
+ lang_site_yml["content"]["sources"].append(source)
|
| |
+
|
| |
+ if lang in keywords:
|
| |
+ if "attributes" in lang_site_yml["asciidoc"]:
|
| |
+ lang_site_yml["asciidoc"]["attributes"].append(keywords[lang])
|
| |
+ else:
|
| |
+ lang_site_yml["asciidoc"]["attributes"] = keywords[lang]
|
| |
+
|
| |
+ filename = os.path.join(docs_repo, "site-{lang}.yml".format(lang=lang))
|
| |
+ with open(filename, "w") as file:
|
| |
+ file.write(yaml.dump(lang_site_yml))
|
| |
+
|
| |
+ log(" {lang} done".format(lang=lang))
|
| |
+
|
| |
+ # Building all the translated sites
|
| |
+ lastlang = list(languages)[-1]
|
| |
+ for lang in languages:
|
| |
+ log("")
|
| |
+ log("===== Building the {lang} site =====".format(lang=lang))
|
| |
+ filename = "site-{lang}.yml".format(lang=lang)
|
| |
+ antora_env["ANTORA_LANGUAGE"] = lang
|
| |
+ result = subprocess.run(["antora", "--html-url-extension-style=indexify", os.path.join(docs_repo, filename)], env=antora_env)
|
| |
+
|
| |
+ if result.returncode != 0:
|
| |
+ log("ERROR building the {lang} site".format(lang=lang))
|
| |
+ sys.exit(1)
|
| |
+
|
| |
+ # Copying the translated site
|
| |
+ log("")
|
| |
+ log(f"=== Copying the {lang} site ===")
|
| |
+
|
| |
+ results_dir = os.path.join(docs_repo, "public", lang)
|
| |
+ publish_dir = f"/antora/output/{lang}"
|
| |
+ copying_dir = f"/antora/output/{lang}.tmp"
|
| |
+ lastlang_dir = f"/antora/output/{lastlang}"
|
| |
+
|
| |
+ # I have:
|
| |
+ # docs_repo/public/xx <- results_dir (local partition)
|
| |
+ # /antora/output/xx.tmp <- copying_dir (mounted partition)
|
| |
+ # /antora/output/xx <- publish_dir (mounted partition)
|
| |
+ #
|
| |
+ # I need to:
|
| |
+ # 1/ copy from local to mounted
|
| |
+ # 2/ remove old in mounted
|
| |
+ # 3/ move new within mounted
|
| |
+ # 4/ remove the copying dir
|
| |
+
|
| |
+ # Make sure copying_dir doesn't exist
|
| |
+ shutil.rmtree(copying_dir, ignore_errors=True)
|
| |
+
|
| |
+ # Copy results from local partition to a mounted partition
|
| |
+ log(f"Copying from {results_dir} to {copying_dir}")
|
| |
+ subprocess.run(["cp", "-a", results_dir, copying_dir])
|
| |
+
|
| |
+ # Swap the old tree for the new one for each language
|
| |
+ log(f"Moving language to the final place: {copying_dir} to {publish_dir}")
|
| |
+ shutil.rmtree(publish_dir, ignore_errors=True)
|
| |
+ subprocess.run(["mv", copying_dir, publish_dir])
|
| |
+
|
| |
+ # Recreate hardlinks as we go, in case the rsync job
|
| |
+ # start while we are still building
|
| |
+ if (lang != lastlang):
|
| |
+ log(f"hardlinking files: {publish_dir} - {lastlang_dir}")
|
| |
+ subprocess.run(["hardlink", "-cv", publish_dir, lastlang_dir])
|
| |
+
|
| |
+ # Remove local build
|
| |
+ shutil.rmtree(results_dir, ignore_errors=True)
|
| |
+
|
| |
+ # End Building all the translated sites
|
| |
+
|
| |
+ # https://pagure.io/fedora-infrastructure/issue/8964
|
| |
+ log("Consolidate files with hardlink...")
|
| |
+ subprocess.run(["hardlink", "-cv", "/antora/output/"])
|
| |
+
|
| |
+ log("DONE!")
|
| |
+
|
| |
+
|
| |
+
|
| |
+ if __name__ == "__main__":
|
| |
+ main()
|
| |
+
|
| |
advika
commented 2 years ago
While going through user tests and interviews about the fedora docs website. One of the main complaints of the users was that :-
1) Lack of hierarchy :- The users couldn’t understand which topic contained which information. They couldn't navigate through the website.
2) Information was scattered :- The information displayed was not structured and “all over the place”
3) Too Much information :- The users complained that there was too much information and they didn’t understand where to look for what. They had to scan and search for everything which was tedious and time taking, after a few seconds they didn’t feel like completing tasks given to them while user testing.
To tackle these issues , I got to focusing on organizing, structuring, and labeling content in an effective and sustainable way. The goal is to help users find information and complete tasks. To do this, I needed to understand how the pieces fit together to create the larger picture, how items relate to each other within the system. To do that I got to reading all the available topics on the Fedora Docs Website. After a long time of reading I finally created an Information Architecture. The aim of Information Architecture is to describe the flow to the website, under which heading one would find a topic. It shows that in order to reach a specific topic, how the user would need to navigate through the website.
Link to the Information Architecture :- https://miro.com/app/board/o9J_lkoizq8=/?moveToWidget=3458764522412264047&cot=14
Currently the website contains 3 major heading :-
1. Edit this Page
2. User documentation
3. Projects and Community
There were 20 subheadings also on the front Page of the Fedora Docs front page.
I have converted and narrowed it down this to 5 Headings Displayed on the front page :-
1. Introduction
2. User Documentation
3. Community
4. Contribute
5. Guidelines
The users would then need to click these 5 major headings, through which they would access subsequent subheadings (These would not be displayed by default, Please also refer to the Information Architecture). These decisions were made to keep the navigation of the website easier and user friendly. It was also made to make the website more structured and hierarchal.
While this is to the best of my understanding of topics, I would love to gather more views on this Information Architecture, regarding the flow of information, terminologies used etc. To do that, I ask you to comment your views while I run to gather the users perspective on the information Architecture. After this the plan is to redesign the website following the Information Architecture.