From 8512ea786570af52806aab09c692aae8cc210f72 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 13:17:47 +0000 Subject: [PATCH 1/23] remove debug and fix format or antora.yml --- diff --git a/src-to-pot.sh b/src-to-pot.sh index a6d5175..36989b7 100755 --- a/src-to-pot.sh +++ b/src-to-pot.sh @@ -214,7 +214,6 @@ for adoc_file in $(find . -name '*.adoc'); do if ! po4a-gettextize \ --format asciidoc \ - --option debug split_attributelist \ --master "$adoc_file" \ --master-charset "UTF-8" \ --localized-charset "UTF-8" \ @@ -238,7 +237,6 @@ for adoc_file in $(find . -name '*.adoc'); do if ! po4a-updatepo \ --format asciidoc \ - --option debug split_attributelist \ --master "$adoc_file" \ --master-charset "UTF-8" \ --po "$po_file" ; then @@ -259,8 +257,7 @@ popd > /dev/null if [ $module = "ROOT" ]; then pushd "$tmp_english_src/$src_basedir" > /dev/null if ! po4a-gettextize \ - --format asciidoc \ - --option debug split_attributelist \ + --format yaml \ --master "antora.yml" \ --master-charset "UTF-8" \ --po "$module_dir/antora.yml.pot" ; then From 5f772caafd59adba5fda903d38b557eda4a958e1 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 13:46:21 +0000 Subject: [PATCH 2/23] call src-to-pot.sh from compnent list --- diff --git a/all-src-to-pot.py b/all-src-to-pot.py index be4b34a..fb558c2 100755 --- a/all-src-to-pot.py +++ b/all-src-to-pot.py @@ -23,8 +23,17 @@ def main(): os.makedirs(source_dir, exist_ok=True) output_dir = os.path.abspath(output_dir) - parse_yml(output_dir) + # clone_sources() make_component_lists() + # parse_yml(output_dir) + +def call_src_to_po(url, repo_name, branch = 'master', basedir = '/', module = "ROOT"): + print("call_src_to_po: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module) + output = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name + command = "./src-to-pot.sh --src-branch {b} --src-basedir {bd} --module {m} {u} {o}".format(b = branch, bd = basedir, m = module, u = url, o = output) + + args = shlex.split(command) + subprocess.run(args, check=True) def call_podman_and_src_to_po(output_dir, doc): command = "podman run --rm -it -v {o}:/output:z ".format(o=output_dir) @@ -89,11 +98,12 @@ def clone_sources(): # Remove site.yml os.remove("site.yml") -def get_component(dir, branch = 'master'): +def get_component(dir, basedir = '/', branch = 'master'): + print("get_component: "+dir + basedir +" "+branch) subprocess.run(['git', 'fetch'], check=True, cwd = dir) subprocess.run(['git', 'checkout', branch], check=True, cwd = dir) - with open(dir + "/antora.yml", 'r') as stream: + with open(dir +"/"+ basedir + "/antora.yml", 'r') as stream: data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) component = data_loaded['name'] @@ -110,6 +120,7 @@ def get_component(dir, branch = 'master'): else: target_repo_name = component + "-" + module clone_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) + call_src_to_po(url = dir, repo_name = target_repo_name, basedir = basedir, branch = branch, module = module) def make_component_lists(): """List repositories to convert to pot from antora yaml file""" @@ -130,11 +141,11 @@ def make_component_lists(): dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name if 'start_path' in source: - get_component(dir+"/"+source['start_path']) + get_component(dir, basedir = source['start_path']) elif 'branches' in source: for branch in source['branches']: - get_component(dir, branch) + get_component(dir, branch = branch) else: get_component(dir) From 5e95dbb76638d5ac9af91a415f6d98d10a574434 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 14:32:08 +0000 Subject: [PATCH 3/23] follow shellcheck suggestions and all spaces in filenames --- diff --git a/src-to-pot.sh b/src-to-pot.sh index 36989b7..3bb84d4 100755 --- a/src-to-pot.sh +++ b/src-to-pot.sh @@ -80,9 +80,9 @@ fi # TARGET_DIR need to exist if [ -d "$output_dir" ]; then - pushd $output_dir > /dev/null + pushd "$output_dir" > /dev/null || exit output_dir=$(pwd) - popd > /dev/null + popd > /dev/null || exit else echo "Error. The specified output directory doesn't exist" exit 1 @@ -98,7 +98,7 @@ fi tmp_english_src=$(mktemp -d) -if ! git clone -b $src_branch $src_repo $tmp_english_src ; then +if ! git clone -b $src_branch "$src_repo" "$tmp_english_src" ; then echo "" echo "Error cloning the specified git repository." rm -rf "$tmp_english_src" @@ -123,7 +123,7 @@ fi -pushd "$tmp_english_src/$src_basedir" > /dev/null +pushd "$tmp_english_src/$src_basedir" > /dev/null || exit # Get component name from the antora.yml component=$(sed -n 's/^name: *\"*\([[:alnum:]_-]*\).*/\1/p' antora.yml) @@ -136,7 +136,7 @@ version=$(sed -n 's/^version: *\"*\([[:alnum:]_-]*\).*/\1/p' antora.yml) # 2) the user specified it using --module MODULE and it exists # In case there are multiple modules under ./modules/ # and the user hasn't specified one, throw an error. -if [ ! $(ls -1 ./modules | wc -l) = 1 ] && [ -z $module ]; then +if [ ! $(ls -1 ./modules | wc -l) = 1 ] && [ -z "$module" ]; then echo "" echo "Error. This repository contains multiple antora modules," echo "and the script doesn't know which one to choose." @@ -149,7 +149,7 @@ if [ ! $(ls -1 ./modules | wc -l) = 1 ] && [ -z $module ]; then exit 1 fi -if [ -z $module ]; then +if [ -z "$module" ]; then module=$(ls ./modules) fi @@ -165,15 +165,15 @@ if [ ! -d "modules/$module" ]; then exit 1 fi -popd > /dev/null +popd > /dev/null || exit -if [ $module = "ROOT" ]; then +if [ "$module" = "ROOT" ]; then target_repo_name="$component" else target_repo_name="$component-$module" fi -if [ "$clone_target_repo_name" ]; then +if [ "$clone_target_repo_name" = true ]; then if [ -d "$output_dir/$target_repo_name" ]; then echo "" echo "Error. You chose to clone the target repo," @@ -205,10 +205,11 @@ module_dir="$output_dir/$target_repo_name/pot/$version" # Confert all .adoc files to .pot -pushd "$tmp_english_src/$src_basedir/modules/$module" > /dev/null +pushd "$tmp_english_src/$src_basedir/modules/$module" > /dev/null || exit -for adoc_file in $(find . -name '*.adoc'); do - pot_file="$module_dir/$(dirname $adoc_file)/$(basename -s .adoc $adoc_file).pot" +while IFS= read -r -d '' adoc_file +do + pot_file="$module_dir/$(dirname "$adoc_file")/$(basename -s .adoc "$adoc_file").pot" # generate new POT files @@ -228,12 +229,13 @@ for adoc_file in $(find . -name '*.adoc'); do if [ -d "$output_dir/$target_repo_name/po" ] ; then - for lang in $(ls "$output_dir/$target_repo_name/po"); do + for lang in "$output_dir/$target_repo_name/po/"* ; do + [[ -d "$lang" ]] || break # handle the case of lang directory - po_file="$output_dir/$target_repo_name/po/$lang/$version/$(dirname $adoc_file)/$(basename -s .adoc $adoc_file).po" + po_file="$output_dir/$target_repo_name/po/$lang/$version/$(dirname "$adoc_file")/$(basename -s .adoc "$adoc_file").po" # po4a-updatepo would be angry otherwise - sed -i 's/Content-Type: text\/plain; charset=CHARSET/Content-Type: text\/plain; charset=UTF-8/g' $po_file + sed -i 's/Content-Type: text\/plain; charset=CHARSET/Content-Type: text\/plain; charset=UTF-8/g' "$po_file" if ! po4a-updatepo \ --format asciidoc \ @@ -247,15 +249,15 @@ for adoc_file in $(find . -name '*.adoc'); do fi done fi -done +done < <(find . -name '*.adoc' -print0) -popd > /dev/null +popd > /dev/null || exit # Convert the antora.yml to antora.yml.pot # if this is the main module -if [ $module = "ROOT" ]; then - pushd "$tmp_english_src/$src_basedir" > /dev/null +if [ "$module" = "ROOT" ]; then + pushd "$tmp_english_src/$src_basedir" > /dev/null || exit if ! po4a-gettextize \ --format yaml \ --master "antora.yml" \ @@ -266,10 +268,10 @@ if [ $module = "ROOT" ]; then rm -rf "$tmp_english_src" exit 1 fi - popd > /dev/null + popd > /dev/null || exit fi -rm -rf $tmp_english_src +rm -rf "$tmp_english_src" echo -ne "Source: $src_repo basedir:$src_basedir branch:$src_branch is using this repository for pot/po files:https://pagure.io/fedora-l10n/$target_repo_name\n" >> "$output_dir/log.txt" \ No newline at end of file From 329b34027519978dadeae2f5824c9f60a6990b32 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 14:35:54 +0000 Subject: [PATCH 4/23] fix basedir issue --- diff --git a/all-src-to-pot.py b/all-src-to-pot.py index fb558c2..9469851 100755 --- a/all-src-to-pot.py +++ b/all-src-to-pot.py @@ -58,6 +58,7 @@ def clone_source(url, branch = "master"): subprocess.run(args, check=True) def clone_l10n(url): + print("* clone_l10n: " + url) repo_name = url.rsplit('/', 1)[-1].replace('.git', '') dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name @@ -71,6 +72,7 @@ def clone_l10n(url): def clone_sources(): """List repositories to convert to pot from antora yaml file""" + print("* clone_sources:") # download site.yml urllib.request.urlretrieve( @@ -99,7 +101,7 @@ def clone_sources(): os.remove("site.yml") def get_component(dir, basedir = '/', branch = 'master'): - print("get_component: "+dir + basedir +" "+branch) + print("get_component: "+dir + "/" + basedir + " " + branch) subprocess.run(['git', 'fetch'], check=True, cwd = dir) subprocess.run(['git', 'checkout', branch], check=True, cwd = dir) @@ -110,10 +112,13 @@ def get_component(dir, basedir = '/', branch = 'master'): version = data_loaded['version'] f = [] - for (dirpath, dirnames, filenames) in os.walk(dir + "/modules"): + for (dirpath, dirnames, filenames) in os.walk(dir + "/" + basedir + "/modules"): f.extend(dirnames) break + if len(f) < 1: + raise ValueError('There should be at least one module!') + for module in f: if module == "ROOT": target_repo_name = component From a25d9b3d963be947f24161c345be393c0b081b01 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 14:49:27 +0000 Subject: [PATCH 5/23] fix doublepath --- diff --git a/src-to-pot.sh b/src-to-pot.sh index 3bb84d4..52e1f4d 100755 --- a/src-to-pot.sh +++ b/src-to-pot.sh @@ -232,7 +232,7 @@ do for lang in "$output_dir/$target_repo_name/po/"* ; do [[ -d "$lang" ]] || break # handle the case of lang directory - po_file="$output_dir/$target_repo_name/po/$lang/$version/$(dirname "$adoc_file")/$(basename -s .adoc "$adoc_file").po" + po_file="$lang/$version/$(dirname "$adoc_file")/$(basename -s .adoc "$adoc_file").po" # po4a-updatepo would be angry otherwise sed -i 's/Content-Type: text\/plain; charset=CHARSET/Content-Type: text\/plain; charset=UTF-8/g' "$po_file" From 485ad444fdedbc76dd70370f6ed5827da4928927 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 14:50:23 +0000 Subject: [PATCH 6/23] fix output path --- diff --git a/all-src-to-pot.py b/all-src-to-pot.py index 9469851..4f0a554 100755 --- a/all-src-to-pot.py +++ b/all-src-to-pot.py @@ -27,9 +27,9 @@ def main(): make_component_lists() # parse_yml(output_dir) -def call_src_to_po(url, repo_name, branch = 'master', basedir = '/', module = "ROOT"): +def call_src_to_po(url, branch = 'master', basedir = '/', module = "ROOT"): print("call_src_to_po: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module) - output = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name + output = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" command = "./src-to-pot.sh --src-branch {b} --src-basedir {bd} --module {m} {u} {o}".format(b = branch, bd = basedir, m = module, u = url, o = output) args = shlex.split(command) @@ -125,7 +125,7 @@ def get_component(dir, basedir = '/', branch = 'master'): else: target_repo_name = component + "-" + module clone_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) - call_src_to_po(url = dir, repo_name = target_repo_name, basedir = basedir, branch = branch, module = module) + call_src_to_po(url = dir, basedir = basedir, branch = branch, module = module) def make_component_lists(): """List repositories to convert to pot from antora yaml file""" From 93808dac4031a431c7569c41f1dd79934275bd8e Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 14:56:28 +0000 Subject: [PATCH 7/23] don't update po file if not existing --- diff --git a/src-to-pot.sh b/src-to-pot.sh index 52e1f4d..80bb330 100755 --- a/src-to-pot.sh +++ b/src-to-pot.sh @@ -234,6 +234,10 @@ do po_file="$lang/$version/$(dirname "$adoc_file")/$(basename -s .adoc "$adoc_file").po" + if [ ! -e "$po_file" ] ; then + echo "missing po file: $po_file" + break + fi # po4a-updatepo would be angry otherwise sed -i 's/Content-Type: text\/plain; charset=CHARSET/Content-Type: text\/plain; charset=UTF-8/g' "$po_file" From 528a400864ecfb9d92cdd48525a9e0c9c600e655 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 15:33:25 +0000 Subject: [PATCH 8/23] add commit to l10n repositories --- diff --git a/all-po-to-src.sh b/all-po-to-src.sh index 444b435..e592914 100755 --- a/all-po-to-src.sh +++ b/all-po-to-src.sh @@ -40,6 +40,7 @@ fi cmd="podman run --rm -it -v $output_dir:/output:z -v $(pwd)/po-to-src.sh:/scripts/po-to-src.sh:z asamalik/fedora-docs-translations /scripts/po-to-src.sh " output_dir="/output" +po-to-src.sh for language in "cs" "fr" "jp"; do # Convert the homepage diff --git a/all-src-to-pot.py b/all-src-to-pot.py index 4f0a554..699b953 100755 --- a/all-src-to-pot.py +++ b/all-src-to-pot.py @@ -65,11 +65,24 @@ def clone_l10n(url): if os.path.exists(dir): subprocess.run(['git', 'pull'], check=True, cwd = dir) else: - command = "git clone {u} {d}".format(u=url, d=dir) + command = "git clone {u} {d} --config 'remote.origin.pushurl=ssh://git@pagure.io/fedora-l10n/{repo_name}.git'".format(u=url, d=dir, repo_name = repo_name) print(command) args = shlex.split(command) subprocess.run(args, check=True) +def commit_l10n(url): + print("* commit_l10n: " + url) + repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name + + subprocess.run(['git', 'add', '.'], check=True, cwd = dir) + + command = "git commit -m 'automatic update of pot/po files'" + args = shlex.split(command) + subprocess.run(args, check=True, cwd = dir) + + subprocess.run(['git', 'push'], check=True, cwd = dir) + def clone_sources(): """List repositories to convert to pot from antora yaml file""" print("* clone_sources:") @@ -126,6 +139,7 @@ def get_component(dir, basedir = '/', branch = 'master'): target_repo_name = component + "-" + module clone_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) call_src_to_po(url = dir, basedir = basedir, branch = branch, module = module) + commit_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) def make_component_lists(): """List repositories to convert to pot from antora yaml file""" From 7d09f014ca71d21bae2aa4a02d138b2641b232b7 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 15:41:53 +0000 Subject: [PATCH 9/23] apply shellcheck advices --- diff --git a/po-to-src.sh b/po-to-src.sh index e889191..6bfbaf3 100755 --- a/po-to-src.sh +++ b/po-to-src.sh @@ -82,9 +82,9 @@ fi # TARGET_DIR need to exist if [ -d "$output_dir" ]; then - pushd $output_dir > /dev/null + pushd "$output_dir" > /dev/null || exit output_dir=$(pwd) - popd > /dev/null + popd > /dev/null || exit else echo "Error. The specified output directory doesn't exist" exit 1 @@ -106,7 +106,7 @@ delete_tmp() { rm -rf "$tmp_po_pot" } -if ! git clone -b $src_branch $src_repo $tmp_english_src ; then +if ! git clone -b "$src_branch" "$src_repo" "$tmp_english_src" ; then echo "" echo "Error cloning the specified git repository." delete_tmp @@ -131,7 +131,7 @@ fi -pushd "$tmp_english_src/$src_basedir" > /dev/null +pushd "$tmp_english_src/$src_basedir" > /dev/null || exit # Get component name from the antora.yml component=$(sed -n 's/^name: *\"*\([[:alnum:]_-]*\).*/\1/p' antora.yml) @@ -144,7 +144,7 @@ version=$(sed -n 's/^version: *\"*\([[:alnum:]_-]*\).*/\1/p' antora.yml) # 2) the user specified it using --module MODULE and it exists # In case there are multiple modules under ./modules/ # and the user hasn't specified one, throw an error. -if [ ! $(ls -1 ./modules | wc -l) = 1 ] && [ -z $module ]; then +if [ ! $(ls -1 ./modules | wc -l) = 1 ] && [ -z "$module" ]; then echo "" echo "Error. This repository contains multiple antora modules," echo "and the script doesn't know which one to choose." @@ -157,7 +157,7 @@ if [ ! $(ls -1 ./modules | wc -l) = 1 ] && [ -z $module ]; then exit 1 fi -if [ -z $module ]; then +if [ -z "$module" ]; then module=$(ls ./modules) fi @@ -173,9 +173,9 @@ if [ ! -d "modules/$module" ]; then exit 1 fi -popd > /dev/null +popd > /dev/null || exit -if [ $module = "ROOT" ]; then +if [ "$module" = "ROOT" ]; then po_pot_repo_name="$component" else po_pot_repo_name="$component-$module" @@ -219,11 +219,11 @@ if [ ! -d "$tmp_po_pot/po/$language" ] ; then fi # Confert all .adoc files to .pot -pushd "$tmp_english_src/$src_basedir/modules/$module" > /dev/null +pushd "$tmp_english_src/$src_basedir/modules/$module" > /dev/null || exit for source_adoc_file in $(find . -name '*.adoc'); do - po_file="$po_pot_module_dir/$(dirname $source_adoc_file)/$(basename -s .adoc $source_adoc_file).po" - target_adoc_file="$output_module_dir/$(dirname $source_adoc_file)/$(basename -s .adoc $source_adoc_file).adoc" + po_file="$po_pot_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").po" + target_adoc_file="$output_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").adoc" if [ -f "$po_file" ] ; then # TODO: convert file if ! po4a-translate \ @@ -248,13 +248,13 @@ for source_adoc_file in $(find . -name '*.adoc'); do fi done -popd > /dev/null +popd > /dev/null || exit # Convert the antora.yml to antora.yml.pot # if this is the main module -if [ $module = "ROOT" ]; then - pushd "$tmp_english_src/$src_basedir" > /dev/null +if [ "$module" = "ROOT" ]; then + pushd "$tmp_english_src/$src_basedir" > /dev/null || exit source_antora_yml="$tmp_english_src/$src_basedir/antora.yml" po_antora_yml="$tmp_po_pot/po/$language/$version/antora.yml.po" @@ -275,7 +275,7 @@ if [ $module = "ROOT" ]; then echo " Copying it from the original source." cp "$source_antora_yml" "$target_antora_yml" fi - popd > /dev/null + popd > /dev/null || exit fi From 80c61a8b895b25e541bcaafaa34d9d9924bec603 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 12 2019 17:07:08 +0000 Subject: [PATCH 10/23] generate translated sources --- diff --git a/.gitignore b/.gitignore index 8d4e1dc..219c3b9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ l10n/ -sources/ \ No newline at end of file +sources/ +translated-sources/ \ No newline at end of file diff --git a/all-src-to-pot.py b/all-src-to-pot.py index 699b953..91a5441 100755 --- a/all-src-to-pot.py +++ b/all-src-to-pot.py @@ -35,6 +35,14 @@ def call_src_to_po(url, branch = 'master', basedir = '/', module = "ROOT"): args = shlex.split(command) subprocess.run(args, check=True) +def call_po_to_src(url, branch = 'master', basedir = '/', module = "ROOT"): + print("call_po_to_src: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module) + output = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" + for lang in ['fr', 'cs', 'ja']: + command = "./po-to-src.sh --src-branch {b} --src-basedir {bd} --module {m} {l} {u} {o}".format(b = branch, bd = basedir, m = module, l = lang, u = url, o = output) + args = shlex.split(command) + subprocess.run(args, check=True) + def call_podman_and_src_to_po(output_dir, doc): command = "podman run --rm -it -v {o}:/output:z ".format(o=output_dir) command += "-v {c}/src-to-pot.sh:/scripts/src-to-pot.sh:z ".format(c=os.getcwd()) @@ -139,7 +147,8 @@ def get_component(dir, basedir = '/', branch = 'master'): target_repo_name = component + "-" + module clone_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) call_src_to_po(url = dir, basedir = basedir, branch = branch, module = module) - commit_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) + #commit_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) + call_po_to_src(url = dir, basedir = basedir, branch = branch, module = module) def make_component_lists(): """List repositories to convert to pot from antora yaml file""" diff --git a/po-to-src.sh b/po-to-src.sh index 6bfbaf3..3145468 100755 --- a/po-to-src.sh +++ b/po-to-src.sh @@ -99,11 +99,9 @@ if [ -z "$src_basedir" ]; then fi tmp_english_src=$(mktemp -d) -tmp_po_pot=$(mktemp -d) delete_tmp() { rm -rf "$tmp_english_src" - rm -rf "$tmp_po_pot" } if ! git clone -b "$src_branch" "$src_repo" "$tmp_english_src" ; then @@ -181,15 +179,16 @@ else po_pot_repo_name="$component-$module" fi +tmp_po_pot="/home/jean-baptiste/Projects/fedora-l10n/translations-scripts/l10n/$po_pot_repo_name" -if ! git clone "https://pagure.io/fedora-l10n/$po_pot_repo_name.git" "$tmp_po_pot" ; then - echo "" - echo "Error cloning the PO/POT repository." - echo "It might not exist. It should be here:" - echo "https://pagure.io/fedora-l10n/$po_pot_repo_name" - delete_tmp - exit 1 -fi +# if ! git clone "https://pagure.io/fedora-l10n/$po_pot_repo_name.git" "$tmp_po_pot" ; then +# echo "" +# echo "Error cloning the PO/POT repository." +# echo "It might not exist. It should be here:" +# echo "https://pagure.io/fedora-l10n/$po_pot_repo_name" +# delete_tmp +# exit 1 +# fi @@ -212,11 +211,11 @@ po_pot_module_dir="$tmp_po_pot/po/$language/$version" output_module_dir="$output_dir/$language/$component/$version/modules/$module" # Make sure the language exists in the PO/POT repo -if [ ! -d "$tmp_po_pot/po/$language" ] ; then - echo "Error. Locale $language doesn't exist in the PO/POT repo." - delete_tmp - exit 1 -fi +# if [ ! -d "$tmp_po_pot/po/$language" ] ; then +# echo "Error. Locale $language doesn't exist in the PO/POT repo." +# delete_tmp +# exit 1 +# fi # Confert all .adoc files to .pot pushd "$tmp_english_src/$src_basedir/modules/$module" > /dev/null || exit @@ -228,7 +227,6 @@ for source_adoc_file in $(find . -name '*.adoc'); do # TODO: convert file if ! po4a-translate \ --format asciidoc \ - --option debug split_attributelist \ --master "$source_adoc_file" \ --master-charset "UTF-8" \ --po "$po_file" \ @@ -241,9 +239,8 @@ for source_adoc_file in $(find . -name '*.adoc'); do exit 1 fi else - echo "Warning: The following file is in the source, but not in the translations repo:" - echo " $source_adoc_file" - echo " Copying the original over. You might need to update the PO/POT repo." + # "Warning: The following file is in the source, but not in the translations repo:" + mkdir -p "$(dirname "$target_adoc_file")" cp "$source_adoc_file" "$target_adoc_file" fi done @@ -271,8 +268,8 @@ if [ "$module" = "ROOT" ]; then --localized-charset "UTF-8" \ --keep 0 else - echo "Warning: There should be an antora.yml.po, but there isn't." - echo " Copying it from the original source." + # "Warning: There should be an antora.yml.po, but there isn't." + mkdir -p "$(dirname "$target_antora_yml")" cp "$source_antora_yml" "$target_antora_yml" fi popd > /dev/null || exit From 709d64d631834f7be12c9de2270aea8667c35329 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 13 2019 08:37:33 +0000 Subject: [PATCH 11/23] remove old bash script --- diff --git a/all-po-to-src.sh b/all-po-to-src.sh deleted file mode 100755 index e592914..0000000 --- a/all-po-to-src.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash - -usage() { - echo "Calls './po-to-src.sh' for all source repos (hardcoded in this script)" - echo "" - echo "Usage" - echo "$0 TARGET_DIR" - echo "" - echo "TARGET_DIR Output directory" -} - -# Need TARGET_DIR -if [ "$1" != "" ]; then - output_dir="$1" -else - usage - echo "Please specify TARGET_DIR" - exit 1 -fi - -# TARGET_DIR need to exist -if [ -d "$output_dir" ]; then - pushd $output_dir > /dev/null - output_dir=$(pwd) - popd > /dev/null -else - echo "Error. The specified output directory doesn't exist" - exit 1 -fi - -if ! git clone \ - --config "remote.origin.pushurl=ssh://git@pagure.io/fedora-docs/translated-sources.git" \ - "https://pagure.io/fedora-docs/translated-sources.git" $output_dir ; then - echo "Error cloning the translated sources repository." - exit 1 -fi - -# cmd="./po-to-src.sh" -# container -cmd="podman run --rm -it -v $output_dir:/output:z -v $(pwd)/po-to-src.sh:/scripts/po-to-src.sh:z asamalik/fedora-docs-translations /scripts/po-to-src.sh " -output_dir="/output" - -po-to-src.sh - -for language in "cs" "fr" "jp"; do - # Convert the homepage - $cmd \ - --src-basedir "pages/homepage" \ - $language \ - https://pagure.io/fedora-docs/docs-fp-o.git \ - $output_dir - - # Convert the main page of the user guide - # that's pointing to the install guide, - # the release notes, and the sysadmin guide - $cmd \ - --src-branch "f28" \ - $language \ - https://pagure.io/fedora-docs/release-docs-home.git \ - $output_dir - - $cmd \ - --src-branch "f29" \ - $language \ - https://pagure.io/fedora-docs/release-docs-home.git \ - $output_dir - - # Convert the install guide - $cmd \ - --src-branch "f28" \ - $language \ - https://pagure.io/fedora-docs/install-guide.git \ - $output_dir - - $cmd \ - --src-branch "f29" \ - $language \ - https://pagure.io/fedora-docs/install-guide.git \ - $output_dir - - # Convert the release notes - $cmd \ - --src-branch "f28" \ - $language \ - https://pagure.io/fedora-docs/release-notes.git \ - $output_dir - - $cmd \ - --src-branch "f29" \ - $language \ - https://pagure.io/fedora-docs/release-notes.git \ - $output_dir -done diff --git a/all-src-to-pot.sh b/all-src-to-pot.sh deleted file mode 100755 index ce7b339..0000000 --- a/all-src-to-pot.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -usage() { - echo "Calls './src-to-pot.sh' for all source repos (hardcoded in this script)" - echo "" - echo "Usage" - echo "$0 TARGET_DIR" - echo "" - echo "TARGET_DIR Output directory" -} - -# Need TARGET_DIR -if [ "$1" != "" ]; then - output_dir="$1" -else - usage - echo "Please specify TARGET_DIR" - exit 1 -fi - -# TARGET_DIR need to exist -if [ -d "$output_dir" ]; then - pushd $output_dir > /dev/null - output_dir=$(pwd) - popd > /dev/null -else - echo "Error. The specified output directory doesn't exist. Please create it first." - exit 1 -fi - -# TARGET_DIR need to exist -if [ -e "$output_dir/log.txt" ]; then - rm "$output_dir/log.txt" -fi - - -# container -cmd="podman run --rm -it -v $output_dir:/output:z -v $(pwd)/src-to-pot.sh:/scripts/src-to-pot.sh:z asamalik/fedora-docs-translations /scripts/src-to-pot.sh " -output_dir="/output" - - -### === Repository List === ### -# The following is a "configuration" of which repos this script should work with. -# -# Note: Some repos might be listed multiple times, once per branch. In that case, -# only the first entry should include the "--clone-target-repo" option. - -# Convert the homepage -$cmd \ - --clone-target-repo \ - --src-basedir "pages/homepage" \ - https://pagure.io/fedora-docs/docs-fp-o.git \ - $output_dir - -# Convert the "release docs home" page -$cmd \ - --clone-target-repo \ - --src-branch "f28" \ - https://pagure.io/fedora-docs/release-docs-home.git \ - $output_dir - -$cmd \ - --src-branch "f29" \ - https://pagure.io/fedora-docs/release-docs-home.git \ - $output_dir - -# Convert the install guide -$cmd \ - --clone-target-repo \ - --src-branch "f28" \ - https://pagure.io/fedora-docs/install-guide.git \ - $output_dir - -$cmd \ - --src-branch "f29" \ - https://pagure.io/fedora-docs/install-guide.git \ - $output_dir - -# Convert the release notes -$cmd \ - --clone-target-repo \ - --src-branch "f28" \ - https://pagure.io/fedora-docs/release-notes.git \ - $output_dir - -$cmd \ - --src-branch "f29" \ - https://pagure.io/fedora-docs/release-notes.git \ - $output_dir From 594810aec5c8faa39de5bbe25e91318ce643acec Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 13 2019 09:04:43 +0000 Subject: [PATCH 12/23] cleanup --- diff --git a/all-src-to-pot.py b/all-src-to-pot.py deleted file mode 100755 index 91a5441..0000000 --- a/all-src-to-pot.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python3 -"""Calls `./po-to-src.sh` for all source repos""" - -import argparse -import os -import urllib.request -import subprocess -import shlex -import yaml - - -def main(): - """Handle params""" - parser = argparse.ArgumentParser( - description="Calls `./po-to-src.sh` for all source repos") - parser.add_argument("output_dir", help="Output directory") - args = parser.parse_args() - - output_dir = args.output_dir - source_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" - - os.makedirs(output_dir, exist_ok=True) - os.makedirs(source_dir, exist_ok=True) - - output_dir = os.path.abspath(output_dir) - # clone_sources() - make_component_lists() - # parse_yml(output_dir) - -def call_src_to_po(url, branch = 'master', basedir = '/', module = "ROOT"): - print("call_src_to_po: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module) - output = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" - command = "./src-to-pot.sh --src-branch {b} --src-basedir {bd} --module {m} {u} {o}".format(b = branch, bd = basedir, m = module, u = url, o = output) - - args = shlex.split(command) - subprocess.run(args, check=True) - -def call_po_to_src(url, branch = 'master', basedir = '/', module = "ROOT"): - print("call_po_to_src: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module) - output = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" - for lang in ['fr', 'cs', 'ja']: - command = "./po-to-src.sh --src-branch {b} --src-basedir {bd} --module {m} {l} {u} {o}".format(b = branch, bd = basedir, m = module, l = lang, u = url, o = output) - args = shlex.split(command) - subprocess.run(args, check=True) - -def call_podman_and_src_to_po(output_dir, doc): - command = "podman run --rm -it -v {o}:/output:z ".format(o=output_dir) - command += "-v {c}/src-to-pot.sh:/scripts/src-to-pot.sh:z ".format(c=os.getcwd()) - command += "asamalik/fedora-docs-translations /scripts/src-to-pot.sh {d}".format(d=doc) - # print(command) - # args = shlex.split(command) - # subprocess.run(args, check=True) - -def clone_source(url, branch = "master"): - repo_name = url.rsplit('/', 1)[-1].replace('.git', '') - dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name - - if os.path.exists(dir): - subprocess.run(['git', 'fetch'], check=True, cwd = dir) - subprocess.run(['git', 'checkout', branch], check=True, cwd = dir) - subprocess.run(['git', 'pull'], check=True, cwd = dir) - else: - command = "git clone -b {b} {u} {d}".format(b = branch, u=url, d=dir) - print(command) - args = shlex.split(command) - subprocess.run(args, check=True) - -def clone_l10n(url): - print("* clone_l10n: " + url) - repo_name = url.rsplit('/', 1)[-1].replace('.git', '') - dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name - - if os.path.exists(dir): - subprocess.run(['git', 'pull'], check=True, cwd = dir) - else: - command = "git clone {u} {d} --config 'remote.origin.pushurl=ssh://git@pagure.io/fedora-l10n/{repo_name}.git'".format(u=url, d=dir, repo_name = repo_name) - print(command) - args = shlex.split(command) - subprocess.run(args, check=True) - -def commit_l10n(url): - print("* commit_l10n: " + url) - repo_name = url.rsplit('/', 1)[-1].replace('.git', '') - dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name - - subprocess.run(['git', 'add', '.'], check=True, cwd = dir) - - command = "git commit -m 'automatic update of pot/po files'" - args = shlex.split(command) - subprocess.run(args, check=True, cwd = dir) - - subprocess.run(['git', 'push'], check=True, cwd = dir) - -def clone_sources(): - """List repositories to convert to pot from antora yaml file""" - print("* clone_sources:") - - # download site.yml - urllib.request.urlretrieve( - "https://pagure.io/fedora-docs/docs-fp-o/raw/master/f/site.yml", "site.yml" - ) - - # Read site.yml - with open("site.yml", 'r') as stream: - data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) - - # Parse site.yml - for source in data_loaded['content']['sources']: - url = source['url'] - - if 'start_path' in source: - clone_source(url) - - elif 'branches' in source: - for branch in source['branches']: - clone_source(url, branch) - else: - clone_source(url) - - - # Remove site.yml - os.remove("site.yml") - -def get_component(dir, basedir = '/', branch = 'master'): - print("get_component: "+dir + "/" + basedir + " " + branch) - subprocess.run(['git', 'fetch'], check=True, cwd = dir) - subprocess.run(['git', 'checkout', branch], check=True, cwd = dir) - - with open(dir +"/"+ basedir + "/antora.yml", 'r') as stream: - data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) - - component = data_loaded['name'] - version = data_loaded['version'] - - f = [] - for (dirpath, dirnames, filenames) in os.walk(dir + "/" + basedir + "/modules"): - f.extend(dirnames) - break - - if len(f) < 1: - raise ValueError('There should be at least one module!') - - for module in f: - if module == "ROOT": - target_repo_name = component - else: - target_repo_name = component + "-" + module - clone_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) - call_src_to_po(url = dir, basedir = basedir, branch = branch, module = module) - #commit_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) - call_po_to_src(url = dir, basedir = basedir, branch = branch, module = module) - -def make_component_lists(): - """List repositories to convert to pot from antora yaml file""" - - # download site.yml - urllib.request.urlretrieve( - "https://pagure.io/fedora-docs/docs-fp-o/raw/master/f/site.yml", "site.yml" - ) - - # Read site.yml - with open("site.yml", 'r') as stream: - data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) - - # Parse site.yml - for source in data_loaded['content']['sources']: - url = source['url'] - repo_name = url.rsplit('/', 1)[-1].replace('.git', '') - dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name - - if 'start_path' in source: - get_component(dir, basedir = source['start_path']) - - elif 'branches' in source: - for branch in source['branches']: - get_component(dir, branch = branch) - else: - get_component(dir) - - # Remove site.yml - os.remove("site.yml") - -def parse_yml(output_dir): - """List repositories to convert to pot from antora yaml file""" - - # download site.yml - urllib.request.urlretrieve( - "https://pagure.io/fedora-docs/docs-fp-o/raw/master/f/site.yml", "site.yml" - ) - - # Read site.yml - with open("site.yml", 'r') as stream: - data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) - - # Parse site.yml - for source in data_loaded['content']['sources']: - url = source['url'] - - if 'start_path' in source: - src_basedir = source['start_path'] - doc = "--clone-target-repo --src-basedir \"{bd}\" {url} /output".format( - bd=src_basedir, url=url) - - elif 'branches' in source: - for branch in source['branches']: - src_branch = branch - doc = "--clone-target-repo --src-branch \"{br}\" {url} /output".format( - br=src_branch, url=url) - else: - doc = "--clone-target-repo {url} /output".format(url=url) - - call_podman_and_src_to_po(output_dir, doc) - - # Remove site.yml - os.remove("site.yml") - - -if __name__ == '__main__': - main() diff --git a/build.py b/build.py new file mode 100755 index 0000000..4d05655 --- /dev/null +++ b/build.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Calls `./src-to-pot.sh` and `./po-to-src.sh` for all source repos""" + +import argparse +import os +import urllib.request +import subprocess +import shlex +import yaml + + +def main(): + """Handle params""" + parser = argparse.ArgumentParser( + description="Calls `./po-to-src.sh` for all source repos") + parser.add_argument("output_dir", help="Output directory") + args = parser.parse_args() + + output_dir = args.output_dir + source_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + + os.makedirs(output_dir, exist_ok=True) + os.makedirs(source_dir, exist_ok=True) + + output_dir = os.path.abspath(output_dir) + clone_sources() + make_component_lists() + +def call_src_to_pot(url, branch='master', basedir='/', module="ROOT"): + """Call src-to-ot.sh to convert english source (adoc) content to pot files""" + print("call_src_to_pot: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module) + output = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + command = "./src-to-pot.sh --src-branch {b} --src-basedir {bd} --module {m} {u} {o}".format(b=branch, bd=basedir, m=module, u=url, o=output) + + args = shlex.split(command) + subprocess.run(args, check=True) + +def call_po_to_src(url, branch='master', basedir='/', module="ROOT"): + """Call po-to-src.sh to convert po files to translated sources (adoc)""" + print("call_po_to_src: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module) + output = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" + for lang in ['fr', 'cs', 'ja']: + command = "./po-to-src.sh --src-branch {b} --src-basedir {bd} --module {m} {l} {u} {o}".format(b=branch, bd=basedir, m=module, l=lang, u=url, o=output) + args = shlex.split(command) + subprocess.run(args, check=True) + +def clone_source(url, branch="master"): + """Clone english sources in sources folder (update content if existing)""" + repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name + + if os.path.exists(dir): + subprocess.run(['git', 'fetch'], check=True, cwd=dir) + subprocess.run(['git', 'checkout', branch], check=True, cwd=dir) + subprocess.run(['git', 'pull'], check=True, cwd=dir) + else: + command = "git clone -b {b} {u} {d}".format(b=branch, u=url, d=dir) + print(command) + args = shlex.split(command) + subprocess.run(args, check=True) + +def clone_l10n(url): + """Clone localization repos in l10n folder (update content if existing)""" + print("* clone_l10n: " + url) + repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name + + if os.path.exists(dir): + subprocess.run(['git', 'pull'], check=True, cwd=dir) + else: + command = "git clone {u} {d} --config 'remote.origin.pushurl=ssh://git@pagure.io/fedora-l10n/{repo_name}.git'".format(u=url, d=dir, repo_name=repo_name) + print(command) + args = shlex.split(command) + subprocess.run(args, check=True) + +def commit_l10n(url): + """Commit and push changes of a localization repo""" + print("* commit_l10n: " + url) + repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name + + subprocess.run(['git', 'add', '.'], check=True, cwd=dir) + + command = "git commit -m 'automatic update of pot/po files'" + args = shlex.split(command) + subprocess.run(args, check=True, cwd=dir) + + subprocess.run(['git', 'push'], check=True, cwd=dir) + +def clone_sources(): + """List sources repositories from antora yaml file""" + print("* clone_sources:") + + # download site.yml + urllib.request.urlretrieve( + "https://pagure.io/fedora-docs/docs-fp-o/raw/master/f/site.yml", "site.yml" + ) + + # Read site.yml + with open("site.yml", 'r') as stream: + data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) + + # Parse site.yml + for source in data_loaded['content']['sources']: + url = source['url'] + + if 'start_path' in source: + clone_source(url) + + elif 'branches' in source: + for branch in source['branches']: + clone_source(url, branch) + else: + clone_source(url) + + + # Remove site.yml + os.remove("site.yml") + +def get_component(dir, basedir='/', branch='master'): + """Run actions on the component (clone repos, run scripts, commit changes)""" + print("get_component: "+dir + "/" + basedir + " " + branch) + subprocess.run(['git', 'fetch'], check=True, cwd=dir) + subprocess.run(['git', 'checkout', branch], check=True, cwd=dir) + + with open(dir +"/"+ basedir + "/antora.yml", 'r') as stream: + data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) + + component = data_loaded['name'] + version = data_loaded['version'] + + f = [] + for (dirpath, dirnames, filenames) in os.walk(dir + "/" + basedir + "/modules"): + f.extend(dirnames) + break + + if len(f) < 1: + raise ValueError('There should be at least one module!') + + for module in f: + if module == "ROOT": + target_repo_name = component + else: + target_repo_name = component + "-" + module + clone_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) + call_src_to_pot(url=dir, basedir=basedir, branch=branch, module=module) + #commit_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) + call_po_to_src(url=dir, basedir=basedir, branch=branch, module=module) + +def make_component_lists(): + """Deduct the component name from site.yml (l10n repos are per component)""" + + # download site.yml + urllib.request.urlretrieve( + "https://pagure.io/fedora-docs/docs-fp-o/raw/master/f/site.yml", "site.yml" + ) + + # Read site.yml + with open("site.yml", 'r') as stream: + data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) + + # Parse site.yml + for source in data_loaded['content']['sources']: + url = source['url'] + repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name + + if 'start_path' in source: + get_component(dir, basedir=source['start_path']) + + elif 'branches' in source: + for branch in source['branches']: + get_component(dir, branch=branch) + else: + get_component(dir) + + # Remove site.yml + os.remove("site.yml") + +if __name__ == '__main__': + main() From 2b639eb6e22b647bdd07fa4c712b13244a4eb00a Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 13 2019 09:17:05 +0000 Subject: [PATCH 13/23] clean variable names --- diff --git a/build.py b/build.py index 4d05655..2d4c051 100755 --- a/build.py +++ b/build.py @@ -47,14 +47,14 @@ def call_po_to_src(url, branch='master', basedir='/', module="ROOT"): def clone_source(url, branch="master"): """Clone english sources in sources folder (update content if existing)""" repo_name = url.rsplit('/', 1)[-1].replace('.git', '') - dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name + repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name - if os.path.exists(dir): - subprocess.run(['git', 'fetch'], check=True, cwd=dir) - subprocess.run(['git', 'checkout', branch], check=True, cwd=dir) - subprocess.run(['git', 'pull'], check=True, cwd=dir) + if os.path.exists(repo_dir): + subprocess.run(['git', 'fetch'], check=True, cwd=repo_dir) + subprocess.run(['git', 'checkout', branch], check=True, cwd=repo_dir) + subprocess.run(['git', 'pull'], check=True, cwd=repo_dir) else: - command = "git clone -b {b} {u} {d}".format(b=branch, u=url, d=dir) + command = "git clone -b {b} {u} {d}".format(b=branch, u=url, d=repo_dir) print(command) args = shlex.split(command) subprocess.run(args, check=True) @@ -63,10 +63,10 @@ def clone_l10n(url): """Clone localization repos in l10n folder (update content if existing)""" print("* clone_l10n: " + url) repo_name = url.rsplit('/', 1)[-1].replace('.git', '') - dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name + repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name - if os.path.exists(dir): - subprocess.run(['git', 'pull'], check=True, cwd=dir) + if os.path.exists(repo_dir): + subprocess.run(['git', 'pull'], check=True, cwd=repo_dir) else: command = "git clone {u} {d} --config 'remote.origin.pushurl=ssh://git@pagure.io/fedora-l10n/{repo_name}.git'".format(u=url, d=dir, repo_name=repo_name) print(command) @@ -77,15 +77,15 @@ def commit_l10n(url): """Commit and push changes of a localization repo""" print("* commit_l10n: " + url) repo_name = url.rsplit('/', 1)[-1].replace('.git', '') - dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name + repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name - subprocess.run(['git', 'add', '.'], check=True, cwd=dir) + subprocess.run(['git', 'add', '.'], check=True, cwd=repo_dir) command = "git commit -m 'automatic update of pot/po files'" args = shlex.split(command) - subprocess.run(args, check=True, cwd=dir) + subprocess.run(args, check=True, cwd=repo_dir) - subprocess.run(['git', 'push'], check=True, cwd=dir) + subprocess.run(['git', 'push'], check=True, cwd=repo_dir) def clone_sources(): """List sources repositories from antora yaml file""" @@ -117,11 +117,11 @@ def clone_sources(): # Remove site.yml os.remove("site.yml") -def get_component(dir, basedir='/', branch='master'): +def get_component(comp_dir, basedir='/', branch='master'): """Run actions on the component (clone repos, run scripts, commit changes)""" - print("get_component: "+dir + "/" + basedir + " " + branch) - subprocess.run(['git', 'fetch'], check=True, cwd=dir) - subprocess.run(['git', 'checkout', branch], check=True, cwd=dir) + print("get_component: "+ comp_dir + "/" + basedir + " " + branch) + subprocess.run(['git', 'fetch'], check=True, cwd=comp_dir) + subprocess.run(['git', 'checkout', branch], check=True, cwd=comp_dir) with open(dir +"/"+ basedir + "/antora.yml", 'r') as stream: data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) @@ -129,23 +129,23 @@ def get_component(dir, basedir='/', branch='master'): component = data_loaded['name'] version = data_loaded['version'] - f = [] - for (dirpath, dirnames, filenames) in os.walk(dir + "/" + basedir + "/modules"): - f.extend(dirnames) + modules = [] + for (dirpath, dirnames, filenames) in os.walk(comp_dir + "/" + basedir + "/modules"): + modules.extend(dirnames) break - if len(f) < 1: + if not modules: raise ValueError('There should be at least one module!') - for module in f: + for module in modules: if module == "ROOT": target_repo_name = component else: target_repo_name = component + "-" + module clone_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) - call_src_to_pot(url=dir, basedir=basedir, branch=branch, module=module) + call_src_to_pot(url=comp_dir, basedir=basedir, branch=branch, module=module) #commit_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) - call_po_to_src(url=dir, basedir=basedir, branch=branch, module=module) + call_po_to_src(url=comp_dir, basedir=basedir, branch=branch, module=module) def make_component_lists(): """Deduct the component name from site.yml (l10n repos are per component)""" @@ -163,16 +163,16 @@ def make_component_lists(): for source in data_loaded['content']['sources']: url = source['url'] repo_name = url.rsplit('/', 1)[-1].replace('.git', '') - dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name + repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name if 'start_path' in source: - get_component(dir, basedir=source['start_path']) + get_component(repo_dir, basedir=source['start_path']) elif 'branches' in source: for branch in source['branches']: - get_component(dir, branch=branch) + get_component(repo_dir, branch=branch) else: - get_component(dir) + get_component(repo_dir) # Remove site.yml os.remove("site.yml") From 303fecd0eec5a2680ad7f3e572eb2e5407f910ad Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 13 2019 09:55:36 +0000 Subject: [PATCH 14/23] clean subprocesses --- diff --git a/build.py b/build.py index 2d4c051..925d435 100755 --- a/build.py +++ b/build.py @@ -5,7 +5,6 @@ import argparse import os import urllib.request import subprocess -import shlex import yaml @@ -30,19 +29,18 @@ def call_src_to_pot(url, branch='master', basedir='/', module="ROOT"): """Call src-to-ot.sh to convert english source (adoc) content to pot files""" print("call_src_to_pot: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module) output = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" - command = "./src-to-pot.sh --src-branch {b} --src-basedir {bd} --module {m} {u} {o}".format(b=branch, bd=basedir, m=module, u=url, o=output) - args = shlex.split(command) - subprocess.run(args, check=True) + subprocess.run(['./src-to-pot.sh', "--src-branch", branch, + "--src-basedir", basedir, "--module", module, url, output], check=True) def call_po_to_src(url, branch='master', basedir='/', module="ROOT"): """Call po-to-src.sh to convert po files to translated sources (adoc)""" print("call_po_to_src: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module) output = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" for lang in ['fr', 'cs', 'ja']: - command = "./po-to-src.sh --src-branch {b} --src-basedir {bd} --module {m} {l} {u} {o}".format(b=branch, bd=basedir, m=module, l=lang, u=url, o=output) - args = shlex.split(command) - subprocess.run(args, check=True) + subprocess.run(['./po-to-src.sh', "--src-branch", branch, + "--src-basedir", basedir, "--module", module, lang, url, + output], check=True) def clone_source(url, branch="master"): """Clone english sources in sources folder (update content if existing)""" @@ -54,10 +52,7 @@ def clone_source(url, branch="master"): subprocess.run(['git', 'checkout', branch], check=True, cwd=repo_dir) subprocess.run(['git', 'pull'], check=True, cwd=repo_dir) else: - command = "git clone -b {b} {u} {d}".format(b=branch, u=url, d=repo_dir) - print(command) - args = shlex.split(command) - subprocess.run(args, check=True) + subprocess.run(["git", "clone", "-b", branch, url, repo_dir], check=True) def clone_l10n(url): """Clone localization repos in l10n folder (update content if existing)""" @@ -68,10 +63,9 @@ def clone_l10n(url): if os.path.exists(repo_dir): subprocess.run(['git', 'pull'], check=True, cwd=repo_dir) else: - command = "git clone {u} {d} --config 'remote.origin.pushurl=ssh://git@pagure.io/fedora-l10n/{repo_name}.git'".format(u=url, d=dir, repo_name=repo_name) - print(command) - args = shlex.split(command) - subprocess.run(args, check=True) + subprocess.run(["git", "clone", url, repo_dir, "--config", + "remote.origin.pushurl=ssh://git@pagure.io/fedora-l10n/{r}.git".format(r=repo_name)], + check=True) def commit_l10n(url): """Commit and push changes of a localization repo""" @@ -81,9 +75,8 @@ def commit_l10n(url): subprocess.run(['git', 'add', '.'], check=True, cwd=repo_dir) - command = "git commit -m 'automatic update of pot/po files'" - args = shlex.split(command) - subprocess.run(args, check=True, cwd=repo_dir) + subprocess.run(["git", "commit", "-m", "automatic update of pot/po files"], + check=True, cwd=repo_dir) subprocess.run(['git', 'push'], check=True, cwd=repo_dir) @@ -123,7 +116,7 @@ def get_component(comp_dir, basedir='/', branch='master'): subprocess.run(['git', 'fetch'], check=True, cwd=comp_dir) subprocess.run(['git', 'checkout', branch], check=True, cwd=comp_dir) - with open(dir +"/"+ basedir + "/antora.yml", 'r') as stream: + with open(comp_dir +"/"+ basedir + "/antora.yml", 'r') as stream: data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) component = data_loaded['name'] From 74d8533d806643ccb713085e2f48004833d82ce7 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 13 2019 14:00:23 +0000 Subject: [PATCH 15/23] add parameters and translated-sources --- diff --git a/build.py b/build.py index 925d435..288d661 100755 --- a/build.py +++ b/build.py @@ -7,22 +7,34 @@ import urllib.request import subprocess import yaml +commit_changes = 'false' def main(): """Handle params""" + global commit_changes + parser = argparse.ArgumentParser( description="Calls `./po-to-src.sh` for all source repos") - parser.add_argument("output_dir", help="Output directory") + parser.add_argument("--clone_sources", choices=['true', 'false'], required=True, + help="Should we clone or update source or use cache?") + parser.add_argument("--commit_l10n", choices=['true', 'false'], required=True, + help="Should we clone or update source or use cache?") args = parser.parse_args() - output_dir = args.output_dir + commit_changes = args.commit_l10n + source_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + trans_source_dir = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" + l10n_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" - os.makedirs(output_dir, exist_ok=True) os.makedirs(source_dir, exist_ok=True) + os.makedirs(trans_source_dir, exist_ok=True) + os.makedirs(l10n_dir, exist_ok=True) - output_dir = os.path.abspath(output_dir) - clone_sources() + if args.clone_sources == "true": + clone_sources() + + clone_translated_source() make_component_lists() def call_src_to_pot(url, branch='master', basedir='/', module="ROOT"): @@ -67,9 +79,27 @@ def clone_l10n(url): "remote.origin.pushurl=ssh://git@pagure.io/fedora-l10n/{r}.git".format(r=repo_name)], check=True) +def clone_translated_source(): + """Clone translated sources (update content if existing)""" + print("* clone_l10n: " + url) + repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" + + url = "ssh://git@pagure.io/fedora-docs/translated-sources.git" + + if os.path.exists(repo_dir): + subprocess.run(['git', 'pull'], check=True, cwd=repo_dir) + else: + subprocess.run(["git", "clone", url, repo_dir], check=True) + def commit_l10n(url): """Commit and push changes of a localization repo""" + global commit_changes + + if commit_changes != "true": + return + print("* commit_l10n: " + url) + repo_name = url.rsplit('/', 1)[-1].replace('.git', '') repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name @@ -137,7 +167,7 @@ def get_component(comp_dir, basedir='/', branch='master'): target_repo_name = component + "-" + module clone_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) call_src_to_pot(url=comp_dir, basedir=basedir, branch=branch, module=module) - #commit_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) + commit_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) call_po_to_src(url=comp_dir, basedir=basedir, branch=branch, module=module) def make_component_lists(): From 31ec0742c1f259510d2ccaef514fe8b30eb978ed Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 13 2019 16:37:17 +0000 Subject: [PATCH 16/23] split invidual steps --- diff --git a/build.py b/build.py index 288d661..f03e3a3 100755 --- a/build.py +++ b/build.py @@ -81,7 +81,7 @@ def clone_l10n(url): def clone_translated_source(): """Clone translated sources (update content if existing)""" - print("* clone_l10n: " + url) + print("* clone_translated_source") repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" url = "ssh://git@pagure.io/fedora-docs/translated-sources.git" diff --git a/src-to-pot.sh b/src-to-pot.sh index 80bb330..68b3569 100755 --- a/src-to-pot.sh +++ b/src-to-pot.sh @@ -203,6 +203,9 @@ fi module_dir="$output_dir/$target_repo_name/pot/$version" +echo "*************************" +echo "* convert .adoc to .pot" +echo "*************************" # Confert all .adoc files to .pot pushd "$tmp_english_src/$src_basedir/modules/$module" > /dev/null || exit @@ -224,39 +227,57 @@ do rm -rf "$tmp_english_src" exit 1 fi +done < <(find . -name '*.adoc' -print0) - # Update existing PO files - - if [ -d "$output_dir/$target_repo_name/po" ] ; then - - for lang in "$output_dir/$target_repo_name/po/"* ; do - [[ -d "$lang" ]] || break # handle the case of lang directory - - po_file="$lang/$version/$(dirname "$adoc_file")/$(basename -s .adoc "$adoc_file").po" +echo "*************************" +echo "* update po files" +echo "*************************" - if [ ! -e "$po_file" ] ; then - echo "missing po file: $po_file" - break - fi - # po4a-updatepo would be angry otherwise - sed -i 's/Content-Type: text\/plain; charset=CHARSET/Content-Type: text\/plain; charset=UTF-8/g' "$po_file" - - if ! po4a-updatepo \ - --format asciidoc \ - --master "$adoc_file" \ - --master-charset "UTF-8" \ - --po "$po_file" ; then - echo "" - echo "Error updating $lang PO file for: $adoc_file" - #rm -rf "$tmp_english_src" - #exit 1 +while IFS= read -r -d '' adoc_file +do + pot_file="$module_dir/$(dirname "$adoc_file")/$(basename -s .adoc "$adoc_file").pot" + # Update existing PO files + # Make sure there is a po folder + mkdir -p "$output_dir/$target_repo_name/po" + + echo " * $output_dir/$target_repo_name/po/" + for lang in "$output_dir/$target_repo_name/po/"* ; do + if [[ ! -d "$output_dir/$target_repo_name/po/$lang" ]] ; then + # because weblate needs at least one lang folder to detect files + mkdir -p "$output_dir/$target_repo_name/po/fr/" + lang="fr" + fi + + po_file="$output_dir/$target_repo_name/po/$lang/$version/$(dirname "$adoc_file")/$(basename -s .adoc "$adoc_file").po" + + if [ ! -e "$po_file" ] ; then + # because weblate needs at least one po file to detect files + if [ "$lang" = "fr" ] ; then + mkdir -p "$(dirname "$po_file")" + cp "$pot_file" "$po_file" fi - done - fi + # the po file don't exist, of if we just created it don't update it" + break + fi + # po4a-updatepo would be angry otherwise + sed -i 's/Content-Type: text\/plain; charset=CHARSET/Content-Type: text\/plain; charset=UTF-8/g' "$po_file" + + if ! po4a-updatepo \ + --format asciidoc \ + --master "$adoc_file" \ + --master-charset "UTF-8" \ + --po "$po_file" ; then + echo "" + echo "Error updating $lang PO file for: $adoc_file" + fi + done done < <(find . -name '*.adoc' -print0) popd > /dev/null || exit +echo "*************************" +echo "* convert antora.yml to .pot" +echo "*************************" # Convert the antora.yml to antora.yml.pot # if this is the main module @@ -275,6 +296,39 @@ if [ "$module" = "ROOT" ]; then popd > /dev/null || exit fi +echo "*************************" +echo "* update po file" +echo "*************************" + +if [ "$module" = "ROOT" ]; then + pushd "$tmp_english_src/$src_basedir" > /dev/null || exit + pot_file="$module_dir/antora.yml.pot" + + for lang in "$output_dir/$target_repo_name/po/"* ; do + po_file="$output_dir/$target_repo_name/po/$lang/$version/antora.yml.po" + + if [ ! -e "$po_file" ] ; then + # because weblate needs at least one po file to detect files + if [ "$lang" = "fr" ] ; then + cp "$pot_file" "$po_file" + fi + # the po file don't exist, of if we just created it don't update it" + break + fi + # po4a-updatepo would be angry otherwise + sed -i 's/Content-Type: text\/plain; charset=CHARSET/Content-Type: text\/plain; charset=UTF-8/g' "$po_file" + + if ! po4a-updatepo \ + --format asciidoc \ + --master "antora.yml" \ + --master-charset "UTF-8" \ + --po "$po_file" ; then + echo "" + echo "Error updating $lang PO file for: $adoc_file" + fi + done + popd > /dev/null || exit +fi rm -rf "$tmp_english_src" From 6798a824ba103b5026a78b000661a9837fb4dcf7 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Apr 15 2019 20:19:13 +0000 Subject: [PATCH 17/23] pull new translations --- diff --git a/build.py b/build.py index f03e3a3..7a49c33 100755 --- a/build.py +++ b/build.py @@ -145,6 +145,7 @@ def get_component(comp_dir, basedir='/', branch='master'): print("get_component: "+ comp_dir + "/" + basedir + " " + branch) subprocess.run(['git', 'fetch'], check=True, cwd=comp_dir) subprocess.run(['git', 'checkout', branch], check=True, cwd=comp_dir) + subprocess.run(['git', 'pull'], check=True, cwd=comp_dir) with open(comp_dir +"/"+ basedir + "/antora.yml", 'r') as stream: data_loaded = yaml.load(stream, Loader=yaml.SafeLoader) From bd4de217f8c3fa6d7349fbfeb6e7d547be900814 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Apr 15 2019 21:40:06 +0000 Subject: [PATCH 18/23] prevent name conflicts with source repos --- diff --git a/build.py b/build.py index 7a49c33..25df012 100755 --- a/build.py +++ b/build.py @@ -6,12 +6,14 @@ import os import urllib.request import subprocess import yaml +from urllib.parse import urlparse commit_changes = 'false' +selected_component = None def main(): """Handle params""" - global commit_changes + global commit_changes, selected_component parser = argparse.ArgumentParser( description="Calls `./po-to-src.sh` for all source repos") @@ -19,9 +21,12 @@ def main(): help="Should we clone or update source or use cache?") parser.add_argument("--commit_l10n", choices=['true', 'false'], required=True, help="Should we clone or update source or use cache?") + parser.add_argument("--component", required=False, + help="Antora component, all if unset") args = parser.parse_args() commit_changes = args.commit_l10n + selected_component = args.component source_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" trans_source_dir = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" @@ -54,9 +59,9 @@ def call_po_to_src(url, branch='master', basedir='/', module="ROOT"): "--src-basedir", basedir, "--module", module, lang, url, output], check=True) -def clone_source(url, branch="master"): +def clone_source(doc_url, branch="master"): """Clone english sources in sources folder (update content if existing)""" - repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + repo_name = urlparse(doc_url).hostname+urlparse(doc_url).path[1:].replace('.git', '') repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name if os.path.exists(repo_dir): @@ -64,7 +69,7 @@ def clone_source(url, branch="master"): subprocess.run(['git', 'checkout', branch], check=True, cwd=repo_dir) subprocess.run(['git', 'pull'], check=True, cwd=repo_dir) else: - subprocess.run(["git", "clone", "-b", branch, url, repo_dir], check=True) + subprocess.run(["git", "clone", "-b", branch, doc_url, repo_dir], check=True) def clone_l10n(url): """Clone localization repos in l10n folder (update content if existing)""" @@ -125,16 +130,16 @@ def clone_sources(): # Parse site.yml for source in data_loaded['content']['sources']: - url = source['url'] + doc_url = source['url'] if 'start_path' in source: - clone_source(url) + clone_source(doc_url) elif 'branches' in source: for branch in source['branches']: - clone_source(url, branch) + clone_source(doc_url, branch) else: - clone_source(url) + clone_source(doc_url) # Remove site.yml @@ -142,7 +147,7 @@ def clone_sources(): def get_component(comp_dir, basedir='/', branch='master'): """Run actions on the component (clone repos, run scripts, commit changes)""" - print("get_component: "+ comp_dir + "/" + basedir + " " + branch) + print("\n\nget_component: "+ comp_dir + "/" + basedir + " " + branch) subprocess.run(['git', 'fetch'], check=True, cwd=comp_dir) subprocess.run(['git', 'checkout', branch], check=True, cwd=comp_dir) subprocess.run(['git', 'pull'], check=True, cwd=comp_dir) @@ -153,6 +158,10 @@ def get_component(comp_dir, basedir='/', branch='master'): component = data_loaded['name'] version = data_loaded['version'] + if selected_component != None and selected_component != component: + print("--component was set to {s}, ignoring {c}".format(s=selected_component, c=component)) + return + modules = [] for (dirpath, dirnames, filenames) in os.walk(comp_dir + "/" + basedir + "/modules"): modules.extend(dirnames) @@ -185,8 +194,8 @@ def make_component_lists(): # Parse site.yml for source in data_loaded['content']['sources']: - url = source['url'] - repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + doc_url = source['url'] + repo_name = urlparse(doc_url).hostname+urlparse(doc_url).path[1:].replace('.git', '') repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" + repo_name if 'start_path' in source: From bb46ca33fe5529bcdfd5af76620951426c1f8827 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Apr 15 2019 21:55:24 +0000 Subject: [PATCH 19/23] pylint advices --- diff --git a/build.py b/build.py index 25df012..8fe045e 100755 --- a/build.py +++ b/build.py @@ -5,15 +5,17 @@ import argparse import os import urllib.request import subprocess -import yaml + from urllib.parse import urlparse -commit_changes = 'false' -selected_component = None +import yaml + +COMMIT_CHANGES = 'false' +SELECTED_COMPONENT = None def main(): """Handle params""" - global commit_changes, selected_component + global COMMIT_CHANGES, SELECTED_COMPONENT parser = argparse.ArgumentParser( description="Calls `./po-to-src.sh` for all source repos") @@ -25,8 +27,8 @@ def main(): help="Antora component, all if unset") args = parser.parse_args() - commit_changes = args.commit_l10n - selected_component = args.component + COMMIT_CHANGES = args.commit_l10n + SELECTED_COMPONENT = args.component source_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" trans_source_dir = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" @@ -48,7 +50,7 @@ def call_src_to_pot(url, branch='master', basedir='/', module="ROOT"): output = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" subprocess.run(['./src-to-pot.sh', "--src-branch", branch, - "--src-basedir", basedir, "--module", module, url, output], check=True) + "--src-basedir", basedir, "--module", module, url, output], check=True) def call_po_to_src(url, branch='master', basedir='/', module="ROOT"): """Call po-to-src.sh to convert po files to translated sources (adoc)""" @@ -56,8 +58,8 @@ def call_po_to_src(url, branch='master', basedir='/', module="ROOT"): output = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" for lang in ['fr', 'cs', 'ja']: subprocess.run(['./po-to-src.sh', "--src-branch", branch, - "--src-basedir", basedir, "--module", module, lang, url, - output], check=True) + "--src-basedir", basedir, "--module", module, lang, url, + output], check=True) def clone_source(doc_url, branch="master"): """Clone english sources in sources folder (update content if existing)""" @@ -81,8 +83,8 @@ def clone_l10n(url): subprocess.run(['git', 'pull'], check=True, cwd=repo_dir) else: subprocess.run(["git", "clone", url, repo_dir, "--config", - "remote.origin.pushurl=ssh://git@pagure.io/fedora-l10n/{r}.git".format(r=repo_name)], - check=True) + "remote.origin.pushurl=ssh://git@pagure.io/fedora-l10n/{r}.git".format(r=repo_name)], + check=True) def clone_translated_source(): """Clone translated sources (update content if existing)""" @@ -98,9 +100,9 @@ def clone_translated_source(): def commit_l10n(url): """Commit and push changes of a localization repo""" - global commit_changes + global COMMIT_CHANGES - if commit_changes != "true": + if COMMIT_CHANGES != "true": return print("* commit_l10n: " + url) @@ -158,8 +160,8 @@ def get_component(comp_dir, basedir='/', branch='master'): component = data_loaded['name'] version = data_loaded['version'] - if selected_component != None and selected_component != component: - print("--component was set to {s}, ignoring {c}".format(s=selected_component, c=component)) + if SELECTED_COMPONENT not in (None, component): + print("--component was set to {s}, ignoring {c}".format(s=SELECTED_COMPONENT, c=component)) return modules = [] From a3f8ad9b1a3bf8620b337971108700fda7fc7270 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Apr 16 2019 12:54:29 +0000 Subject: [PATCH 20/23] update README and help messages --- diff --git a/README.md b/README.md index 13ebd92..83ab21a 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ -# Docs Translation Prototype +# Fedora Docs internatialization -This is a prototype of a multi-language Fedora Docs build pipeline. Work is tracked in [https://taiga.fedorainfracloud.org/project/asamalik-antora-for-docs/epic/1](https://taiga.fedorainfracloud.org/project/asamalik-antora-for-docs/epic/1). +This is a prototype of a multi-language Fedora Docs build pipeline. Work is tracked in [https://fedoraproject.org/wiki/User:Jibecfed/fedoradoc-antora-localization](https://fedoraproject.org/wiki/User:Jibecfed/fedoradoc-antora-localization). -These scripts work with: +The internatialization work with: -* Fedora Docs sources — sources of the docs in an adoc format. -* Translation repos — [pagure.io/projects/fedora-l10n/*](https://pagure.io/projects/fedora-l10n/%2A) — repositories holding POT and PO files, acting as a bridge between the Fedora Docs site and translators. This way, translators can choose any workflow with a translation engine of their choice. The only requirement is that they need to work with POT and PO files. -* Translated sources - [pagure.io/fedora-docs/translated-sources/](https://pagure.io/fedora-docs/translated-sources/) — all translated adoc files used to build the translated site. +* Fedora Docs sources — sources of the docs in an adoc format. For full list of these, see [https://pagure.io/fedora-docs/docs-fp-o](site.yml in docs-fp-o). +* Translation scripts — [https://pagure.io/fedora-docs/translations-scripts/](https://pagure.io/fedora-docs/translations-scripts/) — scripts to convert from adoc to pot and from pot to adoc. +* Translation repos — [https://pagure.io/projects/fedora-l10n/*](https://pagure.io/projects/fedora-l10n/%2A) — repositories holding POT and PO files, acting as a bridge between the Fedora Docs site and translators. This way, translators can choose any workflow with a translation engine of their choice. The only requirement is that they need to work with POT and PO files. +* Translated sources — [https://pagure.io/fedora-docs/translated-sources/](https://pagure.io/fedora-docs/translated-sources/) — all translated adoc files used to build the translated site. ## How to run @@ -18,61 +19,31 @@ $ sudo dnf install podman ### Submitting sources for translation -The `all-src-to-pot.sh` holds a list of repositories to be translated. +The `buid.py` script will find list of repositories to be translated in the file site.yml from the the docs-fp-o repo. It clones all of the specified source repositories, all of the matching translation repos, and converts the English sources into POT files. The output of this script are local copies of the translation repos, updated with the newly generated POT files. -It takes one argument — an absolute path to an empty directory to store the output. +It takes the following arguments : + --clone_sources {true,false} + Should we clone/update sources or use local content? + --commit_l10n {true,false} + Should we commit changes on l10n repos? + --component COMPONENT + Antora component to convert, all if unset -First, create an empty directory for the output and run the script: +The `buid.py` script will create three folders: -``` -$ mkdir translation-repos -$ ./all-src-to-pot.sh $(pwd)/translation-repos -``` - -Now you have local copies of the translation repos under `./translation-repos`, with the updated POT files. The next step is to push the changes to each repo. Go to the directory, and list all the repos: - -``` -$ cd translation-repos -$ ls -docs fedora fedora-install-guide fedora-release-notes -``` - -Then go to each repo, and push the changes. For example: - -``` -$ cd docs -$ git add . -$ git commit -m "refresh POT files" -$ git push -$ cd .. -``` - -Repeat this for each repo. And you're done. - -### Retrieving the translated sources - -The `$ ./all-po-to-src.sh` holds a list of repositories to be translated and a list of target languages. -The output is a local copy of the translated sources repo, updated with the newly generated adoc files. - -[See the structure of the translated sources](https://pagure.io/fedora-docs/design-documents/blob/master/f/po-pot-repo-structure.md). - -First, create a directory for the output and run the script: - -``` -$ mkdir translated-sources -$ ./all-po-to-src.sh $(pwd)/translated-sources -``` +* $(pwd)/sources — Fedora Docs sources +* $(pwd)/l10n — Translation repos +* $(pwd)/translated-sources — Translated sources -Now you have a local copy of the translated sources repo, right in `./translated-sources`. To push the changes: +### Publish translated sources ``` $ cd translated-sources -$ git add . -$ git commit -m "update translated source" +$ git add. +$ git commit -m "automatic update of translated content" $ git push -$ cd .. ``` ### Rebuilding the translated website @@ -89,5 +60,5 @@ $ ./build-translated.sh $ ./preview.sh ``` -Now you should be able to see, for example, the French site, on [http://localhost:8080/fr-FR/](http://localhost:8080/fr-FR/). +Now you should be able to see, for example, the French site, on [http://localhost:8080/fr/](http://localhost:8080/fr/). diff --git a/build.py b/build.py index 8fe045e..6da3b99 100755 --- a/build.py +++ b/build.py @@ -18,13 +18,13 @@ def main(): global COMMIT_CHANGES, SELECTED_COMPONENT parser = argparse.ArgumentParser( - description="Calls `./po-to-src.sh` for all source repos") + description="Calls `./src-to-pot.sh` and `./po-to-src.sh` for all source repos") parser.add_argument("--clone_sources", choices=['true', 'false'], required=True, - help="Should we clone or update source or use cache?") + help="Should we clone/update sources or use local content?") parser.add_argument("--commit_l10n", choices=['true', 'false'], required=True, - help="Should we clone or update source or use cache?") + help="Should we commit changes on l10n repos?") parser.add_argument("--component", required=False, - help="Antora component, all if unset") + help="Antora component to convert, all if unset") args = parser.parse_args() COMMIT_CHANGES = args.commit_l10n From 90da771cac9cd76c8f658740b5f243d6277e5990 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 17 2019 05:41:10 +0000 Subject: [PATCH 21/23] add translated-sources commit and moved commit_l10n --- diff --git a/README.md b/README.md index 83ab21a..85bbf7f 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ It takes the following arguments : Should we clone/update sources or use local content? --commit_l10n {true,false} Should we commit changes on l10n repos? + --commit_tsources {true,false} + Should we commit translated sources? --component COMPONENT Antora component to convert, all if unset diff --git a/build.py b/build.py index 6da3b99..1150a05 100755 --- a/build.py +++ b/build.py @@ -10,12 +10,11 @@ from urllib.parse import urlparse import yaml -COMMIT_CHANGES = 'false' SELECTED_COMPONENT = None def main(): """Handle params""" - global COMMIT_CHANGES, SELECTED_COMPONENT + global SELECTED_COMPONENT parser = argparse.ArgumentParser( description="Calls `./src-to-pot.sh` and `./po-to-src.sh` for all source repos") @@ -23,11 +22,12 @@ def main(): help="Should we clone/update sources or use local content?") parser.add_argument("--commit_l10n", choices=['true', 'false'], required=True, help="Should we commit changes on l10n repos?") + parser.add_argument("--commit_tsources", choices=['true', 'false'], required=True, + help="Should we commit translated sources?") parser.add_argument("--component", required=False, help="Antora component to convert, all if unset") args = parser.parse_args() - COMMIT_CHANGES = args.commit_l10n SELECTED_COMPONENT = args.component source_dir = os.path.dirname(os.path.abspath(__file__)) + "/sources/" @@ -44,6 +44,12 @@ def main(): clone_translated_source() make_component_lists() + if args.commit_l10n == "true": + commit_l10n_repos() + + if args.commit_tsources == "true": + commit_translated_sources() + def call_src_to_pot(url, branch='master', basedir='/', module="ROOT"): """Call src-to-ot.sh to convert english source (adoc) content to pot files""" print("call_src_to_pot: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module) @@ -73,10 +79,11 @@ def clone_source(doc_url, branch="master"): else: subprocess.run(["git", "clone", "-b", branch, doc_url, repo_dir], check=True) -def clone_l10n(url): +def clone_l10n(repo_name): """Clone localization repos in l10n folder (update content if existing)""" - print("* clone_l10n: " + url) - repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + print("* clone_l10n: " + repo_name) + + url = "https://pagure.io/fedora-l10n/" + repo_name repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name if os.path.exists(repo_dir): @@ -98,16 +105,35 @@ def clone_translated_source(): else: subprocess.run(["git", "clone", url, repo_dir], check=True) -def commit_l10n(url): - """Commit and push changes of a localization repo""" - global COMMIT_CHANGES +def commit_translated_source(): + """Commit translated sources""" + print("* commit_translated_source") + repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" - if COMMIT_CHANGES != "true": - return + subprocess.run(['git', 'add', '.'], check=True, cwd=repo_dir) + + subprocess.run(["git", "commit", "-m", "automatic update of translated content"], + check=True, cwd=repo_dir) + + subprocess.run(['git', 'push'], check=True, cwd=repo_dir) + + +def commit_l10n_repos(): + """Commit and push changes of localization repositories""" + + print("* commit_l10n_repos") + repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + + # commit each folders of the l10n path + [commit_l10n(r) for r in next(os.walk(repo_dir))[1]] + + +def commit_l10n(repo_name): + """Commit and push changes of a localization repo""" - print("* commit_l10n: " + url) + print("* commit_l10n: " + repo_name) - repo_name = url.rsplit('/', 1)[-1].replace('.git', '') + url = "https://pagure.io/fedora-l10n/" + repo_name repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" + repo_name subprocess.run(['git', 'add', '.'], check=True, cwd=repo_dir) @@ -177,9 +203,8 @@ def get_component(comp_dir, basedir='/', branch='master'): target_repo_name = component else: target_repo_name = component + "-" + module - clone_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) + clone_l10n(target_repo_name) call_src_to_pot(url=comp_dir, basedir=basedir, branch=branch, module=module) - commit_l10n("https://pagure.io/fedora-l10n/"+target_repo_name) call_po_to_src(url=comp_dir, basedir=basedir, branch=branch, module=module) def make_component_lists(): From b5ab38641e071311353d089ca6d6cd1db33a493f Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 17 2019 06:03:37 +0000 Subject: [PATCH 22/23] make po-to-src language agnostic --- diff --git a/build.py b/build.py index 1150a05..bbbb6aa 100755 --- a/build.py +++ b/build.py @@ -62,10 +62,10 @@ def call_po_to_src(url, branch='master', basedir='/', module="ROOT"): """Call po-to-src.sh to convert po files to translated sources (adoc)""" print("call_po_to_src: "+url +" b:"+ branch +" bd:"+ basedir +" m:"+ module) output = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" - for lang in ['fr', 'cs', 'ja']: - subprocess.run(['./po-to-src.sh', "--src-branch", branch, - "--src-basedir", basedir, "--module", module, lang, url, - output], check=True) + + subprocess.run(['./po-to-src.sh', "--src-branch", branch, + "--src-basedir", basedir, "--module", module, url, + output], check=True) def clone_source(doc_url, branch="master"): """Clone english sources in sources folder (update content if existing)""" diff --git a/po-to-src.sh b/po-to-src.sh index 3145468..113ea8e 100755 --- a/po-to-src.sh +++ b/po-to-src.sh @@ -9,7 +9,6 @@ usage () { echo "Arguments:" echo " REPO_URL Source repository URL." echo " OUTPUT_DIR Path to an output directory." - echo " LOCALE What language to work with. Such as 'fr-FR'." echo "" echo "Parametres:" echo " Generic:" @@ -51,16 +50,6 @@ while true ; do esac done -# Need LOCALE -if [ "$1" != "" ]; then - language=$1 - shift -else - usage - echo "Please specify LOCALE, REPO_URL and TARGET_DIR" - exit 1 -fi - # Need REPO_URL if [ "$1" != "" ]; then src_repo=$1 @@ -98,6 +87,10 @@ if [ -z "$src_basedir" ]; then src_basedir="" fi +echo "*****************" +echo "* po-to-src" +echo "*****************" + tmp_english_src=$(mktemp -d) delete_tmp() { @@ -204,76 +197,76 @@ tmp_po_pot="/home/jean-baptiste/Projects/fedora-l10n/translations-scripts/l10n/$ # Missing po for .adoc: the .adoc just get s copied # # TODO: stretch idea: if the translations don't exist, I could just copy -# the english sources over... this could simplify automation +# the english sources over... this could simplify automation + +for lang_folder in "$tmp_po_pot/po/"* ; do + [[ -d "$lang_folder" ]] || break # if no result + + language=$(basename $lang_folder) + echo "*****************" + echo "* LANG: $language" + echo "*****************" + #po_pot_module_dir="$output_dir/$po_pot_repo_name/pot/$version" + po_pot_module_dir="$tmp_po_pot/po/$language/$version" + output_module_dir="$output_dir/$language/$component/$version/modules/$module" + + # Confert all .adoc files to .pot + pushd "$tmp_english_src/$src_basedir/modules/$module" > /dev/null || exit + + for source_adoc_file in $(find . -name '*.adoc'); do + po_file="$po_pot_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").po" + target_adoc_file="$output_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").adoc" + if [ -f "$po_file" ] ; then + # TODO: convert file + if ! po4a-translate \ + --format asciidoc \ + --master "$source_adoc_file" \ + --master-charset "UTF-8" \ + --po "$po_file" \ + --localized "$target_adoc_file" \ + --localized-charset "UTF-8" \ + --keep 0 + then + echo "Error converting a PO file: $po_file" + delete_tmp + exit 1 + fi + else + # "Warning: The following file is in the source, but not in the translations repo:" + mkdir -p "$(dirname "$target_adoc_file")" + cp "$source_adoc_file" "$target_adoc_file" + fi + done -#po_pot_module_dir="$output_dir/$po_pot_repo_name/pot/$version" -po_pot_module_dir="$tmp_po_pot/po/$language/$version" -output_module_dir="$output_dir/$language/$component/$version/modules/$module" + popd > /dev/null || exit -# Make sure the language exists in the PO/POT repo -# if [ ! -d "$tmp_po_pot/po/$language" ] ; then -# echo "Error. Locale $language doesn't exist in the PO/POT repo." -# delete_tmp -# exit 1 -# fi -# Confert all .adoc files to .pot -pushd "$tmp_english_src/$src_basedir/modules/$module" > /dev/null || exit - -for source_adoc_file in $(find . -name '*.adoc'); do - po_file="$po_pot_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").po" - target_adoc_file="$output_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").adoc" - if [ -f "$po_file" ] ; then - # TODO: convert file - if ! po4a-translate \ - --format asciidoc \ - --master "$source_adoc_file" \ - --master-charset "UTF-8" \ - --po "$po_file" \ - --localized "$target_adoc_file" \ - --localized-charset "UTF-8" \ - --keep 0 - then - echo "Error converting a PO file: $po_file" - delete_tmp - exit 1 + # Convert the antora.yml to antora.yml.pot + # if this is the main module + if [ "$module" = "ROOT" ]; then + pushd "$tmp_english_src/$src_basedir" > /dev/null || exit + + source_antora_yml="$tmp_english_src/$src_basedir/antora.yml" + po_antora_yml="$tmp_po_pot/po/$language/$version/antora.yml.po" + target_antora_yml="$output_dir/$language/$component/$version/antora.yml" + + if [ -f "$po_antora_yml" ] ; then + po4a-translate \ + --format yaml \ + --option keys="title" \ + --master "$source_antora_yml" \ + --master-charset "UTF-8" \ + --po "$po_antora_yml" \ + --localized "$target_antora_yml" \ + --localized-charset "UTF-8" \ + --keep 0 + else + # "Warning: There should be an antora.yml.po, but there isn't." + mkdir -p "$(dirname "$target_antora_yml")" + cp "$source_antora_yml" "$target_antora_yml" fi - else - # "Warning: The following file is in the source, but not in the translations repo:" - mkdir -p "$(dirname "$target_adoc_file")" - cp "$source_adoc_file" "$target_adoc_file" + popd > /dev/null || exit fi done -popd > /dev/null || exit - - -# Convert the antora.yml to antora.yml.pot -# if this is the main module -if [ "$module" = "ROOT" ]; then - pushd "$tmp_english_src/$src_basedir" > /dev/null || exit - - source_antora_yml="$tmp_english_src/$src_basedir/antora.yml" - po_antora_yml="$tmp_po_pot/po/$language/$version/antora.yml.po" - target_antora_yml="$output_dir/$language/$component/$version/antora.yml" - - if [ -f "$po_antora_yml" ] ; then - po4a-translate \ - --format yaml \ - --option keys="title" \ - --master "$source_antora_yml" \ - --master-charset "UTF-8" \ - --po "$po_antora_yml" \ - --localized "$target_antora_yml" \ - --localized-charset "UTF-8" \ - --keep 0 - else - # "Warning: There should be an antora.yml.po, but there isn't." - mkdir -p "$(dirname "$target_antora_yml")" - cp "$source_antora_yml" "$target_antora_yml" - fi - popd > /dev/null || exit -fi - - delete_tmp From c29d684883720f34d89af7fd47c6af62d6b80908 Mon Sep 17 00:00:00 2001 From: Jibec Date: Apr 18 2019 20:03:56 +0000 Subject: [PATCH 23/23] fix navigation issue --- diff --git a/po-to-src.sh b/po-to-src.sh index 113ea8e..56e0fc5 100755 --- a/po-to-src.sh +++ b/po-to-src.sh @@ -216,10 +216,17 @@ for lang_folder in "$tmp_po_pot/po/"* ; do for source_adoc_file in $(find . -name '*.adoc'); do po_file="$po_pot_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").po" target_adoc_file="$output_module_dir/$(dirname "$source_adoc_file")/$(basename -s .adoc "$source_adoc_file").adoc" + + format="asciidoc" + # nav.adoc is a bullet point list, it isn't really asciidoc + if [ "$(basename -s .adoc "$source_adoc_file")" = "nav" ] ; then + format="text" + fi + if [ -f "$po_file" ] ; then # TODO: convert file if ! po4a-translate \ - --format asciidoc \ + --format "$format" \ --master "$source_adoc_file" \ --master-charset "UTF-8" \ --po "$po_file" \ diff --git a/src-to-pot.sh b/src-to-pot.sh index 68b3569..669f29b 100755 --- a/src-to-pot.sh +++ b/src-to-pot.sh @@ -214,10 +214,15 @@ while IFS= read -r -d '' adoc_file do pot_file="$module_dir/$(dirname "$adoc_file")/$(basename -s .adoc "$adoc_file").pot" + format="asciidoc" + # nav.adoc is a bullet point list, it isn't really asciidoc + if [ "$(basename -s .adoc "$adoc_file")" = "nav" ] ; then + format="text" + fi # generate new POT files if ! po4a-gettextize \ - --format asciidoc \ + --format "$format" \ --master "$adoc_file" \ --master-charset "UTF-8" \ --localized-charset "UTF-8" \ @@ -262,8 +267,14 @@ do # po4a-updatepo would be angry otherwise sed -i 's/Content-Type: text\/plain; charset=CHARSET/Content-Type: text\/plain; charset=UTF-8/g' "$po_file" + format="asciidoc" + # nav.adoc is a bullet point list, it isn't really asciidoc + if [ "$(basename -s .adoc "$adoc_file")" = "nav" ] ; then + format="text" + fi + if ! po4a-updatepo \ - --format asciidoc \ + --format "$format" \ --master "$adoc_file" \ --master-charset "UTF-8" \ --po "$po_file" ; then