From da88756813d479898b5ff9c9250cd1f6dcc57ab0 Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
Date: Jan 25 2021 22:33:48 +0000
Subject: [PATCH 1/2] website: display package progress as a list


goal: translators should be able to filter translation progress and access to upstream translation team.
package pages will be more focused on internal analysis (list of files considered with detailed translations?).

---

diff --git a/build_stats.py b/build_stats.py
index 90cad4b..27807f7 100755
--- a/build_stats.py
+++ b/build_stats.py
@@ -6,7 +6,7 @@ import glob
 import json
 import os
 import shutil
-import tempfile
+import polib
 
 from translate.tools.pocount import calcstats
 
@@ -145,11 +145,32 @@ def get_po_translation_level(files, stats_file):
         for key in keys:
             results[key] = stat.get(key, 0)
 
+        results["team"] = get_language_team(file)
+
         stats[file] = results
 
     return stats
 
 
+def get_language_team(file):
+    metadata = dict()
+    try:
+        metadata = polib.pofile(file).metadata
+    except UnicodeDecodeError:
+        # encoding error, to investigate before using it in TM
+        metadata["Language"] = "error-unicode"
+    except OSError:
+        # maybe a polib bug? to investigate before using it in TM
+        metadata["Language"] = "error-os"
+
+    team = "Unknown..."
+    try:
+        team = metadata["Language-Team"]
+    except KeyError:
+        print("The file {f} have no Language team? Here are the metadata: {m}".format(f=file, m=metadata))
+    return team
+
+
 def extract_release_stats(results, files_stats):
     number_of_packages = results.get("nb_packages", 0)
     number_of_packages += 1
diff --git a/build_website.py b/build_website.py
index 02c6e02..7b38c25 100755
--- a/build_website.py
+++ b/build_website.py
@@ -77,24 +77,18 @@ def main():
         results = consolidate_package_stats(file_stats, log_files)
         store_json_file(results, dest_file)
 
-        langs_results = (
-            results.get("equalsormorethan80percent", [])
-            + results.get("between50and80percent", [])
-            + results.get("lessorequalto50percent", [])
-        )
-
-        for langs in langs_results:
-            val = packages_langs_results.get(langs["lang_code"], [])
-            val.append({"name": name, "progress": langs["progress"]})
-            packages_langs_results[langs["lang_code"]] = val
+        for lang in results.get("stats", []):
+            val = packages_langs_results.get(lang["lang_code"], [])
+            val.append({"name": name, "progress": lang["progress"], "translated": lang["translated"], "team": lang["team"]})
+            packages_langs_results[lang["lang_code"]] = val
 
     print("Prepare json files for languages")
-    langs = [
+    languages = [
         f
         for f in os.listdir(langs_stats)
         if os.path.isfile(os.path.join(langs_stats, f))
     ]
-    for lang in sorted(langs):
+    for lang in sorted(languages):
         if lang.endswith(".json"):
             code = lang[: -len(".json")]
             dest_file = os.path.join(data_langs_folder, code + ".json")
@@ -110,12 +104,12 @@ def main():
 
     # generate static content for languages
     print("Generate static content for languages")
-    langs = [
+    languages = [
         f
         for f in os.listdir(data_langs_folder)
         if os.path.isfile(os.path.join(data_langs_folder, f))
     ]
-    for lang in sorted(langs):
+    for lang in sorted(languages):
         code = lang[: -len(".json")]
         dest_file = os.path.join(static_langs_folder, code + ".md")
 
@@ -146,10 +140,10 @@ def main():
         generate_static_pages_packages(args.results, code, content, dest_file)
 
     print("Copy translation memories")
-    langs = [
+    languages = [
         f for f in os.listdir(tm_folder) if os.path.isfile(os.path.join(tm_folder, f))
     ]
-    for lang in langs:
+    for lang in languages:
         if lang.endswith(".gz"):
             shutil.copyfile(
                 os.path.join(tm_folder, lang), os.path.join(static_tm_folder, lang)
@@ -170,6 +164,7 @@ def consolidate_language_stats(stats_file, distribution_stats):
         "fuzzy": "int",
         "untranslated": "int",
         "translatedtargetwords": "int",
+        "team": "str",
         "totalsourcewords": "int",
     }
 
@@ -214,6 +209,7 @@ def consolidate_package_stats(stats_file, log_files):
         "fuzzy": "int",
         "untranslated": "int",
         "translatedtargetwords": "int",
+        "team": "str",
         "totalsourcewords": "int",
     }
 
@@ -253,7 +249,7 @@ def consolidate_package_stats(stats_file, log_files):
         lambda x: x["_merge"] == "left_only"
     ]
 
-    temp = (
+    temp_translated = (
         stats_df_w_lang.groupby(["lang_code"])
         .agg(
             {
@@ -262,8 +258,10 @@ def consolidate_package_stats(stats_file, log_files):
         )
         .reset_index()
         .droplevel(1, axis=1)
-        .to_dict(orient="records")
     )
+    temp_teams = stats_df_w_lang.groupby("lang_code")["team"].apply(lambda x: ",".join(x))
+    temp = pd.merge(temp_translated, temp_teams, how="inner", on="lang_code").to_dict(orient="records")
+
     for line in temp:
         line["progress"] = 0
         p = 0
@@ -285,27 +283,16 @@ def consolidate_package_stats(stats_file, log_files):
                     tot=total_source_words,
                 )
             )
-
+        line["translated"] = line["translatedsourcewords"]
         line["progress"] = p
 
-    results["totalsourcewords"] = total_source_words
-    results["count_languages"] = len(pd.unique(stats_df_w_lang["lang_code"]))
-
+    results["stats"] = list()
     for line in sorted(temp, key=lambda k: k["progress"], reverse=True):
         del line["translatedsourcewords"]
-        if line["progress"] <= 50:
-            hop = results.get("lessorequalto50percent", [])
-            hop.append(line)
-            results["lessorequalto50percent"] = hop
-        elif line["progress"] < 80:
-            hop = results.get("between50and80percent", [])
-            hop.append(line)
-            results["between50and80percent"] = hop
-        else:
-            hop = results.get("equalsormorethan80percent", [])
-            hop.append(line)
-            results["equalsormorethan80percent"] = hop
+        results["stats"].append(line)
 
+    results["totalsourcewords"] = total_source_words
+    results["count_languages"] = len(pd.unique(stats_df_w_lang["lang_code"]))
     results["no_languages"] = stats_df_no_lang["filename"].tolist()
 
     return results
diff --git a/templates/language.md b/templates/language.md
index a879b1d..9ac40af 100644
--- a/templates/language.md
+++ b/templates/language.md
@@ -21,6 +21,8 @@ Download:
 * {{ "{{%" }} link "/{{ results }}/{{ lang_code }}.tmx.gz" {{ "%}}" }}{{ lang_code }} translation memory{{ "{{%" }} /link {{ "%}}" }} see [tmx](https://en.wikipedia.org/wiki/Translation_Memory_eXchange)
 
 Packages:
+| Name                                                                                           | Translated words         | Progress               | Language team      |
+|------------------------------------------------------------------------------------------------|--------------------------|------------------------|--------------------|
 {% for package in packages -%}
-* [{{ package.name }}]({{ '{{' }}< ref "/{{ results }}/package/{{ package.name }}.md" >{{ '}}' }}) ({{ package.progress }})
+[{{ package.name }}]({{ '{{' }}< ref "/{{ results }}/package/{{ package.name }}.md" >{{ '}}' }}) | {{ package.translated }} | {{ package.progress }} | {{ package.team }} | 
 {% endfor %}
diff --git a/templates/package.md b/templates/package.md
index 0dc57a6..0d905ef 100644
--- a/templates/package.md
+++ b/templates/package.md
@@ -2,35 +2,15 @@
 title: "{{ package }}"
 date: {{ now }}
 ---
-The package {{ package }} is transtlated into {{ count_languages }} languages in Fedora {{ results }}.
+The package {{ package }} is translated into {{ count_languages }} languages in Fedora {{ results }}.
 
-## Languages with ≥80% words translated
-{% if equalsormorethan80percent %} 
-{% for stat in equalsormorethan80percent -%}
-[{{ stat.lang_code }}]({{ '{{' }}< ref "/{{ results }}/language/{{ stat.lang_code }}.md" >{{ '}}' }}) ({{ stat.progress }})
-{% endfor %}
-{% else %}
-None
-{% endif %}
+This packages reprends {{ totalsourcewords }} source words to be translated in each language.
 
-## Languages with >50% and <80% words translated
-{% if between50and80percent %} 
-{% for stat in between50and80percent -%}
-[{{ stat.lang_code }}]({{ '{{' }}< ref "/{{ results }}/language/{{ stat.lang_code }}.md" >{{ '}}' }}) ({{ stat.progress }})
+| Language                                                                                              | Translated words      | Progress            |
+|-------------------------------------------------------------------------------------------------------|-----------------------|---------------------|
+{% for stat in stats -%}
+| [{{ stat.lang_code }}]({{ '{{' }}< ref "/{{ results }}/language/{{ stat.lang_code }}.md" >{{ '}}' }}) | {{ stat.translated }} | {{ stat.progress }} |
 {% endfor %}
-{% else %}
-None
-{% endif %}
-
-
-## Languages with ≤50% words translated
-{% if lessorequalto50percent %} 
-{% for stat in lessorequalto50percent -%}
-[{{ stat.lang_code }}]({{ '{{' }}< ref "/{{ results }}/language/{{ stat.lang_code }}.md" >{{ '}}' }}) ({{ stat.progress }})
-{% endfor %}
-{% else %}
-None
-{% endif %}
 
 ## Errors
 {% if no_languages %}

From 66b22b099cd48870bb2cde17e165495b18f5f8dd Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
Date: Feb 01 2021 20:10:49 +0000
Subject: [PATCH 2/2] ensure translated value always is set


---

diff --git a/build_website.py b/build_website.py
index 7b38c25..b340094 100755
--- a/build_website.py
+++ b/build_website.py
@@ -264,17 +264,18 @@ def consolidate_package_stats(stats_file, log_files):
 
     for line in temp:
         line["progress"] = 0
-        p = 0
+        line["translated"] = line["translatedsourcewords"]
+
         if total_source_words == 0:
             print(
                 " File {f} for file has translatedsourcewords = 0 in line {l}".format(
                     f=stats_file, l=line
                 )
             )
-            line["progress"] = p
+            line["progress"] = 0
             continue
         try:
-            p = round((int(line["translatedsourcewords"]) / total_source_words) * 100)
+            line["progress"] = round((int(line["translatedsourcewords"]) / total_source_words) * 100)
         except OverflowError:
             print(
                 " File {f} has Translated={t} and Source={tot}".format(
@@ -283,8 +284,6 @@ def consolidate_package_stats(stats_file, log_files):
                     tot=total_source_words,
                 )
             )
-        line["translated"] = line["translatedsourcewords"]
-        line["progress"] = p
 
     results["stats"] = list()
     for line in sorted(temp, key=lambda k: k["progress"], reverse=True):

