From aa3c84262ce0dd59d7e10c97e479201729bdbae2 Mon Sep 17 00:00:00 2001
From: Francois Andrieu <darknao@fedoraproject.org>
Date: Feb 25 2021 20:27:40 +0000
Subject: [PATCH 1/10] update Dockerfile for openshift use


---

diff --git a/docker/Dockerfile.33 b/docker/Dockerfile.33
index b2c8d10..5736959 100644
--- a/docker/Dockerfile.33
+++ b/docker/Dockerfile.33
@@ -1,13 +1,18 @@
-FROM registry.fedoraproject.org/fedora:33
-
-RUN dnf install -y lbzip2 unzip xz git cpio translate-toolkit dnf-plugins-core python3-pip rsync vim
+FROM registry.fedoraproject.org/fedora:33 as builder
 
+RUN dnf install -y lbzip2 unzip xz cpio dnf-plugins-core rsync python3-pip hugo gettext git
 COPY requirements.txt /src/requirements.txt
+
 RUN pip install --no-cache -r /src/requirements.txt
 RUN pip install --upgrade https://github.com/WeblateOrg/language-data/archive/master.zip
 RUN pip install charamel
 RUN pip install git+https://github.com/WeblateOrg/translation-finder.git
-
-VOLUME /src
-VOLUME /srpms
+RUN mkdir -p /src/results /srpms; chmod g+rwX /srpms; chmod -R 1777 /tmp
 WORKDIR /src
+
+COPY *.py *.sh /src/
+COPY website /src/website/
+RUN chmod -R g+rwX /src/website
+COPY templates /src/templates/
+VOLUME /src/results
+ENV VERS f33
diff --git a/docker/Dockerfile.latest b/docker/Dockerfile.latest
new file mode 100644
index 0000000..9f73dec
--- /dev/null
+++ b/docker/Dockerfile.latest
@@ -0,0 +1,18 @@
+FROM registry.fedoraproject.org/fedora:34 as builder
+
+RUN dnf install -y lbzip2 unzip xz cpio dnf-plugins-core rsync python3-pip hugo gettext git
+COPY requirements.txt /src/requirements.txt
+
+RUN pip install --no-cache -r /src/requirements.txt
+RUN pip install --upgrade https://github.com/WeblateOrg/language-data/archive/master.zip
+RUN pip install charamel
+RUN pip install git+https://github.com/WeblateOrg/translation-finder.git
+RUN mkdir -p /src/results /srpms; chmod g+rwX /srpms; chmod -R 1777 /tmp
+WORKDIR /src
+
+COPY *.py *.sh /src/
+COPY website /src/website/
+RUN chmod -R g+rwX /src/website
+COPY templates /src/templates/
+VOLUME /src/results
+ENV VERS f34

From aeb576ad0fa7c0a362f79e731f99741d45bc979b Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
Date: Feb 25 2021 20:27:40 +0000
Subject: [PATCH 2/10] recursive search of buggy files


it saves quite a lot of computation time

---

diff --git a/build_tm.py b/build_tm.py
index 4b4eaaa..c672c7f 100755
--- a/build_tm.py
+++ b/build_tm.py
@@ -69,7 +69,7 @@ def main():
     for lang in sorted(langs):
         lang_code = lang[: -len(".json")]
 
-        log.info(" {l}".format(l=lang_code))
+        log.info("Processing {l}".format(l=lang_code))
 
         with open(os.path.join(lang_path, lang), "r") as read_file:
             files = json.load(read_file)["po"]
@@ -79,14 +79,7 @@ def main():
             os.path.dirname(os.path.abspath(__file__)), compendium_file
         )
         if not os.path.isfile(compendium_file):
-            try:
-                process_compendium(files, compendium_file, debug_folder)
-            except Exception as e:
-                log.error(
-                    " Compendium generation triggered an {t} exception: {e}".format(
-                        t=type(e).__name__, e=e
-                    )
-                )
+            process_compendium(files, compendium_file, debug_folder)
 
         tmx_file = os.path.join(tm_folder, lang_code + ".tmx")
         if not os.path.isfile(tmx_file):
@@ -110,6 +103,8 @@ def main():
                     )
                 )
 
+    log.info("All languages are processed".format(l=lang_code))
+
     log.info("Detecting missing files")
     for lang in sorted(langs):
         check_lang(lang[: -len(".json")], tm_folder)
@@ -158,30 +153,15 @@ def process_compendium(langfiles, dest, debug_folder):
 
             count += 1
 
-        # search every file that were successful
-        search_guilty_file(tmp, dest, debug_folder)
+        all_files = [f for f in os.listdir(tmp) if os.path.isfile(os.path.join(tmp, f))]
+        if len(all_files) == 1:
+            shutil.copyfile(os.path.join(tmp, all_files[0]), dest)
+        else:
+            msgcat_recursive(dest, tmp, debug_folder, all_files, list(), list())
 
 
-def search_guilty_file(path, dest, debug_folder):
-    log = logging.getLogger("buildTm.process_compendium.guilty")
-    all_files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
-
-    try_msgcat(all_files, dest, path)
-
-    guilty_file = None
-    while os.path.isfile(dest) is False:
-        guilty_file = all_files.pop()
-        try_msgcat(all_files, dest, path)
-
-    if guilty_file is not None:
-        debug_filename = "tm-msgcat-{lang}-{name}".format(lang=dest.split("/")[-1], name=guilty_file)
-        log.error("the file {f} raised error with msgcat, a copy of this file is into {d} as {n}".format(f=guilty_file, d=debug_folder, n=debug_filename))
-        shutil.move(os.path.join(path, guilty_file), os.path.join(debug_folder, debug_filename))
-        os.remove(dest)
-        search_guilty_file(path, dest, debug_folder)
-
-
-def try_msgcat(files, destination, cwd):
+def msgcat(files, destination, path, doubt=False):
+    log = logging.getLogger("buildTm.msgcat")
     command = [
                   "msgcat",
                   "--force-po",
@@ -191,12 +171,66 @@ def try_msgcat(files, destination, cwd):
               ] + files
 
     try:
-        subprocess.run(command, check=True, cwd=cwd, capture_output=True)
-    except subprocess.CalledProcessError:
+        subprocess.run(command, check=True, cwd=path, capture_output=True)
+    except subprocess.CalledProcessError as e:
         # msgcat often raise exception but continues its processing
+        if doubt is not False:
+            log.error("Error with file {d}: {e}".format(d=doubt, e=e.stderr.decode('utf8')))
         pass
 
 
+def store_debug_file(path, name, file, debug_folder):
+    log = logging.getLogger("buildTm.store_debug_file")
+    debug_filename = "{n}-{f}".format(n=name, f=file)
+    log.error("A copy of the file {f} is into {d} as {n}".format(f=file,
+                                                                 d=debug_folder,
+                                                                 n=debug_filename))
+    shutil.move(os.path.join(path, file), os.path.join(debug_folder, debug_filename))
+
+
+def msgcat_recursive(destination, path, debug_folder, backlog=[], ongoing=[], ok=[]):
+    log = logging.getLogger("buildTm.search_msgcat_buggy_files")
+    doubt = False
+    log.debug("backlog={b}, ongoing={o}, ok={ok}".format(b=len(backlog), o=len(ongoing), ok=len(ok)))
+    if len(ongoing) == 0:
+        ongoing = backlog.copy()
+        backlog = []
+
+    # we can't use msgcat with one single file
+    if len(ongoing) == 1:
+        doubt = ongoing.copy().pop()
+        ongoing.append(ok[0])
+
+    msgcat(ongoing, destination, path, doubt)
+
+    if os.path.isfile(destination) is True:
+        processed = len(ongoing)
+        ok += ongoing
+        # if we added one item from 'ok', we want to make sure it's not duplicated
+        ok = list(set(ok))
+        ongoing = []
+        if len(ok) == processed and len(backlog) == 0:
+            log.debug("First generation worked")
+        else:
+            os.remove(destination)
+    else:
+        if doubt is not False:
+            log.debug("This file raised a msgcat bug: {f}".format(f=doubt))
+            store_debug_file(path, "tm-msgcat-"+destination.split("/")[-1], doubt, debug_folder)
+            ongoing = []
+
+        half = int(len(ongoing) / 2)
+        backlog += ongoing[half:]
+        ongoing = ongoing[:half]
+
+    if len(backlog) + len(ongoing) > 0:
+        msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok)
+    else:
+        if os.path.isfile(destination) is False:
+            log.debug("Generating remaining files")
+            msgcat(ok, destination, path)
+
+
 def process_tmx(lang, source, dest):
     """ Generate a translation memory from a po file """
 
@@ -253,13 +287,13 @@ def check_lang(lang, tm_folder):
     terminology_file = os.path.join(tm_folder, lang + ".terminology.po")
 
     if not os.path.isfile(compendium_file):
-        log.warning(" {l}-compendium is missing".format(l=lang))
+        log.warning("{l}-compendium is missing".format(l=lang))
 
     if not os.path.isfile(tmx_file):
-        log.warning(" {l}-tmx is missing".format(l=lang))
+        log.warning("{l}-tmx is missing".format(l=lang))
 
     if not os.path.isfile(terminology_file):
-        log.warning(" {l}-terminology is missing".format(l=lang))
+        log.warning("{l}-terminology is missing".format(l=lang))
 
 
 def compress(folder):

From 131411a84f35b99b3081db6474d4206633913559 Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
Date: Feb 25 2021 20:27:40 +0000
Subject: [PATCH 3/10] add last check, risk of infinite loop here...


---

diff --git a/build_tm.py b/build_tm.py
index c672c7f..06ea08a 100755
--- a/build_tm.py
+++ b/build_tm.py
@@ -41,7 +41,7 @@ def main():
 
     loglevel = logging.INFO
     if args.verbose:
-      loglevel = logging.DEBUG
+        loglevel = logging.DEBUG
     logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=loglevel)
     log = logging.getLogger("buildTm")
 
@@ -103,7 +103,7 @@ def main():
                     )
                 )
 
-    log.info("All languages are processed".format(l=lang_code))
+    log.info("All languages are processed")
 
     log.info("Detecting missing files")
     for lang in sorted(langs):
@@ -161,6 +161,8 @@ def process_compendium(langfiles, dest, debug_folder):
 
 
 def msgcat(files, destination, path, doubt=False):
+    """ Call the msgcat command on a list of po files
+    Only print output if a bug is suspected """
     log = logging.getLogger("buildTm.msgcat")
     command = [
                   "msgcat",
@@ -180,15 +182,15 @@ def msgcat(files, destination, path, doubt=False):
 
 
 def store_debug_file(path, name, file, debug_folder):
+    """ Move the temporary move file in debug folder """
     log = logging.getLogger("buildTm.store_debug_file")
-    debug_filename = "{n}-{f}".format(n=name, f=file)
-    log.error("A copy of the file {f} is into {d} as {n}".format(f=file,
-                                                                 d=debug_folder,
-                                                                 n=debug_filename))
-    shutil.move(os.path.join(path, file), os.path.join(debug_folder, debug_filename))
+    target = os.path.join(debug_folder, "{n}-{f}".format(n=name, f=file))
+    log.error("The file {f} were moved into {t}".format(f=file, t=target))
+    shutil.move(os.path.join(path, file), target)
 
 
-def msgcat_recursive(destination, path, debug_folder, backlog=[], ongoing=[], ok=[]):
+def msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok):
+    """ Try to call msgcat, retry with half of the files if it fails """
     log = logging.getLogger("buildTm.search_msgcat_buggy_files")
     doubt = False
     log.debug("backlog={b}, ongoing={o}, ok={ok}".format(b=len(backlog), o=len(ongoing), ok=len(ok)))
@@ -230,6 +232,10 @@ def msgcat_recursive(destination, path, debug_folder, backlog=[], ongoing=[], ok
             log.debug("Generating remaining files")
             msgcat(ok, destination, path)
 
+        if os.path.isfile(destination) is False:
+            log.error("weird, some files raising bugs were missed?")
+            msgcat_recursive(destination, path, debug_folder, ok, list(), list())
+
 
 def process_tmx(lang, source, dest):
     """ Generate a translation memory from a po file """

From 52528678ee1ca864be76fd7c6d496aa32a041a32 Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
Date: Feb 25 2021 20:27:40 +0000
Subject: [PATCH 4/10] fix logger name


---

diff --git a/build_tm.py b/build_tm.py
index 06ea08a..3b42928 100755
--- a/build_tm.py
+++ b/build_tm.py
@@ -191,7 +191,7 @@ def store_debug_file(path, name, file, debug_folder):
 
 def msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok):
     """ Try to call msgcat, retry with half of the files if it fails """
-    log = logging.getLogger("buildTm.search_msgcat_buggy_files")
+    log = logging.getLogger("buildTm.msgcat_recursive")
     doubt = False
     log.debug("backlog={b}, ongoing={o}, ok={ok}".format(b=len(backlog), o=len(ongoing), ok=len(ok)))
     if len(ongoing) == 0:

From 2e1678a539f6aaa79d9d70e418d1d06cb6fb0ff5 Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
Date: Feb 25 2021 20:27:40 +0000
Subject: [PATCH 5/10] fix tm and terminology generation


---

diff --git a/build_tm.py b/build_tm.py
index 3b42928..3c7dfe9 100755
--- a/build_tm.py
+++ b/build_tm.py
@@ -124,6 +124,7 @@ def process_compendium(langfiles, dest, debug_folder):
     count = 0
 
     with tempfile.TemporaryDirectory(prefix="l10n-tm") as tmp:
+
         for i in pofiles:
             try:
                 command = [
@@ -240,6 +241,7 @@ def msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok):
 def process_tmx(lang, source, dest):
     """ Generate a translation memory from a po file """
 
+    """
     outputfile = po2tmx.tmxmultifile(dest)
     po2tmx.convertpo(
         inputfile=BytesIO(open(source, "r").read().encode()),
@@ -251,11 +253,16 @@ def process_tmx(lang, source, dest):
     )
 
     outputfile.tmxfile.savefile(dest)
+    """
 
+    command = ["po2tmx", "--language=" + lang, "--progress=none", source, "--output=" + dest]
+    subprocess.run(command, check=True, capture_output=True)
 
 def process_terminology(source, dest):
     """ Generate a termonology from a po file """
 
+
+    """
     extractor = poterminology.TerminologyExtractor()
     options = {
         "inputmin": "1",
@@ -282,6 +289,12 @@ def process_terminology(source, dest):
 
     with open(options["output"], "wb") as fh:
         termfile.serialize(fh)
+    
+    """
+    command = ["poterminology", "--ignore-case", "--fold-titlecase",
+                                "--inputs-needed", "1",
+                                "--progress=none", source, "--output=" + dest]
+    subprocess.run(command, check=True, capture_output=True)
 
 
 def check_lang(lang, tm_folder):

From d8f8e79dbd5475d8a80049073225a72a0b43a129 Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
Date: Feb 25 2021 20:27:40 +0000
Subject: [PATCH 6/10] website: simple display of language list


---

diff --git a/build_website.py b/build_website.py
index 464928a..e2d0af3 100755
--- a/build_website.py
+++ b/build_website.py
@@ -166,6 +166,13 @@ def main():
 
         generate_static_pages_packages(args.results, code, content, dest_file)
 
+    log.info("Generating indexes")
+    dest_file = os.path.join(static_langs_folder, "_index.md")
+    generate_language_index(args.results, dest_file)
+
+    dest_file = os.path.join(static_pkgs_folder, "_index.md")
+    generate_package_index(args.results, dest_file)
+
     log.info("Copy translation memories")
     languages = [
         f for f in os.listdir(tm_folder) if os.path.isfile(os.path.join(tm_folder, f))
@@ -383,6 +390,38 @@ def generate_static_pages_packages(results, code, content, dest_file):
         write_out.write(outputText)
 
 
+def generate_language_index(distribution, dest_file):
+    log = logging.getLogger("buildWebsite.generate_language_index")
+    data = dict()
+    data["distribution"] = distribution
+    data["now"] = datetime.datetime.utcnow()
+
+    templateLoader = jinja2.FileSystemLoader(searchpath="./templates/")
+    templateEnv = jinja2.Environment(loader=templateLoader, undefined=jinja2.Undefined)
+    TEMPLATE_FILE = "_index.language.md"
+    template = templateEnv.get_template(TEMPLATE_FILE)
+    outputText = template.render(data)
+
+    with open(dest_file, "w") as write_out:
+        write_out.write(outputText)
+
+
+def generate_package_index(distribution, dest_file):
+    log = logging.getLogger("buildWebsite.generate_package_index")
+    data = dict()
+    data["distribution"] = distribution
+    data["now"] = datetime.datetime.utcnow()
+
+    templateLoader = jinja2.FileSystemLoader(searchpath="./templates/")
+    templateEnv = jinja2.Environment(loader=templateLoader, undefined=jinja2.Undefined)
+    TEMPLATE_FILE = "_index.package.md"
+    template = templateEnv.get_template(TEMPLATE_FILE)
+    outputText = template.render(data)
+
+    with open(dest_file, "w") as write_out:
+        write_out.write(outputText)
+
+
 def store_json_file(content, dest_file):
     with open(dest_file, "w") as f:
         f.write(json.dumps(content, indent=2))
diff --git a/templates/_index.language.md b/templates/_index.language.md
new file mode 100644
index 0000000..bb7141a
--- /dev/null
+++ b/templates/_index.language.md
@@ -0,0 +1,5 @@
+---
+title: "Languages for {{ distribution }}"
+date: {{ now }}
+layout: "list_languages"
+---
\ No newline at end of file
diff --git a/templates/_index.package.md b/templates/_index.package.md
new file mode 100644
index 0000000..a973b26
--- /dev/null
+++ b/templates/_index.package.md
@@ -0,0 +1,4 @@
+---
+title: "Packages for {{ distribution }}"
+date: {{ now }}
+---
\ No newline at end of file
diff --git a/templates/language.md b/templates/language.md
index 9ac40af..622d9d8 100644
--- a/templates/language.md
+++ b/templates/language.md
@@ -1,6 +1,10 @@
 ---
-title: "{{ lang_name_en }} ({{ lang_name_local }})"
+title: "{{ lang_code }}-{{ lang_name_en }} ({{ lang_name_local }})"
 date: {{ now }}
+code: {{ lang_code }}
+name_english: {{ lang_name_en }}
+name_local: {{ lang_name_local }}
+progress_d: {{ progress_d }}
 ---
 
 Language progress for {{ lang_name_en }} ({{ lang_code }}) in Fedora {{ results }} is:
diff --git a/website/config.toml b/website/config.toml
index c777f70..9beb5cf 100644
--- a/website/config.toml
+++ b/website/config.toml
@@ -1,5 +1,8 @@
 baseURL = "https://jibecfed.fedorapeople.org/partage/fedora-localization-statistics/"
 languageCode = "en-us"
-title = "Temporary demo"
+title = "Fedora localization statistics"
 theme = "beautifulhugo"
 staticDir = "static"
+
+[markup.goldmark.renderer]
+unsafe= true
\ No newline at end of file
diff --git a/website/themes/beautifulhugo/layouts/_default/f33.html b/website/themes/beautifulhugo/layouts/_default/f33.html
new file mode 100644
index 0000000..1a00bdc
--- /dev/null
+++ b/website/themes/beautifulhugo/layouts/_default/f33.html
@@ -0,0 +1,12 @@
+{{ define "main" }}
+  <div class="container" role="main">
+      <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
+      {{ .Content }}
+      <ul class="contents">
+          {{ range sort .Pages }}
+              <li><a href="{{ .Permalink }}">{{ .Title }}</a></li>
+          {{ end }}
+          </ul>
+      </div>
+  </div>
+{{ end }}
diff --git a/website/themes/beautifulhugo/layouts/_default/list.html b/website/themes/beautifulhugo/layouts/_default/list.html
index e08da36..5f7bf09 100644
--- a/website/themes/beautifulhugo/layouts/_default/list.html
+++ b/website/themes/beautifulhugo/layouts/_default/list.html
@@ -7,11 +7,6 @@
             {{.}}
           </div>
         {{ end }}
-        <div class="posts-list">
-          {{ range .Paginator.Pages }}
-            {{ partial "post_preview.html" .}}
-          {{ end }}
-        </div>
         {{ if or (.Paginator.HasPrev) (.Paginator.HasNext) }}
           <ul class="pager main-pager">
             {{ if .Paginator.HasPrev }}
diff --git a/website/themes/beautifulhugo/layouts/_default/list_languages.html b/website/themes/beautifulhugo/layouts/_default/list_languages.html
new file mode 100644
index 0000000..42b9620
--- /dev/null
+++ b/website/themes/beautifulhugo/layouts/_default/list_languages.html
@@ -0,0 +1,24 @@
+{{ define "main" }}
+  <div class="container" role="main">
+      <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
+      {{ .Content }}
+
+      <table class="contents">
+          <tr>
+              <th>code</th>
+              <th>English name</th>
+              <th>Local name</th>
+              <th>Progress</th>
+          </tr>
+          {{ range sort .Pages "Title" "asc" }}
+            <tr>
+                <td><a href="{{ .Permalink }}">{{ .Params.code }}</a></td>
+                <td>{{ .Params.name_english }}</td>
+                <td>{{ .Params.name_local }}</td>
+                <td>{{ .Params.progress_d }}</td>
+            </tr>
+          {{ end }}
+      </table>
+      </div>
+  </div>
+{{ end }}
\ No newline at end of file
diff --git a/website/themes/beautifulhugo/layouts/index.html b/website/themes/beautifulhugo/layouts/index.html
index cc15000..bfd14e7 100644
--- a/website/themes/beautifulhugo/layouts/index.html
+++ b/website/themes/beautifulhugo/layouts/index.html
@@ -7,28 +7,11 @@
             {{.}}
           </div>
         {{ end }}
-
-        <div class="posts-list">
-          {{ $pag := .Paginate (where site.RegularPages "Type" "in" site.Params.mainSections) }}
-          {{ range $pag.Pages }}
-            {{ partial "post_preview" . }}
+        <ul class="">
+          {{ range .Site.Sections }}
+            <li><a href="{{ .Permalink }}">{{ .Title }}</a></li>
           {{ end }}
-        </div>
-
-        {{ if or (.Paginator.HasPrev) (.Paginator.HasNext) }}
-          <ul class="pager main-pager">
-            {{ if .Paginator.HasPrev }}
-              <li class="previous">
-                <a href="{{ .Permalink }}page/{{ .Paginator.Prev.PageNumber }}/">&larr; {{ i18n "newerPosts" }}</a>
-              </li>
-            {{ end }}
-            {{ if .Paginator.HasNext }}
-              <li class="next">
-                <a href="{{ .Permalink }}page/{{ .Paginator.Next.PageNumber }}/">{{ i18n "olderPosts" }} &rarr;</a>
-              </li>
-            {{ end }}
-          </ul>
-        {{ end }}
+        </ul>
       </div>
     </div>
   </div>
diff --git a/website/themes/beautifulhugo/layouts/partials/breadcrumb.html b/website/themes/beautifulhugo/layouts/partials/breadcrumb.html
new file mode 100644
index 0000000..62f1b66
--- /dev/null
+++ b/website/themes/beautifulhugo/layouts/partials/breadcrumb.html
@@ -0,0 +1,13 @@
+<ol  class="nav navbar-nav">
+  {{ template "breadcrumbnav" (dict "p1" . "p2" .) }}
+</ol>
+{{ define "breadcrumbnav" }}
+{{ if .p1.Parent }}
+{{ template "breadcrumbnav" (dict "p1" .p1.Parent "p2" .p2 )  }}
+{{ else if not .p1.IsHome }}
+{{ template "breadcrumbnav" (dict "p1" .p1.Site.Home "p2" .p2 )  }}
+{{ end }}
+<li{{ if eq .p1 .p2 }} class="active"{{ end }}>
+  <a href="{{ .p1.Permalink }}">{{ .p1.Title }}</a>
+</li>
+{{ end }}
\ No newline at end of file

From f46f7bb879c9efd9c113eadc95351a5d075c53d9 Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
Date: Feb 25 2021 20:27:40 +0000
Subject: [PATCH 7/10] website: allow to sort language table


---

diff --git a/website/themes/beautifulhugo/layouts/_default/list_languages.html b/website/themes/beautifulhugo/layouts/_default/list_languages.html
index 42b9620..d108de7 100644
--- a/website/themes/beautifulhugo/layouts/_default/list_languages.html
+++ b/website/themes/beautifulhugo/layouts/_default/list_languages.html
@@ -3,12 +3,13 @@
       <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
       {{ .Content }}
 
-      <table class="contents">
+          <table class="contents" id="languages">
+              <caption>Click on columns headers to sort values</caption>
           <tr>
-              <th>code</th>
-              <th>English name</th>
-              <th>Local name</th>
-              <th>Progress</th>
+              <th onclick="sortTable(0)">code</th>
+              <th onclick="sortTable(1)">English name</th>
+              <th onclick="sortTable(2)">Local name</th>
+              <th onclick="sortTable(3)">Progress</th>
           </tr>
           {{ range sort .Pages "Title" "asc" }}
             <tr>
@@ -21,4 +22,60 @@
       </table>
       </div>
   </div>
+<script>
+function sortTable(n) {
+  var table, rows, switching, i, x, y, shouldSwitch, dir, switchcount = 0;
+  table = document.getElementById("languages");
+  switching = true;
+  // Set the sorting direction to ascending:
+  dir = "asc";
+  /* Make a loop that will continue until
+  no switching has been done: */
+  while (switching) {
+    // Start by saying: no switching is done:
+    switching = false;
+    rows = table.rows;
+    /* Loop through all table rows (except the
+    first, which contains table headers): */
+    for (i = 1; i < (rows.length - 1); i++) {
+      // Start by saying there should be no switching:
+      shouldSwitch = false;
+      /* Get the two elements you want to compare,
+      one from current row and one from the next: */
+      x = rows[i].getElementsByTagName("TD")[n];
+      y = rows[i + 1].getElementsByTagName("TD")[n];
+      /* Check if the two rows should switch place,
+      based on the direction, asc or desc: */
+      if (dir == "asc") {
+        if (x.innerHTML.toLowerCase() > y.innerHTML.toLowerCase()) {
+          // If so, mark as a switch and break the loop:
+          shouldSwitch = true;
+          break;
+        }
+      } else if (dir == "desc") {
+        if (x.innerHTML.toLowerCase() < y.innerHTML.toLowerCase()) {
+          // If so, mark as a switch and break the loop:
+          shouldSwitch = true;
+          break;
+        }
+      }
+    }
+    if (shouldSwitch) {
+      /* If a switch has been marked, make the switch
+      and mark that a switch has been done: */
+      rows[i].parentNode.insertBefore(rows[i + 1], rows[i]);
+      switching = true;
+      // Each time a switch is done, increase this count by 1:
+      switchcount ++;
+    } else {
+      /* If no switching has been done AND the direction is "asc",
+      set the direction to "desc" and run the while loop again. */
+      if (switchcount == 0 && dir == "asc") {
+        dir = "desc";
+        switching = true;
+      }
+    }
+  }
+}
+</script>
 {{ end }}
\ No newline at end of file

From bee3225736f151417a8cbafa9fbe7ec8653dec68 Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
Date: Feb 25 2021 20:27:40 +0000
Subject: [PATCH 8/10] prevent error by using sed, and catch new polib errors


---

diff --git a/.gitignore b/.gitignore
index ab40337..bba57c8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@ website/content/*
 website/public/
 website/static/*
 srpms_*.lst
+log.*
diff --git a/build_language_list.py b/build_language_list.py
index b8402bf..6eb4499 100755
--- a/build_language_list.py
+++ b/build_language_list.py
@@ -120,12 +120,14 @@ def analyze_lang(lang_folder, analized_lang):
         metadata = dict()
         try:
             metadata = polib.pofile(file).metadata
-        except UnicodeDecodeError:
-            # encoding error, to investigate before using it in TM
-            metadata["Language"] = "error-unicode"
         except OSError:
             # maybe a polib bug? to investigate before using it in TM
             metadata["Language"] = "error-os"
+        except TypeError:
+            metadata["Language"] = "error-type"
+        except UnicodeDecodeError:
+            # encoding error, to investigate before using it in TM
+            metadata["Language"] = "error-unicode"
 
         if "Language" not in metadata.keys():
             metadata["Language"] = "zzz_null"
@@ -154,9 +156,7 @@ def analyze_lang(lang_folder, analized_lang):
 
         results[metadata.get("Language")] = language
 
-    results = dict(sorted(results.items(), key=lambda item: item[0]))
-
-    return results
+    return dict(sorted(results.items(), key=lambda item: item[0]))
 
 
 def describe(lang_folder):
@@ -189,8 +189,11 @@ def detect_languages(package_folder, results_folder):
 
     log_file = os.path.join(results_folder, "build_language_list.log")
     file_object = open(log_file, "w")
-
+    count = 0
+    total = len(packages)
     for package in packages:
+        count += 1
+        log.debug("{c}/{t}".format(c=count, t=total))
         discovery_file = os.path.join(package_folder, package, "discover.json")
 
         with open(discovery_file, "r") as read_file:
@@ -212,13 +215,15 @@ def detect_languages(package_folder, results_folder):
                 except UnicodeDecodeError:
                     # encoding error, to investigate before using it in TM
                     error = "error-unicode"
+                except TypeError:
+                    error = "error-type"
                 except OSError:
                     # maybe a polib bug? to investigate before using it in TM
                     error = "error-os"
 
                 lang, decision = choose_lang(lang_code, metadata, error)
 
-                log = ",".join(
+                debug = ",".join(
                     [
                         po,
                         lang_code,
@@ -228,7 +233,7 @@ def detect_languages(package_folder, results_folder):
                         str(decision),
                     ]
                 )
-                file_object.write(log + "\n")
+                file_object.write(debug + "\n")
 
                 lang_result = langs.get(lang, dict())
                 po_results = lang_result.get("po", list())
diff --git a/build_stats.py b/build_stats.py
index fe0970f..02dba08 100755
--- a/build_stats.py
+++ b/build_stats.py
@@ -6,6 +6,8 @@ import glob
 import json
 import os
 import shutil
+import subprocess
+
 import polib
 import logging
 
@@ -71,30 +73,31 @@ def main():
     for package in sorted(packages):
         count += 1
         log.info(" {c}/{t} - {p}".format(c=count, t=len(packages), p=package))
-        with open(os.path.join(packages_folder, package, "discover.json"), "r") as f:
-            discoveries = json.load(f)
 
         src_folder = os.path.join(packages_folder, package)
         stats_file = os.path.join(packages_stats_folder, package + ".json")
 
-        if os.path.isfile(stats_file):
-            continue
+        if os.path.isfile(stats_file) is False:
+            with open(os.path.join(packages_folder, package, "discover.json"), "r") as f:
+                discoveries = json.load(f)
 
-        results = dict()
-        for discover in discoveries:
-            files = glob.glob(os.path.join(src_folder, discover["filemask"]))
+            results = dict()
+            for discover in discoveries:
+                files = glob.glob(os.path.join(src_folder, discover["filemask"]))
 
-            if discover["file_format"] == "po":
-                results[discover["filemask"]] = get_po_translation_level(
-                    files, stats_file
-                )
+                if discover["file_format"] == "po":
+                    results[discover["filemask"]] = get_po_translation_level(
+                        files, stats_file
+                    )
 
-        if len(results) > 0:
-            distribution_stats = extract_release_stats(distribution_stats, results)
+            if len(results) > 0:
+                with open(stats_file, "w") as f:
+                    json.dump(results, f, indent=2)
+        else:
+            with open(stats_file, "r") as f:
+                results = json.load(f)
 
-        if len(results) > 0:
-            with open(stats_file, "w") as f:
-                json.dump(results, f, indent=2)
+        distribution_stats = extract_release_stats(distribution_stats, results)
 
     log.info("Storing distribution stats")
     if not os.path.exists(distribution_stats_folder):
@@ -137,6 +140,11 @@ def get_po_translation_level(files, stats_file):
     stats = dict()
 
     for file in files:
+        # remove non standard comments
+        # taken from: https://github.com/translate/translate/blob/master/tools/pocommentclean
+        command = ["sed", "-i", "/^#$/d;/^#[^\:\~,\.]/d", file]
+        subprocess.run(command, check=True, capture_output=True)
+
         try:
             stat = calcstats(file)
         except Exception as e:
@@ -168,12 +176,15 @@ def get_language_team(file):
     metadata = dict()
     try:
         metadata = polib.pofile(file).metadata
-    except UnicodeDecodeError:
-        # encoding error, to investigate before using it in TM
-        metadata["Language"] = "error-unicode"
     except OSError:
         # maybe a polib bug? to investigate before using it in TM
         metadata["Language"] = "error-os"
+    except UnicodeDecodeError:
+        # encoding error, to investigate before using it in TM
+        metadata["Language"] = "error-unicode"
+    except TypeError:
+        # TypeError: '>' not supported between instances of 'str' and 'int'
+        metadata["Language"] = "error-valuerror"
 
     team = "Unknown..."
     try:
diff --git a/build_tm.py b/build_tm.py
index 3c7dfe9..4e7b280 100755
--- a/build_tm.py
+++ b/build_tm.py
@@ -10,12 +10,6 @@ import shutil
 import tempfile
 import logging
 
-from io import BytesIO
-from translate.convert import po2tmx
-from translate.storage import factory, po
-from translate.tools import poterminology
-
-
 def main():
     """Handle params"""
 
@@ -79,10 +73,12 @@ def main():
             os.path.dirname(os.path.abspath(__file__)), compendium_file
         )
         if not os.path.isfile(compendium_file):
+            log.info("Compendium generation")
             process_compendium(files, compendium_file, debug_folder)
 
         tmx_file = os.path.join(tm_folder, lang_code + ".tmx")
         if not os.path.isfile(tmx_file):
+            log.info("TMX generation")
             try:
                 process_tmx(lang_code, compendium_file, tmx_file)
             except Exception as e:
@@ -94,6 +90,7 @@ def main():
 
         terminology_file = os.path.join(tm_folder, lang_code + ".terminology.po")
         if not os.path.isfile(terminology_file):
+            log.info("Terminology generation")
             try:
                 process_terminology(compendium_file, terminology_file)
             except Exception as e:

From 580d7b0fc43361a61ec497efecd0e882065feec3 Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
Date: Feb 25 2021 20:27:40 +0000
Subject: [PATCH 9/10] remove comments from compendium files


---

diff --git a/build_tm.py b/build_tm.py
index 4e7b280..666665f 100755
--- a/build_tm.py
+++ b/build_tm.py
@@ -75,6 +75,10 @@ def main():
         if not os.path.isfile(compendium_file):
             log.info("Compendium generation")
             process_compendium(files, compendium_file, debug_folder)
+            # remove non standard comments
+            # taken from: https://github.com/translate/translate/blob/master/tools/pocommentclean
+            command = ["sed", "-i", "/^#$/d;/^#[^\:\~,\.]/d", compendium_file]
+            subprocess.run(command, check=True, capture_output=True)
 
         tmx_file = os.path.join(tm_folder, lang_code + ".tmx")
         if not os.path.isfile(tmx_file):

From b114f2f5d10b96dd83bca9f50039e8b22e30b2af Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
Date: Feb 25 2021 20:55:24 +0000
Subject: [PATCH 10/10] reduce space consumption


---

diff --git a/build_tm.py b/build_tm.py
index 666665f..d265757 100755
--- a/build_tm.py
+++ b/build_tm.py
@@ -10,6 +10,7 @@ import shutil
 import tempfile
 import logging
 
+
 def main():
     """Handle params"""
 
@@ -72,7 +73,8 @@ def main():
         compendium_file = os.path.join(
             os.path.dirname(os.path.abspath(__file__)), compendium_file
         )
-        if not os.path.isfile(compendium_file):
+        compendium_archive = compendium_file + ".gz"
+        if os.path.isfile(compendium_file) is False and os.path.isfile(compendium_archive) is False:
             log.info("Compendium generation")
             process_compendium(files, compendium_file, debug_folder)
             # remove non standard comments
@@ -81,7 +83,8 @@ def main():
             subprocess.run(command, check=True, capture_output=True)
 
         tmx_file = os.path.join(tm_folder, lang_code + ".tmx")
-        if not os.path.isfile(tmx_file):
+        tmx_archive = tmx_file + ".gz"
+        if os.path.isfile(tmx_file) is False and os.path.isfile(tmx_archive) is False:
             log.info("TMX generation")
             try:
                 process_tmx(lang_code, compendium_file, tmx_file)
@@ -93,7 +96,8 @@ def main():
                 )
 
         terminology_file = os.path.join(tm_folder, lang_code + ".terminology.po")
-        if not os.path.isfile(terminology_file):
+        terminology_archive = terminology_file + ".gz"
+        if os.path.isfile(terminology_file) is False and os.path.isfile(terminology_archive) is False:
             log.info("Terminology generation")
             try:
                 process_terminology(compendium_file, terminology_file)
@@ -104,15 +108,21 @@ def main():
                     )
                 )
 
+        if args.compress:
+            if os.path.isfile(compendium_file):
+                compress(compendium_file, compendium_archive)
+
+            if os.path.isfile(tmx_file):
+                compress(tmx_file, tmx_archive)
+
+            if os.path.isfile(terminology_file):
+                compress(terminology_file, terminology_archive)
+
     log.info("All languages are processed")
 
     log.info("Detecting missing files")
     for lang in sorted(langs):
-        check_lang(lang[: -len(".json")], tm_folder)
-
-    if args.compress:
-        log.info("Compressing files")
-        compress(tm_folder)
+        check_lang(lang[: -len(".json")], tm_folder, args.compress)
 
 
 def process_compendium(langfiles, dest, debug_folder):
@@ -150,7 +160,9 @@ def process_compendium(langfiles, dest, debug_folder):
                     subprocess.run(command, check=True, cwd=tmp, capture_output=True)
                 except subprocess.CalledProcessError as e:
                     debug_filename = "tm-msguniq-{lang}-{name}".format(lang=dest.split("/")[-1], name=count.__str__())
-                    log.error(" msguniq error with {i} a copy of this file is into {d} as {n}".format(i=i, e=e.output, d=debug_folder, n=debug_filename))
+                    log.error(" msguniq error with {i} a copy of this file is into {d} as {n}".format(i=i, e=e.output,
+                                                                                                      d=debug_folder,
+                                                                                                      n=debug_filename))
                     shutil.copyfile(i, os.path.join(debug_folder, debug_filename))
 
             count += 1
@@ -220,7 +232,7 @@ def msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok):
     else:
         if doubt is not False:
             log.debug("This file raised a msgcat bug: {f}".format(f=doubt))
-            store_debug_file(path, "tm-msgcat-"+destination.split("/")[-1], doubt, debug_folder)
+            store_debug_file(path, "tm-msgcat-" + destination.split("/")[-1], doubt, debug_folder)
             ongoing = []
 
         half = int(len(ongoing) / 2)
@@ -242,63 +254,20 @@ def msgcat_recursive(destination, path, debug_folder, backlog, ongoing, ok):
 def process_tmx(lang, source, dest):
     """ Generate a translation memory from a po file """
 
-    """
-    outputfile = po2tmx.tmxmultifile(dest)
-    po2tmx.convertpo(
-        inputfile=BytesIO(open(source, "r").read().encode()),
-        outputfile=outputfile,
-        templatefile=None,
-        sourcelanguage="en",
-        targetlanguage=lang,
-        comment="source",
-    )
-
-    outputfile.tmxfile.savefile(dest)
-    """
-
     command = ["po2tmx", "--language=" + lang, "--progress=none", source, "--output=" + dest]
     subprocess.run(command, check=True, capture_output=True)
 
+
 def process_terminology(source, dest):
     """ Generate a termonology from a po file """
 
-
-    """
-    extractor = poterminology.TerminologyExtractor()
-    options = {
-        "inputmin": "1",
-        "fullmsgmin": "1",
-        "substrmin": "2",
-        "locmin": "2",
-        "nonstopmin": 1,
-        "sortorders": ["frequency", "dictionary", "length"],
-        "output": dest,
-    }
-
-    with open(source, "rb") as fh:
-        inputfile = factory.getobject(fh)
-
-    extractor.processunits(inputfile.units, source)
-    terms = extractor.extract_terms()
-
-    termfile = po.pofile()
-    termitems = extractor.filter_terms(
-        terms, nonstopmin=options["nonstopmin"], sortorders=options["sortorders"]
-    )
-    for count, unit in termitems:
-        termfile.units.append(unit)
-
-    with open(options["output"], "wb") as fh:
-        termfile.serialize(fh)
-    
-    """
     command = ["poterminology", "--ignore-case", "--fold-titlecase",
-                                "--inputs-needed", "1",
-                                "--progress=none", source, "--output=" + dest]
+               "--inputs-needed", "1",
+               "--progress=none", source, "--output=" + dest]
     subprocess.run(command, check=True, capture_output=True)
 
 
-def check_lang(lang, tm_folder):
+def check_lang(lang, tm_folder, compress):
     """ Check if expected files were generated """
     log = logging.getLogger("buildTm.check_lang")
 
@@ -306,33 +275,31 @@ def check_lang(lang, tm_folder):
     tmx_file = os.path.join(tm_folder, lang + ".tmx")
     terminology_file = os.path.join(tm_folder, lang + ".terminology.po")
 
-    if not os.path.isfile(compendium_file):
+    if compress is True:
+        compendium_file += ".gz"
+        tmx_file += ".gz"
+        terminology_file += ".gz"
+
+    if os.path.isfile(compendium_file) is False:
         log.warning("{l}-compendium is missing".format(l=lang))
 
-    if not os.path.isfile(tmx_file):
+    if os.path.isfile(tmx_file) is False:
         log.warning("{l}-tmx is missing".format(l=lang))
 
-    if not os.path.isfile(terminology_file):
+    if os.path.isfile(terminology_file) is False:
         log.warning("{l}-terminology is missing".format(l=lang))
 
 
-def compress(folder):
+def compress(source, archive):
     """ Compress files uzing gzip """
     log = logging.getLogger("buildTm.compress")
 
-    files = [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))]
-
-    for file in sorted(files):
-        if file.endswith(".gz"):
-            continue
-
-        dest = file + ".gz"
-        if os.path.isfile(os.path.join(folder, dest)):
-            continue
+    log.info("Compressing")
+    with open(source, "rb") as file_in:
+        with gzip.open(archive, "wb") as file_out:
+            file_out.writelines(file_in)
 
-        with open(os.path.join(folder, file), "rb") as file_in:
-            with gzip.open(os.path.join(folder, dest), "wb") as file_out:
-                file_out.writelines(file_in)
+    os.remove(source)
 
 
 if __name__ == "__main__":

