From 0f91badd86f7852bbf25bc227632824f7c7a6ee6 Mon Sep 17 00:00:00 2001
From: Brendan Early <mymindstorm@evermiss.net>
Date: Jan 09 2021 01:11:26 +0000
Subject: [PATCH 1/4] generate diff table in primary db files


---

diff --git a/bin/fetch-repository-dbs.py b/bin/fetch-repository-dbs.py
index 0bbb6ef..afa80a1 100755
--- a/bin/fetch-repository-dbs.py
+++ b/bin/fetch-repository-dbs.py
@@ -83,6 +83,70 @@ def index_db(name, tempdb):
         conn.commit()
         conn.close()
 
+# Adds a table named 'changes' listing if certian packages were changed,
+#  added, or deleted.
+def gen_db_diff(name, new, old):
+    if not os.path.isfile(old) or not new.endswith('primary.sqlite'):
+        return
+    
+    print(f'{name.ljust(padding)} Creating diff for file: {old}')
+    conn = sqlite3.connect(new)
+    conn.execute(f'ATTACH DATABASE \'{old}\' as old')
+    # changes table schema:
+    # name - package name
+    # arch - package arch
+    # version - {epoch}:{package version}-{package release}
+    # change - 'updated', 'removed', or 'added'
+    conn.execute('''
+        CREATE TABLE changes (
+            name TEXT NOT NULL,
+            arch TEXT NOT NULL,
+            version TEXT,
+            change TEXT NOT NULL,
+            UNIQUE(name, arch)
+        )
+        ''')
+    # Insert added packages list to changes table
+    conn.execute('''
+        INSERT INTO changes (name, arch, version, change)
+        SELECT main.packages.name, main.packages.arch,
+            IIF(main.packages.epoch IS NOT NULL, main.packages.epoch || ':', '') ||
+                main.packages.version || '-' || main.packages.release || '.' || main.packages.arch,
+            'added'
+        FROM main.packages LEFT JOIN old.packages ON main.packages.name = old.packages.name
+            AND main.packages.arch = old.packages.arch
+        WHERE old.packages.name IS NULL
+        ''')
+    # Insert removed packages list to changes table
+    conn.execute('''
+        INSERT INTO changes (name, arch, version, change)
+        SELECT old.packages.name, old.packages.arch,
+            IIF(old.packages.epoch IS NOT NULL, old.packages.epoch || ':', '') ||
+                old.packages.version || '-' || old.packages.release,
+            'removed'
+        FROM old.packages LEFT JOIN main.packages ON main.packages.name = old.packages.name
+            AND main.packages.arch = old.packages.arch
+        WHERE main.packages.name IS NULL
+        ''')
+    # Insert changed packages list to changes table
+    conn.execute('''
+        INSERT INTO changes (name, arch, change)
+        SELECT name, arch, 'changed' FROM 
+            (SELECT main.packages.name, main.packages.arch,
+                IIF(main.packages.epoch IS NOT NULL, main.packages.epoch || ':', '') ||
+                main.packages.version || '-' || main.packages.release || '.' || main.packages.arch
+            FROM main.packages
+            UNION
+            SELECT old.packages.name, old.packages.arch,
+                IIF(old.packages.epoch IS NOT NULL, old.packages.epoch || ':', '') ||
+                old.packages.version || '-' || old.packages.release || '.' || old.packages.arch
+            FROM old.packages)
+        GROUP BY name, arch
+        HAVING COUNT(*) > 1
+        ''')
+    conn.commit()
+    conn.close()
+
 def install_db(name, src, dest):
     print(f'{name.ljust(padding)} Installing {src} to {dest}.')
     shutil.move(src, dest)
@@ -148,6 +212,7 @@ def handle(repo, target_dir):
             download_db(name, repomd_url, archive)
             decompress_db(name, archive, tempdb)
             index_db(name, tempdb)
+            gen_db_diff(name, tempdb, destfile)
             install_db(name, tempdb, destfile)
 
 def get_repository_urls_for(product, version):

From 1931856833386eceabdec91530fae0447d485ace Mon Sep 17 00:00:00 2001
From: Brendan Early <mymindstorm@evermiss.net>
Date: Jan 10 2021 21:30:07 +0000
Subject: [PATCH 2/4] * Only generate files for packages in changes table if it exists

* Add env var for SITEMAP_URL
* Clean package folder when updating it

---

diff --git a/.gitignore b/.gitignore
index a7ee933..6847570 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,5 @@ repositories/
 pagure_owner_alias.json
 vue/node_modules
 vue/dist
+.vscode
+bin/__pycache__
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 9be8acb..42f8935 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,6 +24,7 @@ ENV DB_DIR /etc/packages/repositories
 ENV MAINTAINER_MAPPING /etc/packages/pagure_owner_alias.json
 ENV SOLR_CORE packages
 ENV SOLR_URL http://127.0.0.1:8983/
+ENV SITEMAP_URL https://localhost:8080
 
 COPY . .
 RUN chmod -R o+rx assets
@@ -32,7 +33,7 @@ RUN make setup-js \
   && make js
 
 COPY container/nginx.conf /etc/nginx/nginx.conf
-COPY container/update-packages.sh /etc/cron.weekly/
+COPY container/update-packages.sh /etc/cron.daily/
 COPY container/supervisord.conf /etc/supervisord.conf
 
 # TODO: Figure out how to use a read-write volume for
diff --git a/bin/fetch-repository-dbs.py b/bin/fetch-repository-dbs.py
index afa80a1..9778bd7 100755
--- a/bin/fetch-repository-dbs.py
+++ b/bin/fetch-repository-dbs.py
@@ -95,7 +95,7 @@ def gen_db_diff(name, new, old):
     # changes table schema:
     # name - package name
     # arch - package arch
-    # version - {epoch}:{package version}-{package release}
+    # version (optional) - {epoch}:{package version}-{package release}
     # change - 'updated', 'removed', or 'added'
     conn.execute('''
         CREATE TABLE changes (
@@ -131,7 +131,7 @@ def gen_db_diff(name, new, old):
     # Insert changed packages list to changes table
     conn.execute('''
         INSERT INTO changes (name, arch, change)
-        SELECT name, arch, 'changed' FROM 
+        SELECT name, arch, 'updated' FROM 
             (SELECT main.packages.name, main.packages.arch,
                 IIF(main.packages.epoch IS NOT NULL, main.packages.epoch || ':', '') ||
                 main.packages.version || '-' || main.packages.release || '.' || main.packages.arch
@@ -147,6 +147,14 @@ def gen_db_diff(name, new, old):
     conn.commit()
     conn.close()
 
+def clear_diff_table(db):
+    conn = sqlite3.connect(db)
+    result = conn.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'changes'")
+    if result.fetchone() is not None:
+        conn.execute("DELETE FROM changes")
+        conn.commit()
+    conn.close()
+
 def install_db(name, src, dest):
     print(f'{name.ljust(padding)} Installing {src} to {dest}.')
     shutil.move(src, dest)
@@ -200,6 +208,7 @@ def handle(repo, target_dir):
         # Have we downloaded this before?  Did it change?
         destfile = os.path.join(target_dir, db)
         if not needs_update(destfile, shasum, shatype):
+            clear_diff_table(destfile)
             print(f'{name.ljust(padding)} No change of {repomd_url}')
             continue
 
diff --git a/bin/generate-html.py b/bin/generate-html.py
index 0b56330..42b2548 100755
--- a/bin/generate-html.py
+++ b/bin/generate-html.py
@@ -12,6 +12,7 @@ import json
 import shutil
 import sqlite3
 import argparse
+import glob
 
 from datetime import date
 from collections import defaultdict
@@ -22,7 +23,7 @@ TEMPLATE_DIR='../templates'
 DBS_DIR=os.environ.get('DB_DIR') or "repositories"
 ASSETS_DIR='assets'
 SCM_MAINTAINER_MAPPING=os.environ.get('MAINTAINER_MAPPING') or "pagure_owner_alias.json"
-SITEMAP_URL = 'https://pkgs.fedoraproject.org'
+SITEMAP_URL = os.environ.get('SITEMAP_URL') or 'https://localhost:8080'
 
 class Package:
     def __init__(self, name):
@@ -63,6 +64,15 @@ def open_db(db):
 
     return (conn, c)
 
+def clean_dir(path):
+    files = glob.glob(os.path.join(path, '*.html'))
+    for file in files:
+        try:
+            os.remove(file)
+        except:
+            print("Error cleaning directory!")
+            print(sys.exc_info()[0])
+
 def save_to(path, content):
     with open(path, 'w') as fh:
         fh.write(content)
@@ -81,7 +91,6 @@ def main():
     os.makedirs(output_dir, exist_ok=True)
 
     # Initialize templating system.
-    db = os.path.join(DBS_DIR, "koji-primary.sqlite")
     env = Environment(
             loader=PackageLoader('generate-html', TEMPLATE_DIR),
             autoescape=select_autoescape(['html'])
@@ -109,6 +118,7 @@ def main():
 
     # Build internal package metadata structure / cache.
     packages = {}
+    partial_update = False
     srpm_pattern = re.compile("^(.+)-.+-.+.src.rpm$")
     changelog_mail_pattern = re.compile("<(.+@.+)>")
     release_branch_pattern = re.compile("^([fedora|epel]+-[\w|\d]+)-?([a-z|-]+)?$")
@@ -123,6 +133,16 @@ def main():
         (_, filelist) = open_db(databases[release_branch]["filelists"])
         (_, other) = open_db(databases[release_branch]["other"])
 
+        if not partial_update:
+            primary.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'changes'")
+            partial_update = primary.fetchone() is not None
+
+        partial_update_packages = []
+        if partial_update:
+            primary.execute('SELECT name FROM changes')
+            for row in primary.fetchall():
+                partial_update_packages.append(row['name'])
+
         for raw in primary.execute('SELECT * FROM packages'):
             pkg = packages.get(raw["name"])
             revision = "{}-{}".format(raw["version"], raw["release"])
@@ -142,6 +162,12 @@ def main():
                 pkg.license = raw["rpm_license"]
                 pkg.maintainers = maintainer_mapping["rpms"].get(pkg.name, [])
 
+            # Check if package should be updated during a partial update
+            if partial_update and pkg.name in partial_update_packages:
+                pkg.should_update = True
+            elif first_pkg_encounter and partial_update:
+                pkg.should_update = False
+
             # Handle subpackage specific case.
             (srpm_name) = srpm_pattern.findall(raw["rpm_sourcerpm"])[0]
             if pkg.name != srpm_name:
@@ -214,7 +240,10 @@ def main():
     # Generate main pages.
     print(">>> Index pages...")
     for pkg in packages.values():
+        if pkg.should_update == False:
+            continue
         pkg_dir = os.path.join(output_dir, 'pkgs', pkg.name)
+        clean_dir(pkg_dir)
         os.makedirs(pkg_dir, exist_ok=True)
 
         html_path = os.path.join(pkg_dir, 'index.html')
@@ -232,6 +261,8 @@ def main():
     db_conns = {}
     detailed_page_count = 0
     for pkg in packages.values():
+      if pkg.should_update == False:
+        continue
       pkg_dir = os.path.join(output_dir, 'pkgs', pkg.name)
       for release in pkg.releases.keys():
         for branch in pkg.get_release(release).keys():
@@ -299,7 +330,7 @@ def main():
           print("Processed {}/{} pages.".format(detailed_page_count, max_page_count))
 
     print("DONE.")
-    print("> {} packages processed.".format(len(packages)))
+    print("> {} packages processed.".format(page_count))
 
 if __name__ == '__main__':
     main()
diff --git a/container/update-packages.sh b/container/update-packages.sh
index 7e25db1..dafa998 100644
--- a/container/update-packages.sh
+++ b/container/update-packages.sh
@@ -1,7 +1,4 @@
 #!/bin/sh
-OUTPUT_DIR=/tmp/packages/
 cd /usr/local/src/packages
-make all
-rsync -av --progress --delete $OUTPUT_DIR /srv/packages
-make update-solr
-rm -rf $OUTPUT_DIR
+make html-only
+make update-solr
\ No newline at end of file

From e0430721d581190ebfa6287065758242e1b721c2 Mon Sep 17 00:00:00 2001
From: Brendan Early <mymindstorm@evermiss.net>
Date: Jan 13 2021 19:12:45 +0000
Subject: [PATCH 3/4] remove packages from target dir if it has been removed from all repoistories during a partial update


---

diff --git a/bin/generate-html.py b/bin/generate-html.py
index 42b2548..c0e403e 100755
--- a/bin/generate-html.py
+++ b/bin/generate-html.py
@@ -119,6 +119,7 @@ def main():
     # Build internal package metadata structure / cache.
     packages = {}
     partial_update = False
+    removed_packages = set()
     srpm_pattern = re.compile("^(.+)-.+-.+.src.rpm$")
     changelog_mail_pattern = re.compile("<(.+@.+)>")
     release_branch_pattern = re.compile("^([fedora|epel]+-[\w|\d]+)-?([a-z|-]+)?$")
@@ -183,6 +184,17 @@ def main():
                 branch = "base"
 
             pkg.set_release(release, raw["pkgKey"], branch, revision)
+        
+        # Get removed packages to determine if folder needs to be deleted later
+        if partial_update:
+            for removed in primary.execute("SELECT name FROM changes WHERE change = 'removed'"):
+                removed_packages.add(removed["name"])
+
+    # If a package was removed and it was not in any repository, attempt to
+    # delete the folder from the target directory
+    for removed_package in removed_packages:
+        if removed_package not in packages:
+            shutil.rmtree(os.path.join(output_dir, 'pkgs', removed_package), True)
 
     # Set license and maintainers for subpackages. We have to wait for all
     # packages to have been processed since subpackage might have been

From b73a9e644be2b8aa7408b1fbc234b5f2bd74a674 Mon Sep 17 00:00:00 2001
From: Brendan Early <mymindstorm@evermiss.net>
Date: Jan 13 2021 19:27:09 +0000
Subject: [PATCH 4/4] always update packages if a changes table does not exist


---

diff --git a/bin/generate-html.py b/bin/generate-html.py
index c0e403e..10bab4d 100755
--- a/bin/generate-html.py
+++ b/bin/generate-html.py
@@ -134,9 +134,9 @@ def main():
         (_, filelist) = open_db(databases[release_branch]["filelists"])
         (_, other) = open_db(databases[release_branch]["other"])
 
-        if not partial_update:
-            primary.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'changes'")
-            partial_update = primary.fetchone() is not None
+        # Check if this db has a changes table
+        primary.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'changes'")
+        partial_update = primary.fetchone() is not None
 
         partial_update_packages = []
         if partial_update:
@@ -168,6 +168,11 @@ def main():
                 pkg.should_update = True
             elif first_pkg_encounter and partial_update:
                 pkg.should_update = False
+            
+            # If a changes table does not exist, then the package should
+            # always be updated.
+            if not partial_update:
+                pkg.should_update = True
 
             # Handle subpackage specific case.
             (srpm_name) = srpm_pattern.findall(raw["rpm_sourcerpm"])[0]

