mkpkglists improvements

This overlaid MR changes a couple things, namely the format of the meta package archive and only updating archives when needed.

Update When Needed:

We now use the information_schema.tables AUTO_INCREMENT column to keep track of a "serial number" that can be used to tell us whether we need to update or not.

Metadata Format:

    The SQL logic in this file for package metadata now exactly
    reflects RPC's search logic, without searching for specific
    packages.

    Two command line arguments are available:

        --extended | Include License, Keywords and Groups fields

    When one of these arguments are passed, the script will create
    a packages-meta-ext-v1.json.gz, configured via packagesmetaextfile.

    Archive JSON is in the following format: line-separated package objects
    enclosed in a list:

        [
        {...},
        {...},
        {...}
        ]

    This allows users to stream the archive and process packages
    line-by-line, during the local storage process.

    Signed-off-by: Kevin Morris <kevr@0cost.org>

aurweb/scripts/mkpkglists.py

@@ -7,15 +7,14 @@ Archives:
 
     packages.gz               | A line-separated list of package names
     packages-meta-v1.json     | A type=search RPC-formatted JSON dataset
-    packages-meta-ext-v1.json | An --extended or --all archive
+    packages-meta-ext-v1.json | An --extended archive
     pkgbase.gz                | A line-separated list of package base names
     users.gz                  | A line-separated list of user names
 
-This script takes an optional argument: --extended or --all. Based
+This script takes an optional argument: --extended. Based
 on the following, right-hand side fields are added to each item.
 
-    --extended  | License, Keywords, Groups
-    --all       | License, Keywords, Groups, relations and dependencies
+    --extended  | License, Keywords, Groups, relations and dependencies
 
 """
 
@@ -131,34 +130,12 @@ def get_extended_dict(query: str):
         if "Keywords" not in data[pkgid]:
             data[pkgid]["Keywords"] = []
 
-        if "Groups" not in data[pkgid]:
-            data[pkgid]["Groups"] = []
-
         data[pkgid][key].append(output)
 
     conn.close()
     return data
 
 
-def get_limited_extended_fields():
-    # Returns: [ID, Type, Name, Cond]
-    query = """
-    SELECT PackageGroups.PackageID AS ID, 'Groups' AS Type,
-           Groups.Name, '' AS Cond
-    FROM Groups INNER JOIN PackageGroups
-    ON PackageGroups.GroupID = Groups.ID
-    UNION SELECT PackageLicenses.PackageID AS ID, 'License' AS Type,
-          Licenses.Name, '' as Cond
-    FROM Licenses INNER JOIN PackageLicenses
-    ON PackageLicenses.LicenseID = Licenses.ID
-    UNION SELECT Packages.ID AS ID, 'Keywords' AS Type,
-          PackageKeywords.Keyword AS Name, '' as Cond
-    FROM PackageKeywords
-    INNER JOIN Packages ON Packages.PackageBaseID = PackageKeywords.PackageBaseID
-    """
-    return get_extended_dict(query)
-
-
 def get_extended_fields():
     # Returns: [ID, Type, Name, Cond]
     query = """
@@ -190,8 +167,7 @@ def get_extended_fields():
 
 
 EXTENDED_FIELD_HANDLERS = {
-    "--extended": get_limited_extended_fields,
-    "--all": get_extended_fields
+    "--extended": get_extended_fields
 }
 
 
@@ -202,6 +178,17 @@ def is_decimal(column):
     return column
 
 
+def write_archive(archive: str, output: list):
+    with gzip.open(archive, "wb") as f:
+        f.write(b"[\n")
+        for i, item in enumerate(output):
+            f.write(orjson.dumps(item))
+            if i < len(output) - 1:
+                f.write(b",")
+            f.write(b"\n")
+        f.write(b"]")
+
+
 def main():
     conn = aurweb.db.Connection()
 
@@ -235,22 +222,15 @@ def main():
         # Produce packages-meta-v1.json.gz
         output = list()
         snapshot_uri = aurweb.config.get("options", "snapshot_uri")
-        with gzip.open(packagesmetafile, "wb") as f:
-            for result in cur.fetchall():
-                item = {
-                    column[0]: is_decimal(result[i])
-                    for i, column in enumerate(cur.description)
-                }
-                item["URLPath"] = snapshot_uri % item.get("Name")
-                output.append(item)
-
-            f.write(b"[\n")
-            for i, item in enumerate(output):
-                f.write(orjson.dumps(item))
-                if i < len(output) - 1:
-                    f.write(b",")
-                f.write(b"\n")
-            f.write(b"]")
+        for result in cur.fetchall():
+            item = {
+                column[0]: is_decimal(result[i])
+                for i, column in enumerate(cur.description)
+            }
+            item["URLPath"] = snapshot_uri % item.get("Name")
+            output.append(item)
+
+        write_archive(packagesmetafile, output)
 
         # Produce packages-meta-ext-v1.json.gz
         if len(sys.argv) > 1 and sys.argv[1] in EXTENDED_FIELD_HANDLERS:
@@ -262,14 +242,7 @@ def main():
                 data_ = data.get(output[i].get("ID"), default_)
                 output[i].update(data_)
 
-            with gzip.open(packagesmetaextfile, "wb") as f:
-                f.write(b"[\n")
-                for i, item in enumerate(output):
-                    f.write(orjson.dumps(item))
-                    if i < len(output) - 1:
-                        f.write(b",")
-                    f.write(b"\n")
-                f.write(b"]")
+            write_archive(packagesmetaextfile, output)
 
         # Produce packages.gz
         with gzip.open(packagesfile, "wb") as f:
@@ -280,8 +253,6 @@ def main():
             ])
 
         update_state(packages_state, update_time)
-    else:
-        print("Packages have not been updated; skipping.")
 
     updated, update_time = should_update(pkgbases_state, "PackageBases")
     if not updated:
@@ -293,8 +264,6 @@ def main():
                                "WHERE PackagerUID IS NOT NULL")
             f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()])
         update_state(pkgbases_state, update_time)
-    else:
-        print("PackageBases have not been updated; skipping.")
 
     updated, update_time = should_update(users_state, "Users")
     if not updated:
@@ -305,8 +274,6 @@ def main():
             cur = conn.execute("SELECT UserName FROM Users")
             f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()])
         update_state(users_state, update_time)
-    else:
-        print("Users have not been updated; skipping.")
 
     conn.close()