Verified Commit 4f38fcf5 authored by Jan Alexander Steffens (heftig)'s avatar Jan Alexander Steffens (heftig)
Browse files

db-update: Load pkgfiles in parallel

parent 7e058c47
#!/usr/bin/python #!/usr/bin/python
import asyncio import asyncio
import json
import hashlib
import base64 import base64
import hashlib
import json
from asyncio.subprocess import PIPE from asyncio.subprocess import PIPE
from fcntl import LOCK_EX, flock from fcntl import LOCK_EX, flock
from pathlib import Path from pathlib import Path
...@@ -68,7 +68,7 @@ async def get_pkgfiles(pkgfile: Path) -> dict: ...@@ -68,7 +68,7 @@ async def get_pkgfiles(pkgfile: Path) -> dict:
return parse_pkgfiles(await get_output("bsdtar", "-tf", pkgfile)) return parse_pkgfiles(await get_output("bsdtar", "-tf", pkgfile))
async def build_pkgmeta(pkgpath, pkginfo): def build_pkgmeta(pkgpath, pkginfo, pkgfiles):
hash_md5 = hashlib.md5() hash_md5 = hashlib.md5()
hash_sha256 = hashlib.sha256() hash_sha256 = hashlib.sha256()
with pkgpath.open(mode="rb") as f: with pkgpath.open(mode="rb") as f:
...@@ -103,7 +103,7 @@ async def build_pkgmeta(pkgpath, pkginfo): ...@@ -103,7 +103,7 @@ async def build_pkgmeta(pkgpath, pkginfo):
"replaces": pkginfo.get("replace"), "replaces": pkginfo.get("replace"),
"conflicts": pkginfo.get("conflict"), "conflicts": pkginfo.get("conflict"),
"provides": pkginfo.get("provides"), "provides": pkginfo.get("provides"),
"files": await get_pkgfiles(pkgpath), "files": pkgfiles,
} }
) )
...@@ -125,7 +125,7 @@ async def main() -> int: ...@@ -125,7 +125,7 @@ async def main() -> int:
# load pkginfo # load pkginfo
async def load(ps, p): async def load(ps, p):
ps[p] = await get_pkginfo(p) ps[p] = await asyncio.gather(get_pkginfo(p), get_pkgfiles(p))
await asyncio.gather( await asyncio.gather(
*(load(ps, p) for r, ps in packages.items() for p in ps.keys()) *(load(ps, p) for r, ps in packages.items() for p in ps.keys())
...@@ -134,7 +134,7 @@ async def main() -> int: ...@@ -134,7 +134,7 @@ async def main() -> int:
# prepare meta structure # prepare meta structure
pkgbases = {r: {} for r in packages.keys()} pkgbases = {r: {} for r in packages.keys()}
for repo, ps in sorted(packages.items()): for repo, ps in sorted(packages.items()):
for pkgpath, pkginfo in ps.items(): for pkgpath, (pkginfo, pkgfiles) in ps.items():
pkgbase = pkginfo["pkgbase"][0] pkgbase = pkginfo["pkgbase"][0]
if pkgbase not in pkgbases[repo]: if pkgbase not in pkgbases[repo]:
pkgbases[repo][pkgbase] = { pkgbases[repo][pkgbase] = {
...@@ -142,7 +142,7 @@ async def main() -> int: ...@@ -142,7 +142,7 @@ async def main() -> int:
"packages": [], "packages": [],
} }
pkgbases[repo][pkgbase]["packages"].append( pkgbases[repo][pkgbase]["packages"].append(
await build_pkgmeta(pkgpath, pkginfo) build_pkgmeta(pkgpath, pkginfo, pkgfiles)
) )
# save meta info to json files # save meta info to json files
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment