Commit b8e90d34 authored by Florian Pritz's avatar Florian Pritz
Browse files

Add db to track uploaded files


Signed-off-by: Florian Pritz's avatarFlorian Pritz <bluewind@xinu.at>
parent 29b8e1c1
import sqlite3
class DB:
def __init__(self, db_file):
self.db = sqlite3.connect(db_file)
self.init_db()
def init_db(self):
c = self.db.cursor()
c.execute('create table if not exists files (filename text, uploaded int)')
self.db.commit()
def add_file(self, filename):
c = self.db.cursor()
c.execute('insert into files (filename, uploaded) values (?, 1)',
(filename,))
self.db.commit()
def exists(self, filename):
c = self.db.cursor()
c.execute('select uploaded from files where filename = ?', (filename,))
return c.fetchone() is not None
#!/usr/bin/env python
import upload_pkg_internetarchive
import DB
import mock
from unittest.mock import MagicMock
......@@ -10,9 +11,13 @@ class TestUploader(unittest.TestCase):
def test_upload_pkg(self):
mock_uploader = MagicMock()
app = upload_pkg_internetarchive.ArchiveUploader(mock_uploader)
app = upload_pkg_internetarchive.ArchiveUploader(mock_uploader, DB.DB(':memory:'))
app.main('./test-data/archive/packages/f/fb-client')
self.assertTrue(app.db.exists('fb-client-2.0.4-1-any.pkg.tar.xz'))
self.assertTrue(app.db.exists('fb-client-2.0.3-2-any.pkg.tar.xz'))
self.assertFalse(app.db.exists('fb-client-2.0.99-1-any.pkg.tar.xz'))
mock_uploader.upload.assert_called_with('archlinux_pkg_fb-client',
files=['./test-data/archive/packages/f/fb-client/fb-client-2.0.4-1-any.pkg.tar.xz',
'./test-data/archive/packages/f/fb-client/fb-client-2.0.3-2-any.pkg.tar.xz',
......
......@@ -7,6 +7,8 @@ import tarfile
import internetarchive as ia
import DB
class ArchiveUploader:
DESCRIPTION = """{pkgdesc}
......@@ -16,10 +18,9 @@ class ArchiveUploader:
See the <a href="https://wiki.archlinux.org/index.php/Arch_Linux_Archive">Arch Linux Archive documentation</a> for details.
"""
def __init__(self, internetarchive = ia):
def __init__(self, internetarchive = ia, db = DB.DB('archive-uploader.sqlite')):
self.ia = internetarchive
pass
self.db = db
def clean_name(self, name):
"""Remove chars that are not allowed in an Internet Archive identifier: @.+
......@@ -52,7 +53,9 @@ class ArchiveUploader:
"""Upload all versions for package given by [directory]"""
files = []
for f in os.scandir(directory):
files.append(f.path)
filename = os.path.basename(f.path)
if not self.db.exists(filename):
files.append(f.path)
if not files:
return
# Get last package, to extract a description
......@@ -65,7 +68,12 @@ class ArchiveUploader:
#print(metadata)
try:
res = self.ia.upload(identifier, files=files, metadata=metadata)
if not all([x.status_code == 200 for x in res]):
if all([x.status_code == 200 for x in res]):
for f in files:
if not f.endswith('.sig'):
filename = os.path.basename(f)
self.db.add_file(filename)
else:
ok = len([x for x in res if x.status_code == 200])
nok = len([x for x in res if x.status_code != 200])
codes = set([x.status_code for x in res])
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment