Add db to track uploaded files

Signed-off-by: Florian Pritz <bluewind@xinu.at>

DB.py

+import sqlite3
+
+class DB:
+    def __init__(self, db_file):
+        self.db = sqlite3.connect(db_file)
+        self.init_db()
+
+    def init_db(self):
+        c = self.db.cursor()
+        c.execute('create table if not exists files (filename text, uploaded int)')
+        self.db.commit()
+
+    def add_file(self, filename):
+        c = self.db.cursor()
+        c.execute('insert into files (filename, uploaded) values (?, 1)',
+                (filename,))
+        self.db.commit()
+
+    def exists(self, filename):
+        c = self.db.cursor()
+        c.execute('select uploaded from files where filename = ?', (filename,))
+        return c.fetchone() is not None

test.py

 #!/usr/bin/env python
 
 import upload_pkg_internetarchive
+import DB
 
 import mock
 from unittest.mock import MagicMock
@@ -10,9 +11,13 @@ class TestUploader(unittest.TestCase):
 
     def test_upload_pkg(self):
         mock_uploader = MagicMock()
-        app = upload_pkg_internetarchive.ArchiveUploader(mock_uploader)
+        app = upload_pkg_internetarchive.ArchiveUploader(mock_uploader, DB.DB(':memory:'))
         app.main('./test-data/archive/packages/f/fb-client')
 
+        self.assertTrue(app.db.exists('fb-client-2.0.4-1-any.pkg.tar.xz'))
+        self.assertTrue(app.db.exists('fb-client-2.0.3-2-any.pkg.tar.xz'))
+        self.assertFalse(app.db.exists('fb-client-2.0.99-1-any.pkg.tar.xz'))
+
         mock_uploader.upload.assert_called_with('archlinux_pkg_fb-client',
                 files=['./test-data/archive/packages/f/fb-client/fb-client-2.0.4-1-any.pkg.tar.xz',
                     './test-data/archive/packages/f/fb-client/fb-client-2.0.3-2-any.pkg.tar.xz',

upload_pkg_internetarchive.py

@@ -7,6 +7,8 @@ import tarfile
 
 import internetarchive as ia
 
+import DB
+
 class ArchiveUploader:
     DESCRIPTION = """{pkgdesc}
 
@@ -16,10 +18,9 @@ class ArchiveUploader:
     See the <a href="https://wiki.archlinux.org/index.php/Arch_Linux_Archive">Arch Linux Archive documentation</a> for details.
     """
 
-    def __init__(self, internetarchive = ia):
+    def __init__(self, internetarchive = ia, db = DB.DB('archive-uploader.sqlite')):
         self.ia = internetarchive
-
-        pass
+        self.db = db
 
     def clean_name(self, name):
         """Remove chars that are not allowed in an Internet Archive identifier: @.+
@@ -52,7 +53,9 @@ class ArchiveUploader:
         """Upload all versions for package given by [directory]"""
         files = []
         for f in os.scandir(directory):
-            files.append(f.path)
+            filename = os.path.basename(f.path)
+            if not self.db.exists(filename):
+                files.append(f.path)
         if not files:
             return
         # Get last package, to extract a description
@@ -65,7 +68,12 @@ class ArchiveUploader:
         #print(metadata)
         try:
             res = self.ia.upload(identifier, files=files, metadata=metadata)
-            if not all([x.status_code == 200 for x in res]):
+            if all([x.status_code == 200 for x in res]):
+                for f in files:
+                    if not f.endswith('.sig'):
+                        filename = os.path.basename(f)
+                        self.db.add_file(filename)
+            else:
                 ok = len([x for x in res if x.status_code == 200])
                 nok = len([x for x in res if x.status_code != 200])
                 codes = set([x.status_code for x in res])