Merge pull request #129 from jelly/mirrorlog_cleanup

Mirrorlog cleanup

mirrors/management/commands/mirrorcheck.py

@@ -10,7 +10,7 @@
 """
 
 from collections import deque
-from datetime import datetime
+from datetime import datetime, timedelta
 from httplib import HTTPException
 import logging
 import os
@@ -29,6 +29,7 @@
 
 from django.core.management.base import BaseCommand
 from django.db import transaction
+from django.conf import settings
 from django.utils.timezone import now
 
 from mirrors.models import MirrorUrl, MirrorLog, CheckLocation
@@ -84,6 +85,7 @@ def handle(self, **options):
 
         pool = MirrorCheckPool(urls, location, timeout)
         pool.run()
+        pool.cleanup()
         return 0
 
 
@@ -246,7 +248,7 @@ class MirrorCheckPool(object):
     def __init__(self, urls, location, timeout=10, num_threads=10):
         self.tasks = Queue()
         self.logs = deque()
-        for url in list(urls):
+        for url in urls:
             self.tasks.put(url)
         self.threads = []
         for _ in range(num_threads):
@@ -266,4 +268,11 @@ def run(self):
         MirrorLog.objects.bulk_create(self.logs)
         logger.debug("log entries saved")
 
+    def cleanup(self):
+        days = getattr(settings, 'MIRRORLOG_RETENTION_PERIOD', 365)
+        removal_date = now() - timedelta(days=days)
+        logger.info("cleaning up older MirrorLog objects then %s", removal_date.strftime('%Y-%m-%d'))
+        MirrorLog.objects.filter(check_time__lt=removal_date).delete()
+        logger.info('Finished cleaning up old MirrorLog objects')
+
 # vim: set ts=4 sw=4 et:

mirrors/models.py

@@ -150,8 +150,7 @@ def __unicode__(self):
 
     @property
     def family(self):
-        info = socket.getaddrinfo(self.source_ip, None, 0, 0, 0,
-                socket.AI_NUMERICHOST)
+        info = socket.getaddrinfo(self.source_ip, None, 0, 0, 0, socket.AI_NUMERICHOST)
         families = [x[0] for x in info]
         return families[0]
 

mirrors/tests/__init__.py

@@ -2,7 +2,7 @@
 
 
 def create_mirror_url(name='mirror1', country='US',
-                      protocol='http', url='https://archlinux.org'):
+                      protocol='http', url='https://archlinux.org/'):
         mirror = Mirror.objects.create(name=name,
                                        admin_email='admin@archlinux.org')
         mirror_protocol = MirrorProtocol.objects.create(protocol=protocol)

mirrors/tests/test_mirrorcheck.py

+import mock
+import time
+
+
+from django.utils.timezone import now
+from datetime import timedelta
+
+
+from django.test import TestCase
+from django.core.management import call_command
+
+
+from mirrors.tests import create_mirror_url
+from mirrors.models import MirrorLog
+
+
+class MirrorCheckTest(TestCase):
+    def setUp(self):
+        self.mirror_url = create_mirror_url()
+
+    def tearDown(self):
+        self.mirror_url.delete()
+
+    @mock.patch('urllib2.Request')
+    @mock.patch('urllib2.urlopen')
+    def test_invalid(self, urlopen, Request):
+        urlopen.return_value.read.return_value = 'data'
+        Request.get_host.return_value = 'archlinux.org'
+        Request.type.return_value = 'https'
+
+        call_command('mirrorcheck')
+        mirrorlog = MirrorLog.objects.first()
+        self.assertNotEqual(mirrorlog.error, '')
+        self.assertEqual(mirrorlog.is_success, False)
+
+    @mock.patch('urllib2.Request')
+    @mock.patch('urllib2.urlopen')
+    def test_valid(self, urlopen, Request):
+        urlopen.return_value.read.return_value = str(int(time.time()))
+        Request.get_host.return_value = 'archlinux.org'
+        Request.type.return_value = 'https'
+
+        call_command('mirrorcheck')
+        mirrorlog = MirrorLog.objects.first()
+        self.assertEqual(mirrorlog.error, '')
+        self.assertEqual(mirrorlog.is_success, True)
+
+    @mock.patch('urllib2.Request')
+    @mock.patch('urllib2.urlopen')
+    def test_valid(self, urlopen, Request):
+        urlopen.return_value.read.return_value = str(int(time.time()))
+        Request.get_host.return_value = 'archlinux.org'
+        Request.type.return_value = 'https'
+
+        date = now() - timedelta(days=600)
+        MirrorLog.objects.create(url=self.mirror_url, check_time=date)
+        call_command('mirrorcheck')
+        self.assertEqual(len(MirrorLog.objects.all()), 1)

mirrors/tests/test_mirrorresolv.py

+import mock
+
+from django.test import TestCase
+from django.core.management import call_command
+
+from mirrors.tests import create_mirror_url
+
+
+class MirrorCheckTest(TestCase):
+    def setUp(self):
+        self.mirror_url = create_mirror_url()
+
+    def tearDown(self):
+        self.mirror_url.delete()
+
+    @mock.patch('socket.getaddrinfo')
+    def test_ip4_ip6(self, getaddrinfo):
+        getaddrinfo.return_value = [(2, 1, 6, '', ('1.1.1.1', 0)), (10, 1, 6, '', ('1a01:3f8:132:1d96::1', 0, 0, 0))]
+        call_command('mirrorresolv')
+        self.mirror_url.refresh_from_db()
+        self.assertEqual(self.mirror_url.has_ipv4, True)
+        self.assertEqual(self.mirror_url.has_ipv6, True)
+
+    @mock.patch('socket.getaddrinfo')
+    def test_ip4_only(self, getaddrinfo):
+        getaddrinfo.return_value = [(2, 1, 6, '', ('1.1.1.1', 0))]
+        call_command('mirrorresolv')
+        self.mirror_url.refresh_from_db()
+        self.assertEqual(self.mirror_url.has_ipv4, True)
+        self.assertEqual(self.mirror_url.has_ipv6, False)
+
+    @mock.patch('socket.getaddrinfo')
+    def test_running_twice(self, getaddrinfo):
+        getaddrinfo.return_value = [(2, 1, 6, '', ('1.1.1.1', 0)), (10, 1, 6, '', ('1a01:3f8:132:1d96::1', 0, 0, 0))]
+
+        # Check if values changed
+        with mock.patch('mirrors.management.commands.mirrorresolv.logger') as logger:
+            call_command('mirrorresolv', '-v3')
+        self.assertEqual(logger.debug.call_count, 4)
+
+        # running again does not change any values.
+        with mock.patch('mirrors.management.commands.mirrorresolv.logger') as logger:
+            call_command('mirrorresolv', '-v3')
+        self.assertEqual(logger.debug.call_count, 3)

settings.py

@@ -145,6 +145,9 @@
     'http://tracker.archlinux.org:6969/announce',
 )
 
+# How long to keep mirrorlog's in days
+MIRRORLOG_RETENTION_PERIOD = 365
+
 # Shorten some names just a bit
 COUNTRIES_OVERRIDE = {
     'GB': 'United Kingdom',