Unverified Commit 559b8bc7 authored by Jelle van der Waa's avatar Jelle van der Waa 🚧 Committed by GitHub
Browse files

Merge pull request #129 from jelly/mirrorlog_cleanup

Mirrorlog cleanup
parents 196b1fd8 d1abef20
......@@ -10,7 +10,7 @@
"""
from collections import deque
from datetime import datetime
from datetime import datetime, timedelta
from httplib import HTTPException
import logging
import os
......@@ -29,6 +29,7 @@
from django.core.management.base import BaseCommand
from django.db import transaction
from django.conf import settings
from django.utils.timezone import now
from mirrors.models import MirrorUrl, MirrorLog, CheckLocation
......@@ -84,6 +85,7 @@ def handle(self, **options):
pool = MirrorCheckPool(urls, location, timeout)
pool.run()
pool.cleanup()
return 0
......@@ -246,7 +248,7 @@ class MirrorCheckPool(object):
def __init__(self, urls, location, timeout=10, num_threads=10):
self.tasks = Queue()
self.logs = deque()
for url in list(urls):
for url in urls:
self.tasks.put(url)
self.threads = []
for _ in range(num_threads):
......@@ -266,4 +268,11 @@ def run(self):
MirrorLog.objects.bulk_create(self.logs)
logger.debug("log entries saved")
def cleanup(self):
days = getattr(settings, 'MIRRORLOG_RETENTION_PERIOD', 365)
removal_date = now() - timedelta(days=days)
logger.info("cleaning up older MirrorLog objects then %s", removal_date.strftime('%Y-%m-%d'))
MirrorLog.objects.filter(check_time__lt=removal_date).delete()
logger.info('Finished cleaning up old MirrorLog objects')
# vim: set ts=4 sw=4 et:
......@@ -150,8 +150,7 @@ def __unicode__(self):
@property
def family(self):
info = socket.getaddrinfo(self.source_ip, None, 0, 0, 0,
socket.AI_NUMERICHOST)
info = socket.getaddrinfo(self.source_ip, None, 0, 0, 0, socket.AI_NUMERICHOST)
families = [x[0] for x in info]
return families[0]
......
......@@ -2,7 +2,7 @@
def create_mirror_url(name='mirror1', country='US',
protocol='http', url='https://archlinux.org'):
protocol='http', url='https://archlinux.org/'):
mirror = Mirror.objects.create(name=name,
admin_email='admin@archlinux.org')
mirror_protocol = MirrorProtocol.objects.create(protocol=protocol)
......
import mock
import time
from django.utils.timezone import now
from datetime import timedelta
from django.test import TestCase
from django.core.management import call_command
from mirrors.tests import create_mirror_url
from mirrors.models import MirrorLog
class MirrorCheckTest(TestCase):
def setUp(self):
self.mirror_url = create_mirror_url()
def tearDown(self):
self.mirror_url.delete()
@mock.patch('urllib2.Request')
@mock.patch('urllib2.urlopen')
def test_invalid(self, urlopen, Request):
urlopen.return_value.read.return_value = 'data'
Request.get_host.return_value = 'archlinux.org'
Request.type.return_value = 'https'
call_command('mirrorcheck')
mirrorlog = MirrorLog.objects.first()
self.assertNotEqual(mirrorlog.error, '')
self.assertEqual(mirrorlog.is_success, False)
@mock.patch('urllib2.Request')
@mock.patch('urllib2.urlopen')
def test_valid(self, urlopen, Request):
urlopen.return_value.read.return_value = str(int(time.time()))
Request.get_host.return_value = 'archlinux.org'
Request.type.return_value = 'https'
call_command('mirrorcheck')
mirrorlog = MirrorLog.objects.first()
self.assertEqual(mirrorlog.error, '')
self.assertEqual(mirrorlog.is_success, True)
@mock.patch('urllib2.Request')
@mock.patch('urllib2.urlopen')
def test_valid(self, urlopen, Request):
urlopen.return_value.read.return_value = str(int(time.time()))
Request.get_host.return_value = 'archlinux.org'
Request.type.return_value = 'https'
date = now() - timedelta(days=600)
MirrorLog.objects.create(url=self.mirror_url, check_time=date)
call_command('mirrorcheck')
self.assertEqual(len(MirrorLog.objects.all()), 1)
import mock
from django.test import TestCase
from django.core.management import call_command
from mirrors.tests import create_mirror_url
class MirrorCheckTest(TestCase):
def setUp(self):
self.mirror_url = create_mirror_url()
def tearDown(self):
self.mirror_url.delete()
@mock.patch('socket.getaddrinfo')
def test_ip4_ip6(self, getaddrinfo):
getaddrinfo.return_value = [(2, 1, 6, '', ('1.1.1.1', 0)), (10, 1, 6, '', ('1a01:3f8:132:1d96::1', 0, 0, 0))]
call_command('mirrorresolv')
self.mirror_url.refresh_from_db()
self.assertEqual(self.mirror_url.has_ipv4, True)
self.assertEqual(self.mirror_url.has_ipv6, True)
@mock.patch('socket.getaddrinfo')
def test_ip4_only(self, getaddrinfo):
getaddrinfo.return_value = [(2, 1, 6, '', ('1.1.1.1', 0))]
call_command('mirrorresolv')
self.mirror_url.refresh_from_db()
self.assertEqual(self.mirror_url.has_ipv4, True)
self.assertEqual(self.mirror_url.has_ipv6, False)
@mock.patch('socket.getaddrinfo')
def test_running_twice(self, getaddrinfo):
getaddrinfo.return_value = [(2, 1, 6, '', ('1.1.1.1', 0)), (10, 1, 6, '', ('1a01:3f8:132:1d96::1', 0, 0, 0))]
# Check if values changed
with mock.patch('mirrors.management.commands.mirrorresolv.logger') as logger:
call_command('mirrorresolv', '-v3')
self.assertEqual(logger.debug.call_count, 4)
# running again does not change any values.
with mock.patch('mirrors.management.commands.mirrorresolv.logger') as logger:
call_command('mirrorresolv', '-v3')
self.assertEqual(logger.debug.call_count, 3)
......@@ -145,6 +145,9 @@
'http://tracker.archlinux.org:6969/announce',
)
# How long to keep mirrorlog's in days
MIRRORLOG_RETENTION_PERIOD = 365
# Shorten some names just a bit
COUNTRIES_OVERRIDE = {
'GB': 'United Kingdom',
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment