Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Jelle van der Waa
Archweb
Commits
559b8bc7
Unverified
Commit
559b8bc7
authored
Jul 23, 2018
by
Jelle van der Waa
🚧
Committed by
GitHub
Jul 23, 2018
Browse files
Merge pull request #129 from jelly/mirrorlog_cleanup
Mirrorlog cleanup
parents
196b1fd8
d1abef20
Changes
6
Hide whitespace changes
Inline
Side-by-side
mirrors/management/commands/mirrorcheck.py
View file @
559b8bc7
...
...
@@ -10,7 +10,7 @@
"""
from
collections
import
deque
from
datetime
import
datetime
from
datetime
import
datetime
,
timedelta
from
httplib
import
HTTPException
import
logging
import
os
...
...
@@ -29,6 +29,7 @@
from
django.core.management.base
import
BaseCommand
from
django.db
import
transaction
from
django.conf
import
settings
from
django.utils.timezone
import
now
from
mirrors.models
import
MirrorUrl
,
MirrorLog
,
CheckLocation
...
...
@@ -84,6 +85,7 @@ def handle(self, **options):
pool
=
MirrorCheckPool
(
urls
,
location
,
timeout
)
pool
.
run
()
pool
.
cleanup
()
return
0
...
...
@@ -246,7 +248,7 @@ class MirrorCheckPool(object):
def
__init__
(
self
,
urls
,
location
,
timeout
=
10
,
num_threads
=
10
):
self
.
tasks
=
Queue
()
self
.
logs
=
deque
()
for
url
in
list
(
urls
)
:
for
url
in
urls
:
self
.
tasks
.
put
(
url
)
self
.
threads
=
[]
for
_
in
range
(
num_threads
):
...
...
@@ -266,4 +268,11 @@ def run(self):
MirrorLog
.
objects
.
bulk_create
(
self
.
logs
)
logger
.
debug
(
"log entries saved"
)
def
cleanup
(
self
):
days
=
getattr
(
settings
,
'MIRRORLOG_RETENTION_PERIOD'
,
365
)
removal_date
=
now
()
-
timedelta
(
days
=
days
)
logger
.
info
(
"cleaning up older MirrorLog objects then %s"
,
removal_date
.
strftime
(
'%Y-%m-%d'
))
MirrorLog
.
objects
.
filter
(
check_time__lt
=
removal_date
).
delete
()
logger
.
info
(
'Finished cleaning up old MirrorLog objects'
)
# vim: set ts=4 sw=4 et:
mirrors/models.py
View file @
559b8bc7
...
...
@@ -150,8 +150,7 @@ def __unicode__(self):
@
property
def
family
(
self
):
info
=
socket
.
getaddrinfo
(
self
.
source_ip
,
None
,
0
,
0
,
0
,
socket
.
AI_NUMERICHOST
)
info
=
socket
.
getaddrinfo
(
self
.
source_ip
,
None
,
0
,
0
,
0
,
socket
.
AI_NUMERICHOST
)
families
=
[
x
[
0
]
for
x
in
info
]
return
families
[
0
]
...
...
mirrors/tests/__init__.py
View file @
559b8bc7
...
...
@@ -2,7 +2,7 @@
def
create_mirror_url
(
name
=
'mirror1'
,
country
=
'US'
,
protocol
=
'http'
,
url
=
'https://archlinux.org'
):
protocol
=
'http'
,
url
=
'https://archlinux.org
/
'
):
mirror
=
Mirror
.
objects
.
create
(
name
=
name
,
admin_email
=
'admin@archlinux.org'
)
mirror_protocol
=
MirrorProtocol
.
objects
.
create
(
protocol
=
protocol
)
...
...
mirrors/tests/test_mirrorcheck.py
0 → 100644
View file @
559b8bc7
import
mock
import
time
from
django.utils.timezone
import
now
from
datetime
import
timedelta
from
django.test
import
TestCase
from
django.core.management
import
call_command
from
mirrors.tests
import
create_mirror_url
from
mirrors.models
import
MirrorLog
class
MirrorCheckTest
(
TestCase
):
def
setUp
(
self
):
self
.
mirror_url
=
create_mirror_url
()
def
tearDown
(
self
):
self
.
mirror_url
.
delete
()
@
mock
.
patch
(
'urllib2.Request'
)
@
mock
.
patch
(
'urllib2.urlopen'
)
def
test_invalid
(
self
,
urlopen
,
Request
):
urlopen
.
return_value
.
read
.
return_value
=
'data'
Request
.
get_host
.
return_value
=
'archlinux.org'
Request
.
type
.
return_value
=
'https'
call_command
(
'mirrorcheck'
)
mirrorlog
=
MirrorLog
.
objects
.
first
()
self
.
assertNotEqual
(
mirrorlog
.
error
,
''
)
self
.
assertEqual
(
mirrorlog
.
is_success
,
False
)
@
mock
.
patch
(
'urllib2.Request'
)
@
mock
.
patch
(
'urllib2.urlopen'
)
def
test_valid
(
self
,
urlopen
,
Request
):
urlopen
.
return_value
.
read
.
return_value
=
str
(
int
(
time
.
time
()))
Request
.
get_host
.
return_value
=
'archlinux.org'
Request
.
type
.
return_value
=
'https'
call_command
(
'mirrorcheck'
)
mirrorlog
=
MirrorLog
.
objects
.
first
()
self
.
assertEqual
(
mirrorlog
.
error
,
''
)
self
.
assertEqual
(
mirrorlog
.
is_success
,
True
)
@
mock
.
patch
(
'urllib2.Request'
)
@
mock
.
patch
(
'urllib2.urlopen'
)
def
test_valid
(
self
,
urlopen
,
Request
):
urlopen
.
return_value
.
read
.
return_value
=
str
(
int
(
time
.
time
()))
Request
.
get_host
.
return_value
=
'archlinux.org'
Request
.
type
.
return_value
=
'https'
date
=
now
()
-
timedelta
(
days
=
600
)
MirrorLog
.
objects
.
create
(
url
=
self
.
mirror_url
,
check_time
=
date
)
call_command
(
'mirrorcheck'
)
self
.
assertEqual
(
len
(
MirrorLog
.
objects
.
all
()),
1
)
mirrors/tests/test_mirrorresolv.py
0 → 100644
View file @
559b8bc7
import
mock
from
django.test
import
TestCase
from
django.core.management
import
call_command
from
mirrors.tests
import
create_mirror_url
class
MirrorCheckTest
(
TestCase
):
def
setUp
(
self
):
self
.
mirror_url
=
create_mirror_url
()
def
tearDown
(
self
):
self
.
mirror_url
.
delete
()
@
mock
.
patch
(
'socket.getaddrinfo'
)
def
test_ip4_ip6
(
self
,
getaddrinfo
):
getaddrinfo
.
return_value
=
[(
2
,
1
,
6
,
''
,
(
'1.1.1.1'
,
0
)),
(
10
,
1
,
6
,
''
,
(
'1a01:3f8:132:1d96::1'
,
0
,
0
,
0
))]
call_command
(
'mirrorresolv'
)
self
.
mirror_url
.
refresh_from_db
()
self
.
assertEqual
(
self
.
mirror_url
.
has_ipv4
,
True
)
self
.
assertEqual
(
self
.
mirror_url
.
has_ipv6
,
True
)
@
mock
.
patch
(
'socket.getaddrinfo'
)
def
test_ip4_only
(
self
,
getaddrinfo
):
getaddrinfo
.
return_value
=
[(
2
,
1
,
6
,
''
,
(
'1.1.1.1'
,
0
))]
call_command
(
'mirrorresolv'
)
self
.
mirror_url
.
refresh_from_db
()
self
.
assertEqual
(
self
.
mirror_url
.
has_ipv4
,
True
)
self
.
assertEqual
(
self
.
mirror_url
.
has_ipv6
,
False
)
@
mock
.
patch
(
'socket.getaddrinfo'
)
def
test_running_twice
(
self
,
getaddrinfo
):
getaddrinfo
.
return_value
=
[(
2
,
1
,
6
,
''
,
(
'1.1.1.1'
,
0
)),
(
10
,
1
,
6
,
''
,
(
'1a01:3f8:132:1d96::1'
,
0
,
0
,
0
))]
# Check if values changed
with
mock
.
patch
(
'mirrors.management.commands.mirrorresolv.logger'
)
as
logger
:
call_command
(
'mirrorresolv'
,
'-v3'
)
self
.
assertEqual
(
logger
.
debug
.
call_count
,
4
)
# running again does not change any values.
with
mock
.
patch
(
'mirrors.management.commands.mirrorresolv.logger'
)
as
logger
:
call_command
(
'mirrorresolv'
,
'-v3'
)
self
.
assertEqual
(
logger
.
debug
.
call_count
,
3
)
settings.py
View file @
559b8bc7
...
...
@@ -145,6 +145,9 @@
'http://tracker.archlinux.org:6969/announce'
,
)
# How long to keep mirrorlog's in days
MIRRORLOG_RETENTION_PERIOD
=
365
# Shorten some names just a bit
COUNTRIES_OVERRIDE
=
{
'GB'
:
'United Kingdom'
,
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment