Commit 822898e5 authored by Dan McGee's avatar Dan McGee
Browse files

Merge branch 'django14'

Conflicts:
	templates/releng/result_section.html
parents ea87160c 8e10699d
......@@ -22,6 +22,7 @@
import logging
from datetime import datetime
from optparse import make_option
from pytz import utc
from django.core.management.base import BaseCommand, CommandError
from django.db import connections, router, transaction
......@@ -29,8 +30,10 @@
from devel.utils import UserFinder
from main.models import Arch, Package, PackageDepend, PackageFile, Repo
from main.utils import utc_now
from packages.models import Conflict, Provision, Replacement
logging.basicConfig(
level=logging.WARNING,
format='%(asctime)s -> %(levelname)s: %(message)s',
......@@ -113,7 +116,8 @@ def populate(self, values):
self.epoch = int(match.group(2))
elif k == 'builddate':
try:
self.builddate = datetime.utcfromtimestamp(int(v[0]))
builddate = datetime.utcfromtimestamp(int(v[0]))
self.builddate = builddate.replace(tzinfo=utc)
except ValueError:
try:
self.builddate = datetime.strptime(v[0],
......@@ -270,33 +274,21 @@ def populate_files(dbpkg, repopkg, force=False):
delete_pkg_files(dbpkg)
logger.info("adding %d files for package %s",
len(repopkg.files), dbpkg.pkgname)
pkg_files = []
for f in repopkg.files:
dirname, filename = f.rsplit('/', 1)
if filename == '':
filename = None
# this is basically like calling dbpkg.packagefile_set.create(),
# but much faster as we can skip a lot of the repeated code paths
# TODO use Django 1.4 bulk_create
pkgfile = PackageFile(pkg=dbpkg,
is_directory=(filename is None),
directory=dirname + '/',
filename=filename)
pkgfile.save(force_insert=True)
dbpkg.files_last_update = datetime.utcnow()
pkg_files.append(pkgfile)
PackageFile.objects.bulk_create(pkg_files)
dbpkg.files_last_update = utc_now()
dbpkg.save()
def select_pkg_for_update(dbpkg):
database = router.db_for_write(Package, instance=dbpkg)
connection = connections[database]
if 'sqlite' in connection.settings_dict['ENGINE'].lower():
return dbpkg
new_pkg = Package.objects.raw(
'SELECT * FROM packages WHERE id = %s FOR UPDATE',
[dbpkg.id])
return list(new_pkg)[0]
def update_common(archname, reponame, pkgs, sanity_check=True):
# If isolation level is repeatable-read, we need to ensure each package
# update starts a new transaction and re-queries the database as
......@@ -363,7 +355,7 @@ def db_update(archname, reponame, pkgs, force=False):
dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository)
try:
with transaction.commit_on_success():
populate_pkg(dbpkg, pkg, timestamp=datetime.utcnow())
populate_pkg(dbpkg, pkg, timestamp=utc_now())
except IntegrityError:
logger.warning("Could not add package %s; "
"not fatal if another thread beat us to it.",
......@@ -390,14 +382,13 @@ def db_update(archname, reponame, pkgs, force=False):
if not force and pkg_same_version(pkg, dbpkg):
continue
elif not force:
timestamp = datetime.utcnow()
timestamp = utc_now()
# The odd select_for_update song and dance here are to ensure
# simultaneous updates don't happen on a package, causing
# files/depends/all related items to be double-imported.
with transaction.commit_on_success():
# TODO Django 1.4 select_for_update() will work once released
dbpkg = select_pkg_for_update(dbpkg)
dbpkg = Package.objects.select_for_update().get(id=dbpkg.id)
if not force and pkg_same_version(pkg, dbpkg):
logger.debug("Package %s was already updated", pkg.name)
continue
......@@ -428,8 +419,7 @@ def filesonly_update(archname, reponame, pkgs, force=False):
elif not force and dbpkg.files_last_update > dbpkg.last_update:
logger.debug("Files for %s are up to date", pkg.name)
continue
# TODO Django 1.4 select_for_update() will work once released
dbpkg = select_pkg_for_update(dbpkg)
dbpkg = Package.objects.select_for_update().get(id=dbpkg.id)
logger.debug("Checking files for package %s", pkg.name)
populate_files(dbpkg, pkg, force=force)
......
from django.conf.urls.defaults import patterns
from django.conf.urls import patterns
urlpatterns = patterns('devel.views',
(r'^admin_log/$','admin_log'),
......
from datetime import datetime, timedelta
import operator
import pytz
import random
from string import ascii_letters, digits
import time
from django import forms
from django.http import HttpResponseRedirect
from django.contrib.auth.decorators import \
......@@ -16,19 +23,13 @@
from django.utils.http import http_date
from main.models import Package, PackageDepend, PackageFile, TodolistPkg
from main.models import Arch, Repo
from main.models import UserProfile
from main.models import Arch, Repo, UserProfile
from main.utils import utc_now
from packages.models import PackageRelation
from packages.utils import get_signoff_groups
from todolists.utils import get_annotated_todolists
from .utils import get_annotated_maintainers
from datetime import datetime, timedelta
import operator
import pytz
import random
from string import ascii_letters, digits
import time
@login_required
def index(request):
......@@ -85,22 +86,14 @@ def clock(request):
devs = User.objects.filter(is_active=True).order_by(
'first_name', 'last_name').select_related('userprofile')
now = datetime.now()
utc_now = datetime.utcnow().replace(tzinfo=pytz.utc)
# now annotate each dev object with their current time
for dev in devs:
tz = pytz.timezone(dev.userprofile.time_zone)
dev.current_time = utc_now.astimezone(tz)
now = utc_now()
page_dict = {
'developers': devs,
'now': now,
'utc_now': utc_now,
'utc_now': now,
}
response = direct_to_template(request, 'devel/clock.html', page_dict)
if not response.has_header('Expires'):
# why this works only with the non-UTC date I have no idea...
expire_time = now.replace(second=0, microsecond=0)
expire_time += timedelta(minutes=1)
expire_time = time.mktime(expire_time.timetuple())
......@@ -168,12 +161,12 @@ def report(request, report_name, username=None):
if report_name == 'old':
title = 'Packages last built more than two years ago'
cutoff = datetime.utcnow() - timedelta(days=365 * 2)
cutoff = utc_now() - timedelta(days=365 * 2)
packages = packages.filter(
build_date__lt=cutoff).order_by('build_date')
elif report_name == 'long-out-of-date':
title = 'Packages marked out-of-date more than 90 days ago'
cutoff = datetime.utcnow() - timedelta(days=90)
cutoff = utc_now() - timedelta(days=90)
packages = packages.filter(
flag_date__lt=cutoff).order_by('flag_date')
elif report_name == 'big':
......
import hashlib
import pytz
from django.contrib.sites.models import Site
from django.contrib.syndication.views import Feed
from django.db.models import Q
from django.utils.feedgenerator import Rss201rev2Feed
from django.utils.hashcompat import md5_constructor
from django.views.decorators.http import condition
from main.utils import retrieve_latest
......@@ -32,7 +32,7 @@ def write_items(self, handler):
def package_etag(request, *args, **kwargs):
latest = retrieve_latest(Package)
if latest:
return md5_constructor(str(kwargs) + str(latest)).hexdigest()
return hashlib.md5(str(kwargs) + str(latest)).hexdigest()
return None
def package_last_modified(request, *args, **kwargs):
......@@ -108,7 +108,7 @@ def item_categories(self, item):
def news_etag(request, *args, **kwargs):
latest = retrieve_latest(News)
if latest:
return md5_constructor(str(latest)).hexdigest()
return hashlib.md5(str(latest)).hexdigest()
return None
def news_last_modified(request, *args, **kwargs):
......
# encoding: utf-8
import datetime
from pytz import utc
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
......@@ -9,7 +10,9 @@ class Migration(SchemaMigration):
def forwards(self, orm):
db.alter_column('packages', 'compressed_size', self.gf('main.models.PositiveBigIntegerField')(default=0))
db.alter_column('packages', 'installed_size', self.gf('main.models.PositiveBigIntegerField')(default=0))
db.alter_column('packages', 'last_update', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime(2000, 1, 1)))
old_date = datetime.datetime(2000, 1, 1)
old_date = old_date.replace(tzinfo=utc)
db.alter_column('packages', 'last_update', self.gf('django.db.models.fields.DateTimeField')(default=old_date))
def backwards(self, orm):
db.alter_column('packages', 'compressed_size', self.gf('django.db.models.fields.BigIntegerField')(null=True))
......
# encoding: utf-8
import datetime
from pytz import utc
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
......@@ -8,7 +9,9 @@ class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'Donor.created'
db.add_column('donors', 'created', self.gf('django.db.models.fields.DateTimeField')(default=datetime.date(2000, 1, 1)), keep_default=False)
old_date = datetime.datetime(2000, 1, 1)
old_date = old_date.replace(tzinfo=utc)
db.add_column('donors', 'created', self.gf('django.db.models.fields.DateTimeField')(default=old_date), keep_default=False)
def backwards(self, orm):
......
......@@ -9,7 +9,7 @@
from django.contrib.sites.models import Site
from .fields import PositiveBigIntegerField, PGPKeyField
from .utils import cache_function, make_choice, set_created_field
from .utils import cache_function, make_choice, set_created_field, utc_now
class UserProfile(models.Model):
......@@ -515,7 +515,7 @@ class Meta:
def set_todolist_fields(sender, **kwargs):
todolist = kwargs['instance']
if not todolist.date_added:
todolist.date_added = datetime.utcnow()
todolist.date_added = utc_now()
# connect signals needed to keep cache in line with reality
from main.utils import refresh_latest
......
......@@ -4,20 +4,24 @@
import pickle
from datetime import datetime
import hashlib
from pytz import utc
from django.core.cache import cache
from django.utils.hashcompat import md5_constructor
CACHE_TIMEOUT = 1800
INVALIDATE_TIMEOUT = 10
CACHE_LATEST_PREFIX = 'cache_latest_'
def cache_function_key(func, args, kwargs):
raw = [func.__name__, func.__module__, args, kwargs]
pickled = pickle.dumps(raw, protocol=pickle.HIGHEST_PROTOCOL)
key = md5_constructor(pickled).hexdigest()
key = hashlib.md5(pickled).hexdigest()
return 'cache_function.' + func.__name__ + '.' + key
def cache_function(length):
"""
A variant of the snippet posted by Jeff Wheeler at
......@@ -43,6 +47,7 @@ def inner_func(*args, **kwargs):
return inner_func
return decorator
def clear_cache_function(func, args, kwargs):
key = cache_function_key(func, args, kwargs)
cache.delete(key)
......@@ -50,6 +55,7 @@ def clear_cache_function(func, args, kwargs):
# utility to make a pair of django choices
make_choice = lambda l: [(str(m), str(m)) for m in l]
# These are in here because we would be jumping around in some import circles
# and hoops otherwise. The only thing currently using these keys is the feed
# caching stuff.
......@@ -65,6 +71,7 @@ def refresh_latest(**kwargs):
# will be valid again. See "Scaling Django" by Mike Malone, slide 30.
cache.set(cache_key, None, INVALIDATE_TIMEOUT)
def retrieve_latest(sender):
# we could break this down based on the request url, but it would probably
# cost us more in query time to do so.
......@@ -84,12 +91,19 @@ def retrieve_latest(sender):
pass
return None
def utc_now():
'''Returns a timezone-aware UTC date representing now.'''
return datetime.utcnow().replace(tzinfo=utc)
def set_created_field(sender, **kwargs):
'''This will set the 'created' field on any object to datetime.utcnow() if
it is unset. For use as a pre_save signal handler.'''
'''This will set the 'created' field on any object to the current UTC time
if it is unset. For use as a pre_save signal handler.'''
obj = kwargs['instance']
if hasattr(obj, 'created') and not obj.created:
obj.created = datetime.utcnow()
obj.created = utc_now()
def groupby_preserve_order(iterable, keyfunc):
'''Take an iterable and regroup using keyfunc to determine whether items
......@@ -112,6 +126,7 @@ def groupby_preserve_order(iterable, keyfunc):
return result
class PackageStandin(object):
'''Resembles a Package object, and has a few of the same fields, but is
really a link to a pkgbase that has no package with matching pkgname.'''
......
#!/usr/bin/env python2
import os
import sys
from django.core.management import execute_manager
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
try:
import settings # Assumed to be in the same directory.
except ImportError:
import sys
sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n" % __file__)
sys.exit(1)
from django.core.management import execute_from_command_line
if __name__ == "__main__":
execute_manager(settings)
execute_from_command_line(sys.argv)
# vim: set ts=4 sw=4 et:
......@@ -21,9 +21,11 @@
import time
from threading import Thread
import types
from pytz import utc
from Queue import Queue, Empty
import urllib2
from main.utils import utc_now
from mirrors.models import MirrorUrl, MirrorLog
logging.basicConfig(
......@@ -50,7 +52,7 @@ def handle_noargs(self, **options):
def check_mirror_url(mirror_url):
url = mirror_url.url + 'lastsync'
logger.info("checking URL %s", url)
log = MirrorLog(url=mirror_url, check_time=datetime.utcnow())
log = MirrorLog(url=mirror_url, check_time=utc_now())
try:
start = time.time()
result = urllib2.urlopen(url, timeout=10)
......@@ -61,6 +63,7 @@ def check_mirror_url(mirror_url):
parsed_time = None
try:
parsed_time = datetime.utcfromtimestamp(int(data))
parsed_time = parsed_time.replace(tzinfo=utc)
except ValueError:
# it is bad news to try logging the lastsync value;
# sometimes we get a crazy-encoded web page.
......
from django.conf.urls.defaults import patterns
from django.conf.urls import patterns
urlpatterns = patterns('mirrors.views',
(r'^$', 'mirrors', {}, 'mirror-list'),
......
from django.conf.urls.defaults import patterns
from django.conf.urls import patterns
urlpatterns = patterns('mirrors.views',
(r'^$', 'generate_mirrorlist', {}, 'mirrorlist'),
......
from datetime import timedelta
from django.db.models import Avg, Count, Max, Min, StdDev
from main.utils import cache_function
from main.utils import cache_function, utc_now
from .models import MirrorLog, MirrorProtocol, MirrorUrl
import datetime
default_cutoff = datetime.timedelta(hours=24)
default_cutoff = timedelta(hours=24)
def annotate_url(url, delays):
'''Given a MirrorURL object, add a few more attributes to it regarding
......@@ -13,7 +14,7 @@ def annotate_url(url, delays):
url.completion_pct = float(url.success_count) / url.check_count
if url.id in delays:
url_delays = delays[url.id]
url.delay = sum(url_delays, datetime.timedelta()) / len(url_delays)
url.delay = sum(url_delays, timedelta()) / len(url_delays)
hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0
if url.completion_pct > 0:
......@@ -28,7 +29,7 @@ def annotate_url(url, delays):
@cache_function(123)
def get_mirror_statuses(cutoff=default_cutoff):
cutoff_time = datetime.datetime.utcnow() - cutoff
cutoff_time = utc_now() - cutoff
protocols = list(MirrorProtocol.objects.filter(is_download=True))
# I swear, this actually has decent performance...
urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter(
......@@ -82,7 +83,7 @@ def get_mirror_statuses(cutoff=default_cutoff):
@cache_function(117)
def get_mirror_errors(cutoff=default_cutoff):
cutoff_time = datetime.datetime.utcnow() - cutoff
cutoff_time = utc_now() - cutoff
errors = MirrorLog.objects.filter(
is_success=False, check_time__gte=cutoff_time,
url__mirror__active=True, url__mirror__public=True).values(
......
from datetime import datetime
from django.db import models
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from main.utils import utc_now
class News(models.Model):
slug = models.SlugField(max_length=255, unique=True)
author = models.ForeignKey(User, related_name='news_author',
......@@ -28,7 +29,7 @@ class Meta:
def set_news_fields(sender, **kwargs):
news = kwargs['instance']
now = datetime.utcnow()
now = utc_now()
news.last_modified = now
if not news.postdate:
news.postdate = now
......
from django.conf.urls.defaults import patterns
from django.conf.urls import patterns
urlpatterns = patterns('news.views',
(r'^$', 'news_list', {}, 'news-list'),
......
......@@ -17,12 +17,13 @@
from django.template import loader, Context
from collections import namedtuple
from datetime import datetime, timedelta
from datetime import timedelta
import logging
from operator import attrgetter
import sys
from main.models import Repo
from main.utils import utc_now
from packages.models import Signoff
from packages.utils import get_signoff_groups
......@@ -65,7 +66,7 @@ def generate_report(email, repo_name):
new_hours = 24
old_days = 14
now = datetime.utcnow()
now = utc_now()
new_cutoff = now - timedelta(hours=new_hours)
old_cutoff = now - timedelta(days=old_days)
......
# encoding: utf-8
import datetime
from pytz import utc
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
......@@ -7,7 +8,9 @@
class Migration(SchemaMigration):
def forwards(self, orm):
db.add_column('packages_packagerelation', 'created', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.utcnow()), keep_default=False)
old_date = datetime.datetime(2000, 1, 1)
old_date = old_date.replace(tzinfo=utc)
db.add_column('packages_packagerelation', 'created', self.gf('django.db.models.fields.DateTimeField')(default=old_date), keep_default=False)
def backwards(self, orm):
db.delete_column('packages_packagerelation', 'created')
......
from django.conf.urls.defaults import include, patterns
from django.conf.urls import include, patterns
package_patterns = patterns('packages.views',
(r'^$', 'details'),
......
from django.conf.urls.defaults import patterns
from django.conf.urls import patterns
urlpatterns = patterns('packages.views',
(r'^$', 'groups', {}, 'groups-list'),
......
from datetime import datetime
from django import forms
from django.conf import settings
from django.contrib.auth.decorators import permission_required
......@@ -12,6 +10,7 @@
from ..models import FlagRequest
from main.models import Package
from main.utils import utc_now
def flaghelp(request):
......@@ -61,7 +60,7 @@ def flag(request, name, repo, arch):
@transaction.commit_on_success
def perform_updates():
now = datetime.utcnow()
now = utc_now()
pkgs.update(flag_date=now)
# store our flag request
flag_request = FlagRequest(created=now,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment