git-update.py 13.5 KB
Newer Older
Lukas Fleischer's avatar
Lukas Fleischer committed
1
2
3
4
5
6
7
8
9
#!/usr/bin/python3

import configparser
import mysql.connector
import os
import pygit2
import re
import sys

10
11
import srcinfo.parse
import srcinfo.utils
Lukas Fleischer's avatar
Lukas Fleischer committed
12
13

config = configparser.RawConfigParser()
Lukas Fleischer's avatar
Lukas Fleischer committed
14
config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config")
Lukas Fleischer's avatar
Lukas Fleischer committed
15
16
17
18
19

aur_db_host = config.get('database', 'host')
aur_db_name = config.get('database', 'name')
aur_db_user = config.get('database', 'user')
aur_db_pass = config.get('database', 'password')
20
aur_db_socket = config.get('database', 'socket')
Lukas Fleischer's avatar
Lukas Fleischer committed
21

22
repo_path = config.get('serve', 'repo-path')
23
repo_regex = config.get('serve', 'repo-regex')
24

25

26
27
28
29
30
31
32
33
34
35
36
37
38
39
def extract_arch_fields(pkginfo, field):
    values = []

    if field in pkginfo:
        for val in pkginfo[field]:
            values.append({"value": val, "arch": None})

    for arch in ['i686', 'x86_64']:
        if field + '_' + arch in pkginfo:
            for val in pkginfo[field + '_' + arch]:
                values.append({"value": val, "arch": arch})

    return values

40

41
42
43
44
45
46
47
48
49
50
def parse_dep(depstring):
    dep, _, desc = depstring.partition(': ')
    depname = re.sub(r'(<|=|>).*', '', dep)
    depcond = dep[len(depname):]

    if (desc):
        return (depname + ': ' + desc, depcond)
    else:
        return (depname, depcond)

51

52
def save_metadata(metadata, db, cur, user):
Lukas Fleischer's avatar
Lukas Fleischer committed
53
    # Obtain package base ID and previous maintainer.
54
    pkgbase = metadata['pkgbase']
Lukas Fleischer's avatar
Lukas Fleischer committed
55
56
57
58
59
60
61
62
63
64
65
    cur.execute("SELECT ID, MaintainerUID FROM PackageBases "
                "WHERE Name = %s", [pkgbase])
    (pkgbase_id, maintainer_uid) = cur.fetchone()
    was_orphan = not maintainer_uid

    # Obtain the user ID of the new maintainer.
    cur.execute("SELECT ID FROM Users WHERE Username = %s", [user])
    user_id = int(cur.fetchone()[0])

    # Update package base details and delete current packages.
    cur.execute("UPDATE PackageBases SET ModifiedTS = UNIX_TIMESTAMP(), " +
66
67
68
69
70
                "PackagerUID = %s, OutOfDateTS = NULL WHERE ID = %s",
                [user_id, pkgbase_id])
    cur.execute("UPDATE PackageBases SET MaintainerUID = %s " +
                "WHERE ID = %s AND MaintainerUID IS NULL",
                [user_id, pkgbase_id])
Lukas Fleischer's avatar
Lukas Fleischer committed
71
72
73
    cur.execute("DELETE FROM Packages WHERE PackageBaseID = %s",
                [pkgbase_id])

74
75
    for pkgname in srcinfo.utils.get_package_names(metadata):
        pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
Lukas Fleischer's avatar
Lukas Fleischer committed
76

77
        if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0:
78
79
            ver = '{:d}:{:s}-{:s}'.format(int(pkginfo['epoch']),
                                          pkginfo['pkgver'],
80
                                          pkginfo['pkgrel'])
Lukas Fleischer's avatar
Lukas Fleischer committed
81
        else:
82
            ver = '{:s}-{:s}'.format(pkginfo['pkgver'], pkginfo['pkgrel'])
Lukas Fleischer's avatar
Lukas Fleischer committed
83

84
        for field in ('pkgdesc', 'url'):
85
            if field not in pkginfo:
86
87
                pkginfo[field] = None

Lukas Fleischer's avatar
Lukas Fleischer committed
88
89
90
91
92
93
94
95
96
97
        # Create a new package.
        cur.execute("INSERT INTO Packages (PackageBaseID, Name, " +
                    "Version, Description, URL) " +
                    "VALUES (%s, %s, %s, %s, %s)",
                    [pkgbase_id, pkginfo['pkgname'], ver,
                     pkginfo['pkgdesc'], pkginfo['url']])
        db.commit()
        pkgid = cur.lastrowid

        # Add package sources.
98
99
100
101
        for source_info in extract_arch_fields(pkginfo, 'source'):
            cur.execute("INSERT INTO PackageSources (PackageID, Source, " +
                        "SourceArch) VALUES (%s, %s, %s)",
                        [pkgid, source_info['value'], source_info['arch']])
Lukas Fleischer's avatar
Lukas Fleischer committed
102
103
104
105
106
107
108

        # Add package dependencies.
        for deptype in ('depends', 'makedepends',
                        'checkdepends', 'optdepends'):
            cur.execute("SELECT ID FROM DependencyTypes WHERE Name = %s",
                        [deptype])
            deptypeid = cur.fetchone()[0]
109
            for dep_info in extract_arch_fields(pkginfo, deptype):
110
                depname, depcond = parse_dep(dep_info['value'])
111
                deparch = dep_info['arch']
Lukas Fleischer's avatar
Lukas Fleischer committed
112
                cur.execute("INSERT INTO PackageDepends (PackageID, " +
113
114
115
                            "DepTypeID, DepName, DepCondition, DepArch) " +
                            "VALUES (%s, %s, %s, %s, %s)",
                            [pkgid, deptypeid, depname, depcond, deparch])
Lukas Fleischer's avatar
Lukas Fleischer committed
116
117
118
119
120
121

        # Add package relations (conflicts, provides, replaces).
        for reltype in ('conflicts', 'provides', 'replaces'):
            cur.execute("SELECT ID FROM RelationTypes WHERE Name = %s",
                        [reltype])
            reltypeid = cur.fetchone()[0]
122
            for rel_info in extract_arch_fields(pkginfo, reltype):
123
                relname, relcond = parse_dep(rel_info['value'])
124
                relarch = rel_info['arch']
Lukas Fleischer's avatar
Lukas Fleischer committed
125
                cur.execute("INSERT INTO PackageRelations (PackageID, " +
126
127
128
                            "RelTypeID, RelName, RelCondition, RelArch) " +
                            "VALUES (%s, %s, %s, %s, %s)",
                            [pkgid, reltypeid, relname, relcond, relarch])
Lukas Fleischer's avatar
Lukas Fleischer committed
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162

        # Add package licenses.
        if 'license' in pkginfo:
            for license in pkginfo['license']:
                cur.execute("SELECT ID FROM Licenses WHERE Name = %s",
                            [license])
                if cur.rowcount == 1:
                    licenseid = cur.fetchone()[0]
                else:
                    cur.execute("INSERT INTO Licenses (Name) VALUES (%s)",
                                [license])
                    db.commit()
                    licenseid = cur.lastrowid
                cur.execute("INSERT INTO PackageLicenses (PackageID, " +
                            "LicenseID) VALUES (%s, %s)",
                            [pkgid, licenseid])

        # Add package groups.
        if 'groups' in pkginfo:
            for group in pkginfo['groups']:
                cur.execute("SELECT ID FROM Groups WHERE Name = %s",
                            [group])
                if cur.rowcount == 1:
                    groupid = cur.fetchone()[0]
                else:
                    cur.execute("INSERT INTO Groups (Name) VALUES (%s)",
                                [group])
                    db.commit()
                    groupid = cur.lastrowid
                cur.execute("INSERT INTO PackageGroups (PackageID, "
                            "GroupID) VALUES (%s, %s)", [pkgid, groupid])

    # Add user to notification list on adoption.
    if was_orphan:
163
        cur.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " +
164
165
166
                    "PackageBaseID = %s AND UserID = %s",
                    [pkgbase_id, user_id])
        if cur.fetchone()[0] == 0:
167
            cur.execute("INSERT INTO PackageNotifications (PackageBaseID, UserID) " +
168
                        "VALUES (%s, %s)", [pkgbase_id, user_id])
Lukas Fleischer's avatar
Lukas Fleischer committed
169
170
171

    db.commit()

172

Lukas Fleischer's avatar
Lukas Fleischer committed
173
def die(msg):
174
    sys.stderr.write("error: {:s}\n".format(msg))
Lukas Fleischer's avatar
Lukas Fleischer committed
175
176
    exit(1)

177

178
179
180
def warn(msg):
    sys.stderr.write("warning: {:s}\n".format(msg))

181

Lukas Fleischer's avatar
Lukas Fleischer committed
182
183
184
def die_commit(msg, commit):
    sys.stderr.write("error: The following error " +
                     "occurred when parsing commit\n")
185
186
    sys.stderr.write("error: {:s}:\n".format(commit))
    sys.stderr.write("error: {:s}\n".format(msg))
Lukas Fleischer's avatar
Lukas Fleischer committed
187
188
    exit(1)

189

190
repo = pygit2.Repository(repo_path)
Lukas Fleischer's avatar
Lukas Fleischer committed
191
192
193

user = os.environ.get("AUR_USER")
pkgbase = os.environ.get("AUR_PKGBASE")
194
privileged = (os.environ.get("AUR_PRIVILEGED", '0') == '1')
Lukas Fleischer's avatar
Lukas Fleischer committed
195

196
197
198
199
200
201
if len(sys.argv) == 2 and sys.argv[1] == "restore":
    if 'refs/heads/' + pkgbase not in repo.listall_references():
        die('{:s}: repository not found: {:s}'.format(sys.argv[1], pkgbase))
    refname = "refs/heads/master"
    sha1_old = sha1_new = repo.lookup_reference('refs/heads/' + pkgbase).target
elif len(sys.argv) == 4:
202
    refname, sha1_old, sha1_new = sys.argv[1:4]
203
204
205
else:
    die("invalid arguments")

Lukas Fleischer's avatar
Lukas Fleischer committed
206
207
208
if refname != "refs/heads/master":
    die("pushing to a branch other than master is restricted")

209
210
211
212
213
db = mysql.connector.connect(host=aur_db_host, user=aur_db_user,
                             passwd=aur_db_pass, db=aur_db_name,
                             unix_socket=aur_db_socket, buffered=True)
cur = db.cursor()

214
215
216
217
# Detect and deny non-fast-forwards.
if sha1_old != "0000000000000000000000000000000000000000":
    walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL)
    walker.hide(sha1_new)
218
    if next(walker, None) is not None:
219
220
221
222
        cur.execute("SELECT AccountTypeID FROM Users WHERE UserName = %s ",
                    [user])
        if cur.fetchone()[0] == 1:
            die("denying non-fast-forward (you should pull first)")
223
224

# Prepare the walker that validates new commits.
Lukas Fleischer's avatar
Lukas Fleischer committed
225
226
227
228
walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL)
if sha1_old != "0000000000000000000000000000000000000000":
    walker.hide(sha1_old)

Lukas Fleischer's avatar
Lukas Fleischer committed
229
# Validate all new commits.
Lukas Fleischer's avatar
Lukas Fleischer committed
230
for commit in walker:
231
    for fname in ('.SRCINFO', 'PKGBUILD'):
232
        if fname not in commit.tree:
233
            die_commit("missing {:s}".format(fname), str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
234
235

    for treeobj in commit.tree:
236
237
238
239
        blob = repo[treeobj.id]

        if isinstance(blob, pygit2.Tree):
            die_commit("the repository must not contain subdirectories",
240
                       str(commit.id))
241
242

        if not isinstance(blob, pygit2.Blob):
243
244
            die_commit("not a blob object: {:s}".format(treeobj),
                       str(commit.id))
245
246

        if blob.size > 250000:
247
            die_commit("maximum blob size (250kB) exceeded", str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
248

249
250
    metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode()
    (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
Lukas Fleischer's avatar
Lukas Fleischer committed
251
252
253
    if errors:
        sys.stderr.write("error: The following errors occurred "
                         "when parsing .SRCINFO in commit\n")
254
        sys.stderr.write("error: {:s}:\n".format(str(commit.id)))
Lukas Fleischer's avatar
Lukas Fleischer committed
255
        for error in errors:
256
257
            for err in error['error']:
                sys.stderr.write("error: line {:d}: {:s}\n".format(error['line'], err))
Lukas Fleischer's avatar
Lukas Fleischer committed
258
259
        exit(1)

260
261
262
    metadata_pkgbase = metadata['pkgbase']
    if not re.match(repo_regex, metadata_pkgbase):
        die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase),
263
                   str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
264

265
266
    for pkgname in set(metadata['packages'].keys()):
        pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
Lukas Fleischer's avatar
Lukas Fleischer committed
267

268
        for field in ('pkgver', 'pkgrel', 'pkgname'):
269
270
271
            if field not in pkginfo:
                die_commit('missing mandatory field: {:s}'.format(field),
                           str(commit.id))
272

273
        if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit():
274
275
            die_commit('invalid epoch: {:s}'.format(pkginfo['epoch']),
                       str(commit.id))
276

Lukas Fleischer's avatar
Lukas Fleischer committed
277
        if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']):
278
279
            die_commit('invalid package name: {:s}'.format(pkginfo['pkgname']),
                       str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
280
281

        for field in ('pkgname', 'pkgdesc', 'url'):
282
            if field in pkginfo and len(pkginfo[field]) > 255:
283
284
                die_commit('{:s} field too long: {:s}'.format(field, pkginfo[field]),
                           str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
285

286
287
        for field in ('install', 'changelog'):
            if field in pkginfo and not pkginfo[field] in commit.tree:
288
289
                die_commit('missing {:s} file: {:s}'.format(field, pkginfo[field]),
                           str(commit.id))
290

291
292
        for field in extract_arch_fields(pkginfo, 'source'):
            fname = field['value']
293
294
            if "://" in fname or "lp:" in fname:
                continue
295
296
297
298
            if fname not in commit.tree:
                die_commit('missing source file: {:s}'.format(fname),
                           str(commit.id))

299

300
# Display a warning if .SRCINFO is unchanged.
301
if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new):
302
303
304
305
306
    srcinfo_id_old = repo[sha1_old].tree['.SRCINFO'].id
    srcinfo_id_new = repo[sha1_new].tree['.SRCINFO'].id
    if srcinfo_id_old == srcinfo_id_new:
        warn(".SRCINFO unchanged. The package database will not be updated!")

Lukas Fleischer's avatar
Lukas Fleischer committed
307
# Read .SRCINFO from the HEAD commit.
308
309
metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode()
(metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
Lukas Fleischer's avatar
Lukas Fleischer committed
310

Lukas Fleischer's avatar
Lukas Fleischer committed
311
# Ensure that the package base name matches the repository name.
312
313
314
metadata_pkgbase = metadata['pkgbase']
if metadata_pkgbase != pkgbase:
    die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase, pkgbase))
315

Lukas Fleischer's avatar
Lukas Fleischer committed
316
# Ensure that packages are neither blacklisted nor overwritten.
317
pkgbase = metadata['pkgbase']
318
cur.execute("SELECT ID FROM PackageBases WHERE Name = %s", [pkgbase])
319
pkgbase_id = cur.fetchone()[0] if cur.rowcount == 1 else 0
320

321
322
323
cur.execute("SELECT Name FROM PackageBlacklist")
blacklist = [row[0] for row in cur.fetchall()]

324
325
for pkgname in srcinfo.utils.get_package_names(metadata):
    pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
326
    pkgname = pkginfo['pkgname']
327

328
    if pkgname in blacklist and not privileged:
329
        die('package is blacklisted: {:s}'.format(pkgname))
330

331
332
333
    cur.execute("SELECT COUNT(*) FROM Packages WHERE Name = %s AND " +
                "PackageBaseID <> %s", [pkgname, pkgbase_id])
    if cur.fetchone()[0] > 0:
334
        die('cannot overwrite package: {:s}'.format(pkgname))
335

Lukas Fleischer's avatar
Lukas Fleischer committed
336
# Store package base details in the database.
337
save_metadata(metadata, db, cur, user)
338

Lukas Fleischer's avatar
Lukas Fleischer committed
339
db.close()
340
341
342
343
344
345
346
347
348
349

# Create (or update) a branch with the name of the package base for better
# accessibility.
repo.create_reference('refs/heads/' + pkgbase, sha1_new, True)

# Work around a Git bug: The HEAD ref is not updated when using gitnamespaces.
# This can be removed once the bug fix is included in Git mainline. See
# http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref-namespace-td7632149.html
# for details.
repo.create_reference('refs/namespaces/' + pkgbase + '/HEAD', sha1_new, True)