update.py 16.6 KB
Newer Older
1
#!/usr/bin/env python3
Lukas Fleischer's avatar
Lukas Fleischer committed
2
3
4
5

import os
import pygit2
import re
6
import subprocess
Lukas Fleischer's avatar
Lukas Fleischer committed
7
import sys
8
import time
Lukas Fleischer's avatar
Lukas Fleischer committed
9

10
11
import srcinfo.parse
import srcinfo.utils
Lukas Fleischer's avatar
Lukas Fleischer committed
12

13
14
import aurweb.config
import aurweb.db
15

16
notify_cmd = aurweb.config.get('notifications', 'notify-cmd')
17

18
19
repo_path = aurweb.config.get('serve', 'repo-path')
repo_regex = aurweb.config.get('serve', 'repo-regex')
20

21
max_blob_size = aurweb.config.getint('update', 'max-blob-size')
22
23
24
25
26
27
28
29
30
31
32
33


def size_humanize(num):
    for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB']:
        if abs(num) < 2048.0:
            if isinstance(num, int):
                return "{}{}".format(num, unit)
            else:
                return "{:.2f}{}".format(num, unit)
        num /= 1024.0
    return "{:.2f}{}".format(num, 'YiB')

34

35
36
37
38
39
40
41
42
43
44
45
46
47
48
def extract_arch_fields(pkginfo, field):
    values = []

    if field in pkginfo:
        for val in pkginfo[field]:
            values.append({"value": val, "arch": None})

    for arch in ['i686', 'x86_64']:
        if field + '_' + arch in pkginfo:
            for val in pkginfo[field + '_' + arch]:
                values.append({"value": val, "arch": arch})

    return values

49

50
51
52
53
54
def parse_dep(depstring):
    dep, _, desc = depstring.partition(': ')
    depname = re.sub(r'(<|=|>).*', '', dep)
    depcond = dep[len(depname):]

55
    return (depname, desc, depcond)
56

57

Lukas Fleischer's avatar
Lukas Fleischer committed
58
59
60
61
62
63
def create_pkgbase(conn, pkgbase, user):
    cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user])
    userid = cur.fetchone()[0]

    now = int(time.time())
    cur = conn.execute("INSERT INTO PackageBases (Name, SubmittedTS, " +
64
65
66
                       "ModifiedTS, SubmitterUID, MaintainerUID, " +
                       "FlaggerComment) VALUES (?, ?, ?, ?, ?, '')",
                       [pkgbase, now, now, userid, userid])
Lukas Fleischer's avatar
Lukas Fleischer committed
67
68
69
70
71
72
73
74
75
76
77
    pkgbase_id = cur.lastrowid

    cur = conn.execute("INSERT INTO PackageNotifications " +
                       "(PackageBaseID, UserID) VALUES (?, ?)",
                       [pkgbase_id, userid])

    conn.commit()

    return pkgbase_id


78
def save_metadata(metadata, conn, user):
Lukas Fleischer's avatar
Lukas Fleischer committed
79
    # Obtain package base ID and previous maintainer.
80
    pkgbase = metadata['pkgbase']
81
82
    cur = conn.execute("SELECT ID, MaintainerUID FROM PackageBases "
                       "WHERE Name = ?", [pkgbase])
Lukas Fleischer's avatar
Lukas Fleischer committed
83
84
85
86
    (pkgbase_id, maintainer_uid) = cur.fetchone()
    was_orphan = not maintainer_uid

    # Obtain the user ID of the new maintainer.
87
    cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user])
Lukas Fleischer's avatar
Lukas Fleischer committed
88
89
90
    user_id = int(cur.fetchone()[0])

    # Update package base details and delete current packages.
91
92
    now = int(time.time())
    conn.execute("UPDATE PackageBases SET ModifiedTS = ?, " +
93
                 "PackagerUID = ?, OutOfDateTS = NULL WHERE ID = ?",
94
                 [now, user_id, pkgbase_id])
95
96
97
    conn.execute("UPDATE PackageBases SET MaintainerUID = ? " +
                 "WHERE ID = ? AND MaintainerUID IS NULL",
                 [user_id, pkgbase_id])
98
99
100
101
102
103
104
    for table in ('Sources', 'Depends', 'Relations', 'Licenses', 'Groups'):
        conn.execute("DELETE FROM Package" + table + " WHERE EXISTS (" +
                     "SELECT * FROM Packages " +
                     "WHERE Packages.PackageBaseID = ? AND " +
                     "Package" + table + ".PackageID = Packages.ID)",
                     [pkgbase_id])
    conn.execute("DELETE FROM Packages WHERE PackageBaseID = ?", [pkgbase_id])
Lukas Fleischer's avatar
Lukas Fleischer committed
105

106
107
    for pkgname in srcinfo.utils.get_package_names(metadata):
        pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
Lukas Fleischer's avatar
Lukas Fleischer committed
108

109
        if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0:
110
111
            ver = '{:d}:{:s}-{:s}'.format(int(pkginfo['epoch']),
                                          pkginfo['pkgver'],
112
                                          pkginfo['pkgrel'])
Lukas Fleischer's avatar
Lukas Fleischer committed
113
        else:
114
            ver = '{:s}-{:s}'.format(pkginfo['pkgver'], pkginfo['pkgrel'])
Lukas Fleischer's avatar
Lukas Fleischer committed
115

116
        for field in ('pkgdesc', 'url'):
117
            if field not in pkginfo:
118
119
                pkginfo[field] = None

Lukas Fleischer's avatar
Lukas Fleischer committed
120
        # Create a new package.
121
122
123
124
125
126
        cur = conn.execute("INSERT INTO Packages (PackageBaseID, Name, " +
                           "Version, Description, URL) " +
                           "VALUES (?, ?, ?, ?, ?)",
                           [pkgbase_id, pkginfo['pkgname'], ver,
                            pkginfo['pkgdesc'], pkginfo['url']])
        conn.commit()
Lukas Fleischer's avatar
Lukas Fleischer committed
127
128
129
        pkgid = cur.lastrowid

        # Add package sources.
130
        for source_info in extract_arch_fields(pkginfo, 'source'):
131
132
133
            conn.execute("INSERT INTO PackageSources (PackageID, Source, " +
                         "SourceArch) VALUES (?, ?, ?)",
                         [pkgid, source_info['value'], source_info['arch']])
Lukas Fleischer's avatar
Lukas Fleischer committed
134
135
136
137

        # Add package dependencies.
        for deptype in ('depends', 'makedepends',
                        'checkdepends', 'optdepends'):
138
139
            cur = conn.execute("SELECT ID FROM DependencyTypes WHERE Name = ?",
                               [deptype])
Lukas Fleischer's avatar
Lukas Fleischer committed
140
            deptypeid = cur.fetchone()[0]
141
            for dep_info in extract_arch_fields(pkginfo, deptype):
142
                depname, depdesc, depcond = parse_dep(dep_info['value'])
143
                deparch = dep_info['arch']
144
                conn.execute("INSERT INTO PackageDepends (PackageID, " +
145
146
147
148
                             "DepTypeID, DepName, DepDesc, DepCondition, " +
                             "DepArch) VALUES (?, ?, ?, ?, ?, ?)",
                             [pkgid, deptypeid, depname, depdesc, depcond,
                              deparch])
Lukas Fleischer's avatar
Lukas Fleischer committed
149
150
151

        # Add package relations (conflicts, provides, replaces).
        for reltype in ('conflicts', 'provides', 'replaces'):
152
153
            cur = conn.execute("SELECT ID FROM RelationTypes WHERE Name = ?",
                               [reltype])
Lukas Fleischer's avatar
Lukas Fleischer committed
154
            reltypeid = cur.fetchone()[0]
155
            for rel_info in extract_arch_fields(pkginfo, reltype):
156
                relname, _, relcond = parse_dep(rel_info['value'])
157
                relarch = rel_info['arch']
158
159
160
161
                conn.execute("INSERT INTO PackageRelations (PackageID, " +
                             "RelTypeID, RelName, RelCondition, RelArch) " +
                             "VALUES (?, ?, ?, ?, ?)",
                             [pkgid, reltypeid, relname, relcond, relarch])
Lukas Fleischer's avatar
Lukas Fleischer committed
162
163
164
165

        # Add package licenses.
        if 'license' in pkginfo:
            for license in pkginfo['license']:
166
167
                cur = conn.execute("SELECT ID FROM Licenses WHERE Name = ?",
                                   [license])
168
169
170
                row = cur.fetchone()
                if row:
                    licenseid = row[0]
Lukas Fleischer's avatar
Lukas Fleischer committed
171
                else:
172
173
174
                    cur = conn.execute("INSERT INTO Licenses (Name) " +
                                       "VALUES (?)", [license])
                    conn.commit()
Lukas Fleischer's avatar
Lukas Fleischer committed
175
                    licenseid = cur.lastrowid
176
177
178
                conn.execute("INSERT INTO PackageLicenses (PackageID, " +
                             "LicenseID) VALUES (?, ?)",
                             [pkgid, licenseid])
Lukas Fleischer's avatar
Lukas Fleischer committed
179
180
181
182

        # Add package groups.
        if 'groups' in pkginfo:
            for group in pkginfo['groups']:
183
184
                cur = conn.execute("SELECT ID FROM Groups WHERE Name = ?",
                                   [group])
185
186
187
                row = cur.fetchone()
                if row:
                    groupid = row[0]
Lukas Fleischer's avatar
Lukas Fleischer committed
188
                else:
189
190
191
                    cur = conn.execute("INSERT INTO Groups (Name) VALUES (?)",
                                       [group])
                    conn.commit()
Lukas Fleischer's avatar
Lukas Fleischer committed
192
                    groupid = cur.lastrowid
193
194
                conn.execute("INSERT INTO PackageGroups (PackageID, "
                             "GroupID) VALUES (?, ?)", [pkgid, groupid])
Lukas Fleischer's avatar
Lukas Fleischer committed
195
196
197

    # Add user to notification list on adoption.
    if was_orphan:
198
199
200
        cur = conn.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " +
                           "PackageBaseID = ? AND UserID = ?",
                           [pkgbase_id, user_id])
201
        if cur.fetchone()[0] == 0:
202
203
204
            conn.execute("INSERT INTO PackageNotifications " +
                         "(PackageBaseID, UserID) VALUES (?, ?)",
                         [pkgbase_id, user_id])
Lukas Fleischer's avatar
Lukas Fleischer committed
205

206
    conn.commit()
Lukas Fleischer's avatar
Lukas Fleischer committed
207

208

209
def update_notify(conn, user, pkgbase_id):
210
    # Obtain the user ID of the new maintainer.
211
    cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user])
212
213
214
215
    user_id = int(cur.fetchone()[0])

    # Execute the notification script.
    subprocess.Popen((notify_cmd, 'update', str(user_id), str(pkgbase_id)))
216

217

Lukas Fleischer's avatar
Lukas Fleischer committed
218
def die(msg):
219
    sys.stderr.write("error: {:s}\n".format(msg))
Lukas Fleischer's avatar
Lukas Fleischer committed
220
221
    exit(1)

222

223
224
225
def warn(msg):
    sys.stderr.write("warning: {:s}\n".format(msg))

226

Lukas Fleischer's avatar
Lukas Fleischer committed
227
228
229
def die_commit(msg, commit):
    sys.stderr.write("error: The following error " +
                     "occurred when parsing commit\n")
230
231
    sys.stderr.write("error: {:s}:\n".format(commit))
    sys.stderr.write("error: {:s}\n".format(msg))
Lukas Fleischer's avatar
Lukas Fleischer committed
232
233
    exit(1)

234

235
236
237
238
239
240
def main():
    repo = pygit2.Repository(repo_path)

    user = os.environ.get("AUR_USER")
    pkgbase = os.environ.get("AUR_PKGBASE")
    privileged = (os.environ.get("AUR_PRIVILEGED", '0') == '1')
241
    allow_overwrite = (os.environ.get("AUR_OVERWRITE", '0') == '1')
242
243
244
245
246
247
248
249
250
251
252
253
254
    warn_or_die = warn if privileged else die

    if len(sys.argv) == 2 and sys.argv[1] == "restore":
        if 'refs/heads/' + pkgbase not in repo.listall_references():
            die('{:s}: repository not found: {:s}'.format(sys.argv[1],
                pkgbase))
        refname = "refs/heads/master"
        branchref = 'refs/heads/' + pkgbase
        sha1_old = sha1_new = repo.lookup_reference(branchref).target
    elif len(sys.argv) == 4:
        refname, sha1_old, sha1_new = sys.argv[1:4]
    else:
        die("invalid arguments")
255

256
257
    if refname != "refs/heads/master":
        die("pushing to a branch other than master is restricted")
258

259
    conn = aurweb.db.Connection()
Lukas Fleischer's avatar
Lukas Fleischer committed
260

261
    # Detect and deny non-fast-forwards.
262
    if sha1_old != "0" * 40 and not allow_overwrite:
263
264
265
266
        walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL)
        walker.hide(sha1_new)
        if next(walker, None) is not None:
            die("denying non-fast-forward (you should pull first)")
Lukas Fleischer's avatar
Lukas Fleischer committed
267

268
269
270
271
    # Prepare the walker that validates new commits.
    walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL)
    if sha1_old != "0" * 40:
        walker.hide(sha1_old)
Lukas Fleischer's avatar
Lukas Fleischer committed
272

273
274
275
276
277
    # Validate all new commits.
    for commit in walker:
        for fname in ('.SRCINFO', 'PKGBUILD'):
            if fname not in commit.tree:
                die_commit("missing {:s}".format(fname), str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
278

279
280
281
282
283
        for treeobj in commit.tree:
            blob = repo[treeobj.id]

            if isinstance(blob, pygit2.Tree):
                die_commit("the repository must not contain subdirectories",
284
                           str(commit.id))
285

286
287
288
            if not isinstance(blob, pygit2.Blob):
                die_commit("not a blob object: {:s}".format(treeobj),
                           str(commit.id))
289

290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
            if blob.size > max_blob_size:
                die_commit("maximum blob size ({:s}) exceeded".format(
                           size_humanize(max_blob_size)), str(commit.id))

        metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode()
        (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
        if errors:
            sys.stderr.write("error: The following errors occurred "
                             "when parsing .SRCINFO in commit\n")
            sys.stderr.write("error: {:s}:\n".format(str(commit.id)))
            for error in errors:
                for err in error['error']:
                    sys.stderr.write("error: line {:d}: {:s}\n".format(
                                     error['line'], err))
            exit(1)

        metadata_pkgbase = metadata['pkgbase']
        if not re.match(repo_regex, metadata_pkgbase):
            die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase),
309
                       str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
310

311
312
313
        if not metadata['packages']:
            die_commit('missing pkgname entry', str(commit.id))

314
315
        for pkgname in set(metadata['packages'].keys()):
            pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
Lukas Fleischer's avatar
Lukas Fleischer committed
316

317
318
319
320
            for field in ('pkgver', 'pkgrel', 'pkgname'):
                if field not in pkginfo:
                    die_commit('missing mandatory field: {:s}'.format(field),
                               str(commit.id))
321

322
323
            if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit():
                die_commit('invalid epoch: {:s}'.format(pkginfo['epoch']),
324
325
                           str(commit.id))

326
327
328
329
            if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']):
                die_commit('invalid package name: {:s}'.format(
                           pkginfo['pkgname']), str(commit.id))

330
331
332
            max_len = {'pkgname': 255, 'pkgdesc': 255, 'url': 8000}
            for field in max_len.keys():
                if field in pkginfo and len(pkginfo[field]) > max_len[field]:
333
334
335
336
337
338
339
340
341
342
                    die_commit('{:s} field too long: {:s}'.format(field,
                               pkginfo[field]), str(commit.id))

            for field in ('install', 'changelog'):
                if field in pkginfo and not pkginfo[field] in commit.tree:
                    die_commit('missing {:s} file: {:s}'.format(field,
                               pkginfo[field]), str(commit.id))

            for field in extract_arch_fields(pkginfo, 'source'):
                fname = field['value']
343
344
345
                if len(fname) > 8000:
                    die_commit('source entry too long: {:s}'.format(fname),
                               str(commit.id))
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
                if "://" in fname or "lp:" in fname:
                    continue
                if fname not in commit.tree:
                    die_commit('missing source file: {:s}'.format(fname),
                               str(commit.id))

    # Display a warning if .SRCINFO is unchanged.
    if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new):
        srcinfo_id_old = repo[sha1_old].tree['.SRCINFO'].id
        srcinfo_id_new = repo[sha1_new].tree['.SRCINFO'].id
        if srcinfo_id_old == srcinfo_id_new:
            warn(".SRCINFO unchanged. "
                 "The package database will not be updated!")

    # Read .SRCINFO from the HEAD commit.
    metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode()
    (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
363

364
365
366
367
368
    # Ensure that the package base name matches the repository name.
    metadata_pkgbase = metadata['pkgbase']
    if metadata_pkgbase != pkgbase:
        die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase,
                                                          pkgbase))
369

370
371
372
373
374
    # Ensure that packages are neither blacklisted nor overwritten.
    pkgbase = metadata['pkgbase']
    cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase])
    row = cur.fetchone()
    pkgbase_id = row[0] if row else 0
Lukas Fleischer's avatar
Lukas Fleischer committed
375

376
377
    cur = conn.execute("SELECT Name FROM PackageBlacklist")
    blacklist = [row[0] for row in cur.fetchall()]
378

379
380
    cur = conn.execute("SELECT Name, Repo FROM OfficialProviders")
    providers = dict(cur.fetchall())
381

382
383
384
    for pkgname in srcinfo.utils.get_package_names(metadata):
        pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
        pkgname = pkginfo['pkgname']
385

386
387
388
389
390
        if pkgname in blacklist:
            warn_or_die('package is blacklisted: {:s}'.format(pkgname))
        if pkgname in providers:
            warn_or_die('package already provided by [{:s}]: {:s}'.format(
                        providers[pkgname], pkgname))
391

392
393
394
395
        cur = conn.execute("SELECT COUNT(*) FROM Packages WHERE Name = ? " +
                           "AND PackageBaseID <> ?", [pkgname, pkgbase_id])
        if cur.fetchone()[0] > 0:
            die('cannot overwrite package: {:s}'.format(pkgname))
396

397
398
399
    # Create a new package base if it does not exist yet.
    if pkgbase_id == 0:
        pkgbase_id = create_pkgbase(conn, pkgbase, user)
400

401
402
    # Store package base details in the database.
    save_metadata(metadata, conn, user)
403

404
405
406
407
    # Create (or update) a branch with the name of the package base for better
    # accessibility.
    branchref = 'refs/heads/' + pkgbase
    repo.create_reference(branchref, sha1_new, True)
Lukas Fleischer's avatar
Lukas Fleischer committed
408

409
410
411
412
413
414
415
    # Work around a Git bug: The HEAD ref is not updated when using
    # gitnamespaces. This can be removed once the bug fix is included in Git
    # mainline. See
    # http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref-namespace-td7632149.html
    # for details.
    headref = 'refs/namespaces/' + pkgbase + '/HEAD'
    repo.create_reference(headref, sha1_new, True)
416

417
418
    # Send package update notifications.
    update_notify(conn, user, pkgbase_id)
419

420
421
422
    # Close the database.
    cur.close()
    conn.close()
423
424


425
426
if __name__ == '__main__':
    main()