update.py 16.1 KB
Newer Older
Lukas Fleischer's avatar
Lukas Fleischer committed
1
2
3
4
5
#!/usr/bin/python3

import os
import pygit2
import re
6
import subprocess
Lukas Fleischer's avatar
Lukas Fleischer committed
7
import sys
8
import time
Lukas Fleischer's avatar
Lukas Fleischer committed
9

10
11
import srcinfo.parse
import srcinfo.utils
Lukas Fleischer's avatar
Lukas Fleischer committed
12

13
14
import aurweb.config
import aurweb.db
15

16
notify_cmd = aurweb.config.get('notifications', 'notify-cmd')
17

18
19
repo_path = aurweb.config.get('serve', 'repo-path')
repo_regex = aurweb.config.get('serve', 'repo-regex')
20

21
max_blob_size = aurweb.config.getint('update', 'max-blob-size')
22
23
24
25
26
27
28
29
30
31
32
33


def size_humanize(num):
    for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB']:
        if abs(num) < 2048.0:
            if isinstance(num, int):
                return "{}{}".format(num, unit)
            else:
                return "{:.2f}{}".format(num, unit)
        num /= 1024.0
    return "{:.2f}{}".format(num, 'YiB')

34

35
36
37
38
39
40
41
42
43
44
45
46
47
48
def extract_arch_fields(pkginfo, field):
    values = []

    if field in pkginfo:
        for val in pkginfo[field]:
            values.append({"value": val, "arch": None})

    for arch in ['i686', 'x86_64']:
        if field + '_' + arch in pkginfo:
            for val in pkginfo[field + '_' + arch]:
                values.append({"value": val, "arch": arch})

    return values

49

50
51
52
53
54
55
56
57
58
59
def parse_dep(depstring):
    dep, _, desc = depstring.partition(': ')
    depname = re.sub(r'(<|=|>).*', '', dep)
    depcond = dep[len(depname):]

    if (desc):
        return (depname + ': ' + desc, depcond)
    else:
        return (depname, depcond)

60

Lukas Fleischer's avatar
Lukas Fleischer committed
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
def create_pkgbase(conn, pkgbase, user):
    cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user])
    userid = cur.fetchone()[0]

    now = int(time.time())
    cur = conn.execute("INSERT INTO PackageBases (Name, SubmittedTS, " +
                       "ModifiedTS, SubmitterUID, MaintainerUID) VALUES " +
                       "(?, ?, ?, ?, ?)", [pkgbase, now, now, userid, userid])
    pkgbase_id = cur.lastrowid

    cur = conn.execute("INSERT INTO PackageNotifications " +
                       "(PackageBaseID, UserID) VALUES (?, ?)",
                       [pkgbase_id, userid])

    conn.commit()

    return pkgbase_id


80
def save_metadata(metadata, conn, user):
Lukas Fleischer's avatar
Lukas Fleischer committed
81
    # Obtain package base ID and previous maintainer.
82
    pkgbase = metadata['pkgbase']
83
84
    cur = conn.execute("SELECT ID, MaintainerUID FROM PackageBases "
                       "WHERE Name = ?", [pkgbase])
Lukas Fleischer's avatar
Lukas Fleischer committed
85
86
87
88
    (pkgbase_id, maintainer_uid) = cur.fetchone()
    was_orphan = not maintainer_uid

    # Obtain the user ID of the new maintainer.
89
    cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user])
Lukas Fleischer's avatar
Lukas Fleischer committed
90
91
92
    user_id = int(cur.fetchone()[0])

    # Update package base details and delete current packages.
93
94
    now = int(time.time())
    conn.execute("UPDATE PackageBases SET ModifiedTS = ?, " +
95
                 "PackagerUID = ?, OutOfDateTS = NULL WHERE ID = ?",
96
                 [now, user_id, pkgbase_id])
97
98
99
    conn.execute("UPDATE PackageBases SET MaintainerUID = ? " +
                 "WHERE ID = ? AND MaintainerUID IS NULL",
                 [user_id, pkgbase_id])
100
101
102
103
104
105
106
    for table in ('Sources', 'Depends', 'Relations', 'Licenses', 'Groups'):
        conn.execute("DELETE FROM Package" + table + " WHERE EXISTS (" +
                     "SELECT * FROM Packages " +
                     "WHERE Packages.PackageBaseID = ? AND " +
                     "Package" + table + ".PackageID = Packages.ID)",
                     [pkgbase_id])
    conn.execute("DELETE FROM Packages WHERE PackageBaseID = ?", [pkgbase_id])
Lukas Fleischer's avatar
Lukas Fleischer committed
107

108
109
    for pkgname in srcinfo.utils.get_package_names(metadata):
        pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
Lukas Fleischer's avatar
Lukas Fleischer committed
110

111
        if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0:
112
113
            ver = '{:d}:{:s}-{:s}'.format(int(pkginfo['epoch']),
                                          pkginfo['pkgver'],
114
                                          pkginfo['pkgrel'])
Lukas Fleischer's avatar
Lukas Fleischer committed
115
        else:
116
            ver = '{:s}-{:s}'.format(pkginfo['pkgver'], pkginfo['pkgrel'])
Lukas Fleischer's avatar
Lukas Fleischer committed
117

118
        for field in ('pkgdesc', 'url'):
119
            if field not in pkginfo:
120
121
                pkginfo[field] = None

Lukas Fleischer's avatar
Lukas Fleischer committed
122
        # Create a new package.
123
124
125
126
127
128
        cur = conn.execute("INSERT INTO Packages (PackageBaseID, Name, " +
                           "Version, Description, URL) " +
                           "VALUES (?, ?, ?, ?, ?)",
                           [pkgbase_id, pkginfo['pkgname'], ver,
                            pkginfo['pkgdesc'], pkginfo['url']])
        conn.commit()
Lukas Fleischer's avatar
Lukas Fleischer committed
129
130
131
        pkgid = cur.lastrowid

        # Add package sources.
132
        for source_info in extract_arch_fields(pkginfo, 'source'):
133
134
135
            conn.execute("INSERT INTO PackageSources (PackageID, Source, " +
                         "SourceArch) VALUES (?, ?, ?)",
                         [pkgid, source_info['value'], source_info['arch']])
Lukas Fleischer's avatar
Lukas Fleischer committed
136
137
138
139

        # Add package dependencies.
        for deptype in ('depends', 'makedepends',
                        'checkdepends', 'optdepends'):
140
141
            cur = conn.execute("SELECT ID FROM DependencyTypes WHERE Name = ?",
                               [deptype])
Lukas Fleischer's avatar
Lukas Fleischer committed
142
            deptypeid = cur.fetchone()[0]
143
            for dep_info in extract_arch_fields(pkginfo, deptype):
144
                depname, depcond = parse_dep(dep_info['value'])
145
                deparch = dep_info['arch']
146
147
148
149
                conn.execute("INSERT INTO PackageDepends (PackageID, " +
                             "DepTypeID, DepName, DepCondition, DepArch) " +
                             "VALUES (?, ?, ?, ?, ?)",
                             [pkgid, deptypeid, depname, depcond, deparch])
Lukas Fleischer's avatar
Lukas Fleischer committed
150
151
152

        # Add package relations (conflicts, provides, replaces).
        for reltype in ('conflicts', 'provides', 'replaces'):
153
154
            cur = conn.execute("SELECT ID FROM RelationTypes WHERE Name = ?",
                               [reltype])
Lukas Fleischer's avatar
Lukas Fleischer committed
155
            reltypeid = cur.fetchone()[0]
156
            for rel_info in extract_arch_fields(pkginfo, reltype):
157
                relname, relcond = parse_dep(rel_info['value'])
158
                relarch = rel_info['arch']
159
160
161
162
                conn.execute("INSERT INTO PackageRelations (PackageID, " +
                             "RelTypeID, RelName, RelCondition, RelArch) " +
                             "VALUES (?, ?, ?, ?, ?)",
                             [pkgid, reltypeid, relname, relcond, relarch])
Lukas Fleischer's avatar
Lukas Fleischer committed
163
164
165
166

        # Add package licenses.
        if 'license' in pkginfo:
            for license in pkginfo['license']:
167
168
                cur = conn.execute("SELECT ID FROM Licenses WHERE Name = ?",
                                   [license])
169
170
171
                row = cur.fetchone()
                if row:
                    licenseid = row[0]
Lukas Fleischer's avatar
Lukas Fleischer committed
172
                else:
173
174
175
                    cur = conn.execute("INSERT INTO Licenses (Name) " +
                                       "VALUES (?)", [license])
                    conn.commit()
Lukas Fleischer's avatar
Lukas Fleischer committed
176
                    licenseid = cur.lastrowid
177
178
179
                conn.execute("INSERT INTO PackageLicenses (PackageID, " +
                             "LicenseID) VALUES (?, ?)",
                             [pkgid, licenseid])
Lukas Fleischer's avatar
Lukas Fleischer committed
180
181
182
183

        # Add package groups.
        if 'groups' in pkginfo:
            for group in pkginfo['groups']:
184
185
                cur = conn.execute("SELECT ID FROM Groups WHERE Name = ?",
                                   [group])
186
187
188
                row = cur.fetchone()
                if row:
                    groupid = row[0]
Lukas Fleischer's avatar
Lukas Fleischer committed
189
                else:
190
191
192
                    cur = conn.execute("INSERT INTO Groups (Name) VALUES (?)",
                                       [group])
                    conn.commit()
Lukas Fleischer's avatar
Lukas Fleischer committed
193
                    groupid = cur.lastrowid
194
195
                conn.execute("INSERT INTO PackageGroups (PackageID, "
                             "GroupID) VALUES (?, ?)", [pkgid, groupid])
Lukas Fleischer's avatar
Lukas Fleischer committed
196
197
198

    # Add user to notification list on adoption.
    if was_orphan:
199
200
201
        cur = conn.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " +
                           "PackageBaseID = ? AND UserID = ?",
                           [pkgbase_id, user_id])
202
        if cur.fetchone()[0] == 0:
203
204
205
            conn.execute("INSERT INTO PackageNotifications " +
                         "(PackageBaseID, UserID) VALUES (?, ?)",
                         [pkgbase_id, user_id])
Lukas Fleischer's avatar
Lukas Fleischer committed
206

207
    conn.commit()
Lukas Fleischer's avatar
Lukas Fleischer committed
208

209

210
def update_notify(conn, user, pkgbase_id):
211
    # Obtain the user ID of the new maintainer.
212
    cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user])
213
214
215
216
    user_id = int(cur.fetchone()[0])

    # Execute the notification script.
    subprocess.Popen((notify_cmd, 'update', str(user_id), str(pkgbase_id)))
217

218

Lukas Fleischer's avatar
Lukas Fleischer committed
219
def die(msg):
220
    sys.stderr.write("error: {:s}\n".format(msg))
Lukas Fleischer's avatar
Lukas Fleischer committed
221
222
    exit(1)

223

224
225
226
def warn(msg):
    sys.stderr.write("warning: {:s}\n".format(msg))

227

Lukas Fleischer's avatar
Lukas Fleischer committed
228
229
230
def die_commit(msg, commit):
    sys.stderr.write("error: The following error " +
                     "occurred when parsing commit\n")
231
232
    sys.stderr.write("error: {:s}:\n".format(commit))
    sys.stderr.write("error: {:s}\n".format(msg))
Lukas Fleischer's avatar
Lukas Fleischer committed
233
234
    exit(1)

235

236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
def main():
    repo = pygit2.Repository(repo_path)

    user = os.environ.get("AUR_USER")
    pkgbase = os.environ.get("AUR_PKGBASE")
    privileged = (os.environ.get("AUR_PRIVILEGED", '0') == '1')
    warn_or_die = warn if privileged else die

    if len(sys.argv) == 2 and sys.argv[1] == "restore":
        if 'refs/heads/' + pkgbase not in repo.listall_references():
            die('{:s}: repository not found: {:s}'.format(sys.argv[1],
                pkgbase))
        refname = "refs/heads/master"
        branchref = 'refs/heads/' + pkgbase
        sha1_old = sha1_new = repo.lookup_reference(branchref).target
    elif len(sys.argv) == 4:
        refname, sha1_old, sha1_new = sys.argv[1:4]
    else:
        die("invalid arguments")
255

256
257
    if refname != "refs/heads/master":
        die("pushing to a branch other than master is restricted")
258

259
    conn = aurweb.db.Connection()
Lukas Fleischer's avatar
Lukas Fleischer committed
260

261
262
263
264
265
266
    # Detect and deny non-fast-forwards.
    if sha1_old != "0" * 40 and not privileged:
        walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL)
        walker.hide(sha1_new)
        if next(walker, None) is not None:
            die("denying non-fast-forward (you should pull first)")
Lukas Fleischer's avatar
Lukas Fleischer committed
267

268
269
270
271
    # Prepare the walker that validates new commits.
    walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL)
    if sha1_old != "0" * 40:
        walker.hide(sha1_old)
Lukas Fleischer's avatar
Lukas Fleischer committed
272

273
274
275
276
277
    # Validate all new commits.
    for commit in walker:
        for fname in ('.SRCINFO', 'PKGBUILD'):
            if fname not in commit.tree:
                die_commit("missing {:s}".format(fname), str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
278

279
280
281
282
283
        for treeobj in commit.tree:
            blob = repo[treeobj.id]

            if isinstance(blob, pygit2.Tree):
                die_commit("the repository must not contain subdirectories",
284
                           str(commit.id))
285

286
287
288
            if not isinstance(blob, pygit2.Blob):
                die_commit("not a blob object: {:s}".format(treeobj),
                           str(commit.id))
289

290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
            if blob.size > max_blob_size:
                die_commit("maximum blob size ({:s}) exceeded".format(
                           size_humanize(max_blob_size)), str(commit.id))

        metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode()
        (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
        if errors:
            sys.stderr.write("error: The following errors occurred "
                             "when parsing .SRCINFO in commit\n")
            sys.stderr.write("error: {:s}:\n".format(str(commit.id)))
            for error in errors:
                for err in error['error']:
                    sys.stderr.write("error: line {:d}: {:s}\n".format(
                                     error['line'], err))
            exit(1)

        metadata_pkgbase = metadata['pkgbase']
        if not re.match(repo_regex, metadata_pkgbase):
            die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase),
309
                       str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
310

311
312
        for pkgname in set(metadata['packages'].keys()):
            pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
Lukas Fleischer's avatar
Lukas Fleischer committed
313

314
315
316
317
            for field in ('pkgver', 'pkgrel', 'pkgname'):
                if field not in pkginfo:
                    die_commit('missing mandatory field: {:s}'.format(field),
                               str(commit.id))
318

319
320
            if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit():
                die_commit('invalid epoch: {:s}'.format(pkginfo['epoch']),
321
322
                           str(commit.id))

323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
            if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']):
                die_commit('invalid package name: {:s}'.format(
                           pkginfo['pkgname']), str(commit.id))

            for field in ('pkgname', 'pkgdesc', 'url'):
                if field in pkginfo and len(pkginfo[field]) > 255:
                    die_commit('{:s} field too long: {:s}'.format(field,
                               pkginfo[field]), str(commit.id))

            for field in ('install', 'changelog'):
                if field in pkginfo and not pkginfo[field] in commit.tree:
                    die_commit('missing {:s} file: {:s}'.format(field,
                               pkginfo[field]), str(commit.id))

            for field in extract_arch_fields(pkginfo, 'source'):
                fname = field['value']
                if "://" in fname or "lp:" in fname:
                    continue
                if fname not in commit.tree:
                    die_commit('missing source file: {:s}'.format(fname),
                               str(commit.id))

    # Display a warning if .SRCINFO is unchanged.
    if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new):
        srcinfo_id_old = repo[sha1_old].tree['.SRCINFO'].id
        srcinfo_id_new = repo[sha1_new].tree['.SRCINFO'].id
        if srcinfo_id_old == srcinfo_id_new:
            warn(".SRCINFO unchanged. "
                 "The package database will not be updated!")

    # Read .SRCINFO from the HEAD commit.
    metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode()
    (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
356

357
358
359
360
361
    # Ensure that the package base name matches the repository name.
    metadata_pkgbase = metadata['pkgbase']
    if metadata_pkgbase != pkgbase:
        die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase,
                                                          pkgbase))
362

363
364
365
366
367
    # Ensure that packages are neither blacklisted nor overwritten.
    pkgbase = metadata['pkgbase']
    cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase])
    row = cur.fetchone()
    pkgbase_id = row[0] if row else 0
Lukas Fleischer's avatar
Lukas Fleischer committed
368

369
370
    cur = conn.execute("SELECT Name FROM PackageBlacklist")
    blacklist = [row[0] for row in cur.fetchall()]
371

372
373
    cur = conn.execute("SELECT Name, Repo FROM OfficialProviders")
    providers = dict(cur.fetchall())
374

375
376
377
    for pkgname in srcinfo.utils.get_package_names(metadata):
        pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
        pkgname = pkginfo['pkgname']
378

379
380
381
382
383
        if pkgname in blacklist:
            warn_or_die('package is blacklisted: {:s}'.format(pkgname))
        if pkgname in providers:
            warn_or_die('package already provided by [{:s}]: {:s}'.format(
                        providers[pkgname], pkgname))
384

385
386
387
388
        cur = conn.execute("SELECT COUNT(*) FROM Packages WHERE Name = ? " +
                           "AND PackageBaseID <> ?", [pkgname, pkgbase_id])
        if cur.fetchone()[0] > 0:
            die('cannot overwrite package: {:s}'.format(pkgname))
389

390
391
392
    # Create a new package base if it does not exist yet.
    if pkgbase_id == 0:
        pkgbase_id = create_pkgbase(conn, pkgbase, user)
393

394
395
    # Store package base details in the database.
    save_metadata(metadata, conn, user)
396

397
398
399
400
    # Create (or update) a branch with the name of the package base for better
    # accessibility.
    branchref = 'refs/heads/' + pkgbase
    repo.create_reference(branchref, sha1_new, True)
Lukas Fleischer's avatar
Lukas Fleischer committed
401

402
403
404
405
406
407
408
    # Work around a Git bug: The HEAD ref is not updated when using
    # gitnamespaces. This can be removed once the bug fix is included in Git
    # mainline. See
    # http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref-namespace-td7632149.html
    # for details.
    headref = 'refs/namespaces/' + pkgbase + '/HEAD'
    repo.create_reference(headref, sha1_new, True)
409

410
411
    # Send package update notifications.
    update_notify(conn, user, pkgbase_id)
412

413
414
415
    # Close the database.
    cur.close()
    conn.close()
416
417


418
419
if __name__ == '__main__':
    main()