git-update.py 14.3 KB
Newer Older
Lukas Fleischer's avatar
Lukas Fleischer committed
1
2
3
4
5
6
#!/usr/bin/python3

import configparser
import os
import pygit2
import re
7
import subprocess
Lukas Fleischer's avatar
Lukas Fleischer committed
8
9
import sys

10
11
import srcinfo.parse
import srcinfo.utils
Lukas Fleischer's avatar
Lukas Fleischer committed
12

13
14
import db

Lukas Fleischer's avatar
Lukas Fleischer committed
15
config = configparser.RawConfigParser()
Lukas Fleischer's avatar
Lukas Fleischer committed
16
config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config")
Lukas Fleischer's avatar
Lukas Fleischer committed
17

18
19
notify_cmd = config.get('notifications', 'notify-cmd')

20
repo_path = config.get('serve', 'repo-path')
21
repo_regex = config.get('serve', 'repo-regex')
22

23
24
25
26
27
28
29
30
31
32
33
34
35
max_blob_size = config.getint('update', 'max-blob-size')


def size_humanize(num):
    for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB']:
        if abs(num) < 2048.0:
            if isinstance(num, int):
                return "{}{}".format(num, unit)
            else:
                return "{:.2f}{}".format(num, unit)
        num /= 1024.0
    return "{:.2f}{}".format(num, 'YiB')

36

37
38
39
40
41
42
43
44
45
46
47
48
49
50
def extract_arch_fields(pkginfo, field):
    values = []

    if field in pkginfo:
        for val in pkginfo[field]:
            values.append({"value": val, "arch": None})

    for arch in ['i686', 'x86_64']:
        if field + '_' + arch in pkginfo:
            for val in pkginfo[field + '_' + arch]:
                values.append({"value": val, "arch": arch})

    return values

51

52
53
54
55
56
57
58
59
60
61
def parse_dep(depstring):
    dep, _, desc = depstring.partition(': ')
    depname = re.sub(r'(<|=|>).*', '', dep)
    depcond = dep[len(depname):]

    if (desc):
        return (depname + ': ' + desc, depcond)
    else:
        return (depname, depcond)

62

63
def save_metadata(metadata, conn, user):
Lukas Fleischer's avatar
Lukas Fleischer committed
64
    # Obtain package base ID and previous maintainer.
65
    pkgbase = metadata['pkgbase']
66
67
    cur = conn.execute("SELECT ID, MaintainerUID FROM PackageBases "
                       "WHERE Name = ?", [pkgbase])
Lukas Fleischer's avatar
Lukas Fleischer committed
68
69
70
71
    (pkgbase_id, maintainer_uid) = cur.fetchone()
    was_orphan = not maintainer_uid

    # Obtain the user ID of the new maintainer.
72
    cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user])
Lukas Fleischer's avatar
Lukas Fleischer committed
73
74
75
    user_id = int(cur.fetchone()[0])

    # Update package base details and delete current packages.
76
77
78
79
80
81
82
83
    conn.execute("UPDATE PackageBases SET ModifiedTS = UNIX_TIMESTAMP(), " +
                 "PackagerUID = ?, OutOfDateTS = NULL WHERE ID = ?",
                 [user_id, pkgbase_id])
    conn.execute("UPDATE PackageBases SET MaintainerUID = ? " +
                 "WHERE ID = ? AND MaintainerUID IS NULL",
                 [user_id, pkgbase_id])
    conn.execute("DELETE FROM Packages WHERE PackageBaseID = ?",
                 [pkgbase_id])
Lukas Fleischer's avatar
Lukas Fleischer committed
84

85
86
    for pkgname in srcinfo.utils.get_package_names(metadata):
        pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
Lukas Fleischer's avatar
Lukas Fleischer committed
87

88
        if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0:
89
90
            ver = '{:d}:{:s}-{:s}'.format(int(pkginfo['epoch']),
                                          pkginfo['pkgver'],
91
                                          pkginfo['pkgrel'])
Lukas Fleischer's avatar
Lukas Fleischer committed
92
        else:
93
            ver = '{:s}-{:s}'.format(pkginfo['pkgver'], pkginfo['pkgrel'])
Lukas Fleischer's avatar
Lukas Fleischer committed
94

95
        for field in ('pkgdesc', 'url'):
96
            if field not in pkginfo:
97
98
                pkginfo[field] = None

Lukas Fleischer's avatar
Lukas Fleischer committed
99
        # Create a new package.
100
101
102
103
104
105
        cur = conn.execute("INSERT INTO Packages (PackageBaseID, Name, " +
                           "Version, Description, URL) " +
                           "VALUES (?, ?, ?, ?, ?)",
                           [pkgbase_id, pkginfo['pkgname'], ver,
                            pkginfo['pkgdesc'], pkginfo['url']])
        conn.commit()
Lukas Fleischer's avatar
Lukas Fleischer committed
106
107
108
        pkgid = cur.lastrowid

        # Add package sources.
109
        for source_info in extract_arch_fields(pkginfo, 'source'):
110
111
112
            conn.execute("INSERT INTO PackageSources (PackageID, Source, " +
                         "SourceArch) VALUES (?, ?, ?)",
                         [pkgid, source_info['value'], source_info['arch']])
Lukas Fleischer's avatar
Lukas Fleischer committed
113
114
115
116

        # Add package dependencies.
        for deptype in ('depends', 'makedepends',
                        'checkdepends', 'optdepends'):
117
118
            cur = conn.execute("SELECT ID FROM DependencyTypes WHERE Name = ?",
                               [deptype])
Lukas Fleischer's avatar
Lukas Fleischer committed
119
            deptypeid = cur.fetchone()[0]
120
            for dep_info in extract_arch_fields(pkginfo, deptype):
121
                depname, depcond = parse_dep(dep_info['value'])
122
                deparch = dep_info['arch']
123
124
125
126
                conn.execute("INSERT INTO PackageDepends (PackageID, " +
                             "DepTypeID, DepName, DepCondition, DepArch) " +
                             "VALUES (?, ?, ?, ?, ?)",
                             [pkgid, deptypeid, depname, depcond, deparch])
Lukas Fleischer's avatar
Lukas Fleischer committed
127
128
129

        # Add package relations (conflicts, provides, replaces).
        for reltype in ('conflicts', 'provides', 'replaces'):
130
131
            cur = conn.execute("SELECT ID FROM RelationTypes WHERE Name = ?",
                               [reltype])
Lukas Fleischer's avatar
Lukas Fleischer committed
132
            reltypeid = cur.fetchone()[0]
133
            for rel_info in extract_arch_fields(pkginfo, reltype):
134
                relname, relcond = parse_dep(rel_info['value'])
135
                relarch = rel_info['arch']
136
137
138
139
                conn.execute("INSERT INTO PackageRelations (PackageID, " +
                             "RelTypeID, RelName, RelCondition, RelArch) " +
                             "VALUES (?, ?, ?, ?, ?)",
                             [pkgid, reltypeid, relname, relcond, relarch])
Lukas Fleischer's avatar
Lukas Fleischer committed
140
141
142
143

        # Add package licenses.
        if 'license' in pkginfo:
            for license in pkginfo['license']:
144
145
                cur = conn.execute("SELECT ID FROM Licenses WHERE Name = ?",
                                   [license])
Lukas Fleischer's avatar
Lukas Fleischer committed
146
147
148
                if cur.rowcount == 1:
                    licenseid = cur.fetchone()[0]
                else:
149
150
151
                    cur = conn.execute("INSERT INTO Licenses (Name) " +
                                       "VALUES (?)", [license])
                    conn.commit()
Lukas Fleischer's avatar
Lukas Fleischer committed
152
                    licenseid = cur.lastrowid
153
154
155
                conn.execute("INSERT INTO PackageLicenses (PackageID, " +
                             "LicenseID) VALUES (?, ?)",
                             [pkgid, licenseid])
Lukas Fleischer's avatar
Lukas Fleischer committed
156
157
158
159

        # Add package groups.
        if 'groups' in pkginfo:
            for group in pkginfo['groups']:
160
161
                cur = conn.execute("SELECT ID FROM Groups WHERE Name = ?",
                                   [group])
Lukas Fleischer's avatar
Lukas Fleischer committed
162
163
164
                if cur.rowcount == 1:
                    groupid = cur.fetchone()[0]
                else:
165
166
167
                    cur = conn.execute("INSERT INTO Groups (Name) VALUES (?)",
                                       [group])
                    conn.commit()
Lukas Fleischer's avatar
Lukas Fleischer committed
168
                    groupid = cur.lastrowid
169
170
                conn.execute("INSERT INTO PackageGroups (PackageID, "
                             "GroupID) VALUES (?, ?)", [pkgid, groupid])
Lukas Fleischer's avatar
Lukas Fleischer committed
171
172
173

    # Add user to notification list on adoption.
    if was_orphan:
174
175
176
        cur = conn.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " +
                           "PackageBaseID = ? AND UserID = ?",
                           [pkgbase_id, user_id])
177
        if cur.fetchone()[0] == 0:
178
179
180
            conn.execute("INSERT INTO PackageNotifications " +
                         "(PackageBaseID, UserID) VALUES (?, ?)",
                         [pkgbase_id, user_id])
Lukas Fleischer's avatar
Lukas Fleischer committed
181

182
    conn.commit()
Lukas Fleischer's avatar
Lukas Fleischer committed
183

184

185
def update_notify(conn, user, pkgbase_id):
186
    # Obtain the user ID of the new maintainer.
187
    cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user])
188
189
190
191
    user_id = int(cur.fetchone()[0])

    # Execute the notification script.
    subprocess.Popen((notify_cmd, 'update', str(user_id), str(pkgbase_id)))
192

193

Lukas Fleischer's avatar
Lukas Fleischer committed
194
def die(msg):
195
    sys.stderr.write("error: {:s}\n".format(msg))
Lukas Fleischer's avatar
Lukas Fleischer committed
196
197
    exit(1)

198

199
200
201
def warn(msg):
    sys.stderr.write("warning: {:s}\n".format(msg))

202

Lukas Fleischer's avatar
Lukas Fleischer committed
203
204
205
def die_commit(msg, commit):
    sys.stderr.write("error: The following error " +
                     "occurred when parsing commit\n")
206
207
    sys.stderr.write("error: {:s}:\n".format(commit))
    sys.stderr.write("error: {:s}\n".format(msg))
Lukas Fleischer's avatar
Lukas Fleischer committed
208
209
    exit(1)

210

211
repo = pygit2.Repository(repo_path)
Lukas Fleischer's avatar
Lukas Fleischer committed
212
213
214

user = os.environ.get("AUR_USER")
pkgbase = os.environ.get("AUR_PKGBASE")
215
privileged = (os.environ.get("AUR_PRIVILEGED", '0') == '1')
216
warn_or_die = warn if privileged else die
Lukas Fleischer's avatar
Lukas Fleischer committed
217

218
219
220
221
222
223
if len(sys.argv) == 2 and sys.argv[1] == "restore":
    if 'refs/heads/' + pkgbase not in repo.listall_references():
        die('{:s}: repository not found: {:s}'.format(sys.argv[1], pkgbase))
    refname = "refs/heads/master"
    sha1_old = sha1_new = repo.lookup_reference('refs/heads/' + pkgbase).target
elif len(sys.argv) == 4:
224
    refname, sha1_old, sha1_new = sys.argv[1:4]
225
226
227
else:
    die("invalid arguments")

Lukas Fleischer's avatar
Lukas Fleischer committed
228
229
230
if refname != "refs/heads/master":
    die("pushing to a branch other than master is restricted")

231
conn = db.Connection()
232

233
# Detect and deny non-fast-forwards.
234
if sha1_old != "0000000000000000000000000000000000000000" and not privileged:
235
236
    walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL)
    walker.hide(sha1_new)
237
    if next(walker, None) is not None:
238
        die("denying non-fast-forward (you should pull first)")
239
240

# Prepare the walker that validates new commits.
Lukas Fleischer's avatar
Lukas Fleischer committed
241
242
243
244
walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL)
if sha1_old != "0000000000000000000000000000000000000000":
    walker.hide(sha1_old)

Lukas Fleischer's avatar
Lukas Fleischer committed
245
# Validate all new commits.
Lukas Fleischer's avatar
Lukas Fleischer committed
246
for commit in walker:
247
    for fname in ('.SRCINFO', 'PKGBUILD'):
248
        if fname not in commit.tree:
249
            die_commit("missing {:s}".format(fname), str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
250
251

    for treeobj in commit.tree:
252
253
254
255
        blob = repo[treeobj.id]

        if isinstance(blob, pygit2.Tree):
            die_commit("the repository must not contain subdirectories",
256
                       str(commit.id))
257
258

        if not isinstance(blob, pygit2.Blob):
259
260
            die_commit("not a blob object: {:s}".format(treeobj),
                       str(commit.id))
261

262
263
        if blob.size > max_blob_size:
            die_commit("maximum blob size ({:s}) exceeded".format(size_humanize(max_blob_size)), str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
264

265
266
    metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode()
    (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
Lukas Fleischer's avatar
Lukas Fleischer committed
267
268
269
    if errors:
        sys.stderr.write("error: The following errors occurred "
                         "when parsing .SRCINFO in commit\n")
270
        sys.stderr.write("error: {:s}:\n".format(str(commit.id)))
Lukas Fleischer's avatar
Lukas Fleischer committed
271
        for error in errors:
272
273
            for err in error['error']:
                sys.stderr.write("error: line {:d}: {:s}\n".format(error['line'], err))
Lukas Fleischer's avatar
Lukas Fleischer committed
274
275
        exit(1)

276
277
278
    metadata_pkgbase = metadata['pkgbase']
    if not re.match(repo_regex, metadata_pkgbase):
        die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase),
279
                   str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
280

281
282
    for pkgname in set(metadata['packages'].keys()):
        pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
Lukas Fleischer's avatar
Lukas Fleischer committed
283

284
        for field in ('pkgver', 'pkgrel', 'pkgname'):
285
286
287
            if field not in pkginfo:
                die_commit('missing mandatory field: {:s}'.format(field),
                           str(commit.id))
288

289
        if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit():
290
291
            die_commit('invalid epoch: {:s}'.format(pkginfo['epoch']),
                       str(commit.id))
292

Lukas Fleischer's avatar
Lukas Fleischer committed
293
        if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']):
294
295
            die_commit('invalid package name: {:s}'.format(pkginfo['pkgname']),
                       str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
296
297

        for field in ('pkgname', 'pkgdesc', 'url'):
298
            if field in pkginfo and len(pkginfo[field]) > 255:
299
300
                die_commit('{:s} field too long: {:s}'.format(field, pkginfo[field]),
                           str(commit.id))
Lukas Fleischer's avatar
Lukas Fleischer committed
301

302
303
        for field in ('install', 'changelog'):
            if field in pkginfo and not pkginfo[field] in commit.tree:
304
305
                die_commit('missing {:s} file: {:s}'.format(field, pkginfo[field]),
                           str(commit.id))
306

307
308
        for field in extract_arch_fields(pkginfo, 'source'):
            fname = field['value']
309
310
            if "://" in fname or "lp:" in fname:
                continue
311
312
313
314
            if fname not in commit.tree:
                die_commit('missing source file: {:s}'.format(fname),
                           str(commit.id))

315

316
# Display a warning if .SRCINFO is unchanged.
317
if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new):
318
319
320
321
322
    srcinfo_id_old = repo[sha1_old].tree['.SRCINFO'].id
    srcinfo_id_new = repo[sha1_new].tree['.SRCINFO'].id
    if srcinfo_id_old == srcinfo_id_new:
        warn(".SRCINFO unchanged. The package database will not be updated!")

Lukas Fleischer's avatar
Lukas Fleischer committed
323
# Read .SRCINFO from the HEAD commit.
324
325
metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode()
(metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
Lukas Fleischer's avatar
Lukas Fleischer committed
326

Lukas Fleischer's avatar
Lukas Fleischer committed
327
# Ensure that the package base name matches the repository name.
328
329
330
metadata_pkgbase = metadata['pkgbase']
if metadata_pkgbase != pkgbase:
    die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase, pkgbase))
331

Lukas Fleischer's avatar
Lukas Fleischer committed
332
# Ensure that packages are neither blacklisted nor overwritten.
333
pkgbase = metadata['pkgbase']
334
cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase])
335
pkgbase_id = cur.fetchone()[0] if cur.rowcount == 1 else 0
336

337
cur = conn.execute("SELECT Name FROM PackageBlacklist")
338
339
blacklist = [row[0] for row in cur.fetchall()]

340
cur = conn.execute("SELECT Name, Repo FROM OfficialProviders")
341
342
providers = dict(cur.fetchall())

343
344
for pkgname in srcinfo.utils.get_package_names(metadata):
    pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
345
    pkgname = pkginfo['pkgname']
346

347
348
349
    if pkgname in blacklist:
        warn_or_die('package is blacklisted: {:s}'.format(pkgname))
    if pkgname in providers:
350
        repo = providers[pkgname]
351
        warn_or_die('package already provided by [{:s}]: {:s}'.format(repo, pkgname))
352

353
354
    cur = conn.execute("SELECT COUNT(*) FROM Packages WHERE Name = ? AND " +
                       "PackageBaseID <> ?", [pkgname, pkgbase_id])
355
    if cur.fetchone()[0] > 0:
356
        die('cannot overwrite package: {:s}'.format(pkgname))
357

Lukas Fleischer's avatar
Lukas Fleischer committed
358
# Store package base details in the database.
359
save_metadata(metadata, conn, user)
360

361
362
363
364
365
366
367
368
369
# Create (or update) a branch with the name of the package base for better
# accessibility.
repo.create_reference('refs/heads/' + pkgbase, sha1_new, True)

# Work around a Git bug: The HEAD ref is not updated when using gitnamespaces.
# This can be removed once the bug fix is included in Git mainline. See
# http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref-namespace-td7632149.html
# for details.
repo.create_reference('refs/namespaces/' + pkgbase + '/HEAD', sha1_new, True)
370
371

# Send package update notifications.
372
update_notify(conn, user, pkgbase_id)
373
374

# Close the database.
375
conn.close()