rpc.py 13.7 KB
Newer Older
1
2
import os

3
from collections import defaultdict
4
from typing import Any, Callable, Dict, List, NewType, Union
5

6
from fastapi.responses import HTMLResponse
7
from sqlalchemy import and_, literal, orm
8

9
10
import aurweb.config as config

11
from aurweb import db, defaults, models, util
12
from aurweb.exceptions import RPCError
13
from aurweb.packages.search import RPCSearch
14

15
16
17
18
19
20
21
22
TYPE_MAPPING = {
    "depends": "Depends",
    "makedepends": "MakeDepends",
    "checkdepends": "CheckDepends",
    "optdepends": "OptDepends",
    "conflicts": "Conflicts",
    "provides": "Provides",
    "replaces": "Replaces",
23
24
}

25
26
27
28
DataGenerator = NewType("DataGenerator",
                        Callable[[models.Package], Dict[str, Any]])


29
30
31
32
33
34
35
36
37
38
39
40
def documentation():
    aurwebdir = config.get("options", "aurwebdir")
    rpc_doc = os.path.join(aurwebdir, "doc", "rpc.html")

    if not os.path.exists(rpc_doc):
        raise OSError("doc/rpc.html could not be read")

    with open(rpc_doc) as f:
        data = f.read()
    return HTMLResponse(data)


41
42
class RPC:
    """ RPC API handler class.
43

44
45
46
47
48
49
    There are various pieces to RPC's process, and encapsulating them
    inside of a class means that external users do not abuse the
    RPC implementation to achieve goals. We call type handlers
    by taking a reference to the callback named "_handle_{type}_type(...)",
    and if the handler does not exist, we return a not implemented
    error to the API user.
50

51
52
    EXPOSED_VERSIONS holds the set of versions that the API
    officially supports.
53

54
55
    EXPOSED_TYPES holds the set of types that the API officially
    supports.
56

57
    ALIASES holds an alias mapping of type -> type strings.
58

59
60
61
    We should focus on privatizing implementation helpers and
    focusing on performance in the code used.
    """
62

63
64
    # A set of RPC versions supported by this API.
    EXPOSED_VERSIONS = {5}
65

66
67
68
69
70
    # A set of RPC types supported by this API.
    EXPOSED_TYPES = {
        "info", "multiinfo",
        "search", "msearch",
        "suggest", "suggest-pkgbase"
71
72
    }

73
74
75
76
77
78
79
80
81
82
    # A mapping of type aliases.
    TYPE_ALIASES = {"info": "multiinfo"}

    EXPOSED_BYS = {
        "name-desc", "name", "maintainer",
        "depends", "makedepends", "optdepends", "checkdepends"
    }

    # A mapping of by aliases.
    BY_ALIASES = {"name-desc": "nd", "name": "n", "maintainer": "m"}
83

84
    def __init__(self, version: int = 0, type: str = None) -> "RPC":
85
        self.version = version
86
        self.type = RPC.TYPE_ALIASES.get(type, type)
87

88
    def error(self, message: str) -> Dict[str, Any]:
89
90
91
92
93
94
95
96
        return {
            "version": self.version,
            "results": [],
            "resultcount": 0,
            "type": "error",
            "error": message
        }

97
    def _verify_inputs(self, by: str = [], args: List[str] = []) -> None:
98
        if self.version is None:
99
100
            raise RPCError("Please specify an API version.")

101
        if self.version not in RPC.EXPOSED_VERSIONS:
102
103
            raise RPCError("Invalid version specified.")

104
105
106
107
        if by not in RPC.EXPOSED_BYS:
            raise RPCError("Incorrect by field specified.")

        if self.type is None:
108
109
            raise RPCError("No request type/data specified.")

110
        if self.type not in RPC.EXPOSED_TYPES:
111
112
            raise RPCError("Incorrect request type specified.")

113
    def _enforce_args(self, args: List[str]) -> None:
114
115
116
        if not args:
            raise RPCError("No request type/data specified.")

117
    def _get_json_data(self, package: models.Package) -> Dict[str, Any]:
118
119
120
121
122
123
124
125
        """ Produce dictionary data of one Package that can be JSON-serialized.

        :param package: Package instance
        :returns: JSON-serializable dictionary
        """

        # Produce RPC API compatible Popularity: If zero, it's an integer
        # 0, otherwise, it's formatted to the 6th decimal place.
126
        pop = package.Popularity
127
128
129
        pop = 0 if not pop else float(util.number_format(pop, 6))

        snapshot_uri = config.get("options", "snapshot_uri")
130
        return {
131
132
133
            "ID": package.ID,
            "Name": package.Name,
            "PackageBaseID": package.PackageBaseID,
134
            "PackageBase": package.PackageBaseName,
135
            # Maintainer should be set following this update if one exists.
136
            "Maintainer": package.Maintainer,
137
138
139
140
            "Version": package.Version,
            "Description": package.Description,
            "URL": package.URL,
            "URLPath": snapshot_uri % package.Name,
141
            "NumVotes": package.NumVotes,
142
            "Popularity": pop,
143
144
145
146
            "OutOfDate": package.OutOfDateTS,
            "FirstSubmitted": package.SubmittedTS,
            "LastModified": package.ModifiedTS
        }
147

148
    def _get_info_json_data(self, package: models.Package) -> Dict[str, Any]:
149
150
        data = self._get_json_data(package)

151
152
        # All info results have _at least_ an empty list of
        # License and Keywords.
153
        data.update({
154
155
            "License": [],
            "Keywords": []
156
157
        })

158
159
160
161
162
        # If we actually got extra_info records, update data with
        # them for this particular package.
        if self.extra_info:
            data.update(self.extra_info.get(package.ID, {}))

163
164
        return data

165
166
167
168
169
170
171
172
173
    def _assemble_json_data(self, packages: List[models.Package],
                            data_generator: DataGenerator) \
            -> List[Dict[str, Any]]:
        """
        Assemble JSON data out of a list of packages.

        :param packages: A list of Package instances or a Package ORM query
        :param data_generator: Generator callable of single-Package JSON data
        """
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
        return [data_generator(pkg) for pkg in packages]

    def _entities(self, query: orm.Query) -> orm.Query:
        """ Select specific RPC columns on `query`. """
        return query.with_entities(
            models.Package.ID,
            models.Package.Name,
            models.Package.Version,
            models.Package.Description,
            models.Package.URL,
            models.Package.PackageBaseID,
            models.PackageBase.Name.label("PackageBaseName"),
            models.PackageBase.NumVotes,
            models.PackageBase.Popularity,
            models.PackageBase.OutOfDateTS,
            models.PackageBase.SubmittedTS,
            models.PackageBase.ModifiedTS,
            models.User.Username.label("Maintainer"),
        ).group_by(models.Package.ID)
193
194
195

    def _handle_multiinfo_type(self, args: List[str] = [], **kwargs) \
            -> List[Dict[str, Any]]:
196
        self._enforce_args(args)
197
        args = set(args)
198

199
200
201
202
203
204
205
        packages = db.query(models.Package).join(models.PackageBase).join(
            models.User,
            models.User.ID == models.PackageBase.MaintainerUID,
            isouter=True
        ).filter(models.Package.Name.in_(args))
        packages = self._entities(packages)

206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
        ids = {pkg.ID for pkg in packages}

        # Aliases for 80-width.
        Package = models.Package
        PackageKeyword = models.PackageKeyword

        subqueries = [
            # PackageDependency
            db.query(
                models.PackageDependency
            ).join(models.DependencyType).filter(
                models.PackageDependency.PackageID.in_(ids)
            ).with_entities(
                models.PackageDependency.PackageID.label("ID"),
                models.DependencyType.Name.label("Type"),
                models.PackageDependency.DepName.label("Name"),
                models.PackageDependency.DepCondition.label("Cond")
223
            ).distinct().order_by("Name"),
224
225
226
227
228
229
230
231
232
233
234

            # PackageRelation
            db.query(
                models.PackageRelation
            ).join(models.RelationType).filter(
                models.PackageRelation.PackageID.in_(ids)
            ).with_entities(
                models.PackageRelation.PackageID.label("ID"),
                models.RelationType.Name.label("Type"),
                models.PackageRelation.RelName.label("Name"),
                models.PackageRelation.RelCondition.label("Cond")
235
            ).distinct().order_by("Name"),
236
237
238
239
240
241
242
243
244
245
246

            # Groups
            db.query(models.PackageGroup).join(
                models.Group,
                and_(models.PackageGroup.GroupID == models.Group.ID,
                     models.PackageGroup.PackageID.in_(ids))
            ).with_entities(
                models.PackageGroup.PackageID.label("ID"),
                literal("Groups").label("Type"),
                models.Group.Name.label("Name"),
                literal(str()).label("Cond")
247
            ).distinct().order_by("Name"),
248
249
250
251
252
253
254
255
256
257
258
259

            # Licenses
            db.query(models.PackageLicense).join(
                models.License,
                models.PackageLicense.LicenseID == models.License.ID
            ).filter(
                models.PackageLicense.PackageID.in_(ids)
            ).with_entities(
                models.PackageLicense.PackageID.label("ID"),
                literal("License").label("Type"),
                models.License.Name.label("Name"),
                literal(str()).label("Cond")
260
            ).distinct().order_by("Name"),
261
262
263
264
265
266
267
268
269
270
271

            # Keywords
            db.query(models.PackageKeyword).join(
                models.Package,
                and_(Package.PackageBaseID == PackageKeyword.PackageBaseID,
                     Package.ID.in_(ids))
            ).with_entities(
                models.Package.ID.label("ID"),
                literal("Keywords").label("Type"),
                models.PackageKeyword.Keyword.label("Name"),
                literal(str()).label("Cond")
272
            ).distinct().order_by("Name")
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
        ]

        # Union all subqueries together.
        query = subqueries[0].union_all(*subqueries[1:])

        # Store our extra information in a class-wise dictionary,
        # which contains package id -> extra info dict mappings.
        self.extra_info = defaultdict(lambda: defaultdict(list))
        for record in query:
            type_ = TYPE_MAPPING.get(record.Type, record.Type)

            name = record.Name
            if record.Cond:
                name += record.Cond

            self.extra_info[record.ID][type_].append(name)

290
        return self._assemble_json_data(packages, self._get_info_json_data)
291

292
    def _handle_search_type(self, by: str = defaults.RPC_SEARCH_BY,
293
                            args: List[str] = []) -> List[Dict[str, Any]]:
294
295
296
297
298
299
        # If `by` isn't maintainer and we don't have any args, raise an error.
        # In maintainer's case, return all orphans if there are no args,
        # so we need args to pass through to the handler without errors.
        if by != "m" and not len(args):
            raise RPCError("No request type/data specified.")

Kevin Morris's avatar
Kevin Morris committed
300
301
        arg = args[0] if args else str()
        if by != "m" and len(arg) < 2:
302
303
304
305
306
307
            raise RPCError("Query arg too small.")

        search = RPCSearch()
        search.search_by(by, arg)

        max_results = config.getint("options", "max_rpc_results")
308
        results = self._entities(search.results()).limit(max_results)
309
        return self._assemble_json_data(results, self._get_json_data)
310

311
312
    def _handle_msearch_type(self, args: List[str] = [], **kwargs)\
            -> List[Dict[str, Any]]:
313
314
        return self._handle_search_type(by="m", args=args)

315
316
    def _handle_suggest_type(self, args: List[str] = [], **kwargs)\
            -> List[str]:
317
318
319
        if not args:
            return []

320
        arg = args[0]
321
322
323
        packages = db.query(models.Package.Name).join(
            models.PackageBase
        ).filter(
324
325
326
327
328
            and_(models.PackageBase.PackagerUID.isnot(None),
                 models.Package.Name.like(f"%{arg}%"))
        ).order_by(models.Package.Name.asc()).limit(20)
        return [pkg.Name for pkg in packages]

329
330
    def _handle_suggest_pkgbase_type(self, args: List[str] = [], **kwargs)\
            -> List[str]:
331
332
333
        if not args:
            return []

334
        packages = db.query(models.PackageBase.Name).filter(
335
336
337
            and_(models.PackageBase.PackagerUID.isnot(None),
                 models.PackageBase.Name.like(f"%{args[0]}%"))
        ).order_by(models.PackageBase.Name.asc()).limit(20)
338
        return [pkg.Name for pkg in packages]
339

340
341
342
343
344
345
346
347
348
349
350
351
352
    def _is_suggestion(self) -> bool:
        return self.type.startswith("suggest")

    def _handle_callback(self, by: str, args: List[str])\
            -> Union[List[Dict[str, Any]], List[str]]:
        # Get a handle to our callback and trap an RPCError with
        # an empty list of results based on callback's execution.
        callback = getattr(self, f"_handle_{self.type.replace('-', '_')}_type")
        results = callback(by=by, args=args)
        return results

    def handle(self, by: str = defaults.RPC_SEARCH_BY, args: List[str] = [])\
            -> Union[List[Dict[str, Any]], Dict[str, Any]]:
353
354
355
356
357
358
359
360
        """ Request entrypoint. A router should pass v, type and args
        to this function and expect an output dictionary to be returned.

        :param v: RPC version argument
        :param type: RPC type argument
        :param args: Deciphered list of arguments based on arg/arg[] inputs
        """
        # Prepare our output data dictionary with some basic keys.
361
        data = {"version": self.version, "type": self.type}
362
363
364

        # Run some verification on our given arguments.
        try:
365
            self._verify_inputs(by=by, args=args)
366
        except RPCError as exc:
367
            return self.error(str(exc))
368

369
        # Convert by to its aliased value if it has one.
370
        by = RPC.BY_ALIASES.get(by, by)
371

372
        # Process the requested handler.
373
        try:
374
            results = self._handle_callback(by, args)
375
376
        except RPCError as exc:
            return self.error(str(exc))
377
378
379

        # These types are special: we produce a different kind of
        # successful JSON output: a list of results.
380
        if self._is_suggestion():
381
382
383
384
385
386
387
388
            return results

        # Return JSON output.
        data.update({
            "resultcount": len(results),
            "results": results
        })
        return data