rpc.py 13.9 KB
Newer Older
1
2
import os

3
from collections import defaultdict
4
from typing import Any, Callable, Dict, List, NewType, Union
5

6
from fastapi.responses import HTMLResponse
7
from sqlalchemy import and_, literal, orm
8

9
10
import aurweb.config as config

11
from aurweb import db, defaults, models
12
from aurweb.exceptions import RPCError
13
from aurweb.filters import number_format
14
from aurweb.packages.search import RPCSearch
15

16
17
18
19
20
21
22
23
TYPE_MAPPING = {
    "depends": "Depends",
    "makedepends": "MakeDepends",
    "checkdepends": "CheckDepends",
    "optdepends": "OptDepends",
    "conflicts": "Conflicts",
    "provides": "Provides",
    "replaces": "Replaces",
24
25
}

26
27
28
29
DataGenerator = NewType("DataGenerator",
                        Callable[[models.Package], Dict[str, Any]])


30
31
32
33
34
35
36
37
38
39
40
41
def documentation():
    aurwebdir = config.get("options", "aurwebdir")
    rpc_doc = os.path.join(aurwebdir, "doc", "rpc.html")

    if not os.path.exists(rpc_doc):
        raise OSError("doc/rpc.html could not be read")

    with open(rpc_doc) as f:
        data = f.read()
    return HTMLResponse(data)


42
43
class RPC:
    """ RPC API handler class.
44

45
46
47
48
49
50
    There are various pieces to RPC's process, and encapsulating them
    inside of a class means that external users do not abuse the
    RPC implementation to achieve goals. We call type handlers
    by taking a reference to the callback named "_handle_{type}_type(...)",
    and if the handler does not exist, we return a not implemented
    error to the API user.
51

52
53
    EXPOSED_VERSIONS holds the set of versions that the API
    officially supports.
54

55
56
    EXPOSED_TYPES holds the set of types that the API officially
    supports.
57

58
    ALIASES holds an alias mapping of type -> type strings.
59

60
61
62
    We should focus on privatizing implementation helpers and
    focusing on performance in the code used.
    """
63

64
65
    # A set of RPC versions supported by this API.
    EXPOSED_VERSIONS = {5}
66

67
68
69
70
71
    # A set of RPC types supported by this API.
    EXPOSED_TYPES = {
        "info", "multiinfo",
        "search", "msearch",
        "suggest", "suggest-pkgbase"
72
73
    }

74
75
76
77
78
79
80
81
82
83
    # A mapping of type aliases.
    TYPE_ALIASES = {"info": "multiinfo"}

    EXPOSED_BYS = {
        "name-desc", "name", "maintainer",
        "depends", "makedepends", "optdepends", "checkdepends"
    }

    # A mapping of by aliases.
    BY_ALIASES = {"name-desc": "nd", "name": "n", "maintainer": "m"}
84

85
    def __init__(self, version: int = 0, type: str = None) -> "RPC":
86
        self.version = version
87
        self.type = RPC.TYPE_ALIASES.get(type, type)
88

89
    def error(self, message: str) -> Dict[str, Any]:
90
91
92
93
94
95
96
97
        return {
            "version": self.version,
            "results": [],
            "resultcount": 0,
            "type": "error",
            "error": message
        }

98
    def _verify_inputs(self, by: str = [], args: List[str] = []) -> None:
99
        if self.version is None:
100
101
            raise RPCError("Please specify an API version.")

102
        if self.version not in RPC.EXPOSED_VERSIONS:
103
104
            raise RPCError("Invalid version specified.")

105
106
107
108
        if by not in RPC.EXPOSED_BYS:
            raise RPCError("Incorrect by field specified.")

        if self.type is None:
109
110
            raise RPCError("No request type/data specified.")

111
        if self.type not in RPC.EXPOSED_TYPES:
112
113
            raise RPCError("Incorrect request type specified.")

114
    def _enforce_args(self, args: List[str]) -> None:
115
116
117
        if not args:
            raise RPCError("No request type/data specified.")

118
    def _get_json_data(self, package: models.Package) -> Dict[str, Any]:
119
120
121
122
123
124
125
126
        """ Produce dictionary data of one Package that can be JSON-serialized.

        :param package: Package instance
        :returns: JSON-serializable dictionary
        """

        # Produce RPC API compatible Popularity: If zero, it's an integer
        # 0, otherwise, it's formatted to the 6th decimal place.
127
        pop = package.Popularity
128
        pop = 0 if not pop else float(number_format(pop, 6))
129
130

        snapshot_uri = config.get("options", "snapshot_uri")
131
        return {
132
133
134
            "ID": package.ID,
            "Name": package.Name,
            "PackageBaseID": package.PackageBaseID,
135
            "PackageBase": package.PackageBaseName,
136
            # Maintainer should be set following this update if one exists.
137
            "Maintainer": package.Maintainer,
138
139
140
141
            "Version": package.Version,
            "Description": package.Description,
            "URL": package.URL,
            "URLPath": snapshot_uri % package.Name,
142
            "NumVotes": package.NumVotes,
143
            "Popularity": pop,
144
145
146
147
            "OutOfDate": package.OutOfDateTS,
            "FirstSubmitted": package.SubmittedTS,
            "LastModified": package.ModifiedTS
        }
148

149
    def _get_info_json_data(self, package: models.Package) -> Dict[str, Any]:
150
151
        data = self._get_json_data(package)

152
153
        # All info results have _at least_ an empty list of
        # License and Keywords.
154
        data.update({
155
156
            "License": [],
            "Keywords": []
157
158
        })

159
160
161
162
163
        # If we actually got extra_info records, update data with
        # them for this particular package.
        if self.extra_info:
            data.update(self.extra_info.get(package.ID, {}))

164
165
        return data

166
167
168
169
170
171
172
173
174
    def _assemble_json_data(self, packages: List[models.Package],
                            data_generator: DataGenerator) \
            -> List[Dict[str, Any]]:
        """
        Assemble JSON data out of a list of packages.

        :param packages: A list of Package instances or a Package ORM query
        :param data_generator: Generator callable of single-Package JSON data
        """
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
        return [data_generator(pkg) for pkg in packages]

    def _entities(self, query: orm.Query) -> orm.Query:
        """ Select specific RPC columns on `query`. """
        return query.with_entities(
            models.Package.ID,
            models.Package.Name,
            models.Package.Version,
            models.Package.Description,
            models.Package.URL,
            models.Package.PackageBaseID,
            models.PackageBase.Name.label("PackageBaseName"),
            models.PackageBase.NumVotes,
            models.PackageBase.Popularity,
            models.PackageBase.OutOfDateTS,
            models.PackageBase.SubmittedTS,
            models.PackageBase.ModifiedTS,
            models.User.Username.label("Maintainer"),
        ).group_by(models.Package.ID)
194
195
196

    def _handle_multiinfo_type(self, args: List[str] = [], **kwargs) \
            -> List[Dict[str, Any]]:
197
        self._enforce_args(args)
198
        args = set(args)
199

200
201
202
203
204
205
206
        packages = db.query(models.Package).join(models.PackageBase).join(
            models.User,
            models.User.ID == models.PackageBase.MaintainerUID,
            isouter=True
        ).filter(models.Package.Name.in_(args))
        packages = self._entities(packages)

207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
        ids = {pkg.ID for pkg in packages}

        # Aliases for 80-width.
        Package = models.Package
        PackageKeyword = models.PackageKeyword

        subqueries = [
            # PackageDependency
            db.query(
                models.PackageDependency
            ).join(models.DependencyType).filter(
                models.PackageDependency.PackageID.in_(ids)
            ).with_entities(
                models.PackageDependency.PackageID.label("ID"),
                models.DependencyType.Name.label("Type"),
                models.PackageDependency.DepName.label("Name"),
                models.PackageDependency.DepCondition.label("Cond")
224
            ).distinct().order_by("Name"),
225
226
227
228
229
230
231
232
233
234
235

            # PackageRelation
            db.query(
                models.PackageRelation
            ).join(models.RelationType).filter(
                models.PackageRelation.PackageID.in_(ids)
            ).with_entities(
                models.PackageRelation.PackageID.label("ID"),
                models.RelationType.Name.label("Type"),
                models.PackageRelation.RelName.label("Name"),
                models.PackageRelation.RelCondition.label("Cond")
236
            ).distinct().order_by("Name"),
237
238
239
240
241
242
243
244
245
246
247

            # Groups
            db.query(models.PackageGroup).join(
                models.Group,
                and_(models.PackageGroup.GroupID == models.Group.ID,
                     models.PackageGroup.PackageID.in_(ids))
            ).with_entities(
                models.PackageGroup.PackageID.label("ID"),
                literal("Groups").label("Type"),
                models.Group.Name.label("Name"),
                literal(str()).label("Cond")
248
            ).distinct().order_by("Name"),
249
250
251
252
253
254
255
256
257
258
259
260

            # Licenses
            db.query(models.PackageLicense).join(
                models.License,
                models.PackageLicense.LicenseID == models.License.ID
            ).filter(
                models.PackageLicense.PackageID.in_(ids)
            ).with_entities(
                models.PackageLicense.PackageID.label("ID"),
                literal("License").label("Type"),
                models.License.Name.label("Name"),
                literal(str()).label("Cond")
261
            ).distinct().order_by("Name"),
262
263
264
265
266
267
268
269
270
271
272

            # Keywords
            db.query(models.PackageKeyword).join(
                models.Package,
                and_(Package.PackageBaseID == PackageKeyword.PackageBaseID,
                     Package.ID.in_(ids))
            ).with_entities(
                models.Package.ID.label("ID"),
                literal("Keywords").label("Type"),
                models.PackageKeyword.Keyword.label("Name"),
                literal(str()).label("Cond")
273
            ).distinct().order_by("Name")
274
275
276
        ]

        # Union all subqueries together.
277
278
        max_results = config.getint("options", "max_rpc_results")
        query = subqueries[0].union_all(*subqueries[1:]).limit(max_results)
279
280
281
282
283
284
285
286
287
288
289
290
291

        # Store our extra information in a class-wise dictionary,
        # which contains package id -> extra info dict mappings.
        self.extra_info = defaultdict(lambda: defaultdict(list))
        for record in query:
            type_ = TYPE_MAPPING.get(record.Type, record.Type)

            name = record.Name
            if record.Cond:
                name += record.Cond

            self.extra_info[record.ID][type_].append(name)

292
        return self._assemble_json_data(packages, self._get_info_json_data)
293

294
    def _handle_search_type(self, by: str = defaults.RPC_SEARCH_BY,
295
                            args: List[str] = []) -> List[Dict[str, Any]]:
296
297
298
299
300
301
        # If `by` isn't maintainer and we don't have any args, raise an error.
        # In maintainer's case, return all orphans if there are no args,
        # so we need args to pass through to the handler without errors.
        if by != "m" and not len(args):
            raise RPCError("No request type/data specified.")

Kevin Morris's avatar
Kevin Morris committed
302
303
        arg = args[0] if args else str()
        if by != "m" and len(arg) < 2:
304
305
306
307
308
309
            raise RPCError("Query arg too small.")

        search = RPCSearch()
        search.search_by(by, arg)

        max_results = config.getint("options", "max_rpc_results")
310
        results = self._entities(search.results()).limit(max_results)
311
        return self._assemble_json_data(results, self._get_json_data)
312

313
314
    def _handle_msearch_type(self, args: List[str] = [], **kwargs)\
            -> List[Dict[str, Any]]:
315
316
        return self._handle_search_type(by="m", args=args)

317
318
    def _handle_suggest_type(self, args: List[str] = [], **kwargs)\
            -> List[str]:
319
320
321
        if not args:
            return []

322
        arg = args[0]
323
324
325
        packages = db.query(models.Package.Name).join(
            models.PackageBase
        ).filter(
326
327
328
329
330
            and_(models.PackageBase.PackagerUID.isnot(None),
                 models.Package.Name.like(f"%{arg}%"))
        ).order_by(models.Package.Name.asc()).limit(20)
        return [pkg.Name for pkg in packages]

331
332
    def _handle_suggest_pkgbase_type(self, args: List[str] = [], **kwargs)\
            -> List[str]:
333
334
335
        if not args:
            return []

336
        packages = db.query(models.PackageBase.Name).filter(
337
338
339
            and_(models.PackageBase.PackagerUID.isnot(None),
                 models.PackageBase.Name.like(f"%{args[0]}%"))
        ).order_by(models.PackageBase.Name.asc()).limit(20)
340
        return [pkg.Name for pkg in packages]
341

342
343
344
345
346
347
348
349
350
351
352
353
354
    def _is_suggestion(self) -> bool:
        return self.type.startswith("suggest")

    def _handle_callback(self, by: str, args: List[str])\
            -> Union[List[Dict[str, Any]], List[str]]:
        # Get a handle to our callback and trap an RPCError with
        # an empty list of results based on callback's execution.
        callback = getattr(self, f"_handle_{self.type.replace('-', '_')}_type")
        results = callback(by=by, args=args)
        return results

    def handle(self, by: str = defaults.RPC_SEARCH_BY, args: List[str] = [])\
            -> Union[List[Dict[str, Any]], Dict[str, Any]]:
355
356
357
358
359
360
361
362
        """ Request entrypoint. A router should pass v, type and args
        to this function and expect an output dictionary to be returned.

        :param v: RPC version argument
        :param type: RPC type argument
        :param args: Deciphered list of arguments based on arg/arg[] inputs
        """
        # Prepare our output data dictionary with some basic keys.
363
        data = {"version": self.version, "type": self.type}
364
365
366

        # Run some verification on our given arguments.
        try:
367
            self._verify_inputs(by=by, args=args)
368
        except RPCError as exc:
369
            return self.error(str(exc))
370

371
        # Convert by to its aliased value if it has one.
372
        by = RPC.BY_ALIASES.get(by, by)
373

374
        # Process the requested handler.
375
        try:
376
            results = self._handle_callback(by, args)
377
378
        except RPCError as exc:
            return self.error(str(exc))
379
380
381

        # These types are special: we produce a different kind of
        # successful JSON output: a list of results.
382
        if self._is_suggestion():
383
384
385
386
387
388
389
390
            return results

        # Return JSON output.
        data.update({
            "resultcount": len(results),
            "results": results
        })
        return data