rpc.py 13.2 KB
Newer Older
1
2
import os

3
from collections import defaultdict
4
from typing import Any, Callable, Dict, List, NewType, Union
5

6
from fastapi.responses import HTMLResponse
7
from sqlalchemy import and_, literal
8

9
10
import aurweb.config as config

11
from aurweb import db, defaults, models, util
12
from aurweb.exceptions import RPCError
13
from aurweb.packages.search import RPCSearch
14

15
16
17
18
19
20
21
22
TYPE_MAPPING = {
    "depends": "Depends",
    "makedepends": "MakeDepends",
    "checkdepends": "CheckDepends",
    "optdepends": "OptDepends",
    "conflicts": "Conflicts",
    "provides": "Provides",
    "replaces": "Replaces",
23
24
}

25
26
27
28
DataGenerator = NewType("DataGenerator",
                        Callable[[models.Package], Dict[str, Any]])


29
30
31
32
33
34
35
36
37
38
39
40
def documentation():
    aurwebdir = config.get("options", "aurwebdir")
    rpc_doc = os.path.join(aurwebdir, "doc", "rpc.html")

    if not os.path.exists(rpc_doc):
        raise OSError("doc/rpc.html could not be read")

    with open(rpc_doc) as f:
        data = f.read()
    return HTMLResponse(data)


41
42
class RPC:
    """ RPC API handler class.
43

44
45
46
47
48
49
    There are various pieces to RPC's process, and encapsulating them
    inside of a class means that external users do not abuse the
    RPC implementation to achieve goals. We call type handlers
    by taking a reference to the callback named "_handle_{type}_type(...)",
    and if the handler does not exist, we return a not implemented
    error to the API user.
50

51
52
    EXPOSED_VERSIONS holds the set of versions that the API
    officially supports.
53

54
55
    EXPOSED_TYPES holds the set of types that the API officially
    supports.
56

57
    ALIASES holds an alias mapping of type -> type strings.
58

59
60
61
    We should focus on privatizing implementation helpers and
    focusing on performance in the code used.
    """
62

63
64
    # A set of RPC versions supported by this API.
    EXPOSED_VERSIONS = {5}
65

66
67
68
69
70
    # A set of RPC types supported by this API.
    EXPOSED_TYPES = {
        "info", "multiinfo",
        "search", "msearch",
        "suggest", "suggest-pkgbase"
71
72
    }

73
74
75
76
77
78
79
80
81
82
    # A mapping of type aliases.
    TYPE_ALIASES = {"info": "multiinfo"}

    EXPOSED_BYS = {
        "name-desc", "name", "maintainer",
        "depends", "makedepends", "optdepends", "checkdepends"
    }

    # A mapping of by aliases.
    BY_ALIASES = {"name-desc": "nd", "name": "n", "maintainer": "m"}
83

84
    def __init__(self, version: int = 0, type: str = None) -> "RPC":
85
        self.version = version
86
        self.type = RPC.TYPE_ALIASES.get(type, type)
87

88
    def error(self, message: str) -> Dict[str, Any]:
89
90
91
92
93
94
95
96
        return {
            "version": self.version,
            "results": [],
            "resultcount": 0,
            "type": "error",
            "error": message
        }

97
    def _verify_inputs(self, by: str = [], args: List[str] = []) -> None:
98
        if self.version is None:
99
100
            raise RPCError("Please specify an API version.")

101
        if self.version not in RPC.EXPOSED_VERSIONS:
102
103
            raise RPCError("Invalid version specified.")

104
105
106
107
        if by not in RPC.EXPOSED_BYS:
            raise RPCError("Incorrect by field specified.")

        if self.type is None:
108
109
            raise RPCError("No request type/data specified.")

110
        if self.type not in RPC.EXPOSED_TYPES:
111
112
            raise RPCError("Incorrect request type specified.")

113
    def _enforce_args(self, args: List[str]) -> None:
114
115
116
        if not args:
            raise RPCError("No request type/data specified.")

117
    def _get_json_data(self, package: models.Package) -> Dict[str, Any]:
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
        """ Produce dictionary data of one Package that can be JSON-serialized.

        :param package: Package instance
        :returns: JSON-serializable dictionary
        """

        # Produce RPC API compatible Popularity: If zero, it's an integer
        # 0, otherwise, it's formatted to the 6th decimal place.
        pop = package.PackageBase.Popularity
        pop = 0 if not pop else float(util.number_format(pop, 6))

        snapshot_uri = config.get("options", "snapshot_uri")
        data = defaultdict(list)
        data.update({
            "ID": package.ID,
            "Name": package.Name,
            "PackageBaseID": package.PackageBaseID,
            "PackageBase": package.PackageBase.Name,
            # Maintainer should be set following this update if one exists.
            "Maintainer": None,
            "Version": package.Version,
            "Description": package.Description,
            "URL": package.URL,
            "URLPath": snapshot_uri % package.Name,
            "NumVotes": package.PackageBase.NumVotes,
            "Popularity": pop,
            "OutOfDate": package.PackageBase.OutOfDateTS,
            "FirstSubmitted": package.PackageBase.SubmittedTS,
146
147
148
149
150
151
152
153
154
            "LastModified": package.PackageBase.ModifiedTS
        })

        if package.PackageBase.Maintainer is not None:
            # We do have a maintainer: set the Maintainer key.
            data["Maintainer"] = package.PackageBase.Maintainer.Username

        return data

155
    def _get_info_json_data(self, package: models.Package) -> Dict[str, Any]:
156
157
        data = self._get_json_data(package)

158
159
        # All info results have _at least_ an empty list of
        # License and Keywords.
160
        data.update({
161
162
            "License": [],
            "Keywords": []
163
164
        })

165
166
167
168
169
        # If we actually got extra_info records, update data with
        # them for this particular package.
        if self.extra_info:
            data.update(self.extra_info.get(package.ID, {}))

170
171
        return data

172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
    def _assemble_json_data(self, packages: List[models.Package],
                            data_generator: DataGenerator) \
            -> List[Dict[str, Any]]:
        """
        Assemble JSON data out of a list of packages.

        :param packages: A list of Package instances or a Package ORM query
        :param data_generator: Generator callable of single-Package JSON data
        """
        output = []
        for pkg in packages:
            db.refresh(pkg)
            output.append(data_generator(pkg))
        return output

    def _handle_multiinfo_type(self, args: List[str] = [], **kwargs) \
            -> List[Dict[str, Any]]:
189
        self._enforce_args(args)
190
        args = set(args)
191
192

        packages = db.query(models.Package).join(models.PackageBase).filter(
193
            models.Package.Name.in_(args))
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
        ids = {pkg.ID for pkg in packages}

        # Aliases for 80-width.
        Package = models.Package
        PackageKeyword = models.PackageKeyword

        subqueries = [
            # PackageDependency
            db.query(
                models.PackageDependency
            ).join(models.DependencyType).filter(
                models.PackageDependency.PackageID.in_(ids)
            ).with_entities(
                models.PackageDependency.PackageID.label("ID"),
                models.DependencyType.Name.label("Type"),
                models.PackageDependency.DepName.label("Name"),
                models.PackageDependency.DepCondition.label("Cond")
211
            ).distinct().order_by("Name"),
212
213
214
215
216
217
218
219
220
221
222

            # PackageRelation
            db.query(
                models.PackageRelation
            ).join(models.RelationType).filter(
                models.PackageRelation.PackageID.in_(ids)
            ).with_entities(
                models.PackageRelation.PackageID.label("ID"),
                models.RelationType.Name.label("Type"),
                models.PackageRelation.RelName.label("Name"),
                models.PackageRelation.RelCondition.label("Cond")
223
            ).distinct().order_by("Name"),
224
225
226
227
228
229
230
231
232
233
234

            # Groups
            db.query(models.PackageGroup).join(
                models.Group,
                and_(models.PackageGroup.GroupID == models.Group.ID,
                     models.PackageGroup.PackageID.in_(ids))
            ).with_entities(
                models.PackageGroup.PackageID.label("ID"),
                literal("Groups").label("Type"),
                models.Group.Name.label("Name"),
                literal(str()).label("Cond")
235
            ).distinct().order_by("Name"),
236
237
238
239
240
241
242
243
244
245
246
247

            # Licenses
            db.query(models.PackageLicense).join(
                models.License,
                models.PackageLicense.LicenseID == models.License.ID
            ).filter(
                models.PackageLicense.PackageID.in_(ids)
            ).with_entities(
                models.PackageLicense.PackageID.label("ID"),
                literal("License").label("Type"),
                models.License.Name.label("Name"),
                literal(str()).label("Cond")
248
            ).distinct().order_by("Name"),
249
250
251
252
253
254
255
256
257
258
259

            # Keywords
            db.query(models.PackageKeyword).join(
                models.Package,
                and_(Package.PackageBaseID == PackageKeyword.PackageBaseID,
                     Package.ID.in_(ids))
            ).with_entities(
                models.Package.ID.label("ID"),
                literal("Keywords").label("Type"),
                models.PackageKeyword.Keyword.label("Name"),
                literal(str()).label("Cond")
260
            ).distinct().order_by("Name")
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
        ]

        # Union all subqueries together.
        query = subqueries[0].union_all(*subqueries[1:])

        # Store our extra information in a class-wise dictionary,
        # which contains package id -> extra info dict mappings.
        self.extra_info = defaultdict(lambda: defaultdict(list))
        for record in query:
            type_ = TYPE_MAPPING.get(record.Type, record.Type)

            name = record.Name
            if record.Cond:
                name += record.Cond

            self.extra_info[record.ID][type_].append(name)

278
        return self._assemble_json_data(packages, self._get_info_json_data)
279

280
    def _handle_search_type(self, by: str = defaults.RPC_SEARCH_BY,
281
                            args: List[str] = []) -> List[Dict[str, Any]]:
282
283
284
285
286
287
        # If `by` isn't maintainer and we don't have any args, raise an error.
        # In maintainer's case, return all orphans if there are no args,
        # so we need args to pass through to the handler without errors.
        if by != "m" and not len(args):
            raise RPCError("No request type/data specified.")

Kevin Morris's avatar
Kevin Morris committed
288
289
        arg = args[0] if args else str()
        if by != "m" and len(arg) < 2:
290
291
292
293
294
295
296
            raise RPCError("Query arg too small.")

        search = RPCSearch()
        search.search_by(by, arg)

        max_results = config.getint("options", "max_rpc_results")
        results = search.results().limit(max_results)
297
        return self._assemble_json_data(results, self._get_json_data)
298

299
300
    def _handle_msearch_type(self, args: List[str] = [], **kwargs)\
            -> List[Dict[str, Any]]:
301
302
        return self._handle_search_type(by="m", args=args)

303
304
    def _handle_suggest_type(self, args: List[str] = [], **kwargs)\
            -> List[str]:
305
306
307
        if not args:
            return []

308
        arg = args[0]
309
310
311
        packages = db.query(models.Package.Name).join(
            models.PackageBase
        ).filter(
312
313
314
315
316
            and_(models.PackageBase.PackagerUID.isnot(None),
                 models.Package.Name.like(f"%{arg}%"))
        ).order_by(models.Package.Name.asc()).limit(20)
        return [pkg.Name for pkg in packages]

317
318
    def _handle_suggest_pkgbase_type(self, args: List[str] = [], **kwargs)\
            -> List[str]:
319
320
321
        if not args:
            return []

322
        packages = db.query(models.PackageBase.Name).filter(
323
324
325
            and_(models.PackageBase.PackagerUID.isnot(None),
                 models.PackageBase.Name.like(f"%{args[0]}%"))
        ).order_by(models.PackageBase.Name.asc()).limit(20)
326
        return [pkg.Name for pkg in packages]
327

328
329
330
331
332
333
334
335
336
337
338
339
340
    def _is_suggestion(self) -> bool:
        return self.type.startswith("suggest")

    def _handle_callback(self, by: str, args: List[str])\
            -> Union[List[Dict[str, Any]], List[str]]:
        # Get a handle to our callback and trap an RPCError with
        # an empty list of results based on callback's execution.
        callback = getattr(self, f"_handle_{self.type.replace('-', '_')}_type")
        results = callback(by=by, args=args)
        return results

    def handle(self, by: str = defaults.RPC_SEARCH_BY, args: List[str] = [])\
            -> Union[List[Dict[str, Any]], Dict[str, Any]]:
341
342
343
344
345
346
347
348
        """ Request entrypoint. A router should pass v, type and args
        to this function and expect an output dictionary to be returned.

        :param v: RPC version argument
        :param type: RPC type argument
        :param args: Deciphered list of arguments based on arg/arg[] inputs
        """
        # Prepare our output data dictionary with some basic keys.
349
        data = {"version": self.version, "type": self.type}
350
351
352

        # Run some verification on our given arguments.
        try:
353
            self._verify_inputs(by=by, args=args)
354
        except RPCError as exc:
355
            return self.error(str(exc))
356

357
        # Convert by to its aliased value if it has one.
358
        by = RPC.BY_ALIASES.get(by, by)
359

360
        # Process the requested handler.
361
        try:
362
            results = self._handle_callback(by, args)
363
364
        except RPCError as exc:
            return self.error(str(exc))
365
366
367

        # These types are special: we produce a different kind of
        # successful JSON output: a list of results.
368
        if self._is_suggestion():
369
370
371
372
373
374
375
376
            return results

        # Return JSON output.
        data.update({
            "resultcount": len(results),
            "results": results
        })
        return data