matrix: Supply a better set of oEmbed providers

The default set is defective and only covers `twitter.com` and
`youtube.com/shorts`.

Take the [official list][1] and filter it to remove providers that
Synapse rejects.

[1]: https://oembed.com/providers.json

roles/matrix/files/get-oembed-providers.py

+#!/usr/bin/env python3
+
+import json
+from collections import OrderedDict
+from sys import stdout
+from urllib.parse import urlparse
+
+import jsonschema
+import requests
+from jsonschema.exceptions import ValidationError
+
+output = requests.get("https://oembed.com/providers.json")
+output.raise_for_status()
+providers = output.json(object_pairs_hook=OrderedDict)
+
+# From synapse/config/oembed.py
+_OEMBED_PROVIDER_SCHEMA = {
+    "type": "array",
+    "items": {
+        "type": "object",
+        "properties": {
+            "provider_name": {"type": "string"},
+            "provider_url": {"type": "string"},
+            "endpoints": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "schemes": {
+                            "type": "array",
+                            "items": {"type": "string"},
+                        },
+                        "url": {"type": "string"},
+                        "formats": {"type": "array", "items": {"type": "string"}},
+                        "discovery": {"type": "boolean"},
+                    },
+                    "required": ["schemes", "url"],
+                },
+            },
+        },
+        "required": ["provider_name", "provider_url", "endpoints"],
+    },
+}
+
+while True:
+    try:
+        jsonschema.validate(providers, _OEMBED_PROVIDER_SCHEMA)
+    except ValidationError as e:
+        del providers[e.absolute_path[0]]
+    else:
+        break
+
+
+def valid_url(url):
+    return urlparse(url).scheme in ["http", "https"]
+
+
+def valid_provider(provider):
+    for endpoint in provider["endpoints"]:
+        if not valid_url(endpoint["url"]):
+            return False
+        for glob in endpoint["schemes"]:
+            if not valid_url(glob):
+                return False
+    return True
+
+
+providers = [p for p in providers if valid_provider(p)]
+json.dump(providers, stdout, indent=4)
+print()

roles/matrix/tasks/main.yml

@@ -258,6 +258,7 @@
   copy: src={{ item }} dest=/etc/synapse/{{ item }} owner=root group=root mode=0644
   with_items:
     - log_config.yaml
+    - oembed-providers.json
     - worker-appservice.yaml
     - worker-federation_reader.yaml
     - worker-federation_sender.yaml

roles/matrix/templates/homeserver.yaml.j2

@@ -93,6 +93,10 @@ url_preview_ip_range_blacklist:
   - 'fec0::/10'
 url_preview_accept_language:
   - 'en'
+oembed:
+  disable_default_providers: true
+  additional_providers:
+    - "/etc/synapse/oembed-providers.json"
 
 # WebRTC
 turn_uris: