Verified Commit 7485cc23 authored by Kevin Morris's avatar Kevin Morris
Browse files

change: report unhandled tracebacks to a repository



As repeats of these traceback notifications were annoying some of
the devops staff, and it took coordination to share tracebacks with
developers, this commit removes that responsibility off of devops
by reporting tracebacks to Gitlab repositories in the form of issues.

- removed ServerErrorNotification
- removed notifications.postmaster configuration option
- added notifications.gitlab-instance option
- added notifications.error-project option
- added notifications.error-token option
- added aurweb.exceptions.handle_form_exceptions, a POST route decorator

Issues are filed confidentially. This change will need updates
in infrastructure's ansible configuration before this can be
applied to aur.archlinux.org.
Signed-off-by: Kevin Morris's avatarKevin Morris <kevr@0cost.org>
parent e2eb3a7d
Pipeline #15804 passed with stages
in 3 minutes and 3 seconds
......@@ -9,6 +9,8 @@ import typing
from urllib.parse import quote_plus
import requests
from fastapi import FastAPI, HTTPException, Request, Response
from fastapi.responses import RedirectResponse
from fastapi.staticfiles import StaticFiles
......@@ -33,7 +35,6 @@ from aurweb.packages.util import get_pkg_or_base
from aurweb.prometheus import instrumentator
from aurweb.redis import redis_connection
from aurweb.routers import APP_ROUTES
from aurweb.scripts import notify
from aurweb.templates import make_context, render_template
logger = logging.get_logger(__name__)
......@@ -109,6 +110,10 @@ async def internal_server_error(request: Request, exc: Exception) -> Response:
:param request: FastAPI Request
:return: Rendered 500.html template with status_code 500
"""
repo = aurweb.config.get("notifications", "gitlab-instance")
project = aurweb.config.get("notifications", "error-project")
token = aurweb.config.get("notifications", "error-token")
context = make_context(request, "Internal Server Error")
# Print out the exception via `traceback` and store the value
......@@ -120,39 +125,80 @@ async def internal_server_error(request: Request, exc: Exception) -> Response:
# Produce a SHA1 hash of the traceback string.
tb_hash = hashlib.sha1(tb.encode()).hexdigest()
# Use the first 7 characters of the sha1 for the traceback id.
# We will use this to log and include in the notification.
tb_id = tb_hash[:7]
redis = redis_connection()
pipe = redis.pipeline()
key = f"tb:{tb_hash}"
pipe.get(key)
retval, = pipe.execute()
retval = redis.get(key)
if not retval:
# Expire in one hour; this is just done to make sure we
# don't infinitely store these values, but reduce the number
# of automated reports (notification below). At this time of
# writing, unexpected exceptions are not common, thus this
# will not produce a large memory footprint in redis.
pipe = redis.pipeline()
pipe.set(key, tb)
pipe.expire(key, 3600)
pipe.expire(key, 86400) # One day.
pipe.execute()
# Send out notification about it.
notif = notify.ServerErrorNotification(
tb_id, context.get("version"), context.get("utcnow"))
notif.send()
retval = tb
if "set-me" not in (project, token):
proj = quote_plus(project)
endp = f"{repo}/api/v4/projects/{proj}/issues"
base = f"{request.url.scheme}://{request.url.netloc}"
title = f"Traceback [{tb_id}]: {base}{request.url.path}"
desc = [
"DISCLAIMER",
"----------",
"**This issue is confidential** and should be sanitized "
"before sharing with users or developers. Please ensure "
"you've completed the following tasks:",
"- [ ] I have removed any sensitive data and "
"the description history.",
"",
"Exception Details",
"-----------------",
f"- Route: `{request.url.path}`",
f"- User: `{request.user.Username}`",
f"- Email: `{request.user.Email}`",
]
# Add method-specific information to the description.
if request.method.lower() == "get":
# get
if request.url.query:
desc = desc + [f"- Query: `{request.url.query}`"]
desc += ["", f"```{tb}```"]
else:
# post
form_data = str(dict(request.state.form_data))
desc = desc + [
f"- Data: `{form_data}`"
] + ["", f"```{tb}```"]
headers = {"Authorization": f"Bearer {token}"}
data = {
"title": title,
"description": "\n".join(desc),
"labels": ["triage"],
"confidential": True,
}
logger.info(endp)
resp = requests.post(endp, json=data, headers=headers)
if resp.status_code != http.HTTPStatus.CREATED:
logger.error(
f"Unable to report exception to {repo}: {resp.text}")
else:
logger.warning("Unable to report an exception found due to "
"unset notifications.error-{{project,token}}")
# Log details about the exception traceback.
logger.error(f"FATAL[{tb_id}]: An unexpected exception has occurred.")
logger.error(tb)
else:
retval = retval.decode()
# Log details about the exception traceback.
logger.error(f"FATAL[{tb_id}]: An unexpected exception has occurred.")
logger.error(retval)
return render_template(request, "errors/500.html", context,
status_code=http.HTTPStatus.INTERNAL_SERVER_ERROR)
......
......@@ -33,6 +33,8 @@ class AnonymousUser:
makes a request against FastAPI. """
# Stub attributes used to mimic a real user.
ID = 0
Username = "N/A"
Email = "N/A"
class AccountType:
""" A stubbed AccountType static class. In here, we use an ID
......
from typing import Any
import functools
from typing import Any, Callable
import fastapi
class AurwebException(Exception):
......@@ -90,3 +94,19 @@ class ValidationError(AurwebException):
class InvariantError(AurwebException):
pass
def handle_form_exceptions(route: Callable) -> fastapi.Response:
"""
A decorator required when fastapi POST routes are defined.
This decorator populates fastapi's `request.state` with a `form_data`
attribute, which is then used to report form data when exceptions
are caught and reported.
"""
@functools.wraps(route)
async def wrapper(request: fastapi.Request, *args, **kwargs):
request.state.form_data = await request.form()
return await route(request, *args, **kwargs)
return wrapper
......@@ -13,7 +13,7 @@ import aurweb.config
from aurweb import cookies, db, l10n, logging, models, util
from aurweb.auth import account_type_required, requires_auth, requires_guest
from aurweb.captcha import get_captcha_salts
from aurweb.exceptions import ValidationError
from aurweb.exceptions import ValidationError, handle_form_exceptions
from aurweb.l10n import get_translator_for_request
from aurweb.models import account_type as at
from aurweb.models.ssh_pub_key import get_fingerprint
......@@ -35,6 +35,7 @@ async def passreset(request: Request):
@router.post("/passreset", response_class=HTMLResponse)
@handle_form_exceptions
@requires_guest
async def passreset_post(request: Request,
user: str = Form(...),
......@@ -253,6 +254,7 @@ async def account_register(request: Request,
@router.post("/register", response_class=HTMLResponse)
@handle_form_exceptions
@requires_guest
async def account_register_post(request: Request,
U: str = Form(default=str()), # Username
......@@ -369,6 +371,7 @@ async def account_edit(request: Request, username: str):
@router.post("/account/{username}/edit", response_class=HTMLResponse)
@handle_form_exceptions
@requires_auth
async def account_edit_post(request: Request,
username: str,
......@@ -492,6 +495,7 @@ async def accounts(request: Request):
@router.post("/accounts")
@handle_form_exceptions
@requires_auth
@account_type_required({at.TRUSTED_USER,
at.DEVELOPER,
......@@ -601,6 +605,7 @@ async def terms_of_service(request: Request):
@router.post("/tos")
@handle_form_exceptions
@requires_auth
async def terms_of_service_post(request: Request,
accept: bool = Form(default=False)):
......
......@@ -8,6 +8,7 @@ import aurweb.config
from aurweb import cookies, db, time
from aurweb.auth import requires_auth, requires_guest
from aurweb.exceptions import handle_form_exceptions
from aurweb.l10n import get_translator_for_request
from aurweb.models import User
from aurweb.templates import make_variable_context, render_template
......@@ -29,6 +30,7 @@ async def login_get(request: Request, next: str = "/"):
@router.post("/login", response_class=HTMLResponse)
@handle_form_exceptions
@requires_guest
async def login_post(request: Request,
next: str = Form(...),
......@@ -82,6 +84,7 @@ async def login_post(request: Request,
@router.post("/logout")
@handle_form_exceptions
@requires_auth
async def logout(request: Request, next: str = Form(default="/")):
if request.user.is_authenticated():
......
......@@ -15,6 +15,7 @@ import aurweb.models.package_request
from aurweb import cookies, db, models, time, util
from aurweb.cache import db_count_cache
from aurweb.exceptions import handle_form_exceptions
from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID
from aurweb.models.package_request import PENDING_ID
from aurweb.packages.util import query_notified, query_voted, updated_packages
......@@ -31,6 +32,7 @@ async def favicon(request: Request):
@router.post("/language", response_class=RedirectResponse)
@handle_form_exceptions
async def language(request: Request,
set_lang: str = Form(...),
next: str = Form(...),
......
......@@ -8,7 +8,7 @@ import aurweb.filters # noqa: F401
from aurweb import config, db, defaults, logging, models, util
from aurweb.auth import creds, requires_auth
from aurweb.exceptions import InvariantError
from aurweb.exceptions import InvariantError, handle_form_exceptions
from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID, REPLACES_ID
from aurweb.packages import util as pkgutil
from aurweb.packages.search import PackageSearch
......@@ -416,6 +416,7 @@ PACKAGE_ACTIONS = {
@router.post("/packages")
@handle_form_exceptions
@requires_auth
async def packages_post(request: Request,
IDs: List[int] = Form(default=[]),
......
......@@ -6,7 +6,7 @@ from sqlalchemy import and_
from aurweb import config, db, l10n, logging, templates, time, util
from aurweb.auth import creds, requires_auth
from aurweb.exceptions import InvariantError, ValidationError
from aurweb.exceptions import InvariantError, ValidationError, handle_form_exceptions
from aurweb.models import PackageBase
from aurweb.models.package_comment import PackageComment
from aurweb.models.package_keyword import PackageKeyword
......@@ -88,6 +88,7 @@ async def pkgbase_flag_comment(request: Request, name: str):
@router.post("/pkgbase/{name}/keywords")
@handle_form_exceptions
async def pkgbase_keywords(request: Request, name: str,
keywords: str = Form(default=str())):
pkgbase = get_pkg_or_base(name, PackageBase)
......@@ -130,6 +131,7 @@ async def pkgbase_flag_get(request: Request, name: str):
@router.post("/pkgbase/{name}/flag")
@handle_form_exceptions
@requires_auth
async def pkgbase_flag_post(request: Request, name: str,
comments: str = Form(default=str())):
......@@ -158,6 +160,7 @@ async def pkgbase_flag_post(request: Request, name: str,
@router.post("/pkgbase/{name}/comments")
@handle_form_exceptions
@requires_auth
async def pkgbase_comments_post(
request: Request, name: str,
......@@ -254,6 +257,7 @@ async def pkgbase_comment_edit(request: Request, name: str, id: int,
@router.post("/pkgbase/{name}/comments/{id}")
@handle_form_exceptions
@requires_auth
async def pkgbase_comment_post(
request: Request, name: str, id: int,
......@@ -294,6 +298,7 @@ async def pkgbase_comment_post(
@router.post("/pkgbase/{name}/comments/{id}/pin")
@handle_form_exceptions
@requires_auth
async def pkgbase_comment_pin(request: Request, name: str, id: int,
next: str = Form(default=None)):
......@@ -328,6 +333,7 @@ async def pkgbase_comment_pin(request: Request, name: str, id: int,
@router.post("/pkgbase/{name}/comments/{id}/unpin")
@handle_form_exceptions
@requires_auth
async def pkgbase_comment_unpin(request: Request, name: str, id: int,
next: str = Form(default=None)):
......@@ -361,6 +367,7 @@ async def pkgbase_comment_unpin(request: Request, name: str, id: int,
@router.post("/pkgbase/{name}/comments/{id}/delete")
@handle_form_exceptions
@requires_auth
async def pkgbase_comment_delete(request: Request, name: str, id: int,
next: str = Form(default=None)):
......@@ -400,6 +407,7 @@ async def pkgbase_comment_delete(request: Request, name: str, id: int,
@router.post("/pkgbase/{name}/comments/{id}/undelete")
@handle_form_exceptions
@requires_auth
async def pkgbase_comment_undelete(request: Request, name: str, id: int,
next: str = Form(default=None)):
......@@ -438,6 +446,7 @@ async def pkgbase_comment_undelete(request: Request, name: str, id: int,
@router.post("/pkgbase/{name}/vote")
@handle_form_exceptions
@requires_auth
async def pkgbase_vote(request: Request, name: str):
pkgbase = get_pkg_or_base(name, PackageBase)
......@@ -462,6 +471,7 @@ async def pkgbase_vote(request: Request, name: str):
@router.post("/pkgbase/{name}/unvote")
@handle_form_exceptions
@requires_auth
async def pkgbase_unvote(request: Request, name: str):
pkgbase = get_pkg_or_base(name, PackageBase)
......@@ -482,6 +492,7 @@ async def pkgbase_unvote(request: Request, name: str):
@router.post("/pkgbase/{name}/notify")
@handle_form_exceptions
@requires_auth
async def pkgbase_notify(request: Request, name: str):
pkgbase = get_pkg_or_base(name, PackageBase)
......@@ -491,6 +502,7 @@ async def pkgbase_notify(request: Request, name: str):
@router.post("/pkgbase/{name}/unnotify")
@handle_form_exceptions
@requires_auth
async def pkgbase_unnotify(request: Request, name: str):
pkgbase = get_pkg_or_base(name, PackageBase)
......@@ -500,6 +512,7 @@ async def pkgbase_unnotify(request: Request, name: str):
@router.post("/pkgbase/{name}/unflag")
@handle_form_exceptions
@requires_auth
async def pkgbase_unflag(request: Request, name: str):
pkgbase = get_pkg_or_base(name, PackageBase)
......@@ -527,6 +540,7 @@ async def pkgbase_disown_get(request: Request, name: str,
@router.post("/pkgbase/{name}/disown")
@handle_form_exceptions
@requires_auth
async def pkgbase_disown_post(request: Request, name: str,
comments: str = Form(default=str()),
......@@ -565,6 +579,7 @@ async def pkgbase_disown_post(request: Request, name: str,
@router.post("/pkgbase/{name}/adopt")
@handle_form_exceptions
@requires_auth
async def pkgbase_adopt_post(request: Request, name: str):
pkgbase = get_pkg_or_base(name, PackageBase)
......@@ -607,6 +622,7 @@ async def pkgbase_comaintainers(request: Request, name: str) -> Response:
@router.post("/pkgbase/{name}/comaintainers")
@handle_form_exceptions
@requires_auth
async def pkgbase_comaintainers_post(request: Request, name: str,
users: str = Form(default=str())) \
......@@ -660,6 +676,7 @@ async def pkgbase_request(request: Request, name: str,
@router.post("/pkgbase/{name}/request")
@handle_form_exceptions
@requires_auth
async def pkgbase_request_post(request: Request, name: str,
type: str = Form(...),
......@@ -755,6 +772,7 @@ async def pkgbase_delete_get(request: Request, name: str,
@router.post("/pkgbase/{name}/delete")
@handle_form_exceptions
@requires_auth
async def pkgbase_delete_post(request: Request, name: str,
confirm: bool = Form(default=False),
......@@ -819,6 +837,7 @@ async def pkgbase_merge_get(request: Request, name: str,
@router.post("/pkgbase/{name}/merge")
@handle_form_exceptions
@requires_auth
async def pkgbase_merge_post(request: Request, name: str,
into: str = Form(default=str()),
......
......@@ -6,6 +6,7 @@ from sqlalchemy import case
from aurweb import db, defaults, time, util
from aurweb.auth import creds, requires_auth
from aurweb.exceptions import handle_form_exceptions
from aurweb.models import PackageRequest, User
from aurweb.models.package_request import PENDING_ID, REJECTED_ID
from aurweb.requests.util import get_pkgreq_by_id
......@@ -63,6 +64,7 @@ async def request_close(request: Request, id: int):
@router.post("/requests/{id}/close")
@handle_form_exceptions
@requires_auth
async def request_close_post(request: Request, id: int,
comments: str = Form(default=str())):
......
......@@ -11,6 +11,7 @@ from fastapi import APIRouter, Form, Query, Request, Response
from fastapi.responses import JSONResponse
from aurweb import defaults
from aurweb.exceptions import handle_form_exceptions
from aurweb.ratelimit import check_ratelimit
from aurweb.rpc import RPC, documentation
......@@ -150,6 +151,7 @@ async def rpc(request: Request,
@router.get("/rpc.php") # Temporary! Remove on 03/04
@router.post("/rpc/")
@router.post("/rpc")
@handle_form_exceptions
async def rpc_post(request: Request,
v: Optional[int] = Form(default=None),
type: Optional[str] = Form(default=None),
......
......@@ -9,6 +9,7 @@ from sqlalchemy import and_, func, or_
from aurweb import db, l10n, logging, models, time
from aurweb.auth import creds, requires_auth
from aurweb.exceptions import handle_form_exceptions
from aurweb.models import User
from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID
from aurweb.templates import make_context, make_variable_context, render_template
......@@ -173,6 +174,7 @@ async def trusted_user_proposal(request: Request, proposal: int):
@router.post("/tu/{proposal}")
@handle_form_exceptions
@requires_auth
async def trusted_user_proposal_post(request: Request, proposal: int,
decision: str = Form(...)):
......@@ -245,6 +247,7 @@ async def trusted_user_addvote(request: Request, user: str = str(),
@router.post("/addvote")
@handle_form_exceptions
@requires_auth
async def trusted_user_addvote_post(request: Request,
user: str = Form(default=str()),
......
......@@ -7,8 +7,6 @@ import subprocess
import sys
import textwrap
from typing import List, Tuple
from sqlalchemy import and_, or_
import aurweb.config
......@@ -16,7 +14,7 @@ import aurweb.db
import aurweb.filters
import aurweb.l10n
from aurweb import db, l10n, logging
from aurweb import db, logging
from aurweb.models import PackageBase, User
from aurweb.models.package_comaintainer import PackageComaintainer
from aurweb.models.package_comment import PackageComment
......@@ -130,48 +128,6 @@ class Notification:
logger.error(str(exc))
class ServerErrorNotification(Notification):
""" A notification used to represent an internal server error. """
def __init__(self, traceback_id: int, version: str, utc: int):
"""
Construct a ServerErrorNotification.
:param traceback_id: Traceback ID
:param version: aurweb version
:param utc: UTC timestamp
"""
self._tb_id = traceback_id
self._version = version
self._utc = utc
postmaster = aurweb.config.get("notifications", "postmaster")
self._to = postmaster
super().__init__()
def get_recipients(self) -> List[Tuple[str, str]]:
from aurweb.auth import AnonymousUser
user = (db.query(User).filter(User.Email == self._to).first()
or AnonymousUser())
return [(self._to, user.LangPreference)]
def get_subject(self, lang: str) -> str:
return l10n.translator.translate("AUR Server Error", lang)
def get_body(self, lang: str) -> str:
""" A forcibly English email body. """
dt = aurweb.filters.timestamp_to_datetime(self._utc)
dts = dt.strftime("%Y-%m-%d %H:%M")
return (f"Traceback ID: {self._tb_id}\n"
f"Location: {aur_location}\n"
f"Version: {self._version}\n"
f"Datetime: {dts} UTC\n")
def get_refs(self):
return (aur_location,)
class ResetKeyNotification(Notification):
def __init__(self, uid):
......
......@@ -67,9 +67,24 @@ smtp-user =
smtp-password =
sender = notify@aur.archlinux.org
reply-to = noreply@aur.archlinux.org
; Administration email which will receive notifications about
; Gitlab instance base URL. We use this instance to report
; server errors in the form of confidential issues (see error-project).
gitlab-instance = https://gitlab.archlinux.org
; Project URI which will received confidential issues about
; various server details like uncaught exceptions.
postmaster = admin@example.org
; Errors reported will be filed using the 'triage' label, and so
; the 'triage' label must exist in any project URI given.
;
; - must be a valid project URI on notifications.error-repository
; - must contain a 'triage' label
;
error-project = set-me
; Gitlab access token with API privileges to post
; notifications.error-project issues.
error-token = set-me
[fingerprints]
Ed25519 = SHA256:HQ03dn6EasJHNDlt51KpQpFkT3yBX83x7BoIkA1iv2k
......
......@@ -2,6 +2,7 @@ import http
import os
import re
from typing import Callable
from unittest import mock
import fastapi
......@@ -14,13 +15,39 @@ import aurweb.asgi
import aurweb.config
import aurweb.redis
from aurweb.testing.email import Email
from aurweb.exceptions import handle_form_exceptions
from aurweb.testing.requests import Request
@pytest.fixture
def setup(db_test, email_test):
return
aurweb.redis.redis_connection().flushall()
yield
aurweb.redis.redis_connection().flushall()
@pytest.fixture
def mock_glab_request(monkeypatch):
def wrapped(return_value=None, side_effect=None):
def what_to_return(*args, **kwargs):
if side_effect:
return side_effect # pragma: no cover
return return_value
monkeypatch.setattr("requests.post", what_to_return)
return wrapped
def mock_glab_config(project: str = "test/project", token: str = "test-token"):
config_get = aurweb.config.get
def wrapper(section: str, key: str) -> str:
if section == "notifications":
if key == "error-project":
return project
elif key == "error-token":
return token
return config_get(section, key)
return wrapper
@pytest.mark.asyncio
......@@ -77,8 +104,8 @@ async def test_asgi_app_unsupported_backends():
await aurweb.asgi.app_startup()