loki/nginx: Add recording rule for tracking upstream cache status[1]

This enables us to calculate the cache hit ratio, which may help
determine whether more caching would be beneficial.

Please note that this only counts requests for which caching is enabled
(e.g. {fastcgi,proxy}_cache is configured), e.g. for static served files
cache_status will be "".

[1] http://nginx.org/en/docs/http/ngx_http_upstream_module.html#var_upstream_cache_status

roles/loki/files/rules.yaml

@@ -4,3 +4,5 @@ groups:
     rules:
       - record: instance_http_version_tls_version_tls_cipher:requests:rate1m
         expr: 'sum by (instance, http_version, tls_version, tls_cipher) (rate({job="nginx"}[1m] | json http_version="server_protocol", tls_version="ssl_protocol", tls_cipher="ssl_cipher"))'
+      - record: instance_cache_status:requests:rate1m
+        expr: 'sum by (instance, cache_status) (rate({job="nginx"}[1m] | json cache_status="upstream_cache_status"))'

roles/nginx/templates/nginx.conf.j2

@@ -28,13 +28,14 @@ http {
         '$remote_addr $host $remote_user [$time_local] "$request" '
         '$status $body_bytes_sent "$http_referer" '
         '"$http_user_agent" "$http_x_forwarded_for" $request_time '
-        '$server_protocol $ssl_protocol $ssl_cipher';
+        '$server_protocol $ssl_protocol $ssl_cipher '
+        '$upstream_cache_status';
 
     log_format reduced
         '$host [$time_local] "$request" '
         '$status $body_bytes_sent "$http_referer" '
         '"$http_user_agent" $server_protocol $ssl_protocol '
-        '$ssl_cipher';
+        '$ssl_cipher $upstream_cache_status';
 
     log_format json_main escape=json
         '{'
@@ -53,6 +54,7 @@ http {
         '"server_protocol":"$server_protocol",'
         '"ssl_protocol":"$ssl_protocol",'
         '"ssl_cipher":"$ssl_cipher",'
+        '"upstream_cache_status":"$upstream_cache_status",'
         # This was added to keep every log line unique as Loki drops
         # log line with the same timestamp and log text:
         # https://grafana.com/docs/loki/latest/overview/#timestamp-ordering
@@ -73,6 +75,7 @@ http {
         '"server_protocol":"$server_protocol",'
         '"ssl_protocol":"$ssl_protocol",'
         '"ssl_cipher":"$ssl_cipher",'
+        '"upstream_cache_status":"$upstream_cache_status",'
         # This was added to keep every log line unique as Loki drops
         # log line with the same timestamp and log text:
         # https://grafana.com/docs/loki/latest/overview/#timestamp-ordering