loki/nginx: Add recording rule for tracking upstream cache status[1]

This enables us to calculate the cache hit ratio. Please note that this
only counts requests for which caching is enabled (e.g.
{fastcgi,proxy}_cache is configured), e.g. static served files
cache_status will be "".

[1] http://nginx.org/en/docs/http/ngx_http_upstream_module.html#var_upstream_cache_status

roles/loki/files/rules.yaml

@@ -4,3 +4,5 @@ groups:
     rules:
       - record: instance_http_version_tls_version_tls_cipher:requests:rate1m
         expr: 'sum by (instance, http_version, tls_version, tls_cipher) (rate({job="nginx"}[1m] | json http_version="server_protocol", tls_version="ssl_protocol", tls_cipher="ssl_cipher"))'
+      - record: instance_cache_status:requests:rate1m
+        expr: 'sum by (instance, cache_status) (rate({job="nginx"}[1m] | json cache_status="upstream_cache_status"))'

roles/nginx/templates/nginx.conf.j2

@@ -28,13 +28,14 @@ http {
         '$remote_addr $host $remote_user [$time_local] "$request" '
         '$status $body_bytes_sent "$http_referer" '
         '"$http_user_agent" "$http_x_forwarded_for" $request_time '
-        '$server_protocol $ssl_protocol $ssl_cipher';
+        '$server_protocol $ssl_protocol $ssl_cipher '
+        '$upstream_cache_status';
 
     log_format reduced
         '$host [$time_local] "$request" '
         '$status $body_bytes_sent "$http_referer" '
         '"$http_user_agent" $server_protocol $ssl_protocol '
-        '$ssl_cipher';
+        '$ssl_cipher $upstream_cache_status';
 
     log_format json_main escape=json
         '{'
@@ -53,6 +54,7 @@ http {
         '"server_protocol":"$server_protocol",'
         '"ssl_protocol":"$ssl_protocol",'
         '"ssl_cipher":"$ssl_cipher",'
+        '"upstream_cache_status":"$upstream_cache_status",'
         # This was added to keep every log line unique as Loki drops
         # log line with the same timestamp and log text:
         # https://grafana.com/docs/loki/latest/overview/#timestamp-ordering
@@ -73,6 +75,7 @@ http {
         '"server_protocol":"$server_protocol",'
         '"ssl_protocol":"$ssl_protocol",'
         '"ssl_cipher":"$ssl_cipher",'
+        '"upstream_cache_status":"$upstream_cache_status",'
         # This was added to keep every log line unique as Loki drops
         # log line with the same timestamp and log text:
         # https://grafana.com/docs/loki/latest/overview/#timestamp-ordering