loki/nginx: Add recording rule for tracking upstream cache status[1]

This enables us to calculate the cache hit ratio. Please note that the
variable is only set when caching is enabled (e.g. proxy_cache is
configured), so

FIXME: the sum of the two metrics are too high..

correlate

[1] http://nginx.org/en/docs/http/ngx_http_upstream_module.html#var_upstream_cache_status

roles/loki/files/rules.yaml

@@ -4,3 +4,5 @@ groups:
     rules:
       - record: nginx:requests:rate1m
         expr: 'sum by (instance, server_protocol, ssl_protocol, ssl_cipher) (rate({job="nginx"}[1m] | json))'
+      - record: nginx:requests:rate1m
+        expr: 'sum by (instance, upstream_cache_status) (rate({job="nginx"}[1m] | json))'

roles/nginx/templates/nginx.conf.j2

@@ -28,13 +28,14 @@ http {
         '$remote_addr $host $remote_user [$time_local] "$request" '
         '$status $body_bytes_sent "$http_referer" '
         '"$http_user_agent" "$http_x_forwarded_for" $request_time '
-        '$server_protocol $ssl_protocol $ssl_cipher';
+        '$server_protocol $ssl_protocol $ssl_cipher '
+        '$upstream_cache_status';
 
     log_format reduced
         '$host [$time_local] "$request" '
         '$status $body_bytes_sent "$http_referer" '
         '"$http_user_agent" $server_protocol $ssl_protocol '
-        '$ssl_cipher';
+        '$ssl_cipher $upstream_cache_status';
 
     log_format json_main escape=json
         '{'
@@ -53,6 +54,7 @@ http {
         '"server_protocol":"$server_protocol",'
         '"ssl_protocol":"$ssl_protocol",'
         '"ssl_cipher":"$ssl_cipher",'
+        '"upstream_cache_status":"$upstream_cache_status",'
         # This was added to keep every log line unique as Loki drops
         # log line with the same timestamp and log text:
         # https://grafana.com/docs/loki/latest/overview/#timestamp-ordering
@@ -73,6 +75,7 @@ http {
         '"server_protocol":"$server_protocol",'
         '"ssl_protocol":"$ssl_protocol",'
         '"ssl_cipher":"$ssl_cipher",'
+        '"upstream_cache_status":"$upstream_cache_status",'
         # This was added to keep every log line unique as Loki drops
         # log line with the same timestamp and log text:
         # https://grafana.com/docs/loki/latest/overview/#timestamp-ordering