% kubectl logs appmesh-prometheus-5f9b989c88-gz5mr -n appmesh-system
level=warn ts=2019-10-23T07:02:45.771Z caller=main.go:282 deprecation_notice="'storage.tsdb.retention' flag is deprecated use 'storage.tsdb.retention.time' instead."
level=info ts=2019-10-23T07:02:45.771Z caller=main.go:329 msg="Starting Prometheus" version="(version=2.12.0, branch=HEAD, revision=43acd0e2e93f9f70c49b2267efa0124f1e759e86)"
level=info ts=2019-10-23T07:02:45.771Z caller=main.go:330 build_context="(go=go1.12.8, user=root@7a9dbdbe0cc7, date=20190818-13:53:16)"
level=info ts=2019-10-23T07:02:45.771Z caller=main.go:331 host_details="(Linux 4.14.146-119.123.amzn2.x86_64 #1 SMP Mon Sep 23 16:58:43 UTC 2019 x86_64 appmesh-prometheus-5f9b989c88-gz5mr (none))"
level=info ts=2019-10-23T07:02:45.771Z caller=main.go:332 fd_limits="(soft=65536, hard=65536)"
level=info ts=2019-10-23T07:02:45.771Z caller=main.go:333 vm_limits="(soft=unlimited, hard=unlimited)"
level=error ts=2019-10-23T07:02:45.771Z caller=query_logger.go:82 component=activeQueryTracker msg="Error opening query log file" file=data/queries.active err="open data/queries.active: permission denied"
panic: Unable to create mmap-ed active query log

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: prometheus
  namespace: appmesh-system
  labels:
    app.kubernetes.io/name: appmesh-prometheus
spec:
  storageClassName: gp2
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 100Gi
EOF

helm3 upgrade -i appmesh-prometheus eks/appmesh-prometheus \
--namespace appmesh-system \
--set retention=12h \
--set persistentVolumeClaim.claimName=prometheus