diff --git a/etcd-inmemory-monitoring/Dockerfile b/etcd-inmemory-monitoring/Dockerfile new file mode 100644 index 0000000..90de72e --- /dev/null +++ b/etcd-inmemory-monitoring/Dockerfile @@ -0,0 +1,6 @@ + +FROM python:3.8 +RUN pip install prometheus_client + +COPY exporter.py / + diff --git a/etcd-inmemory-monitoring/README b/etcd-inmemory-monitoring/README new file mode 100644 index 0000000..4f7f85a --- /dev/null +++ b/etcd-inmemory-monitoring/README @@ -0,0 +1,3 @@ +https://github.com/prometheus/client_python +pip install prometheus-client + diff --git a/etcd-inmemory-monitoring/alertingrules.yaml b/etcd-inmemory-monitoring/alertingrules.yaml new file mode 100644 index 0000000..aa4d4d3 --- /dev/null +++ b/etcd-inmemory-monitoring/alertingrules.yaml @@ -0,0 +1,64 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: wamblee-cluster-monitoring + namespace: monitoring +spec: + groups: + - name: wamblee-org + partial_response_strategy: "" + rules: + - alert: etcdBackupAbsentOrTooOld + annotations: + description: 'etcd backup is too old or not present' + summary: etcd cluster backup is too old or not present + expr: |- + time() - + max(max_over_time(file_time_seconds{job="pantherbackupexporter",type="backup"}[60m])) + > 3600 + for: 5m + labels: + severity: critical + - alert: etcdBackupFileTooSmall + annotations: + description: 'etcd backup is too small or not present' + summary: etcd cluster backup is too small or not present + expr: |- + max(max_over_time(file_size{job="pantherbackupexporter",type="backup"}[60m])) + < + 10000000 + for: 5m + labels: + severity: critical + - alert: etcdImageFileTooOld + annotations: + description: 'etcd image file is too old or not present' + summary: etcd image file is too old or not present + expr: |- + time() - + max(max_over_time(file_time_seconds{job="pantherbackupexporter",type="image"}[60m])) + > 3600 + for: 5m + labels: + severity: critical + - alert: etcdImageFileTooSmall + annotations: + description: 'etcd image file is too small or not present' + summary: etcd cluster image file is too small or not present + expr: |- + max(max_over_time(file_size{job="pantherbackupexporter",type="image"}[60m])) + < + 10 + for: 5m + labels: + severity: critical + - alert: etcdDiskAlmostFull + annotations: + description: 'etcd high disk usage' + summary: etcd high disk usage + expr: |- + node_filesystem_free_bytes{mountpoint="/var/lib/etcd"}/ + node_filesystem_size_bytes{mountpoint="/var/lib/etcd"} < 0.30 + for: 5m + labels: + severity: critical diff --git a/etcd-inmemory-monitoring/deployment.yaml b/etcd-inmemory-monitoring/deployment.yaml new file mode 100644 index 0000000..c0846c6 --- /dev/null +++ b/etcd-inmemory-monitoring/deployment.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pantherbackupmonitoring + namespace: monitoring +spec: + selector: + matchLabels: + app: pantherbackupmonitoring + template: + metadata: + labels: + app: pantherbackupmonitoring + prometheus-scrapable: "true" + spec: + terminationGracePeriodSeconds: 0 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + nodeSelector: + kubernetes.io/hostname: panther + containers: + - name: exporter + image: docker.example.com/filemonitor:1.0 + args: + - python3 + - -u + - /exporter.py + - backup:/backup/etcd-snapshot-latest.db + - image:/backup/etcdimage + ports: + - containerPort: 8080 + protocol: TCP + name: http + volumeMounts: + - name: pantherbackup + mountPath: /backup + readOnly: true + volumes: + - name: pantherbackup + hostPath: + path: /var/lib/wamblee/etcd + diff --git a/etcd-inmemory-monitoring/docker-compose.yaml b/etcd-inmemory-monitoring/docker-compose.yaml new file mode 100644 index 0000000..30aec21 --- /dev/null +++ b/etcd-inmemory-monitoring/docker-compose.yaml @@ -0,0 +1,19 @@ +services: + filemonitor: + image: docker.example.com/filemonitor:1.0 + build: + context: . + command: + - python3 + - /exporter.py + # some paths for testing + - /data/x.txt + - /data/y.txt + ports: + - "8080:8080" + volumes: + # for testing. + - /home/user/downloads:/data + + + diff --git a/etcd-inmemory-monitoring/exporter.py b/etcd-inmemory-monitoring/exporter.py new file mode 100644 index 0000000..fbbd5c9 --- /dev/null +++ b/etcd-inmemory-monitoring/exporter.py @@ -0,0 +1,81 @@ +import time + +import prometheus_client +from prometheus_client import start_http_server, Gauge, Counter +import sys +import argparse +import os +import stat +from http.server import HTTPServer + +prometheus_client.REGISTRY.unregister(prometheus_client.GC_COLLECTOR) +prometheus_client.REGISTRY.unregister(prometheus_client.PLATFORM_COLLECTOR) +prometheus_client.REGISTRY.unregister(prometheus_client.PROCESS_COLLECTOR) + +# Create a metric to track time spent and requests made. +FILE_TIME = Gauge("file_time_seconds", "File last modification time", labelnames=['path', 'type']) +FILE_SIZE = Gauge("file_size", "File size in bytes", labelnames=['path', 'type']) + + +class Handler(prometheus_client.MetricsHandler): + def do_GET(self) -> None: + for file in FILES.keys(): + type = FILES[file] + try: + FILE_TIME.labels(file, type).set(0) + FILE_SIZE.labels(file, type).set(0) + # follow symlinks + stats = os.stat(path=file) + if stat.S_ISREG(stats.st_mode): + FILE_TIME.labels(file, type).set(stats.st_mtime) + FILE_SIZE.labels(file, type).set(stats.st_size) + except: + pass + + return super().do_GET() + + +# map of filename to type +FILES = {} + +if __name__ == '__main__': + DEFAULT_PORT = 8080 + parser = argparse.ArgumentParser(prog=sys.argv[0], + description=f"""Statistics on a (backup) file, + + Usage: ${sys.argv[0]} [-p|--port ] : .... : + + Listens on port {DEFAULT_PORT} by default. It exposes statistics + on the monitored files to prometheus. Current metrics are + + {FILE_TIME._name}{{path="/path/to/file"}}: file modification time in seconds since 1970 + {FILE_SIZE._name}{{path="/path/to/file"}}: file size in bytes + + If a path does not exist or is not a regular file then the value 0 is returned. + The exporter follow symlinks. + + The synax of each file is fo theform