diff --git a/config.EXAMPLE/monitoring.env.EXAMPLE b/config.EXAMPLE/monitoring.env.EXAMPLE index a71eb10..7b6ef99 100644 --- a/config.EXAMPLE/monitoring.env.EXAMPLE +++ b/config.EXAMPLE/monitoring.env.EXAMPLE @@ -1,7 +1,7 @@ # This is an EXAMPLE file, do not put real secrets in here. # Copy it to ../config/monitoring.env and then DELETE THIS COMMENT. # When does the monitoring job run -MONITORING_CRON_SCHEDULE="*/5 * * * *" +MONITORING_CRON_SCHEDULE="*/15 * * * *" # OpenTelemetry OTLP/HTTP endpoint of the LGTM collector. OTEL_EXPORTER_OTLP_ENDPOINT="http://lgtm:4318" OTEL_SERVICE_NAME=waveform-monitoring diff --git a/monitoring/monitor.py b/monitoring/monitor.py index 76e91f8..24524ae 100644 --- a/monitoring/monitor.py +++ b/monitoring/monitor.py @@ -6,6 +6,7 @@ import logging import os +import shutil import sys import time from pathlib import Path @@ -173,6 +174,18 @@ def _bytes_in_regular_files(tld: Path): return byte_count +def report_disk_free_space(meter: Meter): + disk_free = meter.create_gauge( + "waveform.monitoring.gae_disk_free", + unit="By", + description="Disk free on the /gae partition", + ) + # We assume here that we are mounted onto the partition we + # care about measuring (on the GAE this would be /gae) + gae_free_bytes = shutil.disk_usage(SAVED_MESSAGES_DIR) + disk_free.set(gae_free_bytes.free) + + def main() -> int: service_name = _env("OTEL_SERVICE_NAME") otlp_endpoint = _env("OTEL_EXPORTER_OTLP_ENDPOINT") @@ -184,6 +197,7 @@ def main() -> int: # things to measure scan_hl7_bz2(meter) scan_waveform_exporter_files(meter) + report_disk_free_space(meter) # shutdown, flush data provider = metrics.get_meter_provider()