From 1e866c86d9072a5f85f46a286daf729310be26f78340a2376d5d1e6008d86fee Mon Sep 17 00:00:00 2001 From: localhorst Date: Thu, 13 Aug 2026 18:00:55 +0200 Subject: [PATCH] export empty jsons too --- README.md | 10 ++++-- config.py | 10 +++--- vw-eu-data-act-exporter.py | 64 ++++++++++++++++++++++++++++++-------- 3 files changed, 64 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index e309361..3ddb162 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,11 @@ a shared `/metrics` endpoint (no Docker, plain systemd deployment). scrape-interval timing in the background - **No persistent state by default**: no ZIPs, no JSON, no values are written to disk - everything lives in process memory and is empty again - after a restart. Optionally, raw JSON datasets can be persisted to disk via - `config.persist_raw_json` (see [Configuration](#configuration-configpy)) + after a restart. Optionally, the JSON extracted from every downloaded + dataset - including "_no_content_found.zip" placeholders, if they carry a + JSON payload - can be persisted to disk via `config.persist_raw_json` + (see [Configuration](#configuration-configpy)). ZIP files themselves are + never written to disk - Fields are only updated when they are present in the current dataset - if a field is missing, the last known value is kept - The `health` metric cleanly distinguishes between "portal ok, but no new @@ -52,6 +55,7 @@ All metrics carry the `vin` label; all names carry the configurable prefix | `uptime_seconds` | Gauge | Uptime of the exporter process | | `last_successful_scrape_timestamp_seconds` | Gauge | Last scrape with **new** data | | `health` | Gauge (bool) | `0` initially and after a failure, `1` once new data has been received at least once | +| `http_requests_total` | Counter | Total number of HTTP requests made to the portal (login + API calls) | **Note on enum metrics:** The state space (e.g. possible values of `drivingMode` or `chargingStatus.currentChargeState`) is compiled @@ -98,7 +102,7 @@ sudo systemctl status vw-eu-data-act-exporter | `vw_account` | E-mail + password of the VW account under which **all** configured VINs are registered | | `vins` | List of vehicle VINs to scrape | | `scrape_interval_minutes` | Poll interval per vehicle. The portal only delivers new datasets for "continuous" data requests roughly every ~15 min anyway - a shorter interval will not yield fresher data, but will increase the login frequency against the account | -| `persist_raw_json` | `False` by default. When set to `True`, the raw JSON of every downloaded dataset is written to disk (one file per VIN per scrape) - useful while tuning the field mapping in `vw-eu-data-act-exporter.py`. Not needed for normal operation | +| `persist_raw_json` | `False` by default. When set to `True`, the JSON extracted from every downloaded dataset is written to disk (one file per VIN per scrape), including the JSON extracted from `_no_content_found.zip` placeholders when they carry one. ZIP files are never written to disk, only the extracted JSON - useful while tuning the field mapping in `vw-eu-data-act-exporter.py`. Not needed for normal operation | | `persist_raw_json_dir` | Directory the raw JSON files are written to when `persist_raw_json` is enabled (default: `./raw_json`, relative to the working directory) | Prerequisite in the portal (one-time, in a browser): connect vehicle -> diff --git a/config.py b/config.py index 3c9df15..ef11a60 100644 --- a/config.py +++ b/config.py @@ -41,10 +41,12 @@ user_agents = [ scrape_interval_minutes = 15 # --- Raw JSON persistence --------------------------------------------------- -# Optional: write every downloaded raw JSON dataset to disk (one file per VIN -# per scrape), e.g. to inspect the portal's payload while improving the field -# mapping in vw-eu-data-act-exporter.py. Disabled by default, since the -# exporter is designed to be stateless (see README.md). +# Optional: write the JSON extracted from every downloaded dataset to disk +# (one file per VIN per scrape), e.g. to inspect the portal's payload while +# improving the field mapping in vw-eu-data-act-exporter.py. Also extracts +# and persists the JSON from "_no_content_found.zip" placeholders, if they +# carry one. ZIP files themselves are never written to disk. Disabled by +# default, since the exporter is designed to be stateless (see README.md). persist_raw_json = False persist_raw_json_dir = "./raw_json" diff --git a/vw-eu-data-act-exporter.py b/vw-eu-data-act-exporter.py index fae7b3a..545a27f 100644 --- a/vw-eu-data-act-exporter.py +++ b/vw-eu-data-act-exporter.py @@ -12,9 +12,11 @@ processes). No raw data or ZIP files are written to disk - all state lives purely in process memory and is lost on restart (health/last-scrape metrics then start -at 0 again). Optionally, the raw JSON of every downloaded dataset can be -persisted to disk via ``config.persist_raw_json`` to help refine the field -mapping below. +at 0 again). Optionally, the JSON extracted from every downloaded dataset - +including "_no_content_found.zip" placeholders, if they carry a JSON payload +- can be persisted to disk via ``config.persist_raw_json`` to help refine the +field mapping below. ZIP files themselves are never written to disk, only +the JSON extracted from them. """ from __future__ import annotations @@ -34,7 +36,7 @@ from urllib.parse import urlencode, urljoin, urlparse import requests from bs4 import BeautifulSoup -from prometheus_client import Enum, Gauge, start_http_server +from prometheus_client import Counter, Enum, Gauge, start_http_server from prometheus_client.core import GaugeMetricFamily from prometheus_client.registry import REGISTRY @@ -304,14 +306,16 @@ def flatten_dataset(record: dict) -> dict[str, str]: return fields -def persist_raw_dataset(vin: str, record: dict) -> None: - """Writes the raw dataset JSON to disk, so it can be inspected later to - refine the field mapping below. Only called when - ``config.persist_raw_json`` is enabled.""" +def persist_raw_dataset(vin: str, record: dict, *, suffix: str = "") -> None: + """Writes the JSON extracted from a downloaded dataset to disk, so it + can be inspected later to refine the field mapping below. Only the + extracted JSON is ever written - never the ZIP it came from. Only + called when ``config.persist_raw_json`` is enabled.""" out_dir = Path(config.persist_raw_json_dir) out_dir.mkdir(parents=True, exist_ok=True) timestamp = datetime.utcnow().strftime("%Y%m%d%H%M%S") - dest = out_dir / f"{vin}_{timestamp}.json" + tag = f"_{suffix}" if suffix else "" + dest = out_dir / f"{vin}{tag}_{timestamp}.json" dest.write_text(json.dumps(record, indent=2, ensure_ascii=False), encoding="utf-8") log.debug("Persisted raw dataset for VIN %s to %s", vin, dest) @@ -324,6 +328,9 @@ def fetch_latest_dataset_fields(vin: str) -> dict[str, str] | None: session = requests.Session() session.headers.update({"User-Agent": random.choice(config.user_agents)}) + session.hooks["response"].append( + lambda resp, *args, **kwargs: c_http_requests_total.labels(vin=vin).inc() + ) login(session, config.vw_account["email"], config.vw_account["password"]) @@ -332,12 +339,38 @@ def fetch_latest_dataset_fields(vin: str) -> dict[str, str] | None: if not identifier: raise RuntimeError(f"No data-request identifier in metadata for VIN {vin}.") - datasets = list_datasets(session, vin, identifier) - datasets = [ + all_datasets = list_datasets(session, vin, identifier) + no_content_datasets = [ d - for d in datasets - if not (d.get("name") or d.get("fileName") or "").endswith(NO_CONTENT_SUFFIX) + for d in all_datasets + if (d.get("name") or d.get("fileName") or "").endswith(NO_CONTENT_SUFFIX) ] + datasets = [d for d in all_datasets if d not in no_content_datasets] + + if config.persist_raw_json and no_content_datasets: + nc_latest = pick_latest_dataset(no_content_datasets) + nc_name = nc_latest.get("name") or nc_latest.get("fileName") or str(nc_latest) + try: + nc_raw = download_zip(session, vin, identifier, nc_name) + nc_record = unzip_json(nc_raw, nc_name) + except RuntimeError as exc: + log.warning( + "Could not download no-content placeholder for VIN %s: %s", vin, exc + ) + except ValueError: + log.debug( + "No-content placeholder %s for VIN %s has no JSON payload - nothing to persist.", + nc_name, + vin, + ) + else: + try: + persist_raw_dataset(vin, nc_record, suffix="no_content") + except OSError as exc: + log.warning( + "Could not persist no-content dataset for VIN %s: %s", vin, exc + ) + if not datasets: return None @@ -448,6 +481,11 @@ g_health = Gauge( "0 initially and after every failure (not a failure: portal ok but no new data)", ["vin"], ) +c_http_requests_total = Counter( + f"{P}http_requests_total", + "Total number of HTTP requests made to the EU Data Act portal (login + API calls)", + ["vin"], +) # Enum metrics. The state space is best-effort (compiled from the sample # dataset and comparable projects) plus an "UNKNOWN" fallback for unknown raw