export empty jsons too

README.md

@@ -15,8 +15,11 @@ a shared `/metrics` endpoint (no Docker, plain systemd deployment).
   scrape-interval timing in the background
 - **No persistent state by default**: no ZIPs, no JSON, no values are
   written to disk - everything lives in process memory and is empty again
-  after a restart. Optionally, raw JSON datasets can be persisted to disk via
+  after a restart. Optionally, the JSON extracted from every downloaded
-  `config.persist_raw_json` (see [Configuration](#configuration-configpy))
+  dataset - including "_no_content_found.zip" placeholders, if they carry a
+  JSON payload - can be persisted to disk via `config.persist_raw_json`
+  (see [Configuration](#configuration-configpy)). ZIP files themselves are
+  never written to disk
 - Fields are only updated when they are present in the current dataset - if
   a field is missing, the last known value is kept
 - The `health` metric cleanly distinguishes between "portal ok, but no new
@@ -52,6 +55,7 @@ All metrics carry the `vin` label; all names carry the configurable prefix
 | `uptime_seconds` | Gauge | Uptime of the exporter process |
 | `last_successful_scrape_timestamp_seconds` | Gauge | Last scrape with **new** data |
 | `health` | Gauge (bool) | `0` initially and after a failure, `1` once new data has been received at least once |
+| `http_requests_total` | Counter | Total number of HTTP requests made to the portal (login + API calls) |
 
 **Note on enum metrics:** The state space (e.g. possible values of
 `drivingMode` or `chargingStatus.currentChargeState`) is compiled
@@ -98,7 +102,7 @@ sudo systemctl status vw-eu-data-act-exporter
 | `vw_account` | E-mail + password of the VW account under which **all** configured VINs are registered |
 | `vins` | List of vehicle VINs to scrape |
 | `scrape_interval_minutes` | Poll interval per vehicle. The portal only delivers new datasets for "continuous" data requests roughly every ~15 min anyway - a shorter interval will not yield fresher data, but will increase the login frequency against the account |
-| `persist_raw_json` | `False` by default. When set to `True`, the raw JSON of every downloaded dataset is written to disk (one file per VIN per scrape) - useful while tuning the field mapping in `vw-eu-data-act-exporter.py`. Not needed for normal operation |
+| `persist_raw_json` | `False` by default. When set to `True`, the JSON extracted from every downloaded dataset is written to disk (one file per VIN per scrape), including the JSON extracted from `_no_content_found.zip` placeholders when they carry one. ZIP files are never written to disk, only the extracted JSON - useful while tuning the field mapping in `vw-eu-data-act-exporter.py`. Not needed for normal operation |
 | `persist_raw_json_dir` | Directory the raw JSON files are written to when `persist_raw_json` is enabled (default: `./raw_json`, relative to the working directory) |
 
 Prerequisite in the portal (one-time, in a browser): connect vehicle ->

config.py

@@ -41,10 +41,12 @@ user_agents = [
 scrape_interval_minutes = 15
 
 # --- Raw JSON persistence ---------------------------------------------------
-# Optional: write every downloaded raw JSON dataset to disk (one file per VIN
+# Optional: write the JSON extracted from every downloaded dataset to disk
-# per scrape), e.g. to inspect the portal's payload while improving the field
+# (one file per VIN per scrape), e.g. to inspect the portal's payload while
-# mapping in vw-eu-data-act-exporter.py. Disabled by default, since the
+# improving the field mapping in vw-eu-data-act-exporter.py. Also extracts
-# exporter is designed to be stateless (see README.md).
+# and persists the JSON from "_no_content_found.zip" placeholders, if they
+# carry one. ZIP files themselves are never written to disk. Disabled by
+# default, since the exporter is designed to be stateless (see README.md).
 persist_raw_json = False
 persist_raw_json_dir = "./raw_json"
 

vw-eu-data-act-exporter.py

@@ -12,9 +12,11 @@ processes).
 
 No raw data or ZIP files are written to disk - all state lives purely in
 process memory and is lost on restart (health/last-scrape metrics then start
-at 0 again). Optionally, the raw JSON of every downloaded dataset can be
+at 0 again). Optionally, the JSON extracted from every downloaded dataset -
-persisted to disk via ``config.persist_raw_json`` to help refine the field
+including "_no_content_found.zip" placeholders, if they carry a JSON payload
-mapping below.
+- can be persisted to disk via ``config.persist_raw_json`` to help refine the
+field mapping below. ZIP files themselves are never written to disk, only
+the JSON extracted from them.
 """
 
 from __future__ import annotations
@@ -34,7 +36,7 @@ from urllib.parse import urlencode, urljoin, urlparse
 
 import requests
 from bs4 import BeautifulSoup
-from prometheus_client import Enum, Gauge, start_http_server
+from prometheus_client import Counter, Enum, Gauge, start_http_server
 from prometheus_client.core import GaugeMetricFamily
 from prometheus_client.registry import REGISTRY
 
@@ -304,14 +306,16 @@ def flatten_dataset(record: dict) -> dict[str, str]:
     return fields
 
 
-def persist_raw_dataset(vin: str, record: dict) -> None:
+def persist_raw_dataset(vin: str, record: dict, *, suffix: str = "") -> None:
-    """Writes the raw dataset JSON to disk, so it can be inspected later to
+    """Writes the JSON extracted from a downloaded dataset to disk, so it
-    refine the field mapping below. Only called when
+    can be inspected later to refine the field mapping below. Only the
-    ``config.persist_raw_json`` is enabled."""
+    extracted JSON is ever written - never the ZIP it came from. Only
+    called when ``config.persist_raw_json`` is enabled."""
     out_dir = Path(config.persist_raw_json_dir)
     out_dir.mkdir(parents=True, exist_ok=True)
     timestamp = datetime.utcnow().strftime("%Y%m%d%H%M%S")
-    dest = out_dir / f"{vin}_{timestamp}.json"
+    tag = f"_{suffix}" if suffix else ""
+    dest = out_dir / f"{vin}{tag}_{timestamp}.json"
     dest.write_text(json.dumps(record, indent=2, ensure_ascii=False), encoding="utf-8")
     log.debug("Persisted raw dataset for VIN %s to %s", vin, dest)
 
@@ -324,6 +328,9 @@ def fetch_latest_dataset_fields(vin: str) -> dict[str, str] | None:
 
     session = requests.Session()
     session.headers.update({"User-Agent": random.choice(config.user_agents)})
+    session.hooks["response"].append(
+        lambda resp, *args, **kwargs: c_http_requests_total.labels(vin=vin).inc()
+    )
 
     login(session, config.vw_account["email"], config.vw_account["password"])
 
@@ -332,12 +339,38 @@ def fetch_latest_dataset_fields(vin: str) -> dict[str, str] | None:
     if not identifier:
         raise RuntimeError(f"No data-request identifier in metadata for VIN {vin}.")
 
-    datasets = list_datasets(session, vin, identifier)
+    all_datasets = list_datasets(session, vin, identifier)
-    datasets = [
+    no_content_datasets = [
         d
-        for d in datasets
+        for d in all_datasets
-        if not (d.get("name") or d.get("fileName") or "").endswith(NO_CONTENT_SUFFIX)
+        if (d.get("name") or d.get("fileName") or "").endswith(NO_CONTENT_SUFFIX)
     ]
+    datasets = [d for d in all_datasets if d not in no_content_datasets]
+
+    if config.persist_raw_json and no_content_datasets:
+        nc_latest = pick_latest_dataset(no_content_datasets)
+        nc_name = nc_latest.get("name") or nc_latest.get("fileName") or str(nc_latest)
+        try:
+            nc_raw = download_zip(session, vin, identifier, nc_name)
+            nc_record = unzip_json(nc_raw, nc_name)
+        except RuntimeError as exc:
+            log.warning(
+                "Could not download no-content placeholder for VIN %s: %s", vin, exc
+            )
+        except ValueError:
+            log.debug(
+                "No-content placeholder %s for VIN %s has no JSON payload - nothing to persist.",
+                nc_name,
+                vin,
+            )
+        else:
+            try:
+                persist_raw_dataset(vin, nc_record, suffix="no_content")
+            except OSError as exc:
+                log.warning(
+                    "Could not persist no-content dataset for VIN %s: %s", vin, exc
+                )
+
     if not datasets:
         return None
 
@@ -448,6 +481,11 @@ g_health = Gauge(
     "0 initially and after every failure (not a failure: portal ok but no new data)",
     ["vin"],
 )
+c_http_requests_total = Counter(
+    f"{P}http_requests_total",
+    "Total number of HTTP requests made to the EU Data Act portal (login + API calls)",
+    ["vin"],
+)
 
 # Enum metrics. The state space is best-effort (compiled from the sample
 # dataset and comparable projects) plus an "UNKNOWN" fallback for unknown raw