export empty jsons too
This commit is contained in:
@@ -15,8 +15,11 @@ a shared `/metrics` endpoint (no Docker, plain systemd deployment).
|
||||
scrape-interval timing in the background
|
||||
- **No persistent state by default**: no ZIPs, no JSON, no values are
|
||||
written to disk - everything lives in process memory and is empty again
|
||||
after a restart. Optionally, raw JSON datasets can be persisted to disk via
|
||||
`config.persist_raw_json` (see [Configuration](#configuration-configpy))
|
||||
after a restart. Optionally, the JSON extracted from every downloaded
|
||||
dataset - including "_no_content_found.zip" placeholders, if they carry a
|
||||
JSON payload - can be persisted to disk via `config.persist_raw_json`
|
||||
(see [Configuration](#configuration-configpy)). ZIP files themselves are
|
||||
never written to disk
|
||||
- Fields are only updated when they are present in the current dataset - if
|
||||
a field is missing, the last known value is kept
|
||||
- The `health` metric cleanly distinguishes between "portal ok, but no new
|
||||
@@ -52,6 +55,7 @@ All metrics carry the `vin` label; all names carry the configurable prefix
|
||||
| `uptime_seconds` | Gauge | Uptime of the exporter process |
|
||||
| `last_successful_scrape_timestamp_seconds` | Gauge | Last scrape with **new** data |
|
||||
| `health` | Gauge (bool) | `0` initially and after a failure, `1` once new data has been received at least once |
|
||||
| `http_requests_total` | Counter | Total number of HTTP requests made to the portal (login + API calls) |
|
||||
|
||||
**Note on enum metrics:** The state space (e.g. possible values of
|
||||
`drivingMode` or `chargingStatus.currentChargeState`) is compiled
|
||||
@@ -98,7 +102,7 @@ sudo systemctl status vw-eu-data-act-exporter
|
||||
| `vw_account` | E-mail + password of the VW account under which **all** configured VINs are registered |
|
||||
| `vins` | List of vehicle VINs to scrape |
|
||||
| `scrape_interval_minutes` | Poll interval per vehicle. The portal only delivers new datasets for "continuous" data requests roughly every ~15 min anyway - a shorter interval will not yield fresher data, but will increase the login frequency against the account |
|
||||
| `persist_raw_json` | `False` by default. When set to `True`, the raw JSON of every downloaded dataset is written to disk (one file per VIN per scrape) - useful while tuning the field mapping in `vw-eu-data-act-exporter.py`. Not needed for normal operation |
|
||||
| `persist_raw_json` | `False` by default. When set to `True`, the JSON extracted from every downloaded dataset is written to disk (one file per VIN per scrape), including the JSON extracted from `_no_content_found.zip` placeholders when they carry one. ZIP files are never written to disk, only the extracted JSON - useful while tuning the field mapping in `vw-eu-data-act-exporter.py`. Not needed for normal operation |
|
||||
| `persist_raw_json_dir` | Directory the raw JSON files are written to when `persist_raw_json` is enabled (default: `./raw_json`, relative to the working directory) |
|
||||
|
||||
Prerequisite in the portal (one-time, in a browser): connect vehicle ->
|
||||
|
||||
@@ -41,10 +41,12 @@ user_agents = [
|
||||
scrape_interval_minutes = 15
|
||||
|
||||
# --- Raw JSON persistence ---------------------------------------------------
|
||||
# Optional: write every downloaded raw JSON dataset to disk (one file per VIN
|
||||
# per scrape), e.g. to inspect the portal's payload while improving the field
|
||||
# mapping in vw-eu-data-act-exporter.py. Disabled by default, since the
|
||||
# exporter is designed to be stateless (see README.md).
|
||||
# Optional: write the JSON extracted from every downloaded dataset to disk
|
||||
# (one file per VIN per scrape), e.g. to inspect the portal's payload while
|
||||
# improving the field mapping in vw-eu-data-act-exporter.py. Also extracts
|
||||
# and persists the JSON from "_no_content_found.zip" placeholders, if they
|
||||
# carry one. ZIP files themselves are never written to disk. Disabled by
|
||||
# default, since the exporter is designed to be stateless (see README.md).
|
||||
persist_raw_json = False
|
||||
persist_raw_json_dir = "./raw_json"
|
||||
|
||||
|
||||
+51
-13
@@ -12,9 +12,11 @@ processes).
|
||||
|
||||
No raw data or ZIP files are written to disk - all state lives purely in
|
||||
process memory and is lost on restart (health/last-scrape metrics then start
|
||||
at 0 again). Optionally, the raw JSON of every downloaded dataset can be
|
||||
persisted to disk via ``config.persist_raw_json`` to help refine the field
|
||||
mapping below.
|
||||
at 0 again). Optionally, the JSON extracted from every downloaded dataset -
|
||||
including "_no_content_found.zip" placeholders, if they carry a JSON payload
|
||||
- can be persisted to disk via ``config.persist_raw_json`` to help refine the
|
||||
field mapping below. ZIP files themselves are never written to disk, only
|
||||
the JSON extracted from them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -34,7 +36,7 @@ from urllib.parse import urlencode, urljoin, urlparse
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from prometheus_client import Enum, Gauge, start_http_server
|
||||
from prometheus_client import Counter, Enum, Gauge, start_http_server
|
||||
from prometheus_client.core import GaugeMetricFamily
|
||||
from prometheus_client.registry import REGISTRY
|
||||
|
||||
@@ -304,14 +306,16 @@ def flatten_dataset(record: dict) -> dict[str, str]:
|
||||
return fields
|
||||
|
||||
|
||||
def persist_raw_dataset(vin: str, record: dict) -> None:
|
||||
"""Writes the raw dataset JSON to disk, so it can be inspected later to
|
||||
refine the field mapping below. Only called when
|
||||
``config.persist_raw_json`` is enabled."""
|
||||
def persist_raw_dataset(vin: str, record: dict, *, suffix: str = "") -> None:
|
||||
"""Writes the JSON extracted from a downloaded dataset to disk, so it
|
||||
can be inspected later to refine the field mapping below. Only the
|
||||
extracted JSON is ever written - never the ZIP it came from. Only
|
||||
called when ``config.persist_raw_json`` is enabled."""
|
||||
out_dir = Path(config.persist_raw_json_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
timestamp = datetime.utcnow().strftime("%Y%m%d%H%M%S")
|
||||
dest = out_dir / f"{vin}_{timestamp}.json"
|
||||
tag = f"_{suffix}" if suffix else ""
|
||||
dest = out_dir / f"{vin}{tag}_{timestamp}.json"
|
||||
dest.write_text(json.dumps(record, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
log.debug("Persisted raw dataset for VIN %s to %s", vin, dest)
|
||||
|
||||
@@ -324,6 +328,9 @@ def fetch_latest_dataset_fields(vin: str) -> dict[str, str] | None:
|
||||
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": random.choice(config.user_agents)})
|
||||
session.hooks["response"].append(
|
||||
lambda resp, *args, **kwargs: c_http_requests_total.labels(vin=vin).inc()
|
||||
)
|
||||
|
||||
login(session, config.vw_account["email"], config.vw_account["password"])
|
||||
|
||||
@@ -332,12 +339,38 @@ def fetch_latest_dataset_fields(vin: str) -> dict[str, str] | None:
|
||||
if not identifier:
|
||||
raise RuntimeError(f"No data-request identifier in metadata for VIN {vin}.")
|
||||
|
||||
datasets = list_datasets(session, vin, identifier)
|
||||
datasets = [
|
||||
all_datasets = list_datasets(session, vin, identifier)
|
||||
no_content_datasets = [
|
||||
d
|
||||
for d in datasets
|
||||
if not (d.get("name") or d.get("fileName") or "").endswith(NO_CONTENT_SUFFIX)
|
||||
for d in all_datasets
|
||||
if (d.get("name") or d.get("fileName") or "").endswith(NO_CONTENT_SUFFIX)
|
||||
]
|
||||
datasets = [d for d in all_datasets if d not in no_content_datasets]
|
||||
|
||||
if config.persist_raw_json and no_content_datasets:
|
||||
nc_latest = pick_latest_dataset(no_content_datasets)
|
||||
nc_name = nc_latest.get("name") or nc_latest.get("fileName") or str(nc_latest)
|
||||
try:
|
||||
nc_raw = download_zip(session, vin, identifier, nc_name)
|
||||
nc_record = unzip_json(nc_raw, nc_name)
|
||||
except RuntimeError as exc:
|
||||
log.warning(
|
||||
"Could not download no-content placeholder for VIN %s: %s", vin, exc
|
||||
)
|
||||
except ValueError:
|
||||
log.debug(
|
||||
"No-content placeholder %s for VIN %s has no JSON payload - nothing to persist.",
|
||||
nc_name,
|
||||
vin,
|
||||
)
|
||||
else:
|
||||
try:
|
||||
persist_raw_dataset(vin, nc_record, suffix="no_content")
|
||||
except OSError as exc:
|
||||
log.warning(
|
||||
"Could not persist no-content dataset for VIN %s: %s", vin, exc
|
||||
)
|
||||
|
||||
if not datasets:
|
||||
return None
|
||||
|
||||
@@ -448,6 +481,11 @@ g_health = Gauge(
|
||||
"0 initially and after every failure (not a failure: portal ok but no new data)",
|
||||
["vin"],
|
||||
)
|
||||
c_http_requests_total = Counter(
|
||||
f"{P}http_requests_total",
|
||||
"Total number of HTTP requests made to the EU Data Act portal (login + API calls)",
|
||||
["vin"],
|
||||
)
|
||||
|
||||
# Enum metrics. The state space is best-effort (compiled from the sample
|
||||
# dataset and comparable projects) plus an "UNKNOWN" fallback for unknown raw
|
||||
|
||||
Reference in New Issue
Block a user