"""The Parquet index of a dataset or package: one row per complete radio map.
The index is derived from the scene and radio map records and is rebuilt from them; it is
never read by generation or resume. Its footer metadata records the dataset format and the
manifest's ``index_revision`` it was built for, so a reader can refuse an index that a later
run has made stale.
"""
from __future__ import annotations
import json
from collections.abc import Iterable, Mapping
from pathlib import Path
from typing import TYPE_CHECKING, Any
import pyarrow as pa
import pyarrow.parquet as pq
from openworld_radio_twin import __version__
from openworld_radio_twin.conventions import DATASET_SCHEMA_VERSION
from openworld_radio_twin.scene_format import SCENE_INFO_PATH, SCENE_RADIO_MAPS_DIRECTORY
if TYPE_CHECKING:
from openworld_radio_twin.batch import PlannedScene
INDEX_PATH = "index.parquet"
_STRING = pa.string()
_DOUBLES = pa.list_(pa.float64())
def _field(name: str, type_: pa.DataType, nullable: bool = False) -> pa.Field:
return pa.field(name, type_, nullable=nullable)
INDEX_SCHEMA = pa.schema(
[
_field("radio_map_id", _STRING),
_field("scene", _STRING),
_field("name", _STRING),
_field("site", _STRING, nullable=True),
_field("label", _STRING),
_field("split", _STRING, nullable=True),
_field("archive", _STRING, nullable=True),
_field("latitude", pa.float64()),
_field("longitude", pa.float64()),
_field("radius_m", pa.float64()),
_field("building_source", _STRING),
_field("terrain_provider", _STRING),
_field("material_profile", _STRING),
_field("engine", _STRING),
_field("seed", pa.int64()),
_field("samples_per_tx", pa.int64()),
_field("max_depth", pa.int32()),
_field("association_metric", _STRING),
_field("frequency_hz", pa.float64()),
_field("bandwidth_hz", pa.float64()),
_field("noise_power_w", pa.float64()),
_field("n_tx", pa.int32()),
_field("rows", pa.int32()),
_field("columns", pa.int32()),
_field("resolution_m", pa.float64()),
_field("row_step_m", pa.float64()),
_field("column_step_m", pa.float64()),
_field("receiver_height_m", pa.float64()),
_field("tx_east_m", _DOUBLES),
_field("tx_north_m", _DOUBLES),
_field("tx_up_m", _DOUBLES),
_field("tx_height_agl_m", _DOUBLES),
_field("tx_power_w", _DOUBLES),
_field("tx_power_dbm", _DOUBLES),
_field("tx_azimuth_deg", _DOUBLES),
_field("tx_downtilt_deg", _DOUBLES),
_field("antenna_pattern", pa.list_(_STRING)),
_field("scene_record", _STRING),
_field("record", _STRING),
_field("path_gain", _STRING),
_field("measurement_surface", _STRING),
_field("building_rasters", _STRING),
]
)
def _row(
scene: PlannedScene,
scene_record: dict[str, Any],
record: dict[str, Any],
split: str | None,
archive: str | None,
) -> dict[str, Any]:
specification = scene.specification
request = record["request"]
radio = record["radio"]
grid = record["array_grid_mapping"]
transmitters = request["transmitters"]
radio_map_id = f"{scene.name}/{record['name']}"
frequencies = {transmitter["frequency_ghz"] for transmitter in transmitters}
if len(frequencies) != 1:
raise ValueError(
f"Radio map {radio_map_id} has {len(frequencies)} carrier frequencies; the index "
"holds one per radio map"
)
n_tx, rows, columns = record["array_contract"]["shape"]
east, north, up = zip(*radio["tx_positions_enu_m"], strict=True)
directory = f"{scene.name}/{SCENE_RADIO_MAPS_DIRECTORY.as_posix()}/{record['name']}"
return {
"radio_map_id": radio_map_id,
"scene": scene.name,
"name": record["name"],
"site": specification.site,
"label": scene.label,
"split": split,
"archive": archive,
"latitude": specification.latitude,
"longitude": specification.longitude,
"radius_m": specification.radius_m,
# The key is absent when building inputs are disabled.
"building_source": record["data_provenance"].get("building_source_resolved", "none"),
"terrain_provider": scene_record["provenance"]["terrain_provider"],
"material_profile": specification.material_profile,
"engine": request["engine"],
"seed": record["seed"],
"samples_per_tx": request["samples_per_tx"],
"max_depth": request["max_depth"],
"association_metric": radio["association_metric"],
# Whole hertz, so equality filters match; the plans round the frequency in GHz.
"frequency_hz": float(round(frequencies.pop() * 1e9)),
"bandwidth_hz": float(round(request["bandwidth_mhz"] * 1e6)),
"noise_power_w": radio["noise_power_w"],
"n_tx": n_tx,
"rows": rows,
"columns": columns,
"resolution_m": request["resolution_m"],
"row_step_m": grid["row_step_m"],
"column_step_m": grid["column_step_m"],
"receiver_height_m": request["receiver_height_m"],
"tx_east_m": list(east),
"tx_north_m": list(north),
"tx_up_m": list(up),
"tx_height_agl_m": [item["position"]["altitude_m"] for item in transmitters],
"tx_power_w": radio["tx_power_w"],
"tx_power_dbm": [item["power_dbm"] for item in transmitters],
"tx_azimuth_deg": [item["azimuth_deg"] for item in transmitters],
"tx_downtilt_deg": [item["downtilt_deg"] for item in transmitters],
"antenna_pattern": [item["antenna_pattern"] for item in transmitters],
"scene_record": f"{scene.name}/{SCENE_INFO_PATH.as_posix()}",
"record": f"{directory}/radio_map_info.json",
"path_gain": f"{directory}/{record['array_contract']['path_gain']['path']}",
"measurement_surface": f"{scene.name}/{radio['measurement_surface']}",
"building_rasters": f"{scene.name}/{radio['building_rasters']}",
}
[docs]
def index_table(
dataset_directory: Path,
scenes: Iterable[tuple[PlannedScene, list[dict[str, Any]]]],
revision: int,
splits: Mapping[str, str] | None = None,
archives: Mapping[str, str] | None = None,
source_manifest_sha256: str | None = None,
) -> pa.Table:
"""The index of the given scenes and complete radio map records, in their order.
``splits`` and ``archives`` map scene names to their split and archive path in a
package; outside a package both columns are null. A package also records the SHA-256
checksum of the dataset manifest it snapshots, ``source_manifest_sha256``, in the
footer. Scene-level values not in the plan come from each scene's ``scene_info.json``.
"""
rows = []
for scene, records in scenes:
if not records:
continue
scene_info = dataset_directory / scene.name / SCENE_INFO_PATH
try:
scene_record = json.loads(scene_info.read_text(encoding="utf-8"))
except FileNotFoundError as exc:
raise ValueError(
f"Scene {scene.name} has complete radio maps but no {SCENE_INFO_PATH}"
) from exc
split = splits[scene.name] if splits is not None else None
archive = archives[scene.name] if archives is not None else None
rows.extend(_row(scene, scene_record, record, split, archive) for record in records)
table = pa.Table.from_pylist(rows, schema=INDEX_SCHEMA)
metadata = {
"owrt.schema_version": str(DATASET_SCHEMA_VERSION),
"owrt.index_revision": str(revision),
"owrt.openworld_radio_twin_version": __version__,
"owrt.pyarrow_version": pa.__version__,
}
if source_manifest_sha256 is not None:
metadata["owrt.source_manifest_sha256"] = source_manifest_sha256
return table.replace_schema_metadata(metadata)
[docs]
def write_index(directory: Path, table: pa.Table) -> Path:
"""Write ``index.parquet`` in ``directory`` through a temporary file and return its path."""
path = directory / INDEX_PATH
temporary = path.with_suffix(f"{path.suffix}.tmp")
try:
pq.write_table(table, temporary, compression="snappy")
temporary.replace(path)
finally:
temporary.unlink(missing_ok=True)
return path
[docs]
def read_index(root: str | Path) -> pa.Table:
"""Read the index of a generated dataset or of a package, refusing a stale one.
The index is stale when its recorded ``index_revision`` is not the manifest's: a run is
writing the dataset, or one stopped before it could rebuild the index. Once no run is
active, ``owrt dataset index DATASET`` rebuilds the index of a generated dataset;
packaging writes the index of a package.
"""
directory = Path(root).expanduser()
path = directory / INDEX_PATH
manifest_path = directory / "dataset_manifest.json"
if not manifest_path.is_file():
raise FileNotFoundError(
f"No dataset_manifest.json in {directory}: not a generated dataset or package"
)
if (directory / "splits.json").is_file():
rebuild = "restore it from the release, or package the dataset again"
stale = "the package's files do not belong together"
else:
rebuild = f"once no run is active, rebuild it with 'owrt dataset index {directory}'"
stale = "a run is in progress, or one stopped before it rebuilt the index"
if not path.is_file():
raise FileNotFoundError(f"No {INDEX_PATH} in {directory}; {rebuild}")
# The index before the manifest: the manifest's revision only increases and an index is
# written only for the current revision, so a match proves the table read is current.
table = pq.read_table(path)
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
metadata = {
key.decode(): value.decode() for key, value in (table.schema.metadata or {}).items()
}
if metadata.get("owrt.schema_version") != str(DATASET_SCHEMA_VERSION):
raise ValueError(
f"{path} has dataset format {metadata.get('owrt.schema_version')!r}; this version "
f"reads format {DATASET_SCHEMA_VERSION}"
)
if metadata.get("owrt.index_revision") != str(manifest.get("index_revision")):
raise ValueError(
f"{path} was built for index revision {metadata.get('owrt.index_revision')!r}, "
f"but the manifest is at revision {manifest.get('index_revision')!r}: {stale}; "
f"{rebuild}"
)
return table