Source code for openworld_radio_twin.radio_map_index

"""The Parquet index of a dataset or package: one row per complete radio map.

The index is derived from the scene and radio map records and is rebuilt from them; it is
never read by generation or resume. Its footer metadata records the dataset format and the
manifest's ``index_revision`` it was built for, so a reader can refuse an index that a later
run has made stale.
"""

from __future__ import annotations

import json
from collections.abc import Iterable, Mapping
from pathlib import Path
from typing import TYPE_CHECKING, Any

import pyarrow as pa
import pyarrow.parquet as pq

from openworld_radio_twin import __version__
from openworld_radio_twin.conventions import DATASET_SCHEMA_VERSION
from openworld_radio_twin.scene_format import SCENE_INFO_PATH, SCENE_RADIO_MAPS_DIRECTORY

if TYPE_CHECKING:
    from openworld_radio_twin.batch import PlannedScene

INDEX_PATH = "index.parquet"

_STRING = pa.string()
_DOUBLES = pa.list_(pa.float64())


def _field(name: str, type_: pa.DataType, nullable: bool = False) -> pa.Field:
    return pa.field(name, type_, nullable=nullable)


INDEX_SCHEMA = pa.schema(
    [
        _field("radio_map_id", _STRING),
        _field("scene", _STRING),
        _field("name", _STRING),
        _field("site", _STRING, nullable=True),
        _field("label", _STRING),
        _field("split", _STRING, nullable=True),
        _field("archive", _STRING, nullable=True),
        _field("latitude", pa.float64()),
        _field("longitude", pa.float64()),
        _field("radius_m", pa.float64()),
        _field("building_source", _STRING),
        _field("terrain_provider", _STRING),
        _field("material_profile", _STRING),
        _field("engine", _STRING),
        _field("seed", pa.int64()),
        _field("samples_per_tx", pa.int64()),
        _field("max_depth", pa.int32()),
        _field("association_metric", _STRING),
        _field("frequency_hz", pa.float64()),
        _field("bandwidth_hz", pa.float64()),
        _field("noise_power_w", pa.float64()),
        _field("n_tx", pa.int32()),
        _field("rows", pa.int32()),
        _field("columns", pa.int32()),
        _field("resolution_m", pa.float64()),
        _field("row_step_m", pa.float64()),
        _field("column_step_m", pa.float64()),
        _field("receiver_height_m", pa.float64()),
        _field("tx_east_m", _DOUBLES),
        _field("tx_north_m", _DOUBLES),
        _field("tx_up_m", _DOUBLES),
        _field("tx_height_agl_m", _DOUBLES),
        _field("tx_power_w", _DOUBLES),
        _field("tx_power_dbm", _DOUBLES),
        _field("tx_azimuth_deg", _DOUBLES),
        _field("tx_downtilt_deg", _DOUBLES),
        _field("antenna_pattern", pa.list_(_STRING)),
        _field("scene_record", _STRING),
        _field("record", _STRING),
        _field("path_gain", _STRING),
        _field("measurement_surface", _STRING),
        _field("building_rasters", _STRING),
    ]
)


def _row(
    scene: PlannedScene,
    scene_record: dict[str, Any],
    record: dict[str, Any],
    split: str | None,
    archive: str | None,
) -> dict[str, Any]:
    specification = scene.specification
    request = record["request"]
    radio = record["radio"]
    grid = record["array_grid_mapping"]
    transmitters = request["transmitters"]
    radio_map_id = f"{scene.name}/{record['name']}"
    frequencies = {transmitter["frequency_ghz"] for transmitter in transmitters}
    if len(frequencies) != 1:
        raise ValueError(
            f"Radio map {radio_map_id} has {len(frequencies)} carrier frequencies; the index "
            "holds one per radio map"
        )
    n_tx, rows, columns = record["array_contract"]["shape"]
    east, north, up = zip(*radio["tx_positions_enu_m"], strict=True)
    directory = f"{scene.name}/{SCENE_RADIO_MAPS_DIRECTORY.as_posix()}/{record['name']}"
    return {
        "radio_map_id": radio_map_id,
        "scene": scene.name,
        "name": record["name"],
        "site": specification.site,
        "label": scene.label,
        "split": split,
        "archive": archive,
        "latitude": specification.latitude,
        "longitude": specification.longitude,
        "radius_m": specification.radius_m,
        # The key is absent when building inputs are disabled.
        "building_source": record["data_provenance"].get("building_source_resolved", "none"),
        "terrain_provider": scene_record["provenance"]["terrain_provider"],
        "material_profile": specification.material_profile,
        "engine": request["engine"],
        "seed": record["seed"],
        "samples_per_tx": request["samples_per_tx"],
        "max_depth": request["max_depth"],
        "association_metric": radio["association_metric"],
        # Whole hertz, so equality filters match; the plans round the frequency in GHz.
        "frequency_hz": float(round(frequencies.pop() * 1e9)),
        "bandwidth_hz": float(round(request["bandwidth_mhz"] * 1e6)),
        "noise_power_w": radio["noise_power_w"],
        "n_tx": n_tx,
        "rows": rows,
        "columns": columns,
        "resolution_m": request["resolution_m"],
        "row_step_m": grid["row_step_m"],
        "column_step_m": grid["column_step_m"],
        "receiver_height_m": request["receiver_height_m"],
        "tx_east_m": list(east),
        "tx_north_m": list(north),
        "tx_up_m": list(up),
        "tx_height_agl_m": [item["position"]["altitude_m"] for item in transmitters],
        "tx_power_w": radio["tx_power_w"],
        "tx_power_dbm": [item["power_dbm"] for item in transmitters],
        "tx_azimuth_deg": [item["azimuth_deg"] for item in transmitters],
        "tx_downtilt_deg": [item["downtilt_deg"] for item in transmitters],
        "antenna_pattern": [item["antenna_pattern"] for item in transmitters],
        "scene_record": f"{scene.name}/{SCENE_INFO_PATH.as_posix()}",
        "record": f"{directory}/radio_map_info.json",
        "path_gain": f"{directory}/{record['array_contract']['path_gain']['path']}",
        "measurement_surface": f"{scene.name}/{radio['measurement_surface']}",
        "building_rasters": f"{scene.name}/{radio['building_rasters']}",
    }


[docs] def index_table( dataset_directory: Path, scenes: Iterable[tuple[PlannedScene, list[dict[str, Any]]]], revision: int, splits: Mapping[str, str] | None = None, archives: Mapping[str, str] | None = None, source_manifest_sha256: str | None = None, ) -> pa.Table: """The index of the given scenes and complete radio map records, in their order. ``splits`` and ``archives`` map scene names to their split and archive path in a package; outside a package both columns are null. A package also records the SHA-256 checksum of the dataset manifest it snapshots, ``source_manifest_sha256``, in the footer. Scene-level values not in the plan come from each scene's ``scene_info.json``. """ rows = [] for scene, records in scenes: if not records: continue scene_info = dataset_directory / scene.name / SCENE_INFO_PATH try: scene_record = json.loads(scene_info.read_text(encoding="utf-8")) except FileNotFoundError as exc: raise ValueError( f"Scene {scene.name} has complete radio maps but no {SCENE_INFO_PATH}" ) from exc split = splits[scene.name] if splits is not None else None archive = archives[scene.name] if archives is not None else None rows.extend(_row(scene, scene_record, record, split, archive) for record in records) table = pa.Table.from_pylist(rows, schema=INDEX_SCHEMA) metadata = { "owrt.schema_version": str(DATASET_SCHEMA_VERSION), "owrt.index_revision": str(revision), "owrt.openworld_radio_twin_version": __version__, "owrt.pyarrow_version": pa.__version__, } if source_manifest_sha256 is not None: metadata["owrt.source_manifest_sha256"] = source_manifest_sha256 return table.replace_schema_metadata(metadata)
[docs] def write_index(directory: Path, table: pa.Table) -> Path: """Write ``index.parquet`` in ``directory`` through a temporary file and return its path.""" path = directory / INDEX_PATH temporary = path.with_suffix(f"{path.suffix}.tmp") try: pq.write_table(table, temporary, compression="snappy") temporary.replace(path) finally: temporary.unlink(missing_ok=True) return path
[docs] def read_index(root: str | Path) -> pa.Table: """Read the index of a generated dataset or of a package, refusing a stale one. The index is stale when its recorded ``index_revision`` is not the manifest's: a run is writing the dataset, or one stopped before it could rebuild the index. Once no run is active, ``owrt dataset index DATASET`` rebuilds the index of a generated dataset; packaging writes the index of a package. """ directory = Path(root).expanduser() path = directory / INDEX_PATH manifest_path = directory / "dataset_manifest.json" if not manifest_path.is_file(): raise FileNotFoundError( f"No dataset_manifest.json in {directory}: not a generated dataset or package" ) if (directory / "splits.json").is_file(): rebuild = "restore it from the release, or package the dataset again" stale = "the package's files do not belong together" else: rebuild = f"once no run is active, rebuild it with 'owrt dataset index {directory}'" stale = "a run is in progress, or one stopped before it rebuilt the index" if not path.is_file(): raise FileNotFoundError(f"No {INDEX_PATH} in {directory}; {rebuild}") # The index before the manifest: the manifest's revision only increases and an index is # written only for the current revision, so a match proves the table read is current. table = pq.read_table(path) manifest = json.loads(manifest_path.read_text(encoding="utf-8")) metadata = { key.decode(): value.decode() for key, value in (table.schema.metadata or {}).items() } if metadata.get("owrt.schema_version") != str(DATASET_SCHEMA_VERSION): raise ValueError( f"{path} has dataset format {metadata.get('owrt.schema_version')!r}; this version " f"reads format {DATASET_SCHEMA_VERSION}" ) if metadata.get("owrt.index_revision") != str(manifest.get("index_revision")): raise ValueError( f"{path} was built for index revision {metadata.get('owrt.index_revision')!r}, " f"but the manifest is at revision {manifest.get('index_revision')!r}: {stale}; " f"{rebuild}" ) return table