Source code for openworld_radio_twin.torch_dataset

"""A PyTorch dataset of the radio maps of a generated dataset or an extracted package.

The dataset code imports PyTorch only here; install it with the ``torch`` extra. One sample
is one whole radio map with all its transmitters, or one transmitter of a radio map, as a
dictionary of tensors on the grid of its radio arrays (rows south to north, columns west to
east). Values are in physical units; nothing is normalised.
"""

from __future__ import annotations

import json
import numbers
from collections import OrderedDict
from collections.abc import Callable, Iterable, Sequence
from pathlib import Path
from typing import Any

import numpy as np
import pyarrow as pa
import pyarrow.compute as pc
import torch
from torch.utils.data import Dataset

from openworld_radio_twin.dataset import (
    _decibels,
    _read_radio_map,
    _rss,
    load_radio_map,
    read_index,
)
from openworld_radio_twin.models import GridMapping
from openworld_radio_twin.simulation.radio_map import unpack_path_gain

__all__ = ["NO_SURFACE_CLASS", "SPATIAL_KEYS", "RadioMapDataset", "RandomCrop"]

Sample = dict[str, Any]

# Surface class of a cell whose scene has no surface class grid.
NO_SURFACE_CLASS = -1
# The sample entries on the radio grid, whose last two dimensions are rows and columns.
SPATIAL_KEYS = (
    "geometry",
    "surface_classes",
    "tx_map",
    "path_gain_db",
    "rss_dbm",
    "sinr_db",
    "coverage",
    "serving_tx",
    "indoor",
)
# The index columns of per-transmitter values, in transmitter order.
_TX_COLUMNS = (
    "tx_east_m",
    "tx_north_m",
    "tx_up_m",
    "tx_height_agl_m",
    "tx_power_dbm",
    "tx_azimuth_deg",
    "tx_downtilt_deg",
)
# Measurement surfaces whose scene inputs are kept per dataset instance, and so per DataLoader
# worker. The index is in scene order, so reading it in order loads each surface once when a
# scene's radio maps use at most two surfaces.
_SCENE_CACHE_SIZE = 2
_INDEX_COLUMNS = (
    "radio_map_id",
    "scene_record",
    "record",
    "measurement_surface",
    "building_rasters",
    "receiver_height_m",
    "rows",
    "columns",
    "frequency_hz",
    *_TX_COLUMNS,
)


[docs] class RadioMapDataset(Dataset): """The radio maps listed in the ``index.parquet`` of ``root``, one sample per radio map, or one per transmitter with ``per_transmitter``. ``root`` is a generated dataset, or a package whose scene archives are extracted into the package directory. The index is read with :func:`~openworld_radio_twin.dataset.read_index`, which refuses a stale one. ``radio_map_ids`` keeps the listed radio maps, and an id that is not in the index raises ``KeyError``. ``split`` then keeps those of one split of a package. A split that the package's ``splits.json`` does not declare, and any split of an index without splits, as in a generated dataset, raise ``ValueError``; a declared split with no radio maps gives an empty dataset. Both keep the index order. A radio map whose scene archive is not extracted raises ``FileNotFoundError``. ``index`` holds the index rows of the selected radio maps, one per radio map, in index order. ``transform`` is applied to each sample. A sample holds, with ``T`` transmitters and ``H`` by ``W`` cells: * ``geometry`` (``float32``, ``(3, H, W)``): roof height above ground in metres (0 where there is no building), building footprint mask, and ground elevation in the scene frame in metres (the measurement cell centre minus the receiver height); * ``surface_classes`` (``int64``, ``(H, W)``): the class of the terrain cell that holds the cell centre, ``NO_SURFACE_CLASS`` when the scene has no surface class grid; * ``indoor`` (``bool``, ``(H, W)``): the receiver point lies inside a building; * ``tx_map`` (``float32``, ``(T, H, W)``): 1 in the cell of each transmitter, all 0 for a transmitter off the grid; ``tx_cell`` (``int64``, ``(T, 2)``): that cell's row and column, which can lie outside the grid; * ``tx_position_m`` (``float32``, ``(T, 3)``): east, north and up in the scene frame; ``tx_height_agl_m``, ``tx_power_dbm``, ``tx_azimuth_deg`` and ``tx_downtilt_deg`` (``float32``, ``(T,)``); ``frequency_hz`` (``float64`` scalar); * ``path_gain_db``, ``rss_dbm``, ``sinr_db`` (``float32``, ``(T, H, W)``): the values of :func:`~openworld_radio_twin.dataset.load_radio_map`, 0 where ``coverage`` is false; ``coverage`` (``bool``, ``(T, H, W)``): path gain, RSS and SINR are all positive; * ``serving_tx`` (``int64``, ``(H, W)``): the serving transmitter, -1 where none serves; * ``radio_map_id`` (``str``). With ``per_transmitter``, a sample is one transmitter of a radio map: the radio maps in index order, and the transmitters of each in record order. The sample keeps the transmitter axis with ``T`` = 1, so every sample has one transmitter. It holds the entries above except ``sinr_db`` and ``serving_tx``, which depend on the other transmitters of the radio map; ``coverage`` is where path gain and RSS are positive. It adds ``transmitter`` (``int64`` scalar), the transmitter's index in the radio map. The sample reads the stored path gain of every transmitter of its radio map and unpacks only that transmitter's. """ def __init__( self, root: str | Path, split: str | None = None, radio_map_ids: Iterable[str] | None = None, transform: Callable[[Sample], Sample] | None = None, per_transmitter: bool = False, ) -> None: self.root = Path(root).expanduser() index = read_index(self.root) if split is not None and index["split"].null_count == len(index): raise ValueError(f"The index of {self.root} has no splits; only a package has") if radio_map_ids is not None: if isinstance(radio_map_ids, str): raise TypeError("radio_map_ids takes a list of ids, such as [radio_map_id]") requested = pa.array(list(radio_map_ids), type=pa.string()) unknown = pc.filter(requested, pc.invert(pc.is_in(requested, index["radio_map_id"]))) if len(unknown): raise KeyError(f"Radio maps not in the index of {self.root}: {unknown.to_pylist()}") index = index.filter(pc.is_in(index["radio_map_id"], requested)) if split is not None: splits = json.loads((self.root / "splits.json").read_text(encoding="utf-8")) declared = list(splits["fractions"]) if split not in declared: raise ValueError( f"The package {self.root} has no split {split!r}; its splits are " f"{', '.join(declared)}" ) index = index.filter(pc.field("split") == split) _check_extracted(self.root, index) self.index = index self.transform = transform self.per_transmitter = per_transmitter self._rows = index.select(list(_INDEX_COLUMNS)).to_pylist() # (index row, transmitter) of each sample; the transmitter is None for a whole map. self._samples = [ (position, transmitter) for position, row in enumerate(self._rows) for transmitter in (range(len(row["tx_east_m"])) if per_transmitter else [None]) ] self._scenes: OrderedDict[str, dict[str, np.ndarray]] = OrderedDict() def __len__(self) -> int: return len(self._samples) def __getstate__(self) -> dict[str, Any]: # Workers started with spawn get an empty cache rather than a copy of this one; # forked workers start with a copy of the parent's. return {**self.__dict__, "_scenes": OrderedDict()} def __getitem__(self, item: int) -> Sample: position, transmitter = self._samples[item] row = self._rows[position] path = self.root / row["record"] if transmitter is None: radio_map = load_radio_map(path) _check_shape(row, radio_map.path_gain.shape) grid = radio_map.grid linear = { "path_gain_db": (radio_map.path_gain, 0.0), "rss_dbm": (radio_map.rss, 30.0), "sinr_db": (radio_map.sinr, 0.0), } extra = {"serving_tx": torch.from_numpy(radio_map.association.astype(np.int64))} else: stored = _read_radio_map(path) _check_shape(row, stored.packed.shape) grid = GridMapping.model_validate(stored.record["array_grid_mapping"]) selected = slice(transmitter, transmitter + 1) path_gain = unpack_path_gain(stored.packed[selected]) rss = _rss(stored.power_w[selected], path_gain) # as load_radio_map computes it linear = {"path_gain_db": (path_gain, 0.0), "rss_dbm": (rss, 30.0)} extra = {"transmitter": torch.tensor(transmitter, dtype=torch.int64)} row = {**row, **{name: row[name][selected] for name in _TX_COLUMNS}} scene = self._scene_inputs(row) sample = { **{name: torch.from_numpy(array.copy()) for name, array in scene.items()}, **_transmitters(row, grid), **_targets(path, linear), **extra, "radio_map_id": row["radio_map_id"], } return self.transform(sample) if self.transform is not None else sample def _scene_inputs(self, row: dict[str, Any]) -> dict[str, np.ndarray]: """The geometry, surface classes and indoor mask of a measurement surface, cached.""" key = row["measurement_surface"] if key in self._scenes: self._scenes.move_to_end(key) return self._scenes[key] shape = (row["rows"], row["columns"]) with np.load((self.root / key).with_suffix(".npz")) as arrays: centres = arrays["cell_centers_enu_m"] with np.load(self.root / row["building_rasters"]) as rasters: roofs = rasters["roof_height_agl_m"] footprint = rasters["footprint_mask"] indoor = rasters["receiver_in_building_mask"] for name, array, expected in ( ("cell centres", centres, (*shape, 3)), ("roof heights", roofs, shape), ("footprint mask", footprint, shape), ("indoor mask", indoor, shape), ): if array.shape != expected: raise ValueError( f"The {name} of {key} have shape {array.shape}, not {expected} on the radio " f"grid {shape}" ) geometry = np.stack( [ np.where(np.isnan(roofs), 0.0, roofs), footprint, centres[..., 2] - row["receiver_height_m"], ] ).astype(np.float32) scene_record = self.root / row["scene_record"] inputs = { "geometry": geometry, "surface_classes": _surface_classes(scene_record, centres[0, :, 0], centres[:, 0, 1]), "indoor": indoor.astype(bool), } self._scenes[key] = inputs if len(self._scenes) > _SCENE_CACHE_SIZE: self._scenes.popitem(last=False) return inputs
def _check_extracted(root: Path, index: pa.Table) -> None: """Raise ``FileNotFoundError`` when the first radio map record of a scene is missing.""" scenes: set[str] = set() for row in index.select(["scene", "record", "archive"]).to_pylist(): if row["scene"] in scenes: continue scenes.add(row["scene"]) path = root / row["record"] if path.is_file(): continue if row["archive"] is not None: raise FileNotFoundError( f"{path} is missing: extract {row['archive']} in {root} first " f"(tar --zstd -xf {row['archive']}; GNU tar before 1.31: tar -I zstd -xf " f"{row['archive']})" ) raise FileNotFoundError(f"{path} is missing") def _check_shape(row: dict[str, Any], shape: tuple[int, ...]) -> None: """Raise ``ValueError`` when stored path gain disagrees with the radio map's index row.""" indexed = (len(row["tx_east_m"]), row["rows"], row["columns"]) if tuple(shape) != indexed: raise ValueError( f"Radio map {row['radio_map_id']} has transmitters and grid {tuple(shape)}; its " f"index row gives {indexed}" ) def _surface_classes(scene_record: Path, east_m: np.ndarray, north_m: np.ndarray) -> np.ndarray: """The class of the terrain cell that holds each cell centre, as ``int64`` rows by columns.""" record = json.loads(scene_record.read_text(encoding="utf-8")) assets = record["shared_assets"] if "environment" not in assets: return np.full((north_m.size, east_m.size), NO_SURFACE_CLASS, dtype=np.int64) directory = scene_record.parent terrain = json.loads((directory / assets["environment"]).read_text(encoding="utf-8")) classes = np.load(directory / assets["environment_arrays"]["surface_classes"]) cells = (len(terrain["north_m"]) - 1, len(terrain["east_m"]) - 1) if classes.shape != cells: raise ValueError( f"The surface classes of {scene_record} have shape {classes.shape}, not the " f"{cells} cells of its terrain axes" ) # Terrain axes are cell edges; a centre on an edge belongs to the cell north or east of it. rows = np.searchsorted(np.asarray(terrain["north_m"]), north_m, side="right") - 1 columns = np.searchsorted(np.asarray(terrain["east_m"]), east_m, side="right") - 1 rows = np.clip(rows, 0, classes.shape[0] - 1) columns = np.clip(columns, 0, classes.shape[1] - 1) return classes[np.ix_(rows, columns)].astype(np.int64) def _transmitters(row: dict[str, Any], grid: GridMapping) -> Sample: """Transmitter parameters, cells and location maps of the transmitters of an index row.""" rows, columns = row["rows"], row["columns"] east = np.asarray(row["tx_east_m"], dtype=np.float64) north = np.asarray(row["tx_north_m"], dtype=np.float64) cells = np.stack( [ _nearest_cell(north, grid.first_sample_north_m, grid.row_step_m, rows), _nearest_cell(east, grid.first_sample_east_m, grid.column_step_m, columns), ], axis=1, ) tx_map = np.zeros((east.size, rows, columns), dtype=np.float32) for transmitter, (row_index, column_index) in enumerate(cells): if 0 <= row_index < rows and 0 <= column_index < columns: tx_map[transmitter, row_index, column_index] = 1.0 position = np.stack([east, north, np.asarray(row["tx_up_m"], dtype=np.float64)], axis=1) return { "tx_map": torch.from_numpy(tx_map), "tx_cell": torch.from_numpy(cells), "tx_position_m": torch.from_numpy(position.astype(np.float32)), **{ name: torch.tensor(row[name], dtype=torch.float32) for name in ("tx_height_agl_m", "tx_power_dbm", "tx_azimuth_deg", "tx_downtilt_deg") }, "frequency_hz": torch.tensor(row["frequency_hz"], dtype=torch.float64), } def _nearest_cell(position: np.ndarray, first: float, step: float, count: int) -> np.ndarray: """The index of the cell whose centre is nearest, unclamped, a tie going to the next cell. The grid's far edge belongs to its last cell, as the near edge belongs to its first. """ # Recorded positions carry geodesy round-off, so the offset is snapped before rounding: # a transmitter on a cell edge then takes the tie rule, not the side the round-off picks. offset = np.round((position - first) / step, 6) cells = np.floor(offset + 0.5) return np.where((cells == count) & (offset <= count - 0.5), count - 1, cells).astype(np.int64) def _targets(path: Path, linear: dict[str, tuple[np.ndarray, float]]) -> Sample: """dB targets, 0 outside ``coverage``, of (linear array, dB offset) pairs keyed by name. ``coverage`` is where every one of the arrays is positive. """ bad = [ name for name, (array, _) in linear.items() if not (np.isfinite(array).all() and (array >= 0).all()) ] if bad: raise ValueError(f"{path} gives negative or non-finite values of {', '.join(bad)}") coverage = np.logical_and.reduce([array > 0 for array, _ in linear.values()]) targets = {"coverage": torch.from_numpy(coverage)} for name, (array, offset_db) in linear.items(): decibels = _decibels(array) + np.float32(offset_db) targets[name] = torch.from_numpy(np.where(coverage, decibels, np.float32(0.0))) return targets
[docs] class RandomCrop: """Crop every entry of ``SPATIAL_KEYS`` of a sample to one random window of ``size`` cells. ``size`` is an ``int`` for a square or ``(rows, columns)``. The window is drawn uniformly from PyTorch's global random number generator, which the ``DataLoader`` seeds for each worker. The crop adds ``crop_origin`` (``int64`` ``(row, column)`` of the window's first cell in the full radio map, accumulated over repeated crops) and shifts ``tx_cell`` to the window. Cropped entries are copies of the window, not views of the whole radio map. Every transmitter is kept; one outside the window has an all-zero ``tx_map``. A size that is not one or two positive integers, and a radio map smaller than the window, raise ``ValueError``. """ def __init__(self, size: int | tuple[int, int]) -> None: pair = (size, size) if isinstance(size, numbers.Integral) else size if not ( isinstance(pair, Sequence) and len(pair) == 2 and all( isinstance(n, numbers.Integral) and not isinstance(n, bool) and n > 0 for n in pair ) ): raise ValueError(f"Crop size must be one or two positive integers, not {size!r}") self.size = (int(pair[0]), int(pair[1])) def __call__(self, sample: Sample) -> Sample: height, width = sample["geometry"].shape[-2:] rows, columns = self.size if rows > height or columns > width: raise ValueError( f"Radio map {sample.get('radio_map_id')} has {height} x {width} cells, fewer " f"than the crop of {rows} x {columns}" ) row = int(torch.randint(height - rows + 1, ())) column = int(torch.randint(width - columns + 1, ())) window = (..., slice(row, row + rows), slice(column, column + columns)) cropped = dict(sample) for key in SPATIAL_KEYS: if key in cropped: cropped[key] = cropped[key][window].clone() origin = torch.tensor([row, column], dtype=torch.int64) cropped["crop_origin"] = ( sample.get("crop_origin", torch.zeros(2, dtype=torch.int64)) + origin ) if "tx_cell" in cropped: cropped["tx_cell"] = cropped["tx_cell"] - origin return cropped