"""A PyTorch dataset of the radio maps of a generated dataset or an extracted package.
The dataset code imports PyTorch only here; install it with the ``torch`` extra. One sample
is one whole radio map with all its transmitters, or one transmitter of a radio map, as a
dictionary of tensors on the grid of its radio arrays (rows south to north, columns west to
east). Values are in physical units; nothing is normalised.
"""
from __future__ import annotations
import json
import numbers
from collections import OrderedDict
from collections.abc import Callable, Iterable, Sequence
from pathlib import Path
from typing import Any
import numpy as np
import pyarrow as pa
import pyarrow.compute as pc
import torch
from torch.utils.data import Dataset
from openworld_radio_twin.dataset import (
_decibels,
_read_radio_map,
_rss,
load_radio_map,
read_index,
)
from openworld_radio_twin.models import GridMapping
from openworld_radio_twin.simulation.radio_map import unpack_path_gain
__all__ = ["NO_SURFACE_CLASS", "SPATIAL_KEYS", "RadioMapDataset", "RandomCrop"]
Sample = dict[str, Any]
# Surface class of a cell whose scene has no surface class grid.
NO_SURFACE_CLASS = -1
# The sample entries on the radio grid, whose last two dimensions are rows and columns.
SPATIAL_KEYS = (
"geometry",
"surface_classes",
"tx_map",
"path_gain_db",
"rss_dbm",
"sinr_db",
"coverage",
"serving_tx",
"indoor",
)
# The index columns of per-transmitter values, in transmitter order.
_TX_COLUMNS = (
"tx_east_m",
"tx_north_m",
"tx_up_m",
"tx_height_agl_m",
"tx_power_dbm",
"tx_azimuth_deg",
"tx_downtilt_deg",
)
# Measurement surfaces whose scene inputs are kept per dataset instance, and so per DataLoader
# worker. The index is in scene order, so reading it in order loads each surface once when a
# scene's radio maps use at most two surfaces.
_SCENE_CACHE_SIZE = 2
_INDEX_COLUMNS = (
"radio_map_id",
"scene_record",
"record",
"measurement_surface",
"building_rasters",
"receiver_height_m",
"rows",
"columns",
"frequency_hz",
*_TX_COLUMNS,
)
[docs]
class RadioMapDataset(Dataset):
"""The radio maps listed in the ``index.parquet`` of ``root``, one sample per radio map, or
one per transmitter with ``per_transmitter``.
``root`` is a generated dataset, or a package whose scene archives are extracted into the
package directory. The index is read with :func:`~openworld_radio_twin.dataset.read_index`,
which refuses a stale one. ``radio_map_ids`` keeps the listed radio maps, and an id that
is not in the index raises ``KeyError``. ``split`` then keeps those of one split of a
package. A split that the package's ``splits.json`` does not declare, and any split of an
index without splits, as in a generated dataset, raise ``ValueError``; a declared split
with no radio maps gives an empty dataset. Both keep the index order. A radio map
whose scene archive is not extracted raises ``FileNotFoundError``. ``index`` holds the
index rows of the selected radio maps, one per radio map, in index order.
``transform`` is applied to each sample.
A sample holds, with ``T`` transmitters and ``H`` by ``W`` cells:
* ``geometry`` (``float32``, ``(3, H, W)``): roof height above ground in metres (0 where
there is no building), building footprint mask, and ground elevation in the scene frame
in metres (the measurement cell centre minus the receiver height);
* ``surface_classes`` (``int64``, ``(H, W)``): the class of the terrain cell that holds
the cell centre, ``NO_SURFACE_CLASS`` when the scene has no surface class grid;
* ``indoor`` (``bool``, ``(H, W)``): the receiver point lies inside a building;
* ``tx_map`` (``float32``, ``(T, H, W)``): 1 in the cell of each transmitter, all 0 for a
transmitter off the grid; ``tx_cell`` (``int64``, ``(T, 2)``): that cell's row and
column, which can lie outside the grid;
* ``tx_position_m`` (``float32``, ``(T, 3)``): east, north and up in the scene frame;
``tx_height_agl_m``, ``tx_power_dbm``, ``tx_azimuth_deg`` and ``tx_downtilt_deg``
(``float32``, ``(T,)``); ``frequency_hz`` (``float64`` scalar);
* ``path_gain_db``, ``rss_dbm``, ``sinr_db`` (``float32``, ``(T, H, W)``): the values of
:func:`~openworld_radio_twin.dataset.load_radio_map`, 0 where ``coverage`` is false;
``coverage`` (``bool``, ``(T, H, W)``): path gain, RSS and SINR are all positive;
* ``serving_tx`` (``int64``, ``(H, W)``): the serving transmitter, -1 where none serves;
* ``radio_map_id`` (``str``).
With ``per_transmitter``, a sample is one transmitter of a radio map: the radio maps in
index order, and the transmitters of each in record order. The sample keeps the
transmitter axis with ``T`` = 1, so every sample has one transmitter. It holds the entries
above except ``sinr_db`` and ``serving_tx``, which depend on the other transmitters of
the radio map; ``coverage`` is where path gain and RSS are positive. It adds
``transmitter`` (``int64`` scalar), the transmitter's index in the radio map. The sample
reads the stored path gain of every transmitter of its radio map and unpacks only that
transmitter's.
"""
def __init__(
self,
root: str | Path,
split: str | None = None,
radio_map_ids: Iterable[str] | None = None,
transform: Callable[[Sample], Sample] | None = None,
per_transmitter: bool = False,
) -> None:
self.root = Path(root).expanduser()
index = read_index(self.root)
if split is not None and index["split"].null_count == len(index):
raise ValueError(f"The index of {self.root} has no splits; only a package has")
if radio_map_ids is not None:
if isinstance(radio_map_ids, str):
raise TypeError("radio_map_ids takes a list of ids, such as [radio_map_id]")
requested = pa.array(list(radio_map_ids), type=pa.string())
unknown = pc.filter(requested, pc.invert(pc.is_in(requested, index["radio_map_id"])))
if len(unknown):
raise KeyError(f"Radio maps not in the index of {self.root}: {unknown.to_pylist()}")
index = index.filter(pc.is_in(index["radio_map_id"], requested))
if split is not None:
splits = json.loads((self.root / "splits.json").read_text(encoding="utf-8"))
declared = list(splits["fractions"])
if split not in declared:
raise ValueError(
f"The package {self.root} has no split {split!r}; its splits are "
f"{', '.join(declared)}"
)
index = index.filter(pc.field("split") == split)
_check_extracted(self.root, index)
self.index = index
self.transform = transform
self.per_transmitter = per_transmitter
self._rows = index.select(list(_INDEX_COLUMNS)).to_pylist()
# (index row, transmitter) of each sample; the transmitter is None for a whole map.
self._samples = [
(position, transmitter)
for position, row in enumerate(self._rows)
for transmitter in (range(len(row["tx_east_m"])) if per_transmitter else [None])
]
self._scenes: OrderedDict[str, dict[str, np.ndarray]] = OrderedDict()
def __len__(self) -> int:
return len(self._samples)
def __getstate__(self) -> dict[str, Any]:
# Workers started with spawn get an empty cache rather than a copy of this one;
# forked workers start with a copy of the parent's.
return {**self.__dict__, "_scenes": OrderedDict()}
def __getitem__(self, item: int) -> Sample:
position, transmitter = self._samples[item]
row = self._rows[position]
path = self.root / row["record"]
if transmitter is None:
radio_map = load_radio_map(path)
_check_shape(row, radio_map.path_gain.shape)
grid = radio_map.grid
linear = {
"path_gain_db": (radio_map.path_gain, 0.0),
"rss_dbm": (radio_map.rss, 30.0),
"sinr_db": (radio_map.sinr, 0.0),
}
extra = {"serving_tx": torch.from_numpy(radio_map.association.astype(np.int64))}
else:
stored = _read_radio_map(path)
_check_shape(row, stored.packed.shape)
grid = GridMapping.model_validate(stored.record["array_grid_mapping"])
selected = slice(transmitter, transmitter + 1)
path_gain = unpack_path_gain(stored.packed[selected])
rss = _rss(stored.power_w[selected], path_gain) # as load_radio_map computes it
linear = {"path_gain_db": (path_gain, 0.0), "rss_dbm": (rss, 30.0)}
extra = {"transmitter": torch.tensor(transmitter, dtype=torch.int64)}
row = {**row, **{name: row[name][selected] for name in _TX_COLUMNS}}
scene = self._scene_inputs(row)
sample = {
**{name: torch.from_numpy(array.copy()) for name, array in scene.items()},
**_transmitters(row, grid),
**_targets(path, linear),
**extra,
"radio_map_id": row["radio_map_id"],
}
return self.transform(sample) if self.transform is not None else sample
def _scene_inputs(self, row: dict[str, Any]) -> dict[str, np.ndarray]:
"""The geometry, surface classes and indoor mask of a measurement surface, cached."""
key = row["measurement_surface"]
if key in self._scenes:
self._scenes.move_to_end(key)
return self._scenes[key]
shape = (row["rows"], row["columns"])
with np.load((self.root / key).with_suffix(".npz")) as arrays:
centres = arrays["cell_centers_enu_m"]
with np.load(self.root / row["building_rasters"]) as rasters:
roofs = rasters["roof_height_agl_m"]
footprint = rasters["footprint_mask"]
indoor = rasters["receiver_in_building_mask"]
for name, array, expected in (
("cell centres", centres, (*shape, 3)),
("roof heights", roofs, shape),
("footprint mask", footprint, shape),
("indoor mask", indoor, shape),
):
if array.shape != expected:
raise ValueError(
f"The {name} of {key} have shape {array.shape}, not {expected} on the radio "
f"grid {shape}"
)
geometry = np.stack(
[
np.where(np.isnan(roofs), 0.0, roofs),
footprint,
centres[..., 2] - row["receiver_height_m"],
]
).astype(np.float32)
scene_record = self.root / row["scene_record"]
inputs = {
"geometry": geometry,
"surface_classes": _surface_classes(scene_record, centres[0, :, 0], centres[:, 0, 1]),
"indoor": indoor.astype(bool),
}
self._scenes[key] = inputs
if len(self._scenes) > _SCENE_CACHE_SIZE:
self._scenes.popitem(last=False)
return inputs
def _check_extracted(root: Path, index: pa.Table) -> None:
"""Raise ``FileNotFoundError`` when the first radio map record of a scene is missing."""
scenes: set[str] = set()
for row in index.select(["scene", "record", "archive"]).to_pylist():
if row["scene"] in scenes:
continue
scenes.add(row["scene"])
path = root / row["record"]
if path.is_file():
continue
if row["archive"] is not None:
raise FileNotFoundError(
f"{path} is missing: extract {row['archive']} in {root} first "
f"(tar --zstd -xf {row['archive']}; GNU tar before 1.31: tar -I zstd -xf "
f"{row['archive']})"
)
raise FileNotFoundError(f"{path} is missing")
def _check_shape(row: dict[str, Any], shape: tuple[int, ...]) -> None:
"""Raise ``ValueError`` when stored path gain disagrees with the radio map's index row."""
indexed = (len(row["tx_east_m"]), row["rows"], row["columns"])
if tuple(shape) != indexed:
raise ValueError(
f"Radio map {row['radio_map_id']} has transmitters and grid {tuple(shape)}; its "
f"index row gives {indexed}"
)
def _surface_classes(scene_record: Path, east_m: np.ndarray, north_m: np.ndarray) -> np.ndarray:
"""The class of the terrain cell that holds each cell centre, as ``int64`` rows by columns."""
record = json.loads(scene_record.read_text(encoding="utf-8"))
assets = record["shared_assets"]
if "environment" not in assets:
return np.full((north_m.size, east_m.size), NO_SURFACE_CLASS, dtype=np.int64)
directory = scene_record.parent
terrain = json.loads((directory / assets["environment"]).read_text(encoding="utf-8"))
classes = np.load(directory / assets["environment_arrays"]["surface_classes"])
cells = (len(terrain["north_m"]) - 1, len(terrain["east_m"]) - 1)
if classes.shape != cells:
raise ValueError(
f"The surface classes of {scene_record} have shape {classes.shape}, not the "
f"{cells} cells of its terrain axes"
)
# Terrain axes are cell edges; a centre on an edge belongs to the cell north or east of it.
rows = np.searchsorted(np.asarray(terrain["north_m"]), north_m, side="right") - 1
columns = np.searchsorted(np.asarray(terrain["east_m"]), east_m, side="right") - 1
rows = np.clip(rows, 0, classes.shape[0] - 1)
columns = np.clip(columns, 0, classes.shape[1] - 1)
return classes[np.ix_(rows, columns)].astype(np.int64)
def _transmitters(row: dict[str, Any], grid: GridMapping) -> Sample:
"""Transmitter parameters, cells and location maps of the transmitters of an index row."""
rows, columns = row["rows"], row["columns"]
east = np.asarray(row["tx_east_m"], dtype=np.float64)
north = np.asarray(row["tx_north_m"], dtype=np.float64)
cells = np.stack(
[
_nearest_cell(north, grid.first_sample_north_m, grid.row_step_m, rows),
_nearest_cell(east, grid.first_sample_east_m, grid.column_step_m, columns),
],
axis=1,
)
tx_map = np.zeros((east.size, rows, columns), dtype=np.float32)
for transmitter, (row_index, column_index) in enumerate(cells):
if 0 <= row_index < rows and 0 <= column_index < columns:
tx_map[transmitter, row_index, column_index] = 1.0
position = np.stack([east, north, np.asarray(row["tx_up_m"], dtype=np.float64)], axis=1)
return {
"tx_map": torch.from_numpy(tx_map),
"tx_cell": torch.from_numpy(cells),
"tx_position_m": torch.from_numpy(position.astype(np.float32)),
**{
name: torch.tensor(row[name], dtype=torch.float32)
for name in ("tx_height_agl_m", "tx_power_dbm", "tx_azimuth_deg", "tx_downtilt_deg")
},
"frequency_hz": torch.tensor(row["frequency_hz"], dtype=torch.float64),
}
def _nearest_cell(position: np.ndarray, first: float, step: float, count: int) -> np.ndarray:
"""The index of the cell whose centre is nearest, unclamped, a tie going to the next cell.
The grid's far edge belongs to its last cell, as the near edge belongs to its first.
"""
# Recorded positions carry geodesy round-off, so the offset is snapped before rounding:
# a transmitter on a cell edge then takes the tie rule, not the side the round-off picks.
offset = np.round((position - first) / step, 6)
cells = np.floor(offset + 0.5)
return np.where((cells == count) & (offset <= count - 0.5), count - 1, cells).astype(np.int64)
def _targets(path: Path, linear: dict[str, tuple[np.ndarray, float]]) -> Sample:
"""dB targets, 0 outside ``coverage``, of (linear array, dB offset) pairs keyed by name.
``coverage`` is where every one of the arrays is positive.
"""
bad = [
name
for name, (array, _) in linear.items()
if not (np.isfinite(array).all() and (array >= 0).all())
]
if bad:
raise ValueError(f"{path} gives negative or non-finite values of {', '.join(bad)}")
coverage = np.logical_and.reduce([array > 0 for array, _ in linear.values()])
targets = {"coverage": torch.from_numpy(coverage)}
for name, (array, offset_db) in linear.items():
decibels = _decibels(array) + np.float32(offset_db)
targets[name] = torch.from_numpy(np.where(coverage, decibels, np.float32(0.0)))
return targets
[docs]
class RandomCrop:
"""Crop every entry of ``SPATIAL_KEYS`` of a sample to one random window of ``size`` cells.
``size`` is an ``int`` for a square or ``(rows, columns)``. The window is drawn uniformly
from PyTorch's global random number generator, which the ``DataLoader`` seeds for each
worker. The crop adds ``crop_origin`` (``int64`` ``(row, column)`` of the window's first
cell in the full radio map, accumulated over repeated crops) and shifts ``tx_cell`` to the
window. Cropped entries are copies of the window, not views of the whole radio map. Every
transmitter is kept; one outside the window has an all-zero ``tx_map``. A size that is not
one or two positive integers, and a radio map smaller than the window, raise
``ValueError``.
"""
def __init__(self, size: int | tuple[int, int]) -> None:
pair = (size, size) if isinstance(size, numbers.Integral) else size
if not (
isinstance(pair, Sequence)
and len(pair) == 2
and all(
isinstance(n, numbers.Integral) and not isinstance(n, bool) and n > 0 for n in pair
)
):
raise ValueError(f"Crop size must be one or two positive integers, not {size!r}")
self.size = (int(pair[0]), int(pair[1]))
def __call__(self, sample: Sample) -> Sample:
height, width = sample["geometry"].shape[-2:]
rows, columns = self.size
if rows > height or columns > width:
raise ValueError(
f"Radio map {sample.get('radio_map_id')} has {height} x {width} cells, fewer "
f"than the crop of {rows} x {columns}"
)
row = int(torch.randint(height - rows + 1, ()))
column = int(torch.randint(width - columns + 1, ()))
window = (..., slice(row, row + rows), slice(column, column + columns))
cropped = dict(sample)
for key in SPATIAL_KEYS:
if key in cropped:
cropped[key] = cropped[key][window].clone()
origin = torch.tensor([row, column], dtype=torch.int64)
cropped["crop_origin"] = (
sample.get("crop_origin", torch.zeros(2, dtype=torch.int64)) + origin
)
if "tx_cell" in cropped:
cropped["tx_cell"] = cropped["tx_cell"] - origin
return cropped