import asyncio
import fcntl
import json
import logging
import math
import re
import time
import unicodedata
from collections import Counter
from collections.abc import Callable
from concurrent.futures import Executor
from contextlib import contextmanager
from dataclasses import replace
from functools import partial
from pathlib import Path
from typing import Literal
import numpy as np
from pydantic import BaseModel, ConfigDict, Field, model_validator
from openworld_radio_twin import __version__
from openworld_radio_twin.antennas import (
downlink_channels,
load_antenna_source,
select_antennas,
select_channel,
transmitter_config,
write_antenna_source,
)
from openworld_radio_twin.artifacts import (
RADIO_IMAGE_PATHS,
write_packed_path_gain,
write_radio_images,
)
from openworld_radio_twin.compiler import (
_geometry_request,
compilation_record,
load_building_source,
write_building_source,
)
from openworld_radio_twin.config import Settings
from openworld_radio_twin.conventions import (
DATASET_SCHEMA_VERSION,
HEIGHT_REFERENCE,
PATH_GAIN_ARRAY_PATH,
RADIO_ARRAY_AXIS_ORDER,
RADIO_ENGINE_ARRAY_CONTRACT,
conventions_record,
path_gain_packing,
)
from openworld_radio_twin.environment import RF_MATERIAL_PROFILE_VERSION
from openworld_radio_twin.geodesy import LocalFrame
from openworld_radio_twin.licences import data_licences, licence_notice, scene_licences
from openworld_radio_twin.models import (
MAX_PREVIEW_GRID_CELLS,
MAX_SIONNA_GRID_CELLS,
MAX_TERRAIN_GRID_CELLS,
SIONNA_FREQUENCY_RANGE_GHZ,
SOURCE_ID_PATTERN,
AntennaDefaults,
ChannelRange,
Coordinate,
EngineName,
RadioTechnology,
SceneCenter,
SimulationRequest,
SimulationResponse,
SionnaEngineConfig,
TransmitterConfig,
)
from openworld_radio_twin.providers.antenna_sources import resolve_antenna_source
from openworld_radio_twin.providers.base import (
SCENE_BOUNDARY_POLICY,
AntennaQueryResult,
BuildingQueryResult,
)
from openworld_radio_twin.providers.building_sources import resolve_building_source
from openworld_radio_twin.radio_map_index import index_table, write_index
from openworld_radio_twin.scene_format import (
SCENE_ENVIRONMENT_ARRAY_PATHS,
SCENE_HEIGHT_MAP_PATH,
SCENE_INFO_PATH,
SCENE_MESH_DIRECTORY,
SCENE_PROVENANCE_PATH,
SCENE_RADIO_MAPS_DIRECTORY,
SCENE_SOURCE_ANTENNAS_PATH,
SCENE_SOURCE_BUILDINGS_PATH,
SCENE_TERRAIN_INFO_PATH,
SCENE_XML_PATH,
)
from openworld_radio_twin.simulation.antenna_placement import keep_inside_scene, place_antennas
from openworld_radio_twin.simulation.building_grid import prepare_surface_building_rasters
from openworld_radio_twin.simulation.preview import (
PREVIEW_TEMPERATURE_K,
preview_noise_power_w,
preview_transmitter_positions,
simulate_preview,
)
from openworld_radio_twin.simulation.radio_map import array_grid_mapping, build_coverage_products
from openworld_radio_twin.simulation.scene_builder import (
SceneAssets,
build_scene_assets,
load_environment_assets,
measurement_surface_path,
prepare_measurement_surface,
transmitter_scene_position,
write_voxel_assets,
)
from openworld_radio_twin.simulation.sionna_engine import (
SIONNA_RADIO_MAP_CONTRACT_VERSION,
simulate_sionna_scene,
)
ProgressCallback = Callable[[int, int, str], None]
logger = logging.getLogger(__name__)
# A dataset is resumed only by code that writes radio maps and scene materials the same way.
EXECUTION_CONTRACTS = {
"sionna_radio_map": SIONNA_RADIO_MAP_CONTRACT_VERSION,
"rf_material_profile": RF_MATERIAL_PROFILE_VERSION,
}
MAX_RADIO_MAPS_PER_SCENE = 10_000
[docs]
class FloatRange(BaseModel):
model_config = ConfigDict(extra="forbid")
minimum: float
maximum: float
[docs]
@model_validator(mode="after")
def ordered(self) -> "FloatRange":
if self.maximum < self.minimum:
raise ValueError("maximum must be greater than or equal to minimum")
return self
[docs]
class IntegerRange(BaseModel):
model_config = ConfigDict(extra="forbid")
minimum: int
maximum: int
[docs]
@model_validator(mode="after")
def ordered(self) -> "IntegerRange":
if self.maximum < self.minimum:
raise ValueError("maximum must be greater than or equal to minimum")
return self
[docs]
class BatchOutputSpec(BaseModel):
"""What each radio map writes besides its record and its packed path gain.
``previews`` adds the PNG previews, rendered from the solver's in-memory arrays, for
inspection; packages leave them out.
"""
model_config = ConfigDict(extra="forbid")
previews: bool = False
[docs]
class BatchRadioMapSpace(BaseModel):
model_config = ConfigDict(extra="forbid")
strategy: Literal["fixed", "random"] = "random"
count: int = Field(default=8, ge=1, le=MAX_RADIO_MAPS_PER_SCENE)
engine: EngineName = EngineName.preview
engine_config: dict[str, str | int | float | bool] = Field(default_factory=dict)
transmitter_count: IntegerRange = IntegerRange(minimum=1, maximum=3)
frequency_ghz: FloatRange = FloatRange(minimum=3.5, maximum=3.5)
power_dbm: FloatRange = FloatRange(minimum=24, maximum=36)
altitude_m: FloatRange = FloatRange(minimum=15, maximum=35)
offset_radius_m: FloatRange = FloatRange(minimum=0, maximum=250)
azimuth_deg: FloatRange = FloatRange(minimum=0, maximum=359)
downtilt_deg: FloatRange = FloatRange(minimum=0, maximum=12)
antenna_patterns: list[Literal["sector", "isotropic"]] = Field(
default_factory=lambda: ["sector"]
)
resolutions_m: list[int] = Field(default_factory=lambda: [15])
max_depths: list[int] = Field(default_factory=lambda: [3])
samples_per_tx: list[int] = Field(default_factory=lambda: [1_000_000])
association_metrics: list[Literal["path_gain", "rss", "sinr"]] = Field(
default_factory=lambda: ["sinr"]
)
[docs]
@model_validator(mode="after")
def validate_space(self) -> "BatchRadioMapSpace":
if not 1 <= self.transmitter_count.minimum <= self.transmitter_count.maximum <= 8:
raise ValueError("transmitter count must stay between 1 and 8")
if not self.antenna_patterns:
raise ValueError("at least one antenna pattern is required")
if not self.resolutions_m or any(not 1 <= value <= 200 for value in self.resolutions_m):
raise ValueError("resolutions must stay between 1 and 200 metres")
if not self.max_depths or any(not 0 <= value <= 8 for value in self.max_depths):
raise ValueError("reflection depths must stay between 0 and 8")
if not self.samples_per_tx or any(
not 10_000 <= value <= 100_000_000 for value in self.samples_per_tx
):
raise ValueError("ray samples must stay between 10,000 and 100,000,000")
if not self.association_metrics:
raise ValueError("at least one association metric is required")
ranges = {
"frequency": (self.frequency_ghz, 0.1, 100),
"power": (self.power_dbm, -20, 80),
"altitude": (self.altitude_m, 0, 1000),
"TX offset": (self.offset_radius_m, 0, 5000),
"azimuth": (self.azimuth_deg, 0, 359.999999),
"downtilt": (self.downtilt_deg, -30, 90),
}
for name, (values, lower, upper) in ranges.items():
if values.minimum < lower or values.maximum > upper:
raise ValueError(f"{name} range must stay between {lower:g} and {upper:g}")
return self
[docs]
class BatchAntennaSelection(BaseModel):
"""Transmitters taken from an antenna source instead of sampled positions.
Each radio map simulates one downlink channel with every selected antenna inside
the scene that transmits on it. ``channels`` lists the channels radio maps draw
from; empty means every downlink channel the scene's antennas register that the engine
supports. A source without frequencies, such as OpenCelliD, needs the channels listed.
Values the source records (position, and height or azimuth where given) are kept; the
radio map samples the others from its ranges: height, azimuth, downtilt and
power, as well as the pattern.
The carrier frequency and noise bandwidth come from the channel; each antenna keeps its
own power.
"""
model_config = ConfigDict(extra="forbid")
source: str = Field(default="auto", pattern=SOURCE_ID_PATTERN)
operator: str | None = Field(default=None, max_length=80)
radio: RadioTechnology | None = None
channels: list[ChannelRange] = Field(default_factory=list, max_length=50)
[docs]
class BatchSceneSpec(BaseModel):
model_config = ConfigDict(extra="forbid")
label: str = Field(min_length=1, max_length=80)
scene_mode: Literal["geospatial", "empty"] = "geospatial"
latitude: float = Field(ge=-90, le=90)
longitude: float = Field(ge=-180, le=180)
radius_m: int = Field(default=750, ge=100, le=5000)
receiver_height_m: float = Field(default=1.5, ge=0.1, le=100)
voxel_pitch_m: float = Field(default=3.0, ge=0.5, le=25)
include_buildings: bool = True
building_source: str = Field(default="auto", pattern=SOURCE_ID_PATTERN)
include_terrain: bool = False
material_profile: Literal["itu", "uniform"] = "itu"
terrain_resolution_m: float = Field(default=5.0, ge=1.0, le=100.0)
antennas: BatchAntennaSelection | None = None
# Scenes with the same site share a split when the dataset is packaged, so a scene keeps
# its site once it exists.
site: str | None = Field(default=None, min_length=1, max_length=80)
radio_maps: BatchRadioMapSpace = Field(default_factory=BatchRadioMapSpace)
[docs]
@model_validator(mode="after")
def fit_transmitters_in_scene(self) -> "BatchSceneSpec":
if self.antennas is not None and self.scene_mode == "empty":
raise ValueError("Transmitters from an antenna source need a geospatial scene")
lowest, highest = SIONNA_FREQUENCY_RANGE_GHZ
if self.radio_maps.engine == EngineName.sionna and self.antennas is None:
# A scene taking its transmitters from antennas takes the carrier from a channel.
frequencies = self.radio_maps.frequency_ghz
if frequencies.minimum < lowest or frequencies.maximum > highest:
raise ValueError(
f"Sionna frequency range must stay between {lowest:g} and {highest:g} GHz"
)
if self.radio_maps.engine == EngineName.sionna and self.antennas is not None:
for channel in self.antennas.channels:
if not lowest <= channel.centre_hz / 1e9 <= highest:
raise ValueError(
f"Channel {channel.low_mhz:g}-{channel.high_mhz:g} MHz lies outside "
f"the {lowest:g}-{highest:g} GHz range of Sionna's ITU materials"
)
if self.scene_mode == "empty":
self.include_buildings = False
self.include_terrain = False
self.material_profile = "uniform"
terrain_side = math.ceil(2 * self.radius_m / self.terrain_resolution_m)
if self.include_terrain and terrain_side**2 > MAX_TERRAIN_GRID_CELLS:
raise ValueError(
f"terrain grid exceeds {MAX_TERRAIN_GRID_CELLS:,} cells; "
"increase terrain resolution"
)
# Positions from an antenna source leave the sampled offset unused.
if self.antennas is None and self.radio_maps.offset_radius_m.maximum > self.radius_m:
raise ValueError("maximum TX offset must not exceed the scene radius")
maximum = (
MAX_SIONNA_GRID_CELLS
if self.radio_maps.engine == EngineName.sionna
else MAX_PREVIEW_GRID_CELLS
)
for resolution in self.radio_maps.resolutions_m:
side = math.ceil(2 * self.radius_m / resolution)
if side**2 > maximum:
raise ValueError(
f"{resolution} m resolution creates a {side}x{side} grid, exceeding "
f"the {maximum:,}-cell {self.radio_maps.engine.value} limit"
)
return self
@property
def include_surface_materials(self) -> bool:
"""Return whether semantic surface classes are required by this scene."""
return self.material_profile == "itu"
@property
def include_environment_grid(self) -> bool:
"""Return whether a gridded ground (DEM or flat) is compiled for this scene."""
return self.include_terrain or self.include_surface_materials
[docs]
class BatchDatasetRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
dataset_name: str = Field(min_length=1, max_length=80)
random_seed: int = Field(default=42, ge=0, le=4_294_967_295)
continue_on_error: bool = False
output: BatchOutputSpec = Field(default_factory=BatchOutputSpec)
scenes: list[BatchSceneSpec] = Field(min_length=1)
[docs]
class RegisteredDraw(BaseModel):
"""A radio map's draws for transmitters taken from the scene's antennas.
The scene's channels, and so their number, are known only once its antennas are
acquired, after planning. A random radio map therefore stores ``channel_draw``,
a uniform number in [0, 1) that picks one; a fixed radio map stores
``channel_index`` and takes them in turn. ``defaults`` holds the sampled values for what
the source does not record; ``request`` holds the radio map's request fields
other than the transmitters, carrier and bandwidth.
"""
channel_draw: float | None = Field(default=None, ge=0, lt=1)
channel_index: int | None = Field(default=None, ge=0)
antenna_pattern: Literal["sector", "isotropic"]
defaults: AntennaDefaults
request: dict[str, object]
[docs]
class ExpandedRadioMap(BaseModel):
"""A planned radio map. One taking its transmitters from antennas starts with
only its draws; generation adds the request once the scene's antennas are placed."""
name: str
seed: int
request: SimulationRequest | None = None
registered: RegisteredDraw | None = None
# Notes on the antennas a registered radio map takes, recorded with it.
warnings: list[str] = Field(default_factory=list)
[docs]
@model_validator(mode="after")
def described(self) -> "ExpandedRadioMap":
if self.request is None and self.registered is None:
raise ValueError("A radio map needs a request or registered-antenna draws")
return self
@property
def engine(self) -> str:
if self.request is not None:
return self.request.engine
return str(self.registered.request["engine"])
[docs]
class PlannedScene(BaseModel):
name: str
label: str
specification: BatchSceneSpec
radio_maps: list[ExpandedRadioMap]
[docs]
class BatchPlan(BaseModel):
schema_version: int = DATASET_SCHEMA_VERSION
dataset_name: str
dataset_slug: str
random_seed: int
generator: str = "numpy.random.PCG64"
total_scenes: int
total_radio_maps: int
total_receiver_cells: int
total_initial_ray_samples: int
# Radio maps whose transmitters come from an antenna source; their transmitter
# count, and so their initial rays, is known once the scene's antennas are acquired.
registered_radio_maps: int = 0
scenes: list[PlannedScene]
[docs]
def slugify(value: str, fallback: str) -> str:
ascii_value = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode()
slug = re.sub(r"[^a-zA-Z0-9]+", "-", ascii_value).strip("-").lower()
return (slug or fallback)[:48]
def _sample_float(rng: np.random.Generator, values: FloatRange, random: bool) -> float:
if not random or values.minimum == values.maximum:
return values.minimum
return float(rng.uniform(values.minimum, values.maximum))
def _sample_int(rng: np.random.Generator, values: IntegerRange, random: bool) -> int:
if not random or values.minimum == values.maximum:
return values.minimum
return int(rng.integers(values.minimum, values.maximum + 1))
def _choice(rng: np.random.Generator, values: list, random: bool):
return values[int(rng.integers(0, len(values)))] if random else values[0]
def _expand_radio_map(
dataset_seed: int,
scene_index: int,
radio_map_index: int,
scene: BatchSceneSpec,
) -> ExpandedRadioMap:
space = scene.radio_maps
engine_config = dict(space.engine_config)
if scene.scene_mode == "empty" and space.engine == EngineName.sionna:
engine_config.update(
{
"include_ground": False,
"include_vegetation": False,
"include_paved": False,
"include_water": False,
"include_building_roofs": False,
"include_building_walls": False,
}
)
random = space.strategy == "random"
rng = np.random.default_rng(
np.random.SeedSequence([dataset_seed, scene_index, radio_map_index])
)
radio_map_seed = int(rng.integers(0, 2**32, dtype=np.uint64))
if scene.antennas is not None:
return _expand_registered_radio_map(
rng, radio_map_seed, radio_map_index, scene, engine_config, random
)
tx_count = _sample_int(rng, space.transmitter_count, random)
frequency = round(_sample_float(rng, space.frequency_ghz, random), 6)
pattern = _choice(rng, space.antenna_patterns, random)
frame = LocalFrame.at(scene.longitude, scene.latitude)
transmitters = []
inner = space.offset_radius_m.minimum
outer = space.offset_radius_m.maximum
for _ in range(tx_count):
# Uniform by area in the annulus around the scene centre; the fixed strategy puts
# every transmitter at the inner radius due north.
radial = inner if not random else math.sqrt(rng.uniform(inner**2, outer**2))
angle = 0.0 if not random else rng.uniform(0.0, 2.0 * math.pi)
east, north = radial * math.sin(angle), radial * math.cos(angle)
longitude, latitude, _ = frame.to_geographic(east, north, 0.0)
transmitters.append(
TransmitterConfig(
position=Coordinate(
latitude=latitude,
longitude=longitude,
altitude_m=round(_sample_float(rng, space.altitude_m, random), 6),
),
frequency_ghz=frequency,
power_dbm=round(_sample_float(rng, space.power_dbm, random), 6),
azimuth_deg=round(_sample_float(rng, space.azimuth_deg, random), 6),
downtilt_deg=round(_sample_float(rng, space.downtilt_deg, random), 6),
antenna_pattern=pattern,
)
)
request = SimulationRequest(
transmitters=transmitters,
engine=space.engine,
engine_config=engine_config,
radius_m=scene.radius_m,
resolution_m=int(_choice(rng, space.resolutions_m, random)),
receiver_height_m=scene.receiver_height_m,
max_depth=int(_choice(rng, space.max_depths, random)),
samples_per_tx=int(_choice(rng, space.samples_per_tx, random)),
seed=radio_map_seed,
association_metric=_choice(rng, space.association_metrics, random),
include_buildings=scene.include_buildings,
building_source=scene.building_source,
include_terrain=scene.include_terrain,
material_profile=scene.material_profile,
terrain_resolution_m=scene.terrain_resolution_m,
scene_center=SceneCenter(latitude=scene.latitude, longitude=scene.longitude),
)
frequency_token = f"{frequency:g}".replace(".", "p")
return ExpandedRadioMap(
name=f"radio_map_{radio_map_index:04d}_{pattern}_{tx_count}tx_{frequency_token}ghz",
seed=radio_map_seed,
request=request,
)
def _expand_registered_radio_map(
rng: np.random.Generator,
radio_map_seed: int,
radio_map_index: int,
scene: BatchSceneSpec,
engine_config: dict,
random: bool,
) -> ExpandedRadioMap:
"""Draw what a radio map needs besides the positions its scene's antennas give."""
space = scene.radio_maps
channel_draw = {"channel_draw": float(rng.uniform())} if random else {}
pattern = _choice(rng, space.antenna_patterns, random)
defaults = AntennaDefaults(
height_agl_m=round(_sample_float(rng, space.altitude_m, random), 6),
azimuth_deg=round(_sample_float(rng, space.azimuth_deg, random), 6),
downtilt_deg=round(_sample_float(rng, space.downtilt_deg, random), 6),
power_dbm=round(_sample_float(rng, space.power_dbm, random), 6),
)
request = {
"engine": space.engine.value,
"engine_config": engine_config,
"radius_m": scene.radius_m,
"resolution_m": int(_choice(rng, space.resolutions_m, random)),
"receiver_height_m": scene.receiver_height_m,
"max_depth": int(_choice(rng, space.max_depths, random)),
"samples_per_tx": int(_choice(rng, space.samples_per_tx, random)),
"seed": radio_map_seed,
"association_metric": _choice(rng, space.association_metrics, random),
"include_buildings": scene.include_buildings,
"building_source": scene.building_source,
"include_terrain": scene.include_terrain,
"material_profile": scene.material_profile,
"terrain_resolution_m": scene.terrain_resolution_m,
"scene_center": {"latitude": scene.latitude, "longitude": scene.longitude},
}
return ExpandedRadioMap(
name=f"radio_map_{radio_map_index:04d}_{pattern}_registered",
seed=radio_map_seed,
registered=RegisteredDraw(
**(channel_draw or {"channel_index": radio_map_index - 1}),
antenna_pattern=pattern,
defaults=defaults,
request=request,
),
)
[docs]
def scene_channels(
selection: BatchAntennaSelection, antennas: AntennaQueryResult, engine: str
) -> list[ChannelRange]:
"""The channels a scene's radio maps draw from: the listed ones, each with at
least one antenna on it, else every downlink channel of the scene's antennas that the
engine supports."""
if selection.channels:
for channel in selection.channels:
if not select_channel(antennas, channel).antennas:
raise ValueError(
f"No selected antenna of the scene transmits on {channel.low_mhz:g}-"
f"{channel.high_mhz:g} MHz"
)
return list(selection.channels)
channels = downlink_channels(antennas.antennas)
if engine == EngineName.sionna.value:
channels = [channel for channel in channels if channel.sionna_supported]
if not channels:
raise ValueError(
"The scene's antennas register no downlink channel the engine supports; list "
"the channels to simulate in the scene's antenna selection"
)
return [ChannelRange(low_mhz=item.low_mhz, high_mhz=item.high_mhz) for item in channels]
[docs]
def registered_radio_map(
radio_map: ExpandedRadioMap,
antennas: AntennaQueryResult,
assets: SceneAssets,
channels: list[ChannelRange],
) -> ExpandedRadioMap:
"""The radio map's request: its channel, and the scene's antennas that transmit on it,
placed under the radio map's defaults."""
if radio_map.registered is None:
return radio_map
draw = radio_map.registered
if draw.channel_draw is not None:
channel = channels[int(draw.channel_draw * len(channels))]
else:
channel = channels[draw.channel_index % len(channels)]
placed = place_antennas(select_channel(antennas, channel), assets, draw.defaults)
transmitters = [
transmitter_config(
antenna,
frequency_ghz=round(channel.centre_hz / 1e9, 9),
antenna_pattern=draw.antenna_pattern,
).model_dump(mode="json")
for antenna in placed.antennas
]
request = SimulationRequest.model_validate(
{
**draw.request,
"bandwidth_mhz": channel.width_mhz,
"transmitters": transmitters,
}
)
return radio_map.model_copy(update={"request": request, "warnings": placed.warnings})
def _scene_name(scene_index: int, specification: BatchSceneSpec) -> str:
return f"scene_{scene_index:03d}_{slugify(specification.label, 'scene')}"
[docs]
def build_batch_plan(payload: BatchDatasetRequest) -> BatchPlan:
scenes = []
total_cells = 0
total_samples = 0
registered = 0
for scene_index, specification in enumerate(payload.scenes, start=1):
radio_maps = [
_expand_radio_map(payload.random_seed, scene_index, index, specification)
for index in range(1, specification.radio_maps.count + 1)
]
scenes.append(
PlannedScene(
name=_scene_name(scene_index, specification),
label=specification.label,
specification=specification,
radio_maps=radio_maps,
)
)
for radio_map in radio_maps:
if radio_map.request is None:
registered += 1
resolution = radio_map.registered.request["resolution_m"]
side = math.ceil(2 * specification.radius_m / resolution)
total_cells += side**2
continue
side = math.ceil(2 * specification.radius_m / radio_map.request.resolution_m)
total_cells += side**2
if radio_map.request.engine == EngineName.sionna.value:
total_samples += radio_map.request.samples_per_tx * len(
radio_map.request.transmitters
)
return BatchPlan(
dataset_name=payload.dataset_name,
dataset_slug=slugify(payload.dataset_name, "dataset"),
random_seed=payload.random_seed,
total_scenes=len(scenes),
total_radio_maps=sum(len(scene.radio_maps) for scene in scenes),
total_receiver_cells=total_cells,
total_initial_ray_samples=total_samples,
registered_radio_maps=registered,
scenes=scenes,
)
def _write_text(path: Path, text: str) -> None:
temporary = path.with_suffix(f"{path.suffix}.tmp")
temporary.write_text(text, encoding="utf-8")
temporary.replace(path)
def _write_json(path: Path, value: object) -> None:
_write_text(path, json.dumps(value, indent=2))
def _read_json(path: Path) -> dict:
value = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(value, dict):
raise ValueError(f"Expected a JSON object in {path}")
return value
def _scene_geometry_request(
specification: BatchSceneSpec, settings: Settings
) -> dict[str, object]:
"""The compiler's geometry request for a dataset scene, as ``rt.load_scene`` builds it."""
if specification.include_terrain:
terrain = "elevation"
else:
terrain = "none" if specification.scene_mode == "empty" else "flat"
return _geometry_request(
specification.latitude,
specification.longitude,
specification.radius_m,
"auto" if specification.include_buildings else "none",
terrain,
specification.material_profile,
specification.terrain_resolution_m,
specification.building_source,
settings,
)
def _scene_contract(specification: BatchSceneSpec) -> dict:
"""What resume compares for an existing scene: everything but its radio map count."""
value = specification.model_dump(mode="json")
value["radio_maps"].pop("count")
if value["radio_maps"]["engine"] == EngineName.sionna.value:
value["radio_maps"]["engine_config"] = SionnaEngineConfig.model_validate(
value["radio_maps"]["engine_config"]
).model_dump(mode="json")
return value
def _changed_fields(previous: dict, requested: dict) -> list[str]:
changed = []
for key in previous:
if key == "radio_maps":
changed += [
f"radio_maps.{name}"
for name in previous[key]
if previous[key][name] != requested[key][name]
]
elif previous[key] != requested[key]:
changed.append(key)
return changed
_RESUME_ALLOWS = (
"resume may only raise per-scene radio map counts and append scenes after the existing ones"
)
def _validate_resume_manifest(
manifest: dict,
payload: BatchDatasetRequest,
plan: BatchPlan,
) -> BatchDatasetRequest:
schema_version = manifest.get("schema_version")
if schema_version == 1:
raise ValueError(
"The existing dataset uses dataset format 1 (radio maps stored under "
"cases/) and cannot be resumed; regenerate it in a new directory, with each "
"scene's 'cases' key renamed to 'radio_maps' in the request"
)
if schema_version != DATASET_SCHEMA_VERSION:
raise ValueError(
f"Existing manifest has dataset format {schema_version!r}; this version resumes "
f"format {DATASET_SCHEMA_VERSION} only"
)
if manifest.get("dataset_slug") != plan.dataset_slug:
raise ValueError("Existing manifest dataset identity does not match the requested dataset")
software_version = manifest.get("software", {}).get("openworld_radio_twin")
if software_version != __version__:
raise ValueError(
"Resume requires the same OpenWorld Radio Twin version that created the dataset "
f"({software_version!r} != {__version__!r})"
)
try:
previous = BatchDatasetRequest.model_validate(manifest["request"])
except Exception as exc:
raise ValueError("Existing manifest does not contain a valid resumable request") from exc
if previous.output != payload.output:
raise ValueError(
"output.previews is fixed for the lifetime of a dataset (this dataset was generated "
f"with previews={previous.output.previews}); generate a new dataset to change it"
)
if (previous.dataset_name, previous.random_seed) != (
payload.dataset_name,
payload.random_seed,
):
raise ValueError(
f"The dataset name and random seed must match the existing dataset; {_RESUME_ALLOWS}"
)
if len(payload.scenes) < len(previous.scenes):
raise ValueError(
f"The request has {len(payload.scenes)} scene(s) and the dataset "
f"{len(previous.scenes)}; {_RESUME_ALLOWS}"
)
existing = payload.scenes[: len(previous.scenes)]
for index, (old_scene, new_scene) in enumerate(
zip(previous.scenes, existing, strict=True), start=1
):
name = _scene_name(index, old_scene)
changed = _changed_fields(_scene_contract(old_scene), _scene_contract(new_scene))
if changed:
reason = (
"; the site decides the split groups of its packages" if "site" in changed else ""
)
raise ValueError(
f"Scene {name} changes {', '.join(changed)}{reason}; {_RESUME_ALLOWS}"
)
if new_scene.radio_maps.count < old_scene.radio_maps.count:
raise ValueError(
f"Scene {name} radio map count cannot decrease from "
f"{old_scene.radio_maps.count} to {new_scene.radio_maps.count}; {_RESUME_ALLOWS}"
)
if any(scene.radio_maps.engine == EngineName.sionna.value for scene in payload.scenes):
previous_contract = manifest.get("execution_contracts", {}).get("sionna_radio_map", 1)
if previous_contract != SIONNA_RADIO_MAP_CONTRACT_VERSION:
raise ValueError(
"Cannot resume Sionna data with a different radio-map contract "
f"({previous_contract} != {SIONNA_RADIO_MAP_CONTRACT_VERSION}). "
"Use a new dataset directory or the original generator for this dataset."
)
previous_profile = manifest.get("execution_contracts", {}).get("rf_material_profile", 1)
if previous_profile != RF_MATERIAL_PROFILE_VERSION:
raise ValueError(
"Cannot resume Sionna data with a different RF material profile "
f"({previous_profile} != {RF_MATERIAL_PROFILE_VERSION}). "
"Use a new dataset directory or the original generator for this dataset."
)
return previous
def _required_completed_radio_map_files(output: BatchOutputSpec) -> tuple[str, ...]:
return (PATH_GAIN_ARRAY_PATH, *(RADIO_IMAGE_PATHS if output.previews else ()))
def _completed_radio_map_record(
radio_map_directory: Path,
radio_map: ExpandedRadioMap,
output: BatchOutputSpec,
) -> dict | None:
info_path = radio_map_directory / "radio_map_info.json"
if not info_path.is_file():
return None
try:
record = _read_json(info_path)
except (OSError, ValueError, json.JSONDecodeError):
return None
recorded_request = record.get("request")
if recorded_request is None:
return None
# A radio map taking registered antennas has a planned request only once its
# scene's antennas are placed; before that, its recorded request is taken as it is.
if (
radio_map.request is not None
and recorded_request != radio_map.request.model_dump(mode="json")
):
raise ValueError(
f"Existing radio map {radio_map_directory.name} was generated from an "
"incompatible request"
)
if record.get("status") != "completed":
return None
required_files = _required_completed_radio_map_files(output)
if any(not (radio_map_directory / relative).is_file() for relative in required_files):
return None
return record
def _pending_scene_record(
scene: PlannedScene, previous: dict | None, scanned_completed: int
) -> dict:
previous = previous or {"name": scene.name}
completed = scanned_completed
if any(radio_map.request is None for radio_map in scene.radio_maps):
# The resume scan cannot count radio maps taking registered antennas,
# so keep the previous run's count until the scene is reached.
previous_counts = previous.get("radio_maps")
completed = previous_counts.get("completed", 0) if isinstance(previous_counts, dict) else 0
return {
**previous,
"status": "pending",
"radio_maps": {
"planned": len(scene.radio_maps),
"completed": completed,
"failed": 0,
},
}
def _scan_completed_radio_maps(
dataset_directory: Path,
plan: BatchPlan,
output: BatchOutputSpec,
) -> dict[tuple[str, str], dict]:
completed: dict[tuple[str, str], dict] = {}
for scene in plan.scenes:
radio_maps_directory = dataset_directory / scene.name / SCENE_RADIO_MAPS_DIRECTORY
for radio_map in scene.radio_maps:
if radio_map.request is None:
# Radio maps taking registered antennas are checked once their
# scene is ready.
continue
record = _completed_radio_map_record(
radio_maps_directory / radio_map.name, radio_map, output
)
if record is not None:
completed[(scene.name, radio_map.name)] = record
return completed
[docs]
def completed_radio_map_records(
dataset_directory: Path, plan: BatchPlan, output: BatchOutputSpec
) -> list[tuple[PlannedScene, list[dict]]]:
"""Each planned scene with the records of its complete radio maps, in plan order.
A radio map is complete by the check resume uses; a record generated from another
request than the plan's raises.
"""
scenes = []
for scene in plan.scenes:
radio_maps_directory = dataset_directory / scene.name / SCENE_RADIO_MAPS_DIRECTORY
records = []
for radio_map in scene.radio_maps:
try:
record = _completed_radio_map_record(
radio_maps_directory / radio_map.name, radio_map, output
)
except ValueError as exc:
raise ValueError(f"Scene {scene.name}, radio map {radio_map.name}: {exc}") from exc
if record is not None:
records.append(record)
scenes.append((scene, records))
return scenes
[docs]
def write_dataset_index(
dataset_directory: Path, plan: BatchPlan, output: BatchOutputSpec, revision: int
) -> Path:
"""Rebuild ``index.parquet`` of a dataset for the manifest's ``index_revision``."""
return write_index(
dataset_directory,
index_table(
dataset_directory,
completed_radio_map_records(dataset_directory, plan, output),
revision,
),
)
[docs]
@contextmanager
def exclusive_dataset_lock(dataset_directory: Path):
"""Hold the dataset lock exclusively, refusing while anyone else holds it."""
lock_path = dataset_directory / ".generation.lock"
with lock_path.open("a+", encoding="utf-8") as handle:
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as exc:
raise RuntimeError(
"Another generation run, or a packaging or indexing job, holds the lock of "
f"{dataset_directory}"
) from exc
try:
yield
finally:
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
def _preview_details(request: SimulationRequest, building_count: int, cells: int) -> dict:
return {
"device": "CPU",
"model": "FSPL with directional and building obstruction penalties",
"buildings_considered": building_count,
"transmitter_count": len(request.transmitters),
"array_contract": RADIO_ENGINE_ARRAY_CONTRACT,
"cell_size_m": request.resolution_m,
"receiver_cells": cells,
"solver_seed": request.seed,
"association_metric": request.association_metric,
"noise_bandwidth_hz": request.bandwidth_mhz * 1e6,
"temperature_k": PREVIEW_TEMPERATURE_K,
"thermal_noise_power_w": preview_noise_power_w(request),
"terrain_propagation": False,
}
def _radio_record(
request: SimulationRequest,
engine_details: dict,
assets: SceneAssets,
surface: dict,
) -> dict:
"""What deriving RSS, SINR and the serving transmitter from the path gain needs."""
# Scene ENU metres where the engine placed each transmitter: Sionna at the terrain
# elevation plus the height above ground, the preview at that height above flat ground.
if request.engine == EngineName.sionna.value:
positions = [
transmitter_scene_position(
assets, tx.position.longitude, tx.position.latitude, tx.position.altitude_m
)
for tx in request.transmitters
]
else:
positions = preview_transmitter_positions(request)
return {
# 10 ** ((power_dbm - 30) / 10) W in float64; Sionna applies it in float32.
"tx_power_w": [10.0 ** ((tx.power_dbm - 30.0) / 10.0) for tx in request.transmitters],
"noise_power_w": engine_details["thermal_noise_power_w"],
"tx_positions_enu_m": [list(position) for position in positions],
"association_metric": request.association_metric,
"measurement_surface": surface["path"],
"building_rasters": surface["building_rasters"],
}
def _antenna_provenance(result: AntennaQueryResult) -> dict[str, str | int]:
return {
"antenna_provider": result.provider,
**result.source_provenance,
"antenna_release": result.release or "",
"antenna_license": result.license,
"antenna_attribution": result.attribution,
"antenna_count": len(result.antennas),
}
async def _scene_source_antennas(
scene_directory: Path,
specification: BatchSceneSpec,
assets: SceneAssets,
provider,
started: bool,
) -> tuple[AntennaQueryResult, list[ChannelRange]]:
"""The antennas a scene's radio maps take as transmitters and the channels they
draw from, persisted for a deterministic resume."""
selection = specification.antennas
query = selection.model_dump(mode="json")
expected = resolve_antenna_source(
selection.source,
specification.latitude,
specification.longitude,
provider.settings,
provider.antenna_sources,
)
path = scene_directory / SCENE_SOURCE_ANTENNAS_PATH
if path.is_file():
stored, stored_query, channels = load_antenna_source(path)
if stored.source_selection != expected or stored_query != query or not channels:
raise ValueError(
f"Scene {scene_directory.name} has a different antenna-source selection; use "
"its original source configuration to resume, or choose a new dataset directory"
)
return stored, channels
if started:
raise ValueError(
f"Scene {scene_directory.name} has radio maps but no persisted antennas; "
"its radio maps cannot be extended safely"
)
acquired = await provider.antennas(
specification.latitude,
specification.longitude,
specification.radius_m,
source=selection.source,
)
chosen = select_antennas(
keep_inside_scene(acquired, assets),
operator=selection.operator,
radio=selection.radio,
)
if not chosen.antennas:
wanted = [
f"{name} {value}"
for name, value in (("operator", selection.operator), ("radio", selection.radio))
if value
]
condition = f" match {', '.join(wanted)}" if wanted else ""
reason = f"; {acquired.warnings[0]}" if not acquired.antennas and acquired.warnings else ""
raise ValueError(
f"No antennas of source '{expected['resolved']}' inside scene "
f"{scene_directory.name}{condition}{reason}"
)
channels = scene_channels(selection, chosen, specification.radio_maps.engine.value)
write_antenna_source(path, chosen, query, channels)
return chosen, channels
async def _run_in_worker(executor: Executor, function, *args):
future = executor.submit(partial(function, *args))
job = asyncio.wrap_future(future)
try:
return await asyncio.shield(job)
except asyncio.CancelledError:
# A cancelled run keeps the dataset lock until the job stops writing its files; the
# event loop keeps running meanwhile. A job that has not started is dropped.
future.cancel()
await asyncio.wait([job])
raise
def _index_revision(manifest_path: Path) -> int | None:
"""The manifest's ``index_revision``, or None without a readable manifest."""
try:
return _read_json(manifest_path).get("index_revision")
except (OSError, ValueError):
return None
[docs]
async def generate_batch_dataset(
payload: BatchDatasetRequest,
plan: BatchPlan,
output_root: Path,
provider,
building_limit: int,
sionna_lock: asyncio.Lock,
worker: Executor,
progress: ProgressCallback,
resume: bool = False,
) -> Path:
dataset_directory = output_root.expanduser().resolve() / plan.dataset_slug
output_root.expanduser().resolve().mkdir(parents=True, exist_ok=True)
dataset_directory.mkdir(parents=False, exist_ok=resume)
manifest_path = dataset_directory / "dataset_manifest.json"
with exclusive_dataset_lock(dataset_directory):
revision = _index_revision(manifest_path)
try:
return await _generate_batch_dataset_locked(
payload,
plan,
dataset_directory,
provider,
building_limit,
sionna_lock,
worker,
progress,
resume,
)
except BaseException as exc:
# Only a run that got as far as a new revision is recorded; a refused resume leaves
# the dataset as it was. The original exception propagates in every case.
if _index_revision(manifest_path) != revision:
_finish_stopped_run(dataset_directory, plan, payload.output, exc)
raise
def _write_licences_and_index(
dataset_directory: Path,
manifest: dict,
records: list[tuple[PlannedScene, list[dict]]],
) -> None:
"""Rebuild the manifest's ``data_licences``, ``LICENSE.md`` and ``index.parquet`` from the
persisted scene records and the complete radio map ``records``; the caller writes the
manifest."""
manifest["data_licences"] = data_licences(manifest["scenes"])
_write_text(dataset_directory / "LICENSE.md", licence_notice(manifest))
write_index(
dataset_directory, index_table(dataset_directory, records, manifest["index_revision"])
)
def _finish_stopped_run(
dataset_directory: Path, plan: BatchPlan, output: BatchOutputSpec, error: BaseException
) -> None:
"""Record a run that stopped on ``error`` as failed and rebuild the files derived from the
records. A failure here is logged, so that the caller raises ``error`` itself."""
manifest_path = dataset_directory / "dataset_manifest.json"
try:
manifest = _read_json(manifest_path)
manifest["status"] = "failed"
run_number = len(manifest["generation_runs"])
run = manifest["generation_runs"][-1]
run.update(
status="failed",
# asyncio.run turns Ctrl+C into a cancellation, whose message is empty.
error=(
"interrupted"
if isinstance(error, (asyncio.CancelledError, KeyboardInterrupt))
else str(error) or type(error).__name__
),
)
# Written first, so the failed status survives a failure below.
_write_json(manifest_path, manifest)
# The manifest can lag the records by a radio map, so the counts come from the records.
records = completed_radio_map_records(dataset_directory, plan, output)
completed = {scene.name: len(scene_records) for scene, scene_records in records}
manifest["summary"]["completed_radio_maps"] = sum(completed.values())
run["generated_radio_maps"] = sum(
record.get("generation_run") == run_number
for _, scene_records in records
for record in scene_records
)
for scene_record in manifest["scenes"]:
scene_record["radio_maps"]["completed"] = completed[scene_record["name"]]
if scene_record.get("status") != "pending":
# The manifest is written before scene_info.json, so its entry is the newer.
if scene_record["status"] == "running":
scene_record["status"] = "failed"
scene_info = dataset_directory / scene_record["name"] / SCENE_INFO_PATH
_write_json(scene_info, scene_record)
_write_json(manifest_path, manifest)
_write_licences_and_index(dataset_directory, manifest, records)
_write_json(manifest_path, manifest)
except BaseException:
logger.warning(
"Could not finish the stopped run of %s; run 'owrt dataset index %s' to rebuild "
"its index",
dataset_directory,
dataset_directory,
exc_info=True,
)
async def _generate_batch_dataset_locked(
payload: BatchDatasetRequest,
plan: BatchPlan,
dataset_directory: Path,
provider,
building_limit: int,
sionna_lock: asyncio.Lock,
worker: Executor,
progress: ProgressCallback,
resume: bool,
) -> Path:
manifest_path = dataset_directory / "dataset_manifest.json"
existing_manifest = manifest_path.is_file()
if existing_manifest:
if not resume:
raise FileExistsError(f"Dataset already exists at {dataset_directory}")
manifest = _read_json(manifest_path)
_validate_resume_manifest(manifest, payload, plan)
else:
unexpected = [
path.name for path in dataset_directory.iterdir() if path.name != ".generation.lock"
]
if unexpected:
raise ValueError(
f"Cannot resume {dataset_directory}: dataset_manifest.json is missing from a "
"non-empty directory"
)
manifest = {
"schema_version": plan.schema_version,
"index_revision": 0,
"dataset_name": plan.dataset_name,
"dataset_slug": plan.dataset_slug,
"status": "running",
"software": {"openworld_radio_twin": __version__},
"execution_contracts": dict(EXECUTION_CONTRACTS),
"randomness": {
"dataset_seed": plan.random_seed,
"generator": plan.generator,
"derivation": (
"SeedSequence([dataset_seed, one_based_scene, one_based_radio_map])"
),
"expanded_parameters_recorded": True,
},
"conventions": conventions_record(),
"data_licences": [],
"request": payload.model_dump(mode="json"),
"summary": {},
"scenes": [],
"generation_runs": [],
}
completed_records = _scan_completed_radio_maps(dataset_directory, plan, payload.output)
completed_per_scene = Counter(scene_name for scene_name, _ in completed_records)
existing_scenes = {
item["name"]: item
for item in manifest.get("scenes", [])
if isinstance(item, dict) and "name" in item
}
manifest["scenes"] = [
_pending_scene_record(
scene, existing_scenes.get(scene.name), completed_per_scene[scene.name]
)
for scene in plan.scenes
]
manifest["request"] = payload.model_dump(mode="json")
manifest["status"] = "running"
manifest["summary"] = {
"scenes": plan.total_scenes,
"radio_maps": plan.total_radio_maps,
"completed_radio_maps": len(completed_records),
"failed_radio_maps": 0,
}
generation_runs = manifest.setdefault("generation_runs", [])
run_record = {
"software": {"openworld_radio_twin": __version__},
"resume": existing_manifest,
"requested_radio_maps": plan.total_radio_maps,
"initial_completed_radio_maps": len(completed_records),
"generated_radio_maps": 0,
"failed_radio_maps": 0,
"status": "running",
}
generation_runs.append(run_record)
# Each radio map record names the run that wrote it by this one-based position.
run_number = len(generation_runs)
# Advanced before any radio map is written, so an index built earlier reads as stale
# until this run rebuilds it at its end.
manifest["index_revision"] += 1
_write_json(manifest_path, manifest)
progress(
len(completed_records),
plan.total_radio_maps,
f"Found {len(completed_records)} completed radio map(s); preparing shared "
"scene assets",
)
processed = len(completed_records)
for scene_index, scene in enumerate(plan.scenes):
specification = scene.specification
scene_directory = dataset_directory / scene.name
scene_directory.mkdir(exist_ok=True)
scene_has_completed_radio_maps = any(key[0] == scene.name for key in completed_records)
building_source_path = scene_directory / SCENE_SOURCE_BUILDINGS_PATH
if building_source_path.is_file():
try:
building_result = load_building_source(building_source_path)
except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError):
if scene_has_completed_radio_maps:
raise ValueError(
f"Scene {scene.name} has invalid persisted building inputs required "
"for a deterministic resume"
) from None
building_result = None
else:
building_result = None
if building_result is not None and specification.include_buildings:
expected_source = resolve_building_source(
specification.building_source,
specification.latitude,
specification.longitude,
provider.settings,
provider.building_sources,
)
if building_result.source_selection != expected_source:
raise ValueError(
f"Scene {scene.name} has a different or unrecorded building-source "
"selection/version; use its original software and source configuration "
"to resume, or choose a new dataset directory"
)
if building_result is None and specification.include_buildings:
if scene_has_completed_radio_maps:
raise ValueError(
f"Scene {scene.name} predates resumable source persistence; "
"its completed radio maps cannot be extended safely"
)
building_result = await provider.buildings(
specification.latitude,
specification.longitude,
specification.radius_m,
building_limit,
source=specification.building_source,
)
write_building_source(building_source_path, building_result)
elif building_result is None:
building_result = BuildingQueryResult(buildings=[], provider="disabled")
write_building_source(building_source_path, building_result)
environment = None
environment_warnings = []
if specification.include_environment_grid:
try:
environment = load_environment_assets(scene_directory)
environment_warnings = list(environment.warnings)
except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError):
if scene_has_completed_radio_maps:
raise ValueError(
f"Scene {scene.name} is missing persisted terrain inputs required "
"for a deterministic resume"
) from None
environment, environment_warnings = await provider.environment(
specification.latitude,
specification.longitude,
specification.radius_m,
specification.terrain_resolution_m,
specification.include_surface_materials,
terrain_mode="elevation" if specification.include_terrain else "flat",
)
# Kept in terrain/scene_info.json so a resumed run reports the same warnings.
environment = replace(environment, warnings=tuple(environment_warnings))
assets = await _run_in_worker(
worker,
build_scene_assets,
scene_directory,
building_result.buildings,
specification.longitude,
specification.latitude,
specification.radius_m,
environment,
specification.scene_mode != "empty",
specification.material_profile,
)
await _run_in_worker(
worker,
compilation_record,
scene_directory,
_scene_geometry_request(specification, provider.settings),
building_result,
environment_warnings,
assets,
)
antenna_result = None
planned_radio_maps = scene.radio_maps
if specification.antennas is not None:
radio_maps_directory = scene_directory / SCENE_RADIO_MAPS_DIRECTORY
started = scene_has_completed_radio_maps or any(
(radio_maps_directory / radio_map.name / "radio_map_info.json").is_file()
for radio_map in scene.radio_maps
)
antenna_result, channels = await _scene_source_antennas(
scene_directory, specification, assets, provider, started
)
planned_radio_maps = [
registered_radio_map(radio_map, antenna_result, assets, channels)
for radio_map in scene.radio_maps
]
for radio_map in planned_radio_maps:
if (scene.name, radio_map.name) in completed_records:
continue
record = _completed_radio_map_record(
radio_maps_directory / radio_map.name, radio_map, payload.output
)
if record is not None:
completed_records[(scene.name, radio_map.name)] = record
processed += 1
manifest["summary"]["completed_radio_maps"] += 1
run_record["initial_completed_radio_maps"] += 1
voxels = await _run_in_worker(
worker,
write_voxel_assets,
assets,
specification.receiver_height_m,
specification.voxel_pitch_m,
)
measurement_surfaces = []
receiver_grids = sorted(
{
(radio_map.request.resolution_m, radio_map.request.receiver_height_m)
for radio_map in planned_radio_maps
}
)
for resolution_m, receiver_height_m in receiver_grids:
# Scene-level files are reused when valid and written atomically otherwise, so a
# resume never rewrites what completed radio maps reference.
surface = await _run_in_worker(
worker,
prepare_measurement_surface,
measurement_surface_path(assets, resolution_m, receiver_height_m),
assets,
resolution_m,
receiver_height_m,
)
building_rasters = await _run_in_worker(
worker,
prepare_surface_building_rasters,
building_result.buildings,
surface,
specification.longitude,
specification.latitude,
receiver_height_m,
environment,
)
measurement_surfaces.append(
{
"path": surface.path.relative_to(scene_directory).as_posix(),
"resolution_m": resolution_m,
"receiver_height_agl_m": receiver_height_m,
"cell_shape": list(surface.cell_shape),
"height_reference": HEIGHT_REFERENCE,
"building_rasters": building_rasters.relative_to(scene_directory).as_posix(),
}
)
surfaces_by_grid = {
(entry["resolution_m"], entry["receiver_height_agl_m"]): entry
for entry in measurement_surfaces
}
radio_maps_directory = scene_directory / SCENE_RADIO_MAPS_DIRECTORY
radio_maps_directory.mkdir(exist_ok=True)
# Scene records keep counts only; radio_map_info.json is the one full record.
counts = {
"planned": len(planned_radio_maps),
"completed": sum(
(scene.name, radio_map.name) in completed_records
for radio_map in planned_radio_maps
),
"failed": 0,
}
scene_record = {
"schema_version": DATASET_SCHEMA_VERSION,
"name": scene.name,
"label": scene.label,
"status": "running",
"specification": specification.model_dump(mode="json", exclude={"radio_maps"}),
"provenance": {
"building_provider": building_result.provider,
"building_release": building_result.release,
"scene_boundary_policy": SCENE_BOUNDARY_POLICY,
"source_building_count": building_result.source_count,
"boundary_excluded_building_count": building_result.boundary_excluded_count,
"exported_building_count": assets.building_count,
"triangle_count": assets.triangle_count,
"terrain_provider": (
environment.terrain.source.provider if environment is not None else "disabled"
),
"terrain_spacing_m": (
environment.terrain.spacing_m if environment is not None else 0.0
),
"surface_provider": (
environment.surface_provenance.get("provider", "disabled")
if environment is not None
else "disabled"
),
**(_antenna_provenance(antenna_result) if antenna_result is not None else {}),
"licences": scene_licences(building_result, environment, antenna_result),
},
"shared_assets": {
"scene_xml": SCENE_XML_PATH.as_posix(),
"coordinate_provenance": SCENE_PROVENANCE_PATH.as_posix(),
"mesh": f"{SCENE_MESH_DIRECTORY.as_posix()}/",
"source_buildings": SCENE_SOURCE_BUILDINGS_PATH.as_posix(),
**(
{"source_antennas": SCENE_SOURCE_ANTENNAS_PATH.as_posix()}
if antenna_result is not None
else {}
),
"height_map": SCENE_HEIGHT_MAP_PATH.as_posix(),
"voxels": str(voxels.voxel_path.relative_to(scene_directory)),
"receiver_slice": str(voxels.slice_path.relative_to(scene_directory)),
"voxel_depth": str(voxels.depth_array_path.relative_to(scene_directory)),
"measurement_surfaces": measurement_surfaces,
**(
{
"environment": SCENE_TERRAIN_INFO_PATH.as_posix(),
"environment_arrays": {
name: path.as_posix()
for name, path in SCENE_ENVIRONMENT_ARRAY_PATHS.items()
},
}
if environment is not None
else {}
),
},
"radio_maps": counts,
}
manifest["scenes"][scene_index] = scene_record
manifest["data_licences"] = data_licences(manifest["scenes"])
# The manifest first at every checkpoint, so its scene entry is never the older copy.
_write_json(manifest_path, manifest)
_write_json(scene_directory / SCENE_INFO_PATH, scene_record)
for radio_map in planned_radio_maps:
radio_map_directory = radio_maps_directory / radio_map.name
if (scene.name, radio_map.name) in completed_records:
continue
radio_map_directory.mkdir(exist_ok=True)
progress(
processed, plan.total_radio_maps, f"Running {scene.name}/{radio_map.name}"
)
radio_map_started = time.perf_counter()
_write_json(
radio_map_directory / "radio_map_info.json",
{
"schema_version": DATASET_SCHEMA_VERSION,
"name": radio_map.name,
"status": "running",
"seed": radio_map.seed,
"scene": "../../scene_info.json",
"request": radio_map.request.model_dump(mode="json"),
"generation_run": run_number,
},
)
try:
if radio_map.request.engine == EngineName.sionna.value:
async with sionna_lock:
radio_data, engine_details, warnings = await _run_in_worker(
worker,
simulate_sionna_scene,
radio_map.request,
assets,
len(building_result.buildings),
)
else:
radio_data = await _run_in_worker(
worker,
simulate_preview,
radio_map.request,
building_result.buildings,
)
warnings = [
"Preview mode is analytical and is not a calibrated ray-tracing result."
]
if radio_map.request.include_terrain:
warnings.append(
"Terrain and surface classes are exported but are not propagation "
"inputs to the planar CPU preview; use Sionna RT for terrain-aware "
"radio maps."
)
engine_details = _preview_details(
radio_map.request,
len(building_result.buildings),
radio_data.rows * radio_data.columns,
)
warnings.extend(environment_warnings)
warnings.extend(radio_map.warnings)
# With the scene environment for every engine, the mask of the _masked PNGs
# is the surface's receiver_in_building_mask.
products = build_coverage_products(
radio_map.request,
radio_data,
building_result.buildings,
environment,
)
response = SimulationResponse(
simulation_id=radio_map.name,
engine=radio_map.request.engine,
elapsed_ms=round((time.perf_counter() - radio_map_started) * 1000),
coverage=products.coverage,
buildings=[],
warnings=warnings,
engine_details=engine_details,
data_provenance={
"scene": scene.name,
"shared_geometry": "../../scene_info.json",
"building_provider": building_result.provider,
**building_result.source_provenance,
"building_count": assets.building_count,
**(
_antenna_provenance(antenna_result)
if antenna_result is not None
else {}
),
},
)
await _run_in_worker(
worker, write_packed_path_gain, radio_map_directory, radio_data
)
image_products = None
if payload.output.previews:
image_products = await _run_in_worker(
worker,
write_radio_images,
radio_map_directory,
radio_data,
products.building_mask,
)
radio_surface = surfaces_by_grid[
(radio_map.request.resolution_m, radio_map.request.receiver_height_m)
]
radio_map_record = {
"schema_version": DATASET_SCHEMA_VERSION,
"name": radio_map.name,
"status": "completed",
"seed": radio_map.seed,
"scene": "../../scene_info.json",
"request": radio_map.request.model_dump(mode="json"),
"generation_run": run_number,
"data_provenance": response.data_provenance,
"array_grid_mapping": array_grid_mapping(
radio_map.request, radio_data
).model_dump(mode="json"),
"radio_layers": {
name: layer.model_dump(mode="json", exclude={"values"})
for name, layer in response.coverage.layers.items()
},
"array_contract": {
"shape": list(radio_data.shape),
"axis_order": list(RADIO_ARRAY_AXIS_ORDER),
"path_gain": path_gain_packing(),
},
"radio": _radio_record(
radio_map.request, engine_details, assets, radio_surface
),
**({"image_products": image_products} if image_products else {}),
"engine_details": engine_details,
"warnings": warnings,
}
_write_json(radio_map_directory / "radio_map_info.json", radio_map_record)
except Exception as exc:
radio_map_record = {
"schema_version": DATASET_SCHEMA_VERSION,
"name": radio_map.name,
"status": "failed",
"seed": radio_map.seed,
"scene": "../../scene_info.json",
"request": radio_map.request.model_dump(mode="json"),
"generation_run": run_number,
"error": str(exc),
}
counts["failed"] += 1
manifest["summary"]["failed_radio_maps"] += 1
run_record["failed_radio_maps"] += 1
try:
_write_json(radio_map_directory / "radio_map_info.json", radio_map_record)
if not payload.continue_on_error:
scene_record["status"] = "failed"
_write_json(manifest_path, manifest)
_write_json(scene_directory / SCENE_INFO_PATH, scene_record)
except Exception:
if payload.continue_on_error:
raise
# The error of the radio map, not this one, stops the run.
logger.warning(
"Could not record the failure of %s", radio_map_directory, exc_info=True
)
if not payload.continue_on_error:
# The caller records the run as failed.
raise
else:
counts["completed"] += 1
manifest["summary"]["completed_radio_maps"] += 1
run_record["generated_radio_maps"] += 1
processed += 1
# Written after every radio map, so the counts stay current when a run stops.
_write_json(manifest_path, manifest)
_write_json(scene_directory / SCENE_INFO_PATH, scene_record)
progress(
processed, plan.total_radio_maps, f"Completed {scene.name}/{radio_map.name}"
)
scene_record["status"] = "completed_with_errors" if counts["failed"] else "completed"
_write_json(manifest_path, manifest)
_write_json(scene_directory / SCENE_INFO_PATH, scene_record)
manifest["status"] = (
"completed_with_errors" if manifest["summary"]["failed_radio_maps"] else "completed"
)
run_record["status"] = manifest["status"]
# The completed status is written last, so a failure before it ends as a failed run.
_write_licences_and_index(
dataset_directory,
manifest,
completed_radio_map_records(dataset_directory, plan, payload.output),
)
_write_json(manifest_path, manifest)
return dataset_directory