"""High-level Python interface for reproducible dataset generation."""
from __future__ import annotations
import asyncio
import fcntl
import hashlib
import json
import math
import tarfile
from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from dataclasses import dataclass
from fractions import Fraction
from pathlib import Path
from types import MappingProxyType
from typing import Any, NamedTuple
import httpx
import numpy as np
import pyarrow as pa
import zstandard
from openworld_radio_twin import __version__
from openworld_radio_twin.batch import (
BatchDatasetRequest,
BatchOutputSpec,
BatchPlan,
PlannedScene,
ProgressCallback,
build_batch_plan,
completed_radio_map_records,
exclusive_dataset_lock,
generate_batch_dataset,
write_dataset_index,
)
from openworld_radio_twin.config import Settings, get_settings
from openworld_radio_twin.conventions import (
DATASET_SCHEMA_VERSION,
RADIO_ARRAY_DTYPE,
RADIO_IMAGES_DIRECTORY,
path_gain_packing,
)
from openworld_radio_twin.geodesy import LocalFrame
from openworld_radio_twin.licences import licence_notice, release_problems
from openworld_radio_twin.models import EngineName, GridMapping
from openworld_radio_twin.providers.map_data import MapDataProvider
from openworld_radio_twin.radio_map_index import INDEX_PATH, index_table, read_index, write_index
from openworld_radio_twin.scene_format import SCENE_RADIO_MAPS_DIRECTORY
from openworld_radio_twin.simulation.radio_map import (
serving_transmitter,
sinr_from_rss,
unpack_path_gain,
)
from openworld_radio_twin.simulation.sionna_engine import availability as sionna_availability
DatasetRequestSource = BatchDatasetRequest | Mapping[str, Any] | str | Path
__all__ = [
"DEFAULT_SPLIT_FRACTIONS",
"DatasetGenerationResult",
"DatasetRequestSource",
"RadioMap",
"build_index",
"generate_dataset",
"generate_dataset_sync",
"load_dataset_request",
"load_radio_map",
"package_dataset",
"plan_dataset",
"read_index",
]
[docs]
@dataclass(frozen=True)
class DatasetGenerationResult:
"""Completed dataset location together with its deterministic expanded plan."""
output_directory: Path
plan: BatchPlan
@property
def manifest_path(self) -> Path:
return self.output_directory / "dataset_manifest.json"
@property
def index_path(self) -> Path:
return self.output_directory / INDEX_PATH
[docs]
def read_manifest(self) -> dict[str, Any]:
"""Read the completed dataset manifest."""
value = json.loads(self.manifest_path.read_text(encoding="utf-8"))
if not isinstance(value, dict):
raise ValueError(f"Dataset manifest is not a JSON object: {self.manifest_path}")
return value
[docs]
def load_dataset_request(source: DatasetRequestSource) -> BatchDatasetRequest:
"""Validate a model, mapping, or JSON file as a batch dataset request."""
if isinstance(source, BatchDatasetRequest):
return source
if isinstance(source, (str, Path)):
path = Path(source).expanduser().resolve()
return BatchDatasetRequest.model_validate_json(path.read_text(encoding="utf-8"))
return BatchDatasetRequest.model_validate(source)
[docs]
def plan_dataset(source: DatasetRequestSource) -> BatchPlan:
"""Expand a dataset request without provider access or solver execution."""
return build_batch_plan(load_dataset_request(source))
def _ignore_progress(completed: int, total: int, message: str) -> None:
del completed, total, message
[docs]
async def generate_dataset(
source: DatasetRequestSource,
*,
output_root: str | Path | None = None,
settings: Settings | None = None,
progress: ProgressCallback | None = None,
resume: bool = False,
) -> DatasetGenerationResult:
"""Generate a dataset without starting the web service.
The function owns and closes its HTTP client and worker thread. Use this asynchronous
entry point from notebooks, services, or existing event loops. For ordinary scripts, use
:func:`generate_dataset_sync`. Set ``resume=True`` to skip complete radio maps
and regenerate missing ones in a compatible existing dataset.
"""
runtime_settings = settings or get_settings()
request = load_dataset_request(source)
plan = build_batch_plan(request)
root = Path(output_root or runtime_settings.dataset_root).expanduser().resolve()
target = root / plan.dataset_slug
if target.exists() and not resume:
raise FileExistsError(
f"Dataset '{plan.dataset_slug}' already exists at {target}; "
"choose a new dataset name/output root, or pass resume=True"
)
uses_sionna = any(
radio_map.engine == EngineName.sionna.value
for scene in plan.scenes
for radio_map in scene.radio_maps
)
if uses_sionna:
available, message = sionna_availability()
if not available:
raise RuntimeError(message)
root.mkdir(parents=True, exist_ok=True)
async with httpx.AsyncClient(
follow_redirects=True,
timeout=runtime_settings.http_request_timeout_seconds,
) as client:
provider = MapDataProvider(runtime_settings, client)
with ThreadPoolExecutor(max_workers=1, thread_name_prefix="owrt-dataset") as worker:
output_directory = await generate_batch_dataset(
request,
plan,
root,
provider,
runtime_settings.max_buildings,
asyncio.Lock(),
worker,
progress or _ignore_progress,
resume=resume,
)
return DatasetGenerationResult(output_directory=output_directory, plan=plan)
[docs]
def generate_dataset_sync(
source: DatasetRequestSource,
*,
output_root: str | Path | None = None,
settings: Settings | None = None,
progress: ProgressCallback | None = None,
resume: bool = False,
) -> DatasetGenerationResult:
"""Generate, resume, or deterministically extend a dataset from a normal script."""
try:
asyncio.get_running_loop()
except RuntimeError:
pass
else:
raise RuntimeError(
"generate_dataset_sync() cannot run inside an active event loop; "
"use 'await generate_dataset(...)' instead"
)
# Run outside the handler above, so an error of the run keeps its own context.
return asyncio.run(
generate_dataset(
source,
output_root=output_root,
settings=settings,
progress=progress,
resume=resume,
)
)
def _decibels(values: np.ndarray) -> np.ndarray:
with np.errstate(divide="ignore"):
return 10.0 * np.log10(values)
def _rss(power_w: np.ndarray, path_gain: np.ndarray) -> np.ndarray:
"""RSS in W of transmit powers in W and path gain ``(n_tx, rows, columns)``."""
# RSS is rounded to float32 before the SINR, as the engines compute it.
return (power_w[:, np.newaxis, np.newaxis] * path_gain).astype(RADIO_ARRAY_DTYPE)
[docs]
@dataclass(frozen=True, eq=False)
class RadioMap:
"""A generated radio map: its record and the arrays derived from its stored path gain.
``path_gain``, ``rss`` (W) and ``sinr`` are linear ``float32`` arrays with shape
``(n_tx, rows, columns)``, and ``association`` is the ``int32`` serving transmitter with
shape ``(rows, columns)``, -1 where no transmitter serves the cell. Rows run south to north
on the grid ``grid``. RSS, SINR and the serving transmitter follow the engine's definition
applied to the stored path gain, whose dB value is rounded to 0.01 dB steps (at most
0.005 dB off). RSS shares this bound; SINR can be off by up to twice it, because signal and
interference are rounded independently. The derived values are therefore not the engine's
own values bit for bit, and a near tie can change the serving transmitter. The ``float32``
path gain keeps that bound down to -379.3 dB, the smallest normal ``float32``; below it
``float32`` holds fewer digits, and stored values of -451.55 dB and below load as 0.
"""
directory: Path
record: dict[str, Any]
grid: GridMapping
path_gain: np.ndarray
rss: np.ndarray
sinr: np.ndarray
association: np.ndarray
@property
def path_gain_db(self) -> np.ndarray:
"""Path gain in dB, -inf where it is 0."""
return _decibels(self.path_gain)
@property
def rss_dbm(self) -> np.ndarray:
"""Received power in dBm, -inf where it is 0."""
return _decibels(self.rss) + np.float32(30.0)
@property
def sinr_db(self) -> np.ndarray:
"""SINR in dB, -inf where it is 0."""
return _decibels(self.sinr)
[docs]
def load_radio_map(path: str | Path) -> RadioMap:
"""Load a completed radio map from its directory or its ``radio_map_info.json``.
The path gain is unpacked from its stored ``uint16`` dB values. RSS is the transmit power
times the path gain; the SINR of a transmitter is its RSS over the sum of the other
transmitters' RSS plus the noise power, summed in float64; the serving transmitter has the
largest value under the recorded association metric, the first one on a tie.
"""
stored = _read_radio_map(path)
path_gain = unpack_path_gain(stored.packed)
rss = _rss(stored.power_w, path_gain)
sinr = sinr_from_rss(rss, stored.noise_power_w)
metric = stored.record["radio"]["association_metric"]
association = serving_transmitter(path_gain, rss, sinr, metric)
return RadioMap(
directory=stored.record_path.parent,
record=stored.record,
grid=GridMapping.model_validate(stored.record["array_grid_mapping"]),
path_gain=path_gain,
rss=rss,
sinr=sinr,
association=association,
)
class _StoredRadioMap(NamedTuple):
"""The checked record and stored path gain of a completed radio map."""
record_path: Path
record: dict[str, Any]
packed: np.ndarray # uint16 path gain, (n_tx, rows, columns)
power_w: np.ndarray # transmit powers in W, (n_tx,)
noise_power_w: float
def _read_radio_map(path: str | Path) -> _StoredRadioMap:
"""Read and check the record and stored path gain of a completed radio map."""
location = Path(path).expanduser()
record_path = location / "radio_map_info.json" if location.is_dir() else location
record = json.loads(record_path.read_text(encoding="utf-8"))
if record.get("schema_version") != DATASET_SCHEMA_VERSION:
raise ValueError(
f"{record_path} has dataset format {record.get('schema_version')!r}; this version "
f"reads format {DATASET_SCHEMA_VERSION}"
)
if record.get("status") != "completed":
raise ValueError(f"{record_path} records status {record.get('status')!r}, not completed")
contract = record.get("array_contract")
if not isinstance(contract, dict) or not {"shape", "path_gain"} <= contract.keys():
raise ValueError(f"{record_path} lacks the array_contract shape or path_gain entry")
if contract["path_gain"] != path_gain_packing():
raise ValueError(f"{record_path} records a path-gain encoding this version cannot read")
with np.load(record_path.parent / contract["path_gain"]["path"]) as archive:
packed = archive[contract["path_gain"]["key"]]
if len(contract["shape"]) != 3 or list(packed.shape) != contract["shape"]:
raise ValueError(
f"Path gain of {record_path.parent} has shape {packed.shape}, not {contract['shape']}"
)
radio = record.get("radio", {})
for name in ("tx_power_w", "noise_power_w", "association_metric"):
if name not in radio:
raise ValueError(f"{record_path} lacks the radio block key {name}")
if radio["association_metric"] not in ("path_gain", "rss", "sinr"):
raise ValueError(
f"{record_path} records an unknown association metric {radio['association_metric']!r}"
)
noise_power_w = float(radio["noise_power_w"])
if not math.isfinite(noise_power_w) or noise_power_w <= 0:
raise ValueError(f"{record_path}: thermal noise power must be finite and positive")
power_w = np.asarray(radio["tx_power_w"], dtype=np.float64)
if power_w.shape != packed.shape[:1]:
raise ValueError(f"{record_path} records {power_w.size} powers for {packed.shape[0]} Tx")
if not np.isfinite(power_w).all() or np.any(power_w < 0):
raise ValueError(f"{record_path}: transmit powers must be finite and non-negative")
return _StoredRadioMap(record_path, record, packed, power_w, noise_power_w)
DEFAULT_SPLIT_FRACTIONS = MappingProxyType({"train": 0.8, "validation": 0.1, "test": 0.1})
PACKAGE_SCENES_DIRECTORY = "scenes"
# zstd level of the scene archives, compressed in the calling thread so the bytes depend only
# on the input and the zstandard and zstd library versions.
ARCHIVE_ZSTD_LEVEL = 3
@dataclass(frozen=True)
class _PackagedScene:
"""A scene of the package with its manifest record and the records of its complete
radio maps."""
planned: PlannedScene
record: dict[str, Any]
radio_maps: list[dict[str, Any]]
@dataclass(frozen=True)
class _SceneGroup:
"""A split group with a packaged scene: its id (the name of its earliest scene, packaged
or not), its packaged scenes, its split, and whether it kept the split its scenes have in
a previous package."""
id: str
members: list[str]
split: str
kept: bool
class _HashingWriter:
"""Writes to a file and keeps the SHA-256 digest and size of what was written."""
def __init__(self, handle) -> None:
self.handle = handle
self.digest = hashlib.sha256()
self.size = 0
def write(self, data: bytes) -> int:
self.handle.write(data)
self.digest.update(data)
self.size += len(data)
return len(data)
@contextmanager
def _no_active_generation(dataset_directory: Path):
"""Hold the dataset lock shared, so no generator writes the dataset while it is read."""
lock_path = dataset_directory / ".generation.lock"
if not lock_path.is_file():
yield
return
with lock_path.open("r", encoding="utf-8") as handle:
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
except BlockingIOError as exc:
raise RuntimeError(
f"A generation run or an index rebuild holds the lock of {dataset_directory}; "
"try again once it has finished"
) from exc
try:
yield
finally:
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
def _check_manifest(manifest: dict[str, Any], action: str) -> None:
"""Refuse a manifest of another dataset format or software version."""
if manifest.get("schema_version") != DATASET_SCHEMA_VERSION:
raise ValueError(
f"{action} reads dataset format {DATASET_SCHEMA_VERSION}, not "
f"{manifest.get('schema_version')!r}"
)
# The plan, and with it the name of every radio map, may differ between versions.
software_version = manifest.get("software", {}).get("openworld_radio_twin")
if software_version != __version__:
raise ValueError(
f"{action} requires the same OpenWorld Radio Twin version that created the "
f"dataset ({software_version!r} != {__version__!r})"
)
[docs]
def build_index(dataset_dir: str | Path) -> Path:
"""Rebuild ``index.parquet`` of a generated dataset and return its path.
A run rebuilds the index when it ends; this rebuilds it after a run that was stopped
before, for the manifest's ``index_revision``. It takes the generation lock, so it
refuses while a run, a packaging job or another rebuild holds it, and it requires the
version that created the dataset. A package is refused: packaging writes its index.
"""
dataset_directory = Path(dataset_dir).expanduser().resolve()
manifest_path = dataset_directory / "dataset_manifest.json"
if not manifest_path.is_file():
raise FileNotFoundError(f"No dataset_manifest.json in {dataset_directory}")
if (dataset_directory / "splits.json").is_file():
raise ValueError(
f"{dataset_directory} is a package; package the dataset again to rebuild its index"
)
with exclusive_dataset_lock(dataset_directory):
manifest = json.loads(manifest_path.read_bytes())
_check_manifest(manifest, "Indexing")
request = BatchDatasetRequest.model_validate(manifest["request"])
return write_dataset_index(
dataset_directory,
build_batch_plan(request),
request.output,
manifest["index_revision"],
)
def _split_fractions(fractions: Mapping[str, float]) -> dict[str, float]:
values = {str(name): float(value) for name, value in fractions.items()}
if not values:
raise ValueError("At least one split is required")
for name, value in values.items():
if not name or not name.replace("_", "").replace("-", "").isalnum():
raise ValueError(f"Split name {name!r} may hold only letters, digits, '_' and '-'")
if not math.isfinite(value) or value < 0:
raise ValueError(f"Split fraction of {name!r} must be a non-negative number")
if not math.isclose(sum(values.values()), 1.0, abs_tol=1e-6):
raise ValueError("Split fractions must add up to 1")
return values
def _split_groups(scenes: list[PlannedScene]) -> list[list[int]]:
"""Scene indices grouped so that overlapping scenes and scenes of one site stay together.
Two scenes overlap when their squares, of half-side ``radius_m`` around their centres,
intersect in the local frame of the earlier one. Groups are in scene order.
"""
parent = list(range(len(scenes)))
def root(index: int) -> int:
while parent[index] != index:
parent[index] = parent[parent[index]]
index = parent[index]
return index
def join(first: int, second: int) -> None:
low, high = sorted((root(first), root(second)))
parent[high] = low
specifications = [scene.specification for scene in scenes]
longitudes = np.array([item.longitude for item in specifications], dtype=float)
latitudes = np.array([item.latitude for item in specifications], dtype=float)
radii = np.array([item.radius_m for item in specifications], dtype=float)
for index, specification in enumerate(specifications[:-1]):
later = slice(index + 1, None)
frame = LocalFrame.at(specification.longitude, specification.latitude)
east, north, up = frame.transformer.transform(
longitudes[later], latitudes[later], np.zeros(len(specifications) - index - 1)
)
reach = specification.radius_m + radii[later]
# The height test keeps a point on the far side of the Earth, straight below, apart.
overlapping = (np.abs(east) < reach) & (np.abs(north) < reach) & (np.abs(up) < reach)
for offset in np.flatnonzero(overlapping):
join(index, index + 1 + int(offset))
first_of_site: dict[str, int] = {}
for index, specification in enumerate(specifications):
if specification.site is not None:
join(index, first_of_site.setdefault(specification.site, index))
groups: dict[int, list[int]] = {}
for index in range(len(scenes)):
groups.setdefault(root(index), []).append(index)
return sorted(groups.values(), key=lambda members: members[0])
def _largest_remainder(weights: Mapping[str, float], groups: int) -> list[int]:
"""``groups`` divided in proportion to ``weights`` by the largest remainder method; ties
go to the earlier name."""
# The decimal value of each weight (0.7 is 7/10) in exact arithmetic, so that equal
# remainders tie and the earlier name wins.
values = [Fraction(str(weight)) for weight in weights.values()]
total = sum(values)
quotas = [value * groups / total for value in values]
counts = [math.floor(quota) for quota in quotas]
by_remainder = sorted(range(len(quotas)), key=lambda k: (counts[k] - quotas[k], k))
for k in by_remainder[: groups - sum(counts)]:
counts[k] += 1
return counts
def _split_counts(fractions: dict[str, float], groups: int) -> dict[str, int]:
"""Groups per split by largest remainder; each split with a positive fraction gets at
least one group when there are enough groups."""
names = list(fractions)
counts = _largest_remainder(fractions, groups)
positive = [k for k, name in enumerate(names) if fractions[name] > 0]
if groups >= len(positive):
for k in positive:
if counts[k] == 0:
donor = max(range(len(names)), key=lambda m: (counts[m], -m))
counts[donor] -= 1
counts[k] += 1
return dict(zip(names, counts, strict=True))
def _previous_assignment(
previous: str | Path, plan: BatchPlan, fractions: dict[str, float]
) -> dict[str, str]:
"""The split of each scene in an earlier package of the dataset, from its ``splits.json``
or its directory."""
path = Path(previous).expanduser()
if path.is_dir():
path = path / "splits.json"
if not path.is_file():
raise FileNotFoundError(f"No splits.json of a previous package at {path}")
try:
assignment = json.loads(path.read_text(encoding="utf-8"))["assignment"]
except (UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError) as exc:
raise ValueError(f"{path} is not the splits.json of a package") from exc
if not isinstance(assignment, dict) or not all(
isinstance(name, str) and isinstance(split, str) for name, split in assignment.items()
):
raise ValueError(f"{path} is not the splits.json of a package")
names = {scene.name for scene in plan.scenes}
unknown = sorted(set(assignment) - names)
if unknown:
raise ValueError(
f"Scenes of the previous package are not scenes of this dataset: {', '.join(unknown)}"
)
splits = sorted(set(assignment.values()) - set(fractions))
if splits:
raise ValueError(
f"Splits of the previous package are not among the split fractions: {', '.join(splits)}"
)
return assignment
def _assign_splits(
plan: BatchPlan,
packaged: set[str],
fractions: dict[str, float],
seed: int,
previous: Mapping[str, str],
) -> list[_SceneGroup]:
"""The split groups with a packaged scene, in group order.
Groups are formed over all scenes of the plan. A group with scenes in ``previous`` keeps
their split. The other groups are shuffled with ``seed`` and fill what the kept groups
leave of each split's count for all groups, by largest remainder.
"""
groups: list[tuple[str, list[str]]] = []
kept: list[str | None] = []
for members in _split_groups(plan.scenes):
names = [plan.scenes[index].name for index in members]
earlier = {previous[name] for name in names if name in previous}
if len(earlier) > 1:
scenes = ", ".join(f"{name} ({previous[name]})" for name in names if name in previous)
joining = [name for name in names if name not in previous]
if joining:
scenes += f", joined by {', '.join(joining)}"
raise ValueError(
f"The group of scene {names[0]} joins scenes of different splits of the previous "
f"package: {scenes}. Packaging it would share geometry between splits"
)
if any(name in packaged for name in names):
groups.append((names[0], [name for name in names if name in packaged]))
kept.append(earlier.pop() if earlier else None)
new = [position for position, split in enumerate(kept) if split is None]
target = _split_counts(fractions, len(groups))
need = {name: max(target[name] - kept.count(name), 0) for name in fractions}
counts = _largest_remainder(need, len(new)) if new else [0] * len(need)
order = np.random.Generator(np.random.PCG64(seed)).permutation(len(new))
splits = list(kept)
position = 0
for name, count in zip(need, counts, strict=True):
for group in order[position : position + count]:
splits[new[int(group)]] = name
position += count
return [
_SceneGroup(group_id, members, split, earlier is not None)
for (group_id, members), split, earlier in zip(groups, splits, kept, strict=True)
]
def _tar_members(scene_directory: Path, radio_maps: set[str]) -> list[tuple[str, Path]]:
"""The scene's directories and files, named from the scene directory, in sorted order.
Hidden entries (locks, temporary files and directories), ``*.tmp`` files, radio
maps outside ``radio_maps`` and the PNG previews of radio maps are left out.
"""
radio_maps_directory = scene_directory / SCENE_RADIO_MAPS_DIRECTORY
members = [(scene_directory.name, scene_directory)]
pending = [scene_directory]
while pending:
directory = pending.pop()
for path in directory.iterdir():
if path.name.startswith(".") or path.name.endswith(".tmp"):
continue
if directory == radio_maps_directory and path.name not in radio_maps:
continue
if directory.parent == radio_maps_directory and path.name == RADIO_IMAGES_DIRECTORY:
continue
if path.is_symlink() or not (path.is_dir() or path.is_file()):
raise ValueError(f"Cannot package {path}: only files and directories are packaged")
members.append((path.relative_to(scene_directory.parent).as_posix(), path))
if path.is_dir():
pending.append(path)
return sorted(members)
def _write_scene_archive(path: Path, members: list[tuple[str, Path]]) -> tuple[str, int]:
"""Write a zstd-compressed GNU tar whose bytes depend only on the member names and
contents and the zstandard and zstd library versions; return its SHA-256 digest and
size."""
compressor = zstandard.ZstdCompressor(level=ARCHIVE_ZSTD_LEVEL, threads=0, write_checksum=True)
with path.open("wb") as handle:
writer = _HashingWriter(handle)
with (
compressor.stream_writer(writer, closefd=False) as compressed,
tarfile.open(fileobj=compressed, mode="w", format=tarfile.GNU_FORMAT) as archive,
):
for name, source in members:
info = tarfile.TarInfo(name)
info.mtime = 0
info.uid = info.gid = 0
info.uname = info.gname = ""
if source.is_dir():
info.type = tarfile.DIRTYPE
info.mode = 0o755
archive.addfile(info)
continue
info.mode = 0o644
info.size = source.stat().st_size
with source.open("rb") as data:
archive.addfile(info, data)
return writer.digest.hexdigest(), writer.size
def _package_readme(
manifest: dict[str, Any],
scenes: list[_PackagedScene],
groups: list[_SceneGroup],
fractions: dict[str, float],
archive_sizes: dict[str, int],
left_out_scenes: list[tuple[str, str]],
left_out_radio_maps: int,
manifest_sha256: str,
) -> str:
split_names = list(fractions)
scene_splits = {name: group.split for group in groups for name in group.members}
group_counts = {name: sum(group.split == name for group in groups) for name in split_names}
kept_groups = sum(group.kept for group in groups)
new_groups = len(groups) - kept_groups
lines = [
f"# {manifest.get('dataset_name', 'Dataset')}",
"",
f"Package of the OpenWorld Radio Twin dataset `{manifest.get('dataset_slug')}`, "
f"dataset format {manifest.get('schema_version')}, generated with version "
f"{manifest.get('software', {}).get('openworld_radio_twin')}. The manifest status is "
f"`{manifest.get('status')}`. The package is a snapshot of the dataset at index "
f"revision {manifest.get('index_revision')}, whose `dataset_manifest.json` has the "
f"SHA-256 checksum `{manifest_sha256}`.",
"",
"## Layout",
"",
"| File | Contents |",
"| --- | --- |",
f"| `{PACKAGE_SCENES_DIRECTORY}/<scene>.tar.zst` | One scene directory as the "
"generator wrote it, with its packaged radio maps and without their PNG previews, as "
"a zstd-compressed tar. A record's `image_products` block describes previews that are "
f"not packaged. `tar --zstd -xf {PACKAGE_SCENES_DIRECTORY}/<scene>.tar.zst` in the "
"package directory yields `<scene>/` next to `index.parquet`; GNU tar before 1.31 "
"takes `-I zstd` instead of `--zstd`. |",
"| `dataset_manifest.json` | The dataset manifest: request, seeds, licences and counts. "
"It describes the whole generated dataset, including scenes and radio maps "
"left out; `LICENSE.md`, `splits.json` and `index.parquet` describe the package. Its "
"`data_licences` may name layers of scenes left out; `LICENSE.md` lists the licences "
"of the package. Its `conventions` block states the row order, units, no-data values "
"and data types of the arrays. |",
"| `LICENSE.md` | Data licences and credits of the packaged scenes. |",
"| `splits.json` | Split fractions, seed, groups of scenes, the groups that kept "
"the split of a previous package and the split of each scene. This file is "
"authoritative. |",
"| `index.parquet` | One row per packaged radio map: its identity, `split` and "
"`archive`, scene, radio and transmitter parameters, and the paths of its records "
"and arrays, which resolve in the directory the archives are extracted into, the "
"package directory above. |",
"| `SHA256SUMS` | SHA-256 checksums of the other files; check with "
"`sha256sum -c SHA256SUMS`. |",
"",
"## Splits",
"",
"Scenes are split in groups, never by radio map, because the radio "
"configurations of a scene share its geometry. Scenes whose squares overlap, and "
"scenes with the same `site`, form one group.",
"",
]
if kept_groups:
kept = f"{kept_groups} group(s) kept the split of the previous package"
if new_groups:
kept += (
f"; {new_groups} new group(s), shuffled with the seed, filled what they leave "
"of each split's count"
)
lines += [kept + ".", ""]
lines += [
"| Split | Groups | Scenes | Radio maps |",
"| --- | ---: | ---: | ---: |",
]
for split in split_names:
members = [scene for scene in scenes if scene_splits[scene.planned.name] == split]
radio_maps = sum(len(scene.radio_maps) for scene in members)
lines.append(f"| {split} | {group_counts[split]} | {len(members)} | {radio_maps:,} |")
positive = [split for split in split_names if fractions[split] > 0]
empty = [split for split in positive if not group_counts[split]]
if empty:
named = empty[0] if len(empty) == 1 else f"{', '.join(empty[:-1])} and {empty[-1]}"
lines += ["", f"No group is assigned to {named}."]
lines += [
"",
"## Scenes",
"",
"| Scene | Split | Radio maps | Archive bytes |",
"| --- | --- | ---: | ---: |",
]
for scene in scenes:
name = scene.planned.name
lines.append(
f"| {name} | {scene_splits[name]} | {len(scene.radio_maps):,} | "
f"{archive_sizes[name]:,} |"
)
lines += ["", f"All archives together hold {sum(archive_sizes.values()):,} bytes.", ""]
lines += ["## Left out", ""]
if left_out_scenes:
lines += [f"- {name}: {reason}." for name, reason in left_out_scenes]
else:
lines.append("- No scene of the dataset is left out.")
if left_out_radio_maps:
lines.append(
f"- {left_out_radio_maps:,} planned radio map(s) of the packaged "
"scenes are not packaged because they failed, were interrupted, are incomplete or "
"were not generated yet. "
"The manifest's counts include failed radio maps, whose records are not "
"packaged."
)
else:
lines.append("- Every planned radio map of the packaged scenes is packaged.")
lines += [
"",
"## Reproducibility",
"",
"Packaging the same dataset again with the same split fractions, seed, previous "
"package and options, and the same NumPy, zstandard, zstd library and pyarrow "
"versions, gives the same bytes in every file; NumPy shuffles the new split groups. "
f"This package was written with NumPy {np.__version__}, zstandard "
f"{zstandard.__version__} "
f"(zstd {'.'.join(map(str, zstandard.ZSTD_VERSION))}, level {ARCHIVE_ZSTD_LEVEL}, "
f"single-threaded) and pyarrow {pa.__version__}. A "
"dataset generated again differs, because each `radio_map_info.json` records when "
"its radio map ran.",
]
return "\n".join(lines) + "\n"
[docs]
def package_dataset(
dataset_dir: str | Path,
output_dir: str | Path,
*,
fractions: Mapping[str, float] = DEFAULT_SPLIT_FRACTIONS,
seed: int = 0,
skip_unreleasable: bool = False,
previous: str | Path | None = None,
) -> Path:
"""Package a generated dataset for release: one zstd-compressed tar per scene, splits,
index and checksums.
Only complete radio maps are packaged, by the check resume uses. Scenes are
assigned to the splits in groups: scenes whose squares overlap and scenes with the same
``site`` share a split. With ``previous``, an earlier package of the dataset or its
``splits.json``, groups with scenes of that package keep their split and only new groups
are assigned; a group that joins scenes of different previous splits is refused. A scene
with a layer whose licence is not recorded or does not allow redistribution, or with
CC BY-SA 4.0 and ODbL layers together, is refused; with ``skip_unreleasable=True`` it is
left out and named in the package's ``README.md``. Returns the output directory, which
must not exist or be an empty directory.
"""
dataset_directory = Path(dataset_dir).expanduser().resolve()
output_directory = Path(output_dir).expanduser().resolve()
split_fractions = _split_fractions(fractions)
if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0:
raise ValueError("The split seed must be a non-negative integer")
manifest_path = dataset_directory / "dataset_manifest.json"
if not manifest_path.is_file():
raise FileNotFoundError(f"No dataset_manifest.json in {dataset_directory}")
if output_directory.is_relative_to(dataset_directory):
raise ValueError("The package directory must lie outside the dataset directory")
if output_directory.exists() and (
not output_directory.is_dir() or any(output_directory.iterdir())
):
raise FileExistsError(f"Package directory {output_directory} is not an empty directory")
with _no_active_generation(dataset_directory):
manifest_bytes = manifest_path.read_bytes()
manifest_sha256 = hashlib.sha256(manifest_bytes).hexdigest()
manifest = json.loads(manifest_bytes)
_check_manifest(manifest, "Packaging")
request = BatchDatasetRequest.model_validate(manifest["request"])
plan = build_batch_plan(request)
previous_splits = (
{} if previous is None else _previous_assignment(previous, plan, split_fractions)
)
records = {
item["name"]: item
for item in manifest.get("scenes", [])
if isinstance(item, dict) and "name" in item
}
scenes: list[_PackagedScene] = []
left_out_scenes: list[tuple[str, str]] = []
unreleasable: list[str] = []
left_out_radio_maps = 0
# Previews are never packaged, so a radio map is complete without them.
for planned, complete in completed_radio_map_records(
dataset_directory, plan, BatchOutputSpec(previews=False)
):
if not complete:
left_out_scenes.append((planned.name, "no complete radio map"))
continue
scene_record = records.get(planned.name, {})
licences = scene_record.get("provenance", {}).get("licences")
problems = release_problems(licences) if licences is not None else ["no licence record"]
if problems:
if not skip_unreleasable:
unreleasable.extend(f"{planned.name}: {problem}" for problem in problems)
continue
left_out_scenes.append(
(planned.name, "not redistributable, " + "; ".join(problems))
)
continue
left_out_radio_maps += len(planned.radio_maps) - len(complete)
scenes.append(_PackagedScene(planned, scene_record, complete))
if unreleasable:
raise ValueError(
"Scenes that may not be redistributed: "
+ "; ".join(unreleasable)
+ ". Pass skip_unreleasable=True (--skip-unreleasable) to leave them out"
)
if not scenes:
raise ValueError(f"No scene of {dataset_directory} can be packaged")
groups = _assign_splits(
plan,
{scene.planned.name for scene in scenes},
split_fractions,
seed,
previous_splits,
)
scene_splits = {name: group.split for group in groups for name in group.members}
archives = {
scene.planned.name: f"{PACKAGE_SCENES_DIRECTORY}/{scene.planned.name}.tar.zst"
for scene in scenes
}
# Listed and indexed before anything is written, so a scene that cannot be packaged
# leaves no partial package behind.
members = {
scene.planned.name: _tar_members(
dataset_directory / scene.planned.name,
{record["name"] for record in scene.radio_maps},
)
for scene in scenes
}
table = index_table(
dataset_directory,
[(scene.planned, scene.radio_maps) for scene in scenes],
manifest["index_revision"],
splits=scene_splits,
archives=archives,
source_manifest_sha256=manifest_sha256,
)
scenes_directory = output_directory / PACKAGE_SCENES_DIRECTORY
scenes_directory.mkdir(parents=True, exist_ok=True)
checksums: dict[str, str] = {}
archive_sizes: dict[str, int] = {}
for scene in scenes:
name = scene.planned.name
checksums[archives[name]], archive_sizes[name] = _write_scene_archive(
output_directory / archives[name], members[name]
)
index = write_index(output_directory, table)
checksums[INDEX_PATH] = hashlib.sha256(index.read_bytes()).hexdigest()
sites = {scene.planned.name: scene.planned.specification.site for scene in scenes}
files = {
"dataset_manifest.json": manifest_bytes,
"LICENSE.md": licence_notice(
{**manifest, "scenes": [scene.record for scene in scenes]}
).encode("utf-8"),
"splits.json": json.dumps(
{
"fractions": split_fractions,
"seed": seed,
"generator": "numpy.random.PCG64",
"groups": [
{
"id": group.id,
"members": group.members,
"sites": sorted({site for name in group.members if (site := sites[name])}),
}
for group in groups
],
"kept_from_previous": [group.id for group in groups if group.kept],
"assignment": scene_splits,
},
indent=2,
).encode("utf-8"),
"README.md": _package_readme(
manifest,
scenes,
groups,
split_fractions,
archive_sizes,
left_out_scenes,
left_out_radio_maps,
manifest_sha256,
).encode("utf-8"),
}
for relative, content in files.items():
(output_directory / relative).write_bytes(content)
checksums[relative] = hashlib.sha256(content).hexdigest()
# Written last and moved into place whole: a package with SHA256SUMS is complete.
sums = output_directory / "SHA256SUMS"
temporary = output_directory / "SHA256SUMS.tmp"
try:
temporary.write_text(
"".join(f"{checksums[relative]} {relative}\n" for relative in sorted(checksums)),
encoding="utf-8",
)
temporary.replace(sums)
finally:
temporary.unlink(missing_ok=True)
return output_directory