"""Local, pinned snapshots of downloadable point datasets such as antenna registers.
A source downloads its published export once, records the export's release and SHA-256,
and converts it to a Parquet point index sorted by 0.1-degree tiles. Queries then read
only the row groups of the tiles a scene touches and never use the network. Refreshing a
snapshot is an explicit operation because it changes the data every later query reads.
"""
from __future__ import annotations
import hashlib
import json
import math
import os
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
import httpx
import pyarrow as pa
import pyarrow.compute as pc
import pyarrow.parquet as pq
from openworld_radio_twin.config import Settings
from openworld_radio_twin.geodesy import LocalFrame
TILES_PER_DEGREE = 10
TILE_COLUMNS = 360 * TILES_PER_DEGREE
ROW_GROUP_SIZE = 65_536
[docs]
def source_cache_root(settings: Settings) -> Path:
"""Directory holding one subdirectory of snapshots per source."""
if settings.source_cache_root is not None:
return Path(settings.source_cache_root).expanduser()
return settings.cache_root / "sources"
[docs]
@dataclass(frozen=True)
class Snapshot:
"""The installed snapshot of one source."""
directory: Path
release: str
sha256: str
index: Path
metadata: dict[str, object]
@property
def identity(self) -> str:
return f"{self.release}:{self.sha256[:12]}"
[docs]
def current_snapshot(directory: Path) -> Snapshot | None:
"""The snapshot recorded in ``current.json``, or None before the first download."""
manifest_path = directory / "current.json"
if not manifest_path.is_file():
return None
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
index = directory / str(manifest["index"])
if not index.is_file():
return None
return Snapshot(directory, str(manifest["release"]), str(manifest["sha256"]), index, manifest)
[docs]
def install_snapshot(
directory: Path,
table: pa.Table,
*,
release: str,
sha256: str,
metadata: dict[str, object],
) -> Snapshot:
"""Write the point index, then switch ``current.json`` to it and drop older indexes."""
directory.mkdir(parents=True, exist_ok=True)
index = directory / f"{release}-{sha256[:12]}.parquet"
temporary = index.with_name(f".{index.name}.tmp-{os.getpid()}")
write_point_index(table, temporary)
temporary.replace(index)
manifest = {
**metadata,
"release": release,
"sha256": sha256,
"index": index.name,
"rows": table.num_rows,
"installed_at": datetime.now(timezone.utc).isoformat(),
}
manifest_path = directory / "current.json"
temporary_manifest = manifest_path.with_name(f".current.json.tmp-{os.getpid()}")
temporary_manifest.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
temporary_manifest.replace(manifest_path)
for stale in directory.glob("*.parquet"):
if stale != index:
stale.unlink(missing_ok=True)
return Snapshot(directory, release, sha256, index, manifest)
[docs]
async def download(
client: httpx.AsyncClient,
url: str,
destination: Path,
*,
maximum_bytes: int,
params: dict[str, str] | None = None,
timeout: float = 600,
) -> str:
"""Stream ``url`` to ``destination`` within a size bound; return the file's SHA-256."""
destination.parent.mkdir(parents=True, exist_ok=True)
temporary = destination.with_name(f".{destination.name}.tmp-{os.getpid()}")
digest = hashlib.sha256()
size = 0
try:
async with client.stream("GET", url, params=params, timeout=timeout) as response:
response.raise_for_status()
with temporary.open("wb") as stream:
async for chunk in response.aiter_bytes():
size += len(chunk)
if size > maximum_bytes:
raise ValueError(f"Download exceeds the {maximum_bytes:,}-byte limit")
digest.update(chunk)
stream.write(chunk)
temporary.replace(destination)
finally:
temporary.unlink(missing_ok=True)
return digest.hexdigest()
[docs]
def query_bounds(
latitude: float, longitude: float, radius_m: float
) -> tuple[float, float, float, float]:
"""West, south, east and north around the square ENU scene domain.
The margin covers the curvature of the square's edges; placement cuts records to the
exact square afterwards.
"""
margin_m = radius_m + 25.0
corners = LocalFrame.at(longitude, latitude).corners(-margin_m, -margin_m, margin_m, margin_m)
longitudes = [point[0] for point in corners]
latitudes = [point[1] for point in corners]
return min(longitudes), min(latitudes), max(longitudes), max(latitudes)
def _tiles(latitude: pa.Array, longitude: pa.Array) -> pa.Array:
row = pc.floor(pc.multiply(pc.add(latitude, 90.0), float(TILES_PER_DEGREE)))
column = pc.floor(pc.multiply(pc.add(longitude, 180.0), float(TILES_PER_DEGREE)))
row = pc.min_element_wise(row, 180.0 * TILES_PER_DEGREE - 1)
column = pc.min_element_wise(column, float(TILE_COLUMNS - 1))
return pc.cast(pc.add(pc.multiply(row, float(TILE_COLUMNS)), column), pa.int32())
[docs]
def write_point_index(table: pa.Table, path: Path) -> None:
"""Sort points by tile so a bounding-box read touches few row groups."""
tiles = _tiles(table.column("latitude"), table.column("longitude"))
indexed = table.append_column("tile", tiles).sort_by("tile")
pq.write_table(indexed, path, row_group_size=ROW_GROUP_SIZE, compression="zstd")
[docs]
def read_point_index(
path: Path,
west: float,
south: float,
east: float,
north: float,
) -> pa.Table:
"""Rows whose point lies inside the bounding box, read tile row by tile row."""
first_row = max(0, math.floor((south + 90.0) * TILES_PER_DEGREE))
last_row = min(180 * TILES_PER_DEGREE - 1, math.floor((north + 90.0) * TILES_PER_DEGREE))
first_column = max(0, math.floor((west + 180.0) * TILES_PER_DEGREE))
last_column = min(TILE_COLUMNS - 1, math.floor((east + 180.0) * TILES_PER_DEGREE))
filters = [
[
("tile", ">=", row * TILE_COLUMNS + first_column),
("tile", "<=", row * TILE_COLUMNS + last_column),
]
for row in range(first_row, last_row + 1)
]
table = pq.read_table(path, filters=filters)
inside = pc.and_(
pc.and_(
pc.greater_equal(table.column("latitude"), south),
pc.less_equal(table.column("latitude"), north),
),
pc.and_(
pc.greater_equal(table.column("longitude"), west),
pc.less_equal(table.column("longitude"), east),
),
)
return table.filter(inside).drop_columns(["tile"])