Source code for openworld_radio_twin.providers.antennebureau

"""The Antennebureau's monthly list of mobile antenna installations, the Netherlands.

The Antennebureau of the Dutch government publishes every month a spreadsheet of all
antenna installations for mobile communication taken from the Antenneregister, under CC0.
Each row gives a municipality, a technology (2G, 3G, 4G or 5G), the position in WGS84 to
15 m and the antenna height above ground; it gives no operator, frequency, direction or
power. The adapter reads the newest month of the newest yearly file and joins the rows of
one position and height into one antenna with all its technologies. Without registered
frequencies, these antennas are assumed on whichever channel is selected.

The yearly file changes its name every month, so the adapter finds it on the year's
document page, downloads it once and converts it to a local point index pinned by its
month and SHA-256.
"""

from __future__ import annotations

import asyncio
import re
import zipfile
from datetime import date
from pathlib import Path
from urllib.parse import urljoin, urlsplit

import httpx
import pyarrow as pa
from defusedxml.ElementTree import iterparse

from openworld_radio_twin.config import Settings
from openworld_radio_twin.models import AntennaFeature
from openworld_radio_twin.providers.base import AntennaProviderError, AntennaQueryResult
from openworld_radio_twin.providers.snapshots import (
    Snapshot,
    current_snapshot,
    download,
    install_snapshot,
    query_bounds,
    read_point_index,
    source_cache_root,
)

TITLE = "Antennebureau mobile antenna installations"
ATTRIBUTION = "Antennebureau, Rijksinspectie Digitale Infrastructuur"
LICENSE = "CC0 1.0"
LICENSE_URL = "https://creativecommons.org/publicdomain/zero/1.0/"
SITE = "https://www.antennebureau.nl"
YEAR_PAGE = SITE + "/documenten/{year}/01/01/jaaroverzicht-antenne-installaties-{year}"
MAX_EXPORT_BYTES = 100_000_000
TECHNOLOGIES = {"2G": "GSM", "3G": "UMTS", "4G": "LTE", "5G": "NR"}
MONTHS = {
    "januari": 1,
    "februari": 2,
    "maart": 3,
    "april": 4,
    "mei": 5,
    "juni": 6,
    "juli": 7,
    "augustus": 8,
    "september": 9,
    "oktober": 10,
    "november": 11,
    "december": 12,
}
TABLE = "{urn:oasis:names:tc:opendocument:xmlns:table:1.0}"
OFFICE = "{urn:oasis:names:tc:opendocument:xmlns:office:1.0}"
HEADER = ("Gemeente", "Toepassing")


[docs] def snapshot_directory(settings: Settings) -> Path: return source_cache_root(settings) / "antennebureau"
def _cells(row) -> list[str]: values = [] for cell in row: if cell.get(OFFICE + "value-type") == "float": value = cell.get(OFFICE + "value", "") else: value = "".join(cell.itertext()).strip() repeat = int(cell.get(TABLE + "number-columns-repeated", "1")) values.extend([value] * min(repeat, 8)) if len(values) >= 8: break return values def _number(value: str) -> float: return float(value.replace(",", "."))
[docs] def read_export(path: Path) -> tuple[pa.Table, str]: """One row per antenna position and height from the newest month, and its date.""" newest: tuple[str, list[list[str]]] | None = None title, rows, in_data = "", [], False with zipfile.ZipFile(path) as archive, archive.open("content.xml") as stream: for event, element in iterparse(stream, events=("start", "end")): if event == "start" and element.tag == TABLE + "table": title, rows, in_data = "", [], False elif event == "end" and element.tag == TABLE + "table-row": cells = _cells(element) if not title and cells and "op " in cells[0]: title = cells[0] elif in_data and len(cells) >= 7 and cells[0]: rows.append(cells[:7]) elif tuple(cells[:2]) == HEADER: in_data = True element.clear() elif event == "end" and element.tag == TABLE + "table": if rows: newest = (title, rows) element.clear() if newest is None: raise ValueError("The Antennebureau file holds no month with antennas") title, rows = newest match = re.search(r"op (\d{1,2}) (\w+) (\d{4})", title) if not match or match[2] not in MONTHS: raise ValueError(f"The Antennebureau sheet has no recognisable date: {title[:80]!r}") release = date(int(match[3]), MONTHS[match[2]], int(match[1])).isoformat() antennas: dict[tuple[float, float, float | None], dict] = {} for municipality, technology, latitude, longitude, _x, _y, height in rows: radio = TECHNOLOGIES.get(technology.strip().upper()) if radio is None: continue key = ( round(_number(latitude), 5), round(_number(longitude), 5), _number(height) if height else None, ) entry = antennas.setdefault(key, {"municipality": municipality, "radios": set()}) entry["radios"].add(radio) table = pa.table( { "latitude": [key[0] for key in antennas], "longitude": [key[1] for key in antennas], "height_agl_m": [key[2] for key in antennas], "radios": [sorted(entry["radios"]) for entry in antennas.values()], "municipality": [entry["municipality"] for entry in antennas.values()], }, schema=pa.schema( [ ("latitude", pa.float64()), ("longitude", pa.float64()), ("height_agl_m", pa.float64()), ("radios", pa.list_(pa.string())), ("municipality", pa.string()), ] ), ) return table, release
[docs] def antenna_records(table: pa.Table, release: str) -> list[AntennaFeature]: records = [] for row in table.to_pylist(): site = f"{row['latitude']:.5f},{row['longitude']:.5f}" height = row["height_agl_m"] basis = {"position": "source"} if height is not None: basis["height_agl_m"] = "source" records.append( AntennaFeature( feature_id=f"antennebureau:{site}@{height:g}" if height is not None else (f"antennebureau:{site}"), source="antennebureau", source_release=release, site_id=f"antennebureau-site:{site}", radios=row["radios"], latitude=row["latitude"], longitude=row["longitude"], height_agl_m=height, basis=basis, source_attributes={"municipality": row["municipality"]}, ) ) return records
[docs] async def latest_export(client: httpx.AsyncClient, today: date | None = None) -> str: """URL of the newest yearly file, from this year's document page or last year's.""" year = (today or date.today()).year for candidate in (year, year - 1): response = await client.get(YEAR_PAGE.format(year=candidate), timeout=60) if response.status_code == 404: continue response.raise_for_status() links = re.findall(r'href="([^"]+\.ods)"', response.text) if links: url = urljoin(SITE, links[0]) if urlsplit(url).netloc != urlsplit(SITE).netloc: raise ValueError("The Antennebureau file link left the official site") return url raise ValueError("The Antennebureau document pages list no antenna file")
[docs] class AntennebureauProvider: def __init__(self, settings: Settings, client: httpx.AsyncClient): self.settings = settings self.client = client self.directory = snapshot_directory(settings) self._lock = asyncio.Lock()
[docs] async def snapshot(self, *, refresh: bool = False) -> Snapshot: """The installed snapshot; downloads the newest file first when none is installed.""" async with self._lock: installed = current_snapshot(self.directory) if installed is not None and not refresh: return installed path = self.directory / "antennes.ods" try: url = await latest_export(self.client) sha256 = await download(self.client, url, path, maximum_bytes=MAX_EXPORT_BYTES) table, release = await asyncio.to_thread(read_export, path) return await asyncio.to_thread( install_snapshot, self.directory, table, release=release, sha256=sha256, metadata={"source": "antennebureau", "source_url": url, "license": LICENSE}, ) except httpx.HTTPError as exc: raise AntennaProviderError(f"Antennebureau download failed: {exc}") from exc except (OSError, KeyError, ValueError, zipfile.BadZipFile, pa.ArrowException) as exc: raise AntennaProviderError( f"Antennebureau file could not be installed: {exc}" ) from exc finally: path.unlink(missing_ok=True)
[docs] async def antennas( self, latitude: float, longitude: float, radius_m: float ) -> AntennaQueryResult: snapshot = await self.snapshot() table = await asyncio.to_thread( read_point_index, snapshot.index, *query_bounds(latitude, longitude, radius_m) ) records = await asyncio.to_thread(antenna_records, table, snapshot.identity) return AntennaQueryResult(antennas=records, provider=TITLE, release=snapshot.identity)