"""The Antennebureau's monthly list of mobile antenna installations, the Netherlands.
The Antennebureau of the Dutch government publishes every month a spreadsheet of all
antenna installations for mobile communication taken from the Antenneregister, under CC0.
Each row gives a municipality, a technology (2G, 3G, 4G or 5G), the position in WGS84 to
15 m and the antenna height above ground; it gives no operator, frequency, direction or
power. The adapter reads the newest month of the newest yearly file and joins the rows of
one position and height into one antenna with all its technologies. Without registered
frequencies, these antennas are assumed on whichever channel is selected.
The yearly file changes its name every month, so the adapter finds it on the year's
document page, downloads it once and converts it to a local point index pinned by its
month and SHA-256.
"""
from __future__ import annotations
import asyncio
import re
import zipfile
from datetime import date
from pathlib import Path
from urllib.parse import urljoin, urlsplit
import httpx
import pyarrow as pa
from defusedxml.ElementTree import iterparse
from openworld_radio_twin.config import Settings
from openworld_radio_twin.models import AntennaFeature
from openworld_radio_twin.providers.base import AntennaProviderError, AntennaQueryResult
from openworld_radio_twin.providers.snapshots import (
Snapshot,
current_snapshot,
download,
install_snapshot,
query_bounds,
read_point_index,
source_cache_root,
)
TITLE = "Antennebureau mobile antenna installations"
ATTRIBUTION = "Antennebureau, Rijksinspectie Digitale Infrastructuur"
LICENSE = "CC0 1.0"
LICENSE_URL = "https://creativecommons.org/publicdomain/zero/1.0/"
SITE = "https://www.antennebureau.nl"
YEAR_PAGE = SITE + "/documenten/{year}/01/01/jaaroverzicht-antenne-installaties-{year}"
MAX_EXPORT_BYTES = 100_000_000
TECHNOLOGIES = {"2G": "GSM", "3G": "UMTS", "4G": "LTE", "5G": "NR"}
MONTHS = {
"januari": 1,
"februari": 2,
"maart": 3,
"april": 4,
"mei": 5,
"juni": 6,
"juli": 7,
"augustus": 8,
"september": 9,
"oktober": 10,
"november": 11,
"december": 12,
}
TABLE = "{urn:oasis:names:tc:opendocument:xmlns:table:1.0}"
OFFICE = "{urn:oasis:names:tc:opendocument:xmlns:office:1.0}"
HEADER = ("Gemeente", "Toepassing")
[docs]
def snapshot_directory(settings: Settings) -> Path:
return source_cache_root(settings) / "antennebureau"
def _cells(row) -> list[str]:
values = []
for cell in row:
if cell.get(OFFICE + "value-type") == "float":
value = cell.get(OFFICE + "value", "")
else:
value = "".join(cell.itertext()).strip()
repeat = int(cell.get(TABLE + "number-columns-repeated", "1"))
values.extend([value] * min(repeat, 8))
if len(values) >= 8:
break
return values
def _number(value: str) -> float:
return float(value.replace(",", "."))
[docs]
def read_export(path: Path) -> tuple[pa.Table, str]:
"""One row per antenna position and height from the newest month, and its date."""
newest: tuple[str, list[list[str]]] | None = None
title, rows, in_data = "", [], False
with zipfile.ZipFile(path) as archive, archive.open("content.xml") as stream:
for event, element in iterparse(stream, events=("start", "end")):
if event == "start" and element.tag == TABLE + "table":
title, rows, in_data = "", [], False
elif event == "end" and element.tag == TABLE + "table-row":
cells = _cells(element)
if not title and cells and "op " in cells[0]:
title = cells[0]
elif in_data and len(cells) >= 7 and cells[0]:
rows.append(cells[:7])
elif tuple(cells[:2]) == HEADER:
in_data = True
element.clear()
elif event == "end" and element.tag == TABLE + "table":
if rows:
newest = (title, rows)
element.clear()
if newest is None:
raise ValueError("The Antennebureau file holds no month with antennas")
title, rows = newest
match = re.search(r"op (\d{1,2}) (\w+) (\d{4})", title)
if not match or match[2] not in MONTHS:
raise ValueError(f"The Antennebureau sheet has no recognisable date: {title[:80]!r}")
release = date(int(match[3]), MONTHS[match[2]], int(match[1])).isoformat()
antennas: dict[tuple[float, float, float | None], dict] = {}
for municipality, technology, latitude, longitude, _x, _y, height in rows:
radio = TECHNOLOGIES.get(technology.strip().upper())
if radio is None:
continue
key = (
round(_number(latitude), 5),
round(_number(longitude), 5),
_number(height) if height else None,
)
entry = antennas.setdefault(key, {"municipality": municipality, "radios": set()})
entry["radios"].add(radio)
table = pa.table(
{
"latitude": [key[0] for key in antennas],
"longitude": [key[1] for key in antennas],
"height_agl_m": [key[2] for key in antennas],
"radios": [sorted(entry["radios"]) for entry in antennas.values()],
"municipality": [entry["municipality"] for entry in antennas.values()],
},
schema=pa.schema(
[
("latitude", pa.float64()),
("longitude", pa.float64()),
("height_agl_m", pa.float64()),
("radios", pa.list_(pa.string())),
("municipality", pa.string()),
]
),
)
return table, release
[docs]
def antenna_records(table: pa.Table, release: str) -> list[AntennaFeature]:
records = []
for row in table.to_pylist():
site = f"{row['latitude']:.5f},{row['longitude']:.5f}"
height = row["height_agl_m"]
basis = {"position": "source"}
if height is not None:
basis["height_agl_m"] = "source"
records.append(
AntennaFeature(
feature_id=f"antennebureau:{site}@{height:g}"
if height is not None
else (f"antennebureau:{site}"),
source="antennebureau",
source_release=release,
site_id=f"antennebureau-site:{site}",
radios=row["radios"],
latitude=row["latitude"],
longitude=row["longitude"],
height_agl_m=height,
basis=basis,
source_attributes={"municipality": row["municipality"]},
)
)
return records
[docs]
async def latest_export(client: httpx.AsyncClient, today: date | None = None) -> str:
"""URL of the newest yearly file, from this year's document page or last year's."""
year = (today or date.today()).year
for candidate in (year, year - 1):
response = await client.get(YEAR_PAGE.format(year=candidate), timeout=60)
if response.status_code == 404:
continue
response.raise_for_status()
links = re.findall(r'href="([^"]+\.ods)"', response.text)
if links:
url = urljoin(SITE, links[0])
if urlsplit(url).netloc != urlsplit(SITE).netloc:
raise ValueError("The Antennebureau file link left the official site")
return url
raise ValueError("The Antennebureau document pages list no antenna file")
[docs]
class AntennebureauProvider:
def __init__(self, settings: Settings, client: httpx.AsyncClient):
self.settings = settings
self.client = client
self.directory = snapshot_directory(settings)
self._lock = asyncio.Lock()
[docs]
async def snapshot(self, *, refresh: bool = False) -> Snapshot:
"""The installed snapshot; downloads the newest file first when none is installed."""
async with self._lock:
installed = current_snapshot(self.directory)
if installed is not None and not refresh:
return installed
path = self.directory / "antennes.ods"
try:
url = await latest_export(self.client)
sha256 = await download(self.client, url, path, maximum_bytes=MAX_EXPORT_BYTES)
table, release = await asyncio.to_thread(read_export, path)
return await asyncio.to_thread(
install_snapshot,
self.directory,
table,
release=release,
sha256=sha256,
metadata={"source": "antennebureau", "source_url": url, "license": LICENSE},
)
except httpx.HTTPError as exc:
raise AntennaProviderError(f"Antennebureau download failed: {exc}") from exc
except (OSError, KeyError, ValueError, zipfile.BadZipFile, pa.ArrowException) as exc:
raise AntennaProviderError(
f"Antennebureau file could not be installed: {exc}"
) from exc
finally:
path.unlink(missing_ok=True)
[docs]
async def antennas(
self, latitude: float, longitude: float, radius_m: float
) -> AntennaQueryResult:
snapshot = await self.snapshot()
table = await asyncio.to_thread(
read_point_index, snapshot.index, *query_bounds(latitude, longitude, radius_m)
)
records = await asyncio.to_thread(antenna_records, table, snapshot.identity)
return AntennaQueryResult(antennas=records, provider=TITLE, release=snapshot.identity)