Source code for openworld_radio_twin.providers.anfr

"""The ANFR register of radio installations above 5 W, France and its overseas territories.

ANFR publishes the register monthly on data.gouv.fr under Licence Ouverte 2.0 as linked
tables of stations, supports, antennas, emitters and frequency bands. The adapter keeps
antennas that carry at least one mobile emitter in service (GSM, UMTS, LTE, 5G NR), and
drops antennas indoors, underground or in tunnels, which a street-level scene does not
contain. It records the support position, the antenna's height above ground, its
azimuth when it radiates directionally, its operator and the registered frequency bands.
The register gives no tilt, power or antenna pattern; placement fills those from the
defaults.

The latest export is found through the data.gouv.fr dataset API, downloaded once and
converted to a local point index pinned by its export date and SHA-256.
"""

from __future__ import annotations

import asyncio
import io
import re
import zipfile
from collections import defaultdict
from datetime import datetime
from pathlib import Path

import httpx
import pyarrow as pa
import pyarrow.compute as pc
import pyarrow.csv as pv

from openworld_radio_twin.config import Settings
from openworld_radio_twin.models import AntennaFeature, FrequencyBand
from openworld_radio_twin.providers.base import AntennaProviderError, AntennaQueryResult
from openworld_radio_twin.providers.snapshots import (
    Snapshot,
    current_snapshot,
    download,
    install_snapshot,
    query_bounds,
    read_point_index,
    source_cache_root,
)

TITLE = "ANFR installations > 5 W"
ATTRIBUTION = "Agence nationale des fréquences (ANFR), via data.gouv.fr"
LICENSE = "Licence Ouverte 2.0"
LICENSE_URL = "https://www.etalab.gouv.fr/licence-ouverte-open-licence/"
DATASET_API = (
    "https://www.data.gouv.fr/api/1/datasets/"
    "donnees-sur-les-installations-radioelectriques-de-plus-de-5-watts-1/"
)
EXPORT_PATTERN = re.compile(r"/(\d{8})-export-etalab-(data|ref)\.zip$")
MAX_EXPORT_BYTES = 400_000_000
# Emitter systems of public mobile networks, e.g. "LTE 1800" or "5G NR 3500".
MOBILE_RADIOS = (("5G NR", "NR"), ("LTE", "LTE"), ("UMTS", "UMTS"), ("GSM", "GSM"))
# SUP_NATURE: indoor gallery, indoor underground, tunnel.
INDOOR_SUPPORTS = frozenset({"8", "9", "10"})
# SUP_TYPE_ANTENNE: indoor antenna for mobile telephony, radiating cable.
INDOOR_ANTENNAS = frozenset({"59", "60"})
# Operators of the four mainland networks; overseas networks use other PLMN codes.
MAINLAND_PLMN = {"23": "208-01", "137": "208-10", "240": "208-15", "6": "208-20"}
BAND_UNIT_MHZ = {"K": 1e-3, "M": 1.0, "G": 1e3}


[docs] def snapshot_directory(settings: Settings) -> Path: return source_cache_root(settings) / "anfr"
def _radio(system: str) -> str | None: for prefix, radio in MOBILE_RADIOS: if system.startswith(prefix): return radio return None def _number(value: str | None) -> float | None: if value is None or not value.strip(): return None return float(value.strip().replace(",", ".")) def _date(value: str | None) -> str | None: if value is None or not value.strip(): return None return datetime.strptime(value.strip(), "%d/%m/%Y").date().isoformat() def _degrees(degrees: str, minutes: str, seconds: str, hemisphere: str) -> float: value = int(degrees) + int(minutes) / 60 + float(seconds.replace(",", ".")) / 3600 return value if hemisphere in {"N", "E"} else -value def _mainland(latitude: float, longitude: float) -> bool: return 41.0 <= latitude <= 51.2 and -5.5 <= longitude <= 9.7 def _read(archive: zipfile.ZipFile, name: str, columns: list[str]) -> pa.Table: return pv.read_csv( io.BytesIO(archive.read(name)), read_options=pv.ReadOptions(encoding="latin1", block_size=1 << 24), parse_options=pv.ParseOptions(delimiter=";"), convert_options=pv.ConvertOptions( include_columns=columns, column_types=dict.fromkeys(columns, pa.string()), ), ) def _lookup(table: pa.Table, key: str, value: str) -> dict[str, str]: return dict(zip(table.column(key).to_pylist(), table.column(value).to_pylist(), strict=True))
[docs] def read_export(data_path: Path, reference_path: Path) -> tuple[pa.Table, dict[str, int]]: """Outdoor antennas with a mobile emitter in service, one row per antenna.""" counts: dict[str, int] = defaultdict(int) with zipfile.ZipFile(reference_path) as reference: operators = _lookup( _read(reference, "SUP_EXPLOITANT.txt", ["ADM_ID", "ADM_LB_NOM"]), "ADM_ID", "ADM_LB_NOM", ) natures = _lookup( _read(reference, "SUP_NATURE.txt", ["NAT_ID", "NAT_LB_NOM"]), "NAT_ID", "NAT_LB_NOM" ) antenna_types = _lookup( _read(reference, "SUP_TYPE_ANTENNE.txt", ["TAE_ID", "TAE_LB"]), "TAE_ID", "TAE_LB" ) with zipfile.ZipFile(data_path) as data: emitters = _read( data, "SUP_EMETTEUR.txt", ["EMR_ID", "EMR_LB_SYSTEME", "AER_ID", "EMR_DT_SERVICE"] ) systems = emitters.column("EMR_LB_SYSTEME").to_pylist() radios = [_radio(system or "") for system in systems] started = pc.fill_null(emitters.column("EMR_DT_SERVICE"), "") in_service = pc.greater(pc.utf8_length(started), 0) mobile = pa.array([radio is not None for radio in radios]) counts["mobile_emitters_not_in_service"] = pc.sum( pc.and_(mobile, pc.invert(in_service)) ).as_py() or 0 emitters = emitters.filter(pc.and_(mobile, in_service)) emitter_system = _lookup(emitters, "EMR_ID", "EMR_LB_SYSTEME") antenna_emitters: dict[str, list[tuple[str, str | None]]] = defaultdict(list) for emitter, antenna, started in zip( emitters.column("EMR_ID").to_pylist(), emitters.column("AER_ID").to_pylist(), emitters.column("EMR_DT_SERVICE").to_pylist(), strict=True, ): antenna_emitters[antenna].append((emitter, started)) bands = _read( data, "SUP_BANDE.txt", ["EMR_ID", "BAN_NB_F_DEB", "BAN_NB_F_FIN", "BAN_FG_UNITE"] ) bands = bands.filter(pc.is_in(bands.column("EMR_ID"), emitters.column("EMR_ID"))) emitter_bands: dict[str, list[tuple[float, float]]] = defaultdict(list) for emitter, low, high, unit in zip( bands.column("EMR_ID").to_pylist(), bands.column("BAN_NB_F_DEB").to_pylist(), bands.column("BAN_NB_F_FIN").to_pylist(), bands.column("BAN_FG_UNITE").to_pylist(), strict=True, ): low_value, high_value = _number(low), _number(high) scale = BAND_UNIT_MHZ.get((unit or "").strip()) if low_value is None or high_value is None or scale is None: continue low_mhz, high_mhz = sorted((low_value * scale, high_value * scale)) if low_mhz > 0: emitter_bands[emitter].append((low_mhz, high_mhz)) stations = _lookup( _read(data, "SUP_STATION.txt", ["STA_NM_ANFR", "ADM_ID"]), "STA_NM_ANFR", "ADM_ID" ) supports: dict[str, tuple[float, float, str, float | None]] = {} support_table = _read( data, "SUP_SUPPORT.txt", [ "SUP_ID", "NAT_ID", "COR_NB_DG_LAT", "COR_NB_MN_LAT", "COR_NB_SC_LAT", "COR_CD_NS_LAT", "COR_NB_DG_LON", "COR_NB_MN_LON", "COR_NB_SC_LON", "COR_CD_EW_LON", "SUP_NM_HAUT", ], ) for row in support_table.to_pylist(): if row["SUP_ID"] in supports: continue try: latitude = _degrees( row["COR_NB_DG_LAT"], row["COR_NB_MN_LAT"], row["COR_NB_SC_LAT"], row["COR_CD_NS_LAT"], ) longitude = _degrees( row["COR_NB_DG_LON"], row["COR_NB_MN_LON"], row["COR_NB_SC_LON"], row["COR_CD_EW_LON"], ) except (TypeError, ValueError): continue supports[row["SUP_ID"]] = ( latitude, longitude, row["NAT_ID"] or "0", _number(row["SUP_NM_HAUT"]), ) antennas = _read( data, "SUP_ANTENNE.txt", [ "STA_NM_ANFR", "AER_ID", "TAE_ID", "AER_FG_RAYON", "AER_NB_AZIMUT", "AER_NB_ALT_BAS", "SUP_ID", ], ) antennas = antennas.filter( pc.is_in(antennas.column("AER_ID"), pa.array(list(antenna_emitters))) ) rows: dict[str, list] = defaultdict(list) for antenna in antennas.to_pylist(): support = supports.get(antenna["SUP_ID"]) if support is None: counts["antennas_without_support"] += 1 continue latitude, longitude, nature, support_height = support if nature in INDOOR_SUPPORTS or antenna["TAE_ID"] in INDOOR_ANTENNAS: counts["indoor_antennas"] += 1 continue height = _number(antenna["AER_NB_ALT_BAS"]) if height is not None and height < 0: counts["antennas_below_ground"] += 1 continue directional = (antenna["AER_FG_RAYON"] or "").strip() == "D" azimuth = _number(antenna["AER_NB_AZIMUT"]) if directional else None emitters_here = antenna_emitters[antenna["AER_ID"]] ranges = sorted( { (emitter_system[emitter], low_mhz, high_mhz) for emitter, _ in emitters_here for low_mhz, high_mhz in emitter_bands.get(emitter, ()) } ) station_operator = stations.get(antenna["STA_NM_ANFR"], "") started = [value for _, value in emitters_here if value] rows["antenna_id"].append(antenna["AER_ID"]) rows["support_id"].append(antenna["SUP_ID"]) rows["station_id"].append(antenna["STA_NM_ANFR"]) rows["latitude"].append(latitude) rows["longitude"].append(longitude) rows["height_agl_m"].append(height) rows["azimuth_deg"].append(None if azimuth is None else azimuth % 360.0) rows["directional"].append(directional) rows["antenna_type"].append(antenna_types.get(antenna["TAE_ID"] or "", "")) rows["support_type"].append(natures.get(nature, "")) rows["support_height_m"].append(support_height) rows["operator"].append(operators.get(station_operator, station_operator)) rows["plmn"].append( MAINLAND_PLMN.get(station_operator) if _mainland(latitude, longitude) else None ) rows["systems"].append(sorted({emitter_system[emitter] for emitter, _ in emitters_here})) rows["band_label"].append([label for label, _, _ in ranges]) rows["band_low_mhz"].append([low_mhz for _, low_mhz, _ in ranges]) rows["band_high_mhz"].append([high_mhz for _, _, high_mhz in ranges]) rows["in_service"].append(min((_date(value) for value in started), default=None)) schema = pa.schema( [ ("antenna_id", pa.string()), ("support_id", pa.string()), ("station_id", pa.string()), ("latitude", pa.float64()), ("longitude", pa.float64()), ("height_agl_m", pa.float64()), ("azimuth_deg", pa.float64()), ("directional", pa.bool_()), ("antenna_type", pa.string()), ("support_type", pa.string()), ("support_height_m", pa.float64()), ("operator", pa.string()), ("plmn", pa.string()), ("systems", pa.list_(pa.string())), ("band_label", pa.list_(pa.string())), ("band_low_mhz", pa.list_(pa.float64())), ("band_high_mhz", pa.list_(pa.float64())), ("in_service", pa.string()), ] ) return pa.Table.from_pydict(dict(rows), schema=schema), dict(counts)
[docs] def antenna_records(table: pa.Table, release: str) -> list[AntennaFeature]: records = [] for row in table.to_pylist(): bands = [ FrequencyBand(label=label, low_mhz=low, high_mhz=high) for label, low, high in zip( row["band_label"], row["band_low_mhz"], row["band_high_mhz"], strict=True ) ] radios = sorted({radio for system in row["systems"] if (radio := _radio(system))}) basis = {"position": "source", "bands": "source"} if row["height_agl_m"] is not None: basis["height_agl_m"] = "source" if row["azimuth_deg"] is not None: basis["azimuth_deg"] = "source" records.append( AntennaFeature( feature_id=f"anfr:{row['antenna_id']}", source="anfr", source_release=release, site_id=f"anfr-support:{row['support_id']}", operator=row["operator"] or None, plmn=row["plmn"], radios=radios, bands=bands, latitude=row["latitude"], longitude=row["longitude"], height_agl_m=row["height_agl_m"], azimuth_deg=row["azimuth_deg"], basis=basis, source_attributes={ "station": row["station_id"], "support": row["support_id"], "support_type": row["support_type"], "support_height_m": row["support_height_m"], "antenna_type": row["antenna_type"], "radiation": "directional" if row["directional"] else "omnidirectional", "systems": row["systems"], "in_service": row["in_service"], }, ) ) return records
[docs] async def latest_export(client: httpx.AsyncClient) -> tuple[str, str, str]: """Export date and the data and reference archive URLs of the newest complete export.""" response = await client.get(DATASET_API, timeout=60) response.raise_for_status() exports: dict[str, dict[str, str]] = defaultdict(dict) for resource in response.json().get("resources", []): match = EXPORT_PATTERN.search(str(resource.get("url", ""))) if match: exports[match[1]][match[2]] = resource["url"] complete = sorted(date for date, parts in exports.items() if {"data", "ref"} <= set(parts)) if not complete: raise ValueError("The ANFR dataset lists no complete export") date = complete[-1] return f"{date[:4]}-{date[4:6]}-{date[6:]}", exports[date]["data"], exports[date]["ref"]
[docs] class AnfrProvider: def __init__(self, settings: Settings, client: httpx.AsyncClient): self.settings = settings self.client = client self.directory = snapshot_directory(settings) self._lock = asyncio.Lock()
[docs] async def snapshot(self, *, refresh: bool = False) -> Snapshot: """The installed snapshot; downloads the latest export first when none is installed.""" async with self._lock: installed = current_snapshot(self.directory) if installed is not None and not refresh: return installed data_path = self.directory / "export-data.zip" reference_path = self.directory / "export-ref.zip" try: release, data_url, reference_url = await latest_export(self.client) sha256 = await download( self.client, data_url, data_path, maximum_bytes=MAX_EXPORT_BYTES ) await download( self.client, reference_url, reference_path, maximum_bytes=10_000_000 ) table, counts = await asyncio.to_thread(read_export, data_path, reference_path) return await asyncio.to_thread( install_snapshot, self.directory, table, release=release, sha256=sha256, metadata={ "source": "anfr", "source_url": data_url, "reference_url": reference_url, "license": LICENSE, "excluded": counts, }, ) except httpx.HTTPError as exc: raise AntennaProviderError(f"ANFR export download failed: {exc}") from exc except (OSError, KeyError, ValueError, zipfile.BadZipFile, pa.ArrowException) as exc: raise AntennaProviderError(f"ANFR export could not be installed: {exc}") from exc finally: data_path.unlink(missing_ok=True) reference_path.unlink(missing_ok=True)
[docs] async def antennas( self, latitude: float, longitude: float, radius_m: float ) -> AntennaQueryResult: snapshot = await self.snapshot() table = await asyncio.to_thread( read_point_index, snapshot.index, *query_bounds(latitude, longitude, radius_m) ) records = await asyncio.to_thread(antenna_records, table, snapshot.identity) return AntennaQueryResult(antennas=records, provider=TITLE, release=snapshot.identity)