"""The ANFR register of radio installations above 5 W, France and its overseas territories.
ANFR publishes the register monthly on data.gouv.fr under Licence Ouverte 2.0 as linked
tables of stations, supports, antennas, emitters and frequency bands. The adapter keeps
antennas that carry at least one mobile emitter in service (GSM, UMTS, LTE, 5G NR), and
drops antennas indoors, underground or in tunnels, which a street-level scene does not
contain. It records the support position, the antenna's height above ground, its
azimuth when it radiates directionally, its operator and the registered frequency bands.
The register gives no tilt, power or antenna pattern; placement fills those from the
defaults.
The latest export is found through the data.gouv.fr dataset API, downloaded once and
converted to a local point index pinned by its export date and SHA-256.
"""
from __future__ import annotations
import asyncio
import io
import re
import zipfile
from collections import defaultdict
from datetime import datetime
from pathlib import Path
import httpx
import pyarrow as pa
import pyarrow.compute as pc
import pyarrow.csv as pv
from openworld_radio_twin.config import Settings
from openworld_radio_twin.models import AntennaFeature, FrequencyBand
from openworld_radio_twin.providers.base import AntennaProviderError, AntennaQueryResult
from openworld_radio_twin.providers.snapshots import (
Snapshot,
current_snapshot,
download,
install_snapshot,
query_bounds,
read_point_index,
source_cache_root,
)
TITLE = "ANFR installations > 5 W"
ATTRIBUTION = "Agence nationale des fréquences (ANFR), via data.gouv.fr"
LICENSE = "Licence Ouverte 2.0"
LICENSE_URL = "https://www.etalab.gouv.fr/licence-ouverte-open-licence/"
DATASET_API = (
"https://www.data.gouv.fr/api/1/datasets/"
"donnees-sur-les-installations-radioelectriques-de-plus-de-5-watts-1/"
)
EXPORT_PATTERN = re.compile(r"/(\d{8})-export-etalab-(data|ref)\.zip$")
MAX_EXPORT_BYTES = 400_000_000
# Emitter systems of public mobile networks, e.g. "LTE 1800" or "5G NR 3500".
MOBILE_RADIOS = (("5G NR", "NR"), ("LTE", "LTE"), ("UMTS", "UMTS"), ("GSM", "GSM"))
# SUP_NATURE: indoor gallery, indoor underground, tunnel.
INDOOR_SUPPORTS = frozenset({"8", "9", "10"})
# SUP_TYPE_ANTENNE: indoor antenna for mobile telephony, radiating cable.
INDOOR_ANTENNAS = frozenset({"59", "60"})
# Operators of the four mainland networks; overseas networks use other PLMN codes.
MAINLAND_PLMN = {"23": "208-01", "137": "208-10", "240": "208-15", "6": "208-20"}
BAND_UNIT_MHZ = {"K": 1e-3, "M": 1.0, "G": 1e3}
[docs]
def snapshot_directory(settings: Settings) -> Path:
return source_cache_root(settings) / "anfr"
def _radio(system: str) -> str | None:
for prefix, radio in MOBILE_RADIOS:
if system.startswith(prefix):
return radio
return None
def _number(value: str | None) -> float | None:
if value is None or not value.strip():
return None
return float(value.strip().replace(",", "."))
def _date(value: str | None) -> str | None:
if value is None or not value.strip():
return None
return datetime.strptime(value.strip(), "%d/%m/%Y").date().isoformat()
def _degrees(degrees: str, minutes: str, seconds: str, hemisphere: str) -> float:
value = int(degrees) + int(minutes) / 60 + float(seconds.replace(",", ".")) / 3600
return value if hemisphere in {"N", "E"} else -value
def _mainland(latitude: float, longitude: float) -> bool:
return 41.0 <= latitude <= 51.2 and -5.5 <= longitude <= 9.7
def _read(archive: zipfile.ZipFile, name: str, columns: list[str]) -> pa.Table:
return pv.read_csv(
io.BytesIO(archive.read(name)),
read_options=pv.ReadOptions(encoding="latin1", block_size=1 << 24),
parse_options=pv.ParseOptions(delimiter=";"),
convert_options=pv.ConvertOptions(
include_columns=columns,
column_types=dict.fromkeys(columns, pa.string()),
),
)
def _lookup(table: pa.Table, key: str, value: str) -> dict[str, str]:
return dict(zip(table.column(key).to_pylist(), table.column(value).to_pylist(), strict=True))
[docs]
def read_export(data_path: Path, reference_path: Path) -> tuple[pa.Table, dict[str, int]]:
"""Outdoor antennas with a mobile emitter in service, one row per antenna."""
counts: dict[str, int] = defaultdict(int)
with zipfile.ZipFile(reference_path) as reference:
operators = _lookup(
_read(reference, "SUP_EXPLOITANT.txt", ["ADM_ID", "ADM_LB_NOM"]),
"ADM_ID",
"ADM_LB_NOM",
)
natures = _lookup(
_read(reference, "SUP_NATURE.txt", ["NAT_ID", "NAT_LB_NOM"]), "NAT_ID", "NAT_LB_NOM"
)
antenna_types = _lookup(
_read(reference, "SUP_TYPE_ANTENNE.txt", ["TAE_ID", "TAE_LB"]), "TAE_ID", "TAE_LB"
)
with zipfile.ZipFile(data_path) as data:
emitters = _read(
data, "SUP_EMETTEUR.txt", ["EMR_ID", "EMR_LB_SYSTEME", "AER_ID", "EMR_DT_SERVICE"]
)
systems = emitters.column("EMR_LB_SYSTEME").to_pylist()
radios = [_radio(system or "") for system in systems]
started = pc.fill_null(emitters.column("EMR_DT_SERVICE"), "")
in_service = pc.greater(pc.utf8_length(started), 0)
mobile = pa.array([radio is not None for radio in radios])
counts["mobile_emitters_not_in_service"] = pc.sum(
pc.and_(mobile, pc.invert(in_service))
).as_py() or 0
emitters = emitters.filter(pc.and_(mobile, in_service))
emitter_system = _lookup(emitters, "EMR_ID", "EMR_LB_SYSTEME")
antenna_emitters: dict[str, list[tuple[str, str | None]]] = defaultdict(list)
for emitter, antenna, started in zip(
emitters.column("EMR_ID").to_pylist(),
emitters.column("AER_ID").to_pylist(),
emitters.column("EMR_DT_SERVICE").to_pylist(),
strict=True,
):
antenna_emitters[antenna].append((emitter, started))
bands = _read(
data, "SUP_BANDE.txt", ["EMR_ID", "BAN_NB_F_DEB", "BAN_NB_F_FIN", "BAN_FG_UNITE"]
)
bands = bands.filter(pc.is_in(bands.column("EMR_ID"), emitters.column("EMR_ID")))
emitter_bands: dict[str, list[tuple[float, float]]] = defaultdict(list)
for emitter, low, high, unit in zip(
bands.column("EMR_ID").to_pylist(),
bands.column("BAN_NB_F_DEB").to_pylist(),
bands.column("BAN_NB_F_FIN").to_pylist(),
bands.column("BAN_FG_UNITE").to_pylist(),
strict=True,
):
low_value, high_value = _number(low), _number(high)
scale = BAND_UNIT_MHZ.get((unit or "").strip())
if low_value is None or high_value is None or scale is None:
continue
low_mhz, high_mhz = sorted((low_value * scale, high_value * scale))
if low_mhz > 0:
emitter_bands[emitter].append((low_mhz, high_mhz))
stations = _lookup(
_read(data, "SUP_STATION.txt", ["STA_NM_ANFR", "ADM_ID"]), "STA_NM_ANFR", "ADM_ID"
)
supports: dict[str, tuple[float, float, str, float | None]] = {}
support_table = _read(
data,
"SUP_SUPPORT.txt",
[
"SUP_ID",
"NAT_ID",
"COR_NB_DG_LAT",
"COR_NB_MN_LAT",
"COR_NB_SC_LAT",
"COR_CD_NS_LAT",
"COR_NB_DG_LON",
"COR_NB_MN_LON",
"COR_NB_SC_LON",
"COR_CD_EW_LON",
"SUP_NM_HAUT",
],
)
for row in support_table.to_pylist():
if row["SUP_ID"] in supports:
continue
try:
latitude = _degrees(
row["COR_NB_DG_LAT"],
row["COR_NB_MN_LAT"],
row["COR_NB_SC_LAT"],
row["COR_CD_NS_LAT"],
)
longitude = _degrees(
row["COR_NB_DG_LON"],
row["COR_NB_MN_LON"],
row["COR_NB_SC_LON"],
row["COR_CD_EW_LON"],
)
except (TypeError, ValueError):
continue
supports[row["SUP_ID"]] = (
latitude,
longitude,
row["NAT_ID"] or "0",
_number(row["SUP_NM_HAUT"]),
)
antennas = _read(
data,
"SUP_ANTENNE.txt",
[
"STA_NM_ANFR",
"AER_ID",
"TAE_ID",
"AER_FG_RAYON",
"AER_NB_AZIMUT",
"AER_NB_ALT_BAS",
"SUP_ID",
],
)
antennas = antennas.filter(
pc.is_in(antennas.column("AER_ID"), pa.array(list(antenna_emitters)))
)
rows: dict[str, list] = defaultdict(list)
for antenna in antennas.to_pylist():
support = supports.get(antenna["SUP_ID"])
if support is None:
counts["antennas_without_support"] += 1
continue
latitude, longitude, nature, support_height = support
if nature in INDOOR_SUPPORTS or antenna["TAE_ID"] in INDOOR_ANTENNAS:
counts["indoor_antennas"] += 1
continue
height = _number(antenna["AER_NB_ALT_BAS"])
if height is not None and height < 0:
counts["antennas_below_ground"] += 1
continue
directional = (antenna["AER_FG_RAYON"] or "").strip() == "D"
azimuth = _number(antenna["AER_NB_AZIMUT"]) if directional else None
emitters_here = antenna_emitters[antenna["AER_ID"]]
ranges = sorted(
{
(emitter_system[emitter], low_mhz, high_mhz)
for emitter, _ in emitters_here
for low_mhz, high_mhz in emitter_bands.get(emitter, ())
}
)
station_operator = stations.get(antenna["STA_NM_ANFR"], "")
started = [value for _, value in emitters_here if value]
rows["antenna_id"].append(antenna["AER_ID"])
rows["support_id"].append(antenna["SUP_ID"])
rows["station_id"].append(antenna["STA_NM_ANFR"])
rows["latitude"].append(latitude)
rows["longitude"].append(longitude)
rows["height_agl_m"].append(height)
rows["azimuth_deg"].append(None if azimuth is None else azimuth % 360.0)
rows["directional"].append(directional)
rows["antenna_type"].append(antenna_types.get(antenna["TAE_ID"] or "", ""))
rows["support_type"].append(natures.get(nature, ""))
rows["support_height_m"].append(support_height)
rows["operator"].append(operators.get(station_operator, station_operator))
rows["plmn"].append(
MAINLAND_PLMN.get(station_operator) if _mainland(latitude, longitude) else None
)
rows["systems"].append(sorted({emitter_system[emitter] for emitter, _ in emitters_here}))
rows["band_label"].append([label for label, _, _ in ranges])
rows["band_low_mhz"].append([low_mhz for _, low_mhz, _ in ranges])
rows["band_high_mhz"].append([high_mhz for _, _, high_mhz in ranges])
rows["in_service"].append(min((_date(value) for value in started), default=None))
schema = pa.schema(
[
("antenna_id", pa.string()),
("support_id", pa.string()),
("station_id", pa.string()),
("latitude", pa.float64()),
("longitude", pa.float64()),
("height_agl_m", pa.float64()),
("azimuth_deg", pa.float64()),
("directional", pa.bool_()),
("antenna_type", pa.string()),
("support_type", pa.string()),
("support_height_m", pa.float64()),
("operator", pa.string()),
("plmn", pa.string()),
("systems", pa.list_(pa.string())),
("band_label", pa.list_(pa.string())),
("band_low_mhz", pa.list_(pa.float64())),
("band_high_mhz", pa.list_(pa.float64())),
("in_service", pa.string()),
]
)
return pa.Table.from_pydict(dict(rows), schema=schema), dict(counts)
[docs]
def antenna_records(table: pa.Table, release: str) -> list[AntennaFeature]:
records = []
for row in table.to_pylist():
bands = [
FrequencyBand(label=label, low_mhz=low, high_mhz=high)
for label, low, high in zip(
row["band_label"], row["band_low_mhz"], row["band_high_mhz"], strict=True
)
]
radios = sorted({radio for system in row["systems"] if (radio := _radio(system))})
basis = {"position": "source", "bands": "source"}
if row["height_agl_m"] is not None:
basis["height_agl_m"] = "source"
if row["azimuth_deg"] is not None:
basis["azimuth_deg"] = "source"
records.append(
AntennaFeature(
feature_id=f"anfr:{row['antenna_id']}",
source="anfr",
source_release=release,
site_id=f"anfr-support:{row['support_id']}",
operator=row["operator"] or None,
plmn=row["plmn"],
radios=radios,
bands=bands,
latitude=row["latitude"],
longitude=row["longitude"],
height_agl_m=row["height_agl_m"],
azimuth_deg=row["azimuth_deg"],
basis=basis,
source_attributes={
"station": row["station_id"],
"support": row["support_id"],
"support_type": row["support_type"],
"support_height_m": row["support_height_m"],
"antenna_type": row["antenna_type"],
"radiation": "directional" if row["directional"] else "omnidirectional",
"systems": row["systems"],
"in_service": row["in_service"],
},
)
)
return records
[docs]
async def latest_export(client: httpx.AsyncClient) -> tuple[str, str, str]:
"""Export date and the data and reference archive URLs of the newest complete export."""
response = await client.get(DATASET_API, timeout=60)
response.raise_for_status()
exports: dict[str, dict[str, str]] = defaultdict(dict)
for resource in response.json().get("resources", []):
match = EXPORT_PATTERN.search(str(resource.get("url", "")))
if match:
exports[match[1]][match[2]] = resource["url"]
complete = sorted(date for date, parts in exports.items() if {"data", "ref"} <= set(parts))
if not complete:
raise ValueError("The ANFR dataset lists no complete export")
date = complete[-1]
return f"{date[:4]}-{date[4:6]}-{date[6:]}", exports[date]["data"], exports[date]["ref"]
[docs]
class AnfrProvider:
def __init__(self, settings: Settings, client: httpx.AsyncClient):
self.settings = settings
self.client = client
self.directory = snapshot_directory(settings)
self._lock = asyncio.Lock()
[docs]
async def snapshot(self, *, refresh: bool = False) -> Snapshot:
"""The installed snapshot; downloads the latest export first when none is installed."""
async with self._lock:
installed = current_snapshot(self.directory)
if installed is not None and not refresh:
return installed
data_path = self.directory / "export-data.zip"
reference_path = self.directory / "export-ref.zip"
try:
release, data_url, reference_url = await latest_export(self.client)
sha256 = await download(
self.client, data_url, data_path, maximum_bytes=MAX_EXPORT_BYTES
)
await download(
self.client, reference_url, reference_path, maximum_bytes=10_000_000
)
table, counts = await asyncio.to_thread(read_export, data_path, reference_path)
return await asyncio.to_thread(
install_snapshot,
self.directory,
table,
release=release,
sha256=sha256,
metadata={
"source": "anfr",
"source_url": data_url,
"reference_url": reference_url,
"license": LICENSE,
"excluded": counts,
},
)
except httpx.HTTPError as exc:
raise AntennaProviderError(f"ANFR export download failed: {exc}") from exc
except (OSError, KeyError, ValueError, zipfile.BadZipFile, pa.ArrowException) as exc:
raise AntennaProviderError(f"ANFR export could not be installed: {exc}") from exc
finally:
data_path.unlink(missing_ok=True)
reference_path.unlink(missing_ok=True)
[docs]
async def antennas(
self, latitude: float, longitude: float, radius_m: float
) -> AntennaQueryResult:
snapshot = await self.snapshot()
table = await asyncio.to_thread(
read_point_index, snapshot.index, *query_bounds(latitude, longitude, radius_m)
)
records = await asyncio.to_thread(antenna_records, table, snapshot.identity)
return AntennaQueryResult(antennas=records, provider=TITLE, release=snapshot.identity)