"""The ACMA Register of Radiocommunications Licences, Australia.
ACMA publishes the full register every day as one archive of linked CSV tables. The
adapter keeps the transmitters of spectrum licences, which cover the public mobile networks
and a few other area-wide systems such as NBN fixed wireless, and joins each to its site,
licence and licensee. A record is one carrier at one antenna: transmitters registered
several times with the same site, antenna, direction, height and frequency range, such as
the branches of one MIMO antenna, become one record. The register gives the carrier's
frequency range, the antenna's height, azimuth and transmitter power; the ranges are
transmitter frequencies, so they are marked as downlink. Tilt is kept only as a source
attribute, since the register leaves it empty or zero for almost all transmitters and does
not define its sign.
The archive is downloaded once and converted to a local point index pinned by its date and
SHA-256. Licensees who are natural persons are never recorded, as the ACMA licence requires.
"""
from __future__ import annotations
import asyncio
import math
import zipfile
from pathlib import Path
import httpx
import pyarrow as pa
import pyarrow.compute as pc
import pyarrow.csv as pv
from openworld_radio_twin.config import Settings
from openworld_radio_twin.models import AntennaFeature, FrequencyBand
from openworld_radio_twin.providers.base import AntennaProviderError, AntennaQueryResult
from openworld_radio_twin.providers.snapshots import (
Snapshot,
current_snapshot,
download,
install_snapshot,
query_bounds,
read_point_index,
source_cache_root,
)
TITLE = "ACMA Register of Radiocommunications Licences"
ATTRIBUTION = "Based on Australian Communications and Media Authority information"
LICENSE = "ACMA licence to use the Register of Radiocommunications Licences"
LICENSE_URL = "https://www.acma.gov.au/register-radiocommunication-licences-rrl"
EXPORT_URL = "https://web.acma.gov.au/rrl-updates/spectra_rrl.zip"
MAX_EXPORT_BYTES = 200_000_000
SPECTRUM_LICENCES = "85"
# CLIENT_TYPE_ID of a natural person, whose details the licence forbids reproducing.
PERSON = "7"
# Licensee names of the mobile networks, as the register spells them.
NETWORKS = {
"TELSTRA LIMITED": ("Telstra", "505-01"),
"Telstra 3G Spectrum Holdings Pty Ltd": ("Telstra", "505-01"),
"Optus Mobile Pty Limited": ("Optus", "505-02"),
"OPTUS MOBILE PTY LIMITED": ("Optus", "505-02"),
"Vodafone Hutchison Australia Pty Limited": ("TPG Telecom", "505-03"),
"Vodafone Australia Pty Limited": ("TPG Telecom", "505-03"),
"TPG INTERNET PTY LTD": ("TPG Telecom", "505-03"),
}
DEVICE_COLUMNS = [
"LICENCE_NO",
"DEVICE_REGISTRATION_IDENTIFIER",
"FREQUENCY",
"BANDWIDTH",
"DEVICE_TYPE",
"TRANSMITTER_POWER",
"TRANSMITTER_POWER_UNIT",
"SITE_ID",
"ANTENNA_ID",
"AZIMUTH",
"HEIGHT",
"TILT",
"SV_ID",
]
KEY = ["site_id", "client", "antenna_id", "azimuth_deg", "height_agl_m", "low_mhz", "high_mhz"]
[docs]
def snapshot_directory(settings: Settings) -> Path:
return source_cache_root(settings) / "acma"
def _read(archive: zipfile.ZipFile, name: str, columns: list[str]) -> pa.Table:
with archive.open(name) as stream:
return pv.read_csv(
stream,
read_options=pv.ReadOptions(block_size=1 << 24),
convert_options=pv.ConvertOptions(
include_columns=columns, column_types=dict.fromkeys(columns, pa.string())
),
)
def _number(value: str | None) -> float | None:
if value is None or not value.strip():
return None
number = float(value)
return number if math.isfinite(number) else None
[docs]
def read_export(path: Path) -> tuple[pa.Table, dict[str, int], str]:
"""One row per carrier at an antenna, and the archive's release date."""
counts: dict[str, int] = {}
with zipfile.ZipFile(path) as archive:
release = max(
info.date_time for info in archive.infolist() if info.filename.endswith(".csv")
)
devices = _read(archive, "device_details.csv", DEVICE_COLUMNS)
devices = devices.filter(
pc.and_(
pc.equal(devices["DEVICE_TYPE"], "T"),
pc.equal(devices["SV_ID"], SPECTRUM_LICENCES),
)
)
sites = _read(archive, "site.csv", ["SITE_ID", "LATITUDE", "LONGITUDE", "SITE_PRECISION"])
licences = _read(
archive, "licence.csv", ["LICENCE_NO", "CLIENT_NO", "LICENCE_CATEGORY_NAME"]
)
clients = _read(archive, "client.csv", ["CLIENT_NO", "LICENCEE", "CLIENT_TYPE_ID"])
site = {
row["SITE_ID"]: row for row in sites.to_pylist() if row["LATITUDE"] and row["LONGITUDE"]
}
licence = {row["LICENCE_NO"]: row for row in licences.to_pylist()}
client = {row["CLIENT_NO"]: row for row in clients.to_pylist()}
carriers: dict[tuple, dict] = {}
for row in devices.to_pylist():
place = site.get(row["SITE_ID"])
held = licence.get(row["LICENCE_NO"])
centre, width = _number(row["FREQUENCY"]), _number(row["BANDWIDTH"])
if place is None or held is None or not centre or not width:
counts["skipped_without_site_or_frequency"] = (
counts.get("skipped_without_site_or_frequency", 0) + 1
)
continue
category = held["LICENCE_CATEGORY_NAME"] or ""
if category.startswith("AWL"):
counts["skipped_area_wide_licences"] = counts.get("skipped_area_wide_licences", 0) + 1
continue
holder = client.get(held["CLIENT_NO"]) or {}
height = _number(row["HEIGHT"])
azimuth = _number(row["AZIMUTH"])
record = {
"site_id": row["SITE_ID"],
"client": held["CLIENT_NO"] or "",
"antenna_id": row["ANTENNA_ID"] or "",
"azimuth_deg": None if azimuth is None else azimuth % 360.0,
"height_agl_m": height if height and height > 0 else None,
"low_mhz": round((centre - width / 2) / 1e6, 3),
"high_mhz": round((centre + width / 2) / 1e6, 3),
}
power = _number(row["TRANSMITTER_POWER"])
power_dbm = 10 * math.log10(power) + 30 if power and power > 0 else None
key = tuple(record[name] for name in KEY)
current = carriers.get(key)
if current is not None:
counts["merged_duplicate_registrations"] = (
counts.get("merged_duplicate_registrations", 0) + 1
)
if power_dbm is not None and (
current["power_dbm"] is None or power_dbm > current["power_dbm"]
):
current["power_dbm"] = power_dbm
current["registration"] = min(
current["registration"], row["DEVICE_REGISTRATION_IDENTIFIER"] or ""
)
continue
person = holder.get("CLIENT_TYPE_ID") == PERSON
carriers[key] = {
**record,
"registration": row["DEVICE_REGISTRATION_IDENTIFIER"] or "",
"latitude": float(place["LATITUDE"]),
"longitude": float(place["LONGITUDE"]),
"site_precision": place["SITE_PRECISION"] or "",
"licensee": "" if person else holder.get("LICENCEE") or "",
"category": category,
"power_dbm": power_dbm,
"tilt": row["TILT"] or "",
}
table = pa.Table.from_pylist(
list(carriers.values()),
schema=pa.schema(
[
("site_id", pa.string()),
("client", pa.string()),
("antenna_id", pa.string()),
("azimuth_deg", pa.float64()),
("height_agl_m", pa.float64()),
("low_mhz", pa.float64()),
("high_mhz", pa.float64()),
("registration", pa.string()),
("latitude", pa.float64()),
("longitude", pa.float64()),
("site_precision", pa.string()),
("licensee", pa.string()),
("category", pa.string()),
("power_dbm", pa.float64()),
("tilt", pa.string()),
]
),
)
year, month, day = release[:3]
return table, counts, f"{year:04d}-{month:02d}-{day:02d}"
[docs]
def antenna_records(table: pa.Table, release: str) -> list[AntennaFeature]:
records = []
for row in table.to_pylist():
operator, plmn = NETWORKS.get(row["licensee"], (row["licensee"] or None, None))
basis = {"position": "source", "bands": "source"}
for name in ("height_agl_m", "azimuth_deg", "power_dbm"):
if row[name] is not None:
basis[name] = "source"
records.append(
AntennaFeature(
feature_id=f"acma:{row['registration']}",
source="acma",
source_release=release,
site_id=f"acma-site:{row['site_id']}",
operator=operator,
plmn=plmn,
bands=[
FrequencyBand(
label=row["category"],
low_mhz=row["low_mhz"],
high_mhz=row["high_mhz"],
link="downlink",
)
],
latitude=row["latitude"],
longitude=row["longitude"],
height_agl_m=row["height_agl_m"],
azimuth_deg=row["azimuth_deg"],
power_dbm=row["power_dbm"],
basis=basis,
source_attributes={
"site_id": row["site_id"],
"antenna_id": row["antenna_id"],
"site_precision": row["site_precision"],
"tilt": row["tilt"],
"datum": "GDA94",
},
)
)
return records
[docs]
class AcmaProvider:
def __init__(self, settings: Settings, client: httpx.AsyncClient):
self.settings = settings
self.client = client
self.directory = snapshot_directory(settings)
self._lock = asyncio.Lock()
[docs]
async def snapshot(self, *, refresh: bool = False) -> Snapshot:
"""The installed snapshot; downloads the register first when none is installed."""
async with self._lock:
installed = current_snapshot(self.directory)
if installed is not None and not refresh:
return installed
path = self.directory / "spectra_rrl.zip"
try:
sha256 = await download(
self.client, EXPORT_URL, path, maximum_bytes=MAX_EXPORT_BYTES
)
table, counts, release = await asyncio.to_thread(read_export, path)
return await asyncio.to_thread(
install_snapshot,
self.directory,
table,
release=release,
sha256=sha256,
metadata={
"source": "acma",
"source_url": EXPORT_URL,
"license": LICENSE,
"datum": "GDA94",
"excluded": counts,
},
)
except httpx.HTTPError as exc:
raise AntennaProviderError(f"ACMA register download failed: {exc}") from exc
except (OSError, KeyError, ValueError, zipfile.BadZipFile, pa.ArrowException) as exc:
raise AntennaProviderError(f"ACMA register could not be installed: {exc}") from exc
finally:
path.unlink(missing_ok=True)
[docs]
async def antennas(
self, latitude: float, longitude: float, radius_m: float
) -> AntennaQueryResult:
snapshot = await self.snapshot()
table = await asyncio.to_thread(
read_point_index, snapshot.index, *query_bounds(latitude, longitude, radius_m)
)
records = await asyncio.to_thread(antenna_records, table, snapshot.identity)
return AntennaQueryResult(antennas=records, provider=TITLE, release=snapshot.identity)