Source code for openworld_radio_twin.providers.opencellid

"""OpenCelliD cells from a pinned local snapshot of the daily worldwide export.

OpenCelliD records logical cells, positioned at the centroid of crowdsourced
measurements rather than at the antenna. The export gives no antenna height, azimuth,
tilt, power or band, so placement fills those from the defaults. LTE cells that share an
eNB (``cellid >> 8``) are merged into one site record at their sample-weighted mean;
other radios stay one record per cell, because their cell identifiers do not encode a
site the same way.

The query API cannot serve scenes: it allows 1,000 requests per user and day, returns at
most 50 cells per page and rounds coordinates to three decimals. The adapter therefore
downloads the worldwide export once (about 120 MB, cells observed in the last 18 months)
with the user's token and reads it locally afterwards. The token is used only for that
download and is never recorded.
"""

from __future__ import annotations

import asyncio
import gzip
import struct
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path

import httpx
import pyarrow as pa
import pyarrow.compute as pc
import pyarrow.csv as pv

from openworld_radio_twin.config import Settings
from openworld_radio_twin.models import AntennaFeature
from openworld_radio_twin.providers.base import AntennaProviderError, AntennaQueryResult
from openworld_radio_twin.providers.snapshots import (
    Snapshot,
    current_snapshot,
    download,
    install_snapshot,
    query_bounds,
    read_point_index,
    source_cache_root,
)

TITLE = "OpenCelliD"
ATTRIBUTION = "Cell tower data from OpenCelliD (https://opencellid.org)"
LICENSE = "CC BY-SA 4.0"
LICENSE_URL = "https://creativecommons.org/licenses/by-sa/4.0/"
CONFIGURATION_HINT = "set OWRT_OPENCELLID_TOKEN to an OpenCelliD API token"
EXPORT_URL = "https://opencellid.org/ocid/downloads"
EXPORT_FILE = "cell_towers.csv.gz"
MAX_EXPORT_BYTES = 1_000_000_000
EXPORT_COLUMNS = (
    "radio",
    "mcc",
    "net",
    "area",
    "cell",
    "unit",
    "lon",
    "lat",
    "range",
    "samples",
    "changeable",
    "created",
    "updated",
    "averageSignal",
)
EXPORT_TYPES = {
    "radio": pa.string(),
    "mcc": pa.int32(),
    "net": pa.int32(),
    "area": pa.int64(),
    "cell": pa.int64(),
    "unit": pa.int64(),
    "lon": pa.float64(),
    "lat": pa.float64(),
    "range": pa.int64(),
    "samples": pa.int64(),
    "changeable": pa.int8(),
    "created": pa.int64(),
    "updated": pa.int64(),
    "averageSignal": pa.int64(),
}
INDEX_COLUMNS = ("radio", "mcc", "net", "area", "cell", "lon", "lat", "range", "samples", "updated")
INDEX_NAMES = {"lon": "longitude", "lat": "latitude"}
RADIOS = {"GSM", "UMTS", "LTE", "NR", "NBIOT", "CDMA"}


[docs] def snapshot_directory(settings: Settings) -> Path: return source_cache_root(settings) / "opencellid"
[docs] def configured(settings: Settings) -> bool: """A token allows the first download; an installed snapshot is enough afterwards.""" token = settings.opencellid_token has_token = token is not None and bool(token.get_secret_value().strip()) return has_token or current_snapshot(snapshot_directory(settings)) is not None
def _export_date(path: Path) -> str: """The export's date from the gzip header, which the publisher sets at export time.""" with path.open("rb") as stream: header = stream.read(10) if len(header) < 10 or header[:2] != b"\x1f\x8b": raise ValueError( "OpenCelliD did not return the export; check the token and the limit of two " "downloads per file and day" ) (modified,) = struct.unpack("<I", header[4:8]) moment = datetime.fromtimestamp(modified, timezone.utc) if modified else None return (moment or datetime.now(timezone.utc)).date().isoformat()
[docs] def read_export(path: Path) -> pa.Table: """The export's cells as an index table, with or without a header line.""" with gzip.open(path, "rt", encoding="utf-8") as stream: has_header = stream.readline().split(",", 1)[0].strip() == "radio" table = pv.read_csv( path, read_options=pv.ReadOptions( column_names=None if has_header else list(EXPORT_COLUMNS), block_size=1 << 24, ), convert_options=pv.ConvertOptions( column_types=EXPORT_TYPES, include_columns=list(INDEX_COLUMNS), ), ) table = table.filter(pc.is_in(table.column("radio"), pa.array(sorted(RADIOS)))) return table.rename_columns([INDEX_NAMES.get(name, name) for name in INDEX_COLUMNS])
[docs] def site_records(table: pa.Table, release: str) -> list[AntennaFeature]: """Merge the cells of one LTE eNB into a site; keep other cells as they are.""" groups: dict[tuple, list[dict]] = defaultdict(list) for cell in table.to_pylist(): if cell["radio"] == "LTE": key = (cell["mcc"], cell["net"], "LTE", "enb", cell["cell"] >> 8) else: key = (cell["mcc"], cell["net"], cell["radio"], cell["area"], cell["cell"]) groups[key].append(cell) records = [] for key, cells in sorted(groups.items()): mcc, net, radio = key[0], key[1], key[2] weights = [max(1, int(cell["samples"] or 1)) for cell in cells] total = sum(weights) latitude = sum(cell["latitude"] * w for cell, w in zip(cells, weights, strict=True)) longitude = sum(cell["longitude"] * w for cell, w in zip(cells, weights, strict=True)) if radio == "LTE": site = f"{mcc}-{net}-enb-{key[4]}" else: site = f"{mcc}-{net}-{radio.lower()}-{key[3]}-{key[4]}" updated = max(int(cell["updated"] or 0) for cell in cells) # CDMA system identifiers and some crowdsourced entries exceed the 3-digit MNC range. plmn = f"{mcc:03d}-{net:02d}" if 0 <= net <= 999 else None records.append( AntennaFeature( feature_id=f"opencellid:{site}", source="opencellid", source_release=release, site_id=site, operator=None if plmn else f"{mcc}-{net}", plmn=plmn, radios=[radio], latitude=latitude / total, longitude=longitude / total, basis={"position": "derived"}, source_attributes={ "mcc": mcc, "mnc": net, "cell_ids": sorted(int(cell["cell"]) for cell in cells), "area_codes": sorted({int(cell["area"]) for cell in cells}), "samples": total, "range_m": max(int(cell["range"] or 0) for cell in cells), "last_observed": ( datetime.fromtimestamp(updated, timezone.utc).date().isoformat() if updated else None ), "position_method": "sample-weighted centroid of crowdsourced measurements", }, ) ) return records
[docs] class OpenCellIdProvider: def __init__(self, settings: Settings, client: httpx.AsyncClient): self.settings = settings self.client = client self.directory = snapshot_directory(settings) self._lock = asyncio.Lock()
[docs] async def snapshot(self, *, refresh: bool = False) -> Snapshot: """The installed snapshot; downloads the export first when none is installed.""" async with self._lock: installed = current_snapshot(self.directory) if installed is not None and not refresh: return installed token = self.settings.opencellid_token if token is None or not token.get_secret_value().strip(): raise AntennaProviderError( f"Downloading the OpenCelliD export needs a token; {CONFIGURATION_HINT}" ) archive = self.directory / EXPORT_FILE try: sha256 = await download( self.client, EXPORT_URL, archive, maximum_bytes=MAX_EXPORT_BYTES, params={ "token": token.get_secret_value().strip(), "type": "full", "file": EXPORT_FILE, }, ) release = _export_date(archive) table = await asyncio.to_thread(read_export, archive) return await asyncio.to_thread( install_snapshot, self.directory, table, release=release, sha256=sha256, metadata={ "source": "opencellid", "export": EXPORT_FILE, "source_url": f"{EXPORT_URL}?type=full&file={EXPORT_FILE}", "license": LICENSE, }, ) # httpx messages carry the request URL, which holds the token: report less. except httpx.HTTPStatusError as exc: raise AntennaProviderError( f"OpenCelliD export download failed with HTTP {exc.response.status_code}" ) from None except httpx.HTTPError as exc: raise AntennaProviderError( f"OpenCelliD export download failed: {type(exc).__name__}" ) from None except (OSError, ValueError, pa.ArrowException) as exc: raise AntennaProviderError( f"OpenCelliD export could not be installed: {exc}" ) from exc finally: archive.unlink(missing_ok=True)
[docs] async def antennas( self, latitude: float, longitude: float, radius_m: float ) -> AntennaQueryResult: snapshot = await self.snapshot() cells = await asyncio.to_thread( read_point_index, snapshot.index, *query_bounds(latitude, longitude, radius_m) ) records = await asyncio.to_thread(site_records, cells, snapshot.identity) return AntennaQueryResult( antennas=records, provider=TITLE, release=snapshot.identity, warnings=[ f"OpenCelliD positions are centroids of crowdsourced measurements " f"({cells.num_rows} cells merged into {len(records)} records), not surveyed " "antenna sites; height, azimuth, tilt, power and band are defaults." ] if records else [], )