"""Licences and credits of the data a scene is built from, and the dataset licence notice.
A scene records one entry per data layer and source in its provenance ``licences`` list,
and the dataset manifest holds their union as ``data_licences``. ``LICENSE.md`` at the
dataset root is written from that union. Mapzen terrain has no single licence: its terms
are those of the upstream source of each tile region, so it is used with the full credit
list and is kept apart from the redistributable licences. An upstream dataset that Overture
names for its records is a credit under the Overture entry of the same layer, not a separate
licence.
"""
from __future__ import annotations
import json
from collections.abc import Iterable, Mapping
from typing import TYPE_CHECKING
from openworld_radio_twin.environment import NOT_APPLICABLE
if TYPE_CHECKING:
from openworld_radio_twin.environment import EnvironmentData
from openworld_radio_twin.providers.base import AntennaQueryResult, BuildingQueryResult
ODBL = "ODbL 1.0"
ODBL_URL = "https://opendatacommons.org/licenses/odbl/1-0/"
OVERTURE_PROVIDER = "Overture Maps"
OVERTURE_ATTRIBUTION = (
"Overture Maps Foundation, overturemaps.org; © OpenStreetMap contributors, "
"openstreetmap.org/copyright; further contributors at docs.overturemaps.org/attribution"
)
OSM_ATTRIBUTION = "© OpenStreetMap contributors"
OSM_LICENSE_URL = "https://www.openstreetmap.org/copyright"
OSM_FALLBACK_PROVIDER = "OpenStreetMap Overpass"
# Licence fields of the global building route, keyed by the provider that answered.
GLOBAL_ROUTE_LICENCES = {
OVERTURE_PROVIDER: {
"license": ODBL,
"license_url": ODBL_URL,
"attribution": OVERTURE_ATTRIBUTION,
},
OSM_FALLBACK_PROVIDER: {
"license": ODBL,
"license_url": OSM_LICENSE_URL,
"attribution": OSM_ATTRIBUTION,
},
}
# Credit lines of upstream datasets that Overture names; the others are credited through
# Overture's attribution page.
UPSTREAM_CREDITS = {
"OpenStreetMap": OSM_ATTRIBUTION,
"ESA WorldCover": (
"© ESA WorldCover project 2020 / Contains modified Copernicus Sentinel data (2020) "
"processed by ESA WorldCover consortium"
),
}
OVERTURE_ATTRIBUTION_PAGE_CREDIT = "As listed at docs.overturemaps.org/attribution"
USGS_PUBLIC_DOMAIN = "United States public domain"
USGS_LICENSE_URL = (
"https://www.usgs.gov/information-policies-and-instructions/copyrights-and-credits"
)
USGS_ATTRIBUTION = "3DEP data courtesy of the U.S. Geological Survey"
MAPZEN_TERMS = "Per upstream source for the tile region"
MAPZEN_LICENSE_URL = "https://github.com/tilezen/joerd/blob/master/docs/attribution.md"
# The tile metadata do not name the upstream source of each tile, so the full list applies.
MAPZEN_UPSTREAM_CREDITS = (
"ArcticDEM terrain data DEM(s) were created from DigitalGlobe, Inc., imagery and funded "
"under National Science Foundation awards 1043681, 1559691, and 1542736",
"Australia terrain data © Commonwealth of Australia (Geoscience Australia) 2017",
"Austria terrain data © offene Daten Österreichs – Digitales Geländemodell (DGM) Österreich",
"Canada terrain data contains information licensed under the Open Government Licence – Canada",
"Europe terrain data produced using Copernicus data and information funded by the "
"European Union - EU-DEM layers",
"Global ETOPO1 terrain data U.S. National Oceanic and Atmospheric Administration",
"Mexico terrain data source: INEGI, Continental relief, 2016",
"New Zealand terrain data Copyright 2011 Crown copyright (c) Land Information New Zealand "
"and the New Zealand Government (All rights reserved)",
"Norway terrain data © Kartverket",
"United Kingdom terrain data © Environment Agency copyright and/or database right 2015. "
"All rights reserved",
"United States 3DEP (formerly NED) and global GMTED2010 and SRTM terrain data courtesy of "
"the U.S. Geological Survey",
)
MAPZEN_ATTRIBUTION = "Mapzen; " + "; ".join(MAPZEN_UPSTREAM_CREDITS) + "."
# Licences that let anyone use the data and share data derived from it. A release checks
# every layer against this set; Mapzen terrain is allowed separately, with its credits.
REDISTRIBUTABLE_LICENCES = frozenset(
{
"CC0 1.0",
"CC BY 4.0",
"CC BY-SA 4.0",
ODBL,
"Licence Ouverte 2.0",
"Datenlizenz Deutschland – Zero – Version 2.0",
USGS_PUBLIC_DOMAIN,
}
)
# Overture's ``sources`` column spells licences as SPDX identifiers.
_SPDX_NAMES = {
"ODbL-1.0": ODBL,
"CC-BY-4.0": "CC BY 4.0",
"CC-BY-SA-4.0": "CC BY-SA 4.0",
"CC0-1.0": "CC0 1.0",
}
_LICENCE_URLS = {
ODBL: ODBL_URL,
"CC BY 4.0": "https://creativecommons.org/licenses/by/4.0/",
"CC BY-SA 4.0": "https://creativecommons.org/licenses/by-sa/4.0/",
"CC0 1.0": "https://creativecommons.org/publicdomain/zero/1.0/",
}
[docs]
def licence_name(value: str | None) -> str:
"""The licence as this module names it; an unknown name is kept as given."""
text = (value or "").strip()
return _SPDX_NAMES.get(text, text)
[docs]
def usable_in_release(license: str) -> bool:
"""Whether a layer under ``license`` may be redistributed: a redistributable licence, or
Mapzen terrain with its full credit list."""
return licence_name(license) in REDISTRIBUTABLE_LICENCES or license == MAPZEN_TERMS
[docs]
def via_overture(entry: Mapping[str, object]) -> bool:
"""Whether ``entry`` is an upstream dataset that Overture names for its records.
Such data reach the scene in the Overture release, whose own entry carries the licence;
the upstream entry is a credit, so its licence is not checked for release.
"""
return entry.get("via") == OVERTURE_PROVIDER
def _share_alike(entries: Iterable[Mapping[str, object]]) -> list[Mapping[str, object]]:
"""The CC BY-SA 4.0 layers among ``entries``; their terms rule out an offer under the ODbL."""
return [
entry
for entry in entries
if not via_overture(entry) and licence_name(str(entry.get("license", ""))) == "CC BY-SA 4.0"
]
[docs]
def release_problems(entries: Iterable[Mapping[str, object]]) -> list[str]:
"""Why a scene with these licence entries may not be redistributed; empty when it may.
A layer blocks release when its licence is not recorded or is not redistributable, and a
CC BY-SA 4.0 layer blocks it beside an ODbL layer, because data under one share-alike
licence cannot be offered under the other.
"""
entries = list(entries)
problems = []
for entry in entries:
license_name = str(entry.get("license", ""))
if via_overture(entry) or usable_in_release(license_name):
continue
terms = (
f"is under {license_name}, which is not known to allow redistribution"
if license_name
else "has no recorded licence"
)
problems.append(f"{entry.get('layer')} from {entry.get('source')} {terms}")
odbl = [entry for entry in entries if licence_name(str(entry.get("license", ""))) == ODBL]
if odbl:
problems.extend(
f"{entry.get('layer')} from {entry.get('source')} is under CC BY-SA 4.0 and the "
f"scene also holds {odbl[0].get('layer')} under {ODBL}"
for entry in _share_alike(entries)
)
return problems
def _entry(
layer: str,
source: str,
release: str | None,
license: str,
license_url: str,
attribution: str,
) -> dict[str, object]:
return {
"layer": layer,
"source": source,
"release": release,
"license": license,
"license_url": license_url,
"attribution": attribution,
}
def _upstream_entries(
layer: str, datasets: Iterable[Mapping[str, object]], release: str | None
) -> list[dict[str, object]]:
"""One entry per upstream dataset that Overture names in its ``sources`` column, marked
by ``via``."""
entries = []
for item in datasets:
dataset = str(item["dataset"])
name = licence_name(None if item.get("license") is None else str(item["license"]))
attribution = UPSTREAM_CREDITS.get(dataset, OVERTURE_ATTRIBUTION_PAGE_CREDIT)
entry = _entry(
layer,
f"{dataset}, via {OVERTURE_PROVIDER}",
release,
name,
_LICENCE_URLS.get(name, ""),
attribution,
)
entry["via"] = OVERTURE_PROVIDER
entries.append(entry)
return entries
[docs]
def scene_licences(
buildings: BuildingQueryResult,
environment: EnvironmentData | None,
antennas: AntennaQueryResult | None = None,
) -> list[dict[str, object]]:
"""One entry per data layer and source of a scene; layers without data are left out."""
entries: list[dict[str, object]] = []
if buildings.buildings:
entries.append(
_entry(
"buildings",
buildings.provider,
buildings.release,
buildings.license,
buildings.license_url,
buildings.attribution,
)
)
entries.extend(_upstream_entries("buildings", buildings.source_datasets, buildings.release))
if environment is not None:
terrain = environment.terrain.source
if terrain.license != NOT_APPLICABLE:
entry = _entry(
"terrain",
terrain.provider,
None,
terrain.license,
terrain.license_url,
terrain.attribution,
)
if terrain.license == MAPZEN_TERMS:
tiles = str(terrain.attributes.get("tiles") or "")
entry["tiles"] = [tile for tile in tiles.split(",") if tile]
entries.append(entry)
surfaces = environment.surface_provenance
provider = str(surfaces.get("provider", "disabled"))
if provider != "disabled":
release = surfaces.get("release", environment.overture_release)
entries.append(
_entry(
"surfaces",
provider,
None if release is None else str(release),
str(surfaces.get("license", "")),
str(surfaces.get("license_url", "")),
str(surfaces.get("attribution", "")),
)
)
entries.extend(
_upstream_entries(
"surfaces",
surfaces.get("source_datasets") or [],
None if release is None else str(release),
)
)
if antennas is not None and antennas.antennas:
entries.append(
_entry(
"antennas",
antennas.provider,
antennas.release,
antennas.license,
antennas.license_url,
antennas.attribution,
)
)
return entries
def _licence_groups(
scenes: Iterable[Mapping[str, object]],
) -> list[tuple[dict[str, object], list[str]]]:
"""Licence entries of the scenes, each once, with the scenes that use it.
Entries that differ only in their Mapzen tiles are one entry with the union of tiles.
"""
groups: dict[str, tuple[dict[str, object], list[str]]] = {}
for scene in scenes:
provenance = scene.get("provenance")
if not isinstance(provenance, Mapping):
continue
for item in provenance.get("licences", []):
identity = {key: value for key, value in item.items() if key != "tiles"}
key = json.dumps(identity, sort_keys=True)
entry, names = groups.setdefault(key, (identity, []))
if "tiles" in item:
entry["tiles"] = sorted({*entry.get("tiles", []), *item["tiles"]})
name = str(scene.get("name", ""))
if name and name not in names:
names.append(name)
return list(groups.values())
[docs]
def data_licences(scenes: Iterable[Mapping[str, object]]) -> list[dict[str, object]]:
"""The de-duplicated union of the scenes' licence entries, for the dataset manifest."""
return [entry for entry, _ in _licence_groups(scenes)]
def _odbl_parties(entries: list[dict[str, object]]) -> str:
"""Overture and OpenStreetMap are the only ODbL sources."""
if any(entry["source"] == OVERTURE_PROVIDER or via_overture(entry) for entry in entries):
return "Overture Maps Foundation and © OpenStreetMap contributors"
return "© OpenStreetMap contributors"
[docs]
def licence_notice(manifest: Mapping[str, object]) -> str:
"""The text of the dataset's ``LICENSE.md``, built from the manifest's scene records."""
scenes = manifest.get("scenes", [])
groups = _licence_groups(scenes)
lines = [f"# Data licences of {manifest.get('dataset_name', 'this dataset')}", ""]
odbl = [entry for entry, _ in groups if entry["license"] == ODBL]
# The scenes left out of the dataset's offer, with the reasons. Beside an ODbL offer, a
# share-alike layer keeps its scene out even when the scene holds no ODbL layer itself.
left_out: dict[str, list[str]] = {}
for scene in scenes:
provenance = scene.get("provenance")
entries = list(provenance.get("licences", [])) if isinstance(provenance, Mapping) else []
reasons = release_problems(entries)
if odbl and not reasons:
reasons = [
f"{entry.get('layer')} from {entry.get('source')} is under CC BY-SA 4.0, whose "
"share-alike terms do not allow an offer under the ODbL"
for entry in _share_alike(entries)
]
if reasons:
left_out[str(scene.get("name", ""))] = reasons
if odbl:
lines += [
f"Contains information from {_odbl_parties(odbl)}, made available under the "
f"Open Database License (ODbL) 1.0 ({ODBL_URL}).",
"",
]
if not left_out:
lines.append(
"This dataset is a derivative database of those data and is offered under the "
"ODbL 1.0. Its radio arrays and images are produced from that database. The "
"other sources below are credited as their publishers require."
)
elif len(left_out) < len(scenes):
lines.append(
"Apart from the scenes named in the next paragraphs, this dataset is a "
"derivative database of those data and is offered under the ODbL 1.0. Its "
"radio arrays and images are produced from that database. The other sources "
"below are credited as their publishers require."
)
else:
lines.append(
"No scene of this dataset is offered under the ODbL 1.0; the next paragraphs "
"say why. The other sources below are credited as their publishers require."
)
elif groups:
lines.append(
"No single licence covers this dataset. Each source below keeps the terms under "
"which its publisher released it."
)
else:
lines.append("The scenes of this dataset use no provider data.")
# Without an ODbL layer, only a layer that may not be redistributed leaves a scene out.
outcome = "is not part of the dataset's ODbL offer" if odbl else "may not be redistributed"
for name, reasons in left_out.items():
lines += ["", f"Scene {name} {outcome}: {'; '.join(reasons)}."]
if groups:
introduction = (
"Each source is listed with the terms under which its publisher released it, the "
"credit it requires and the scenes that use it."
)
if any(via_overture(entry) for entry, _ in groups):
introduction += (
" An upstream dataset named by Overture reaches the scenes through the Overture "
"release listed for the same layer and is credited here."
)
lines += ["", "## Sources", "", introduction]
for entry, names in groups:
lines += ["", f"### {str(entry['layer']).capitalize()}: {entry['source']}", ""]
if entry.get("release"):
lines.append(f"- Release: {entry['release']}")
license_name = str(entry["license"])
url = f" ({entry['license_url']})" if entry.get("license_url") else ""
if via_overture(entry):
lines.append(
f"- Licence named by Overture: {license_name}{url}"
if license_name
else "- Licence: not named by Overture"
)
elif license_name == MAPZEN_TERMS:
lines.append(f"- Terms: {MAPZEN_TERMS.lower()}{url}")
elif license_name:
lines.append(f"- Licence: {license_name}{url}")
else:
page = f"; terms page: {entry['license_url']}" if entry.get("license_url") else ""
lines.append(f"- Licence: not recorded{page}")
if entry.get("attribution"):
lines.append(f"- Credit: {entry['attribution']}")
if entry.get("tiles"):
lines.append(f"- Tiles: {', '.join(entry['tiles'])}")
lines.append(f"- Scenes: {', '.join(names)}")
return "\n".join(lines) + "\n"