Serve map tiles from our own origin

The origin-zone map drew its background from a public tile server, which made it
the only part of the admin page that depended on a third-party domain — and the
part that breaks. Where a viewer's network, proxy or browser filter refuses
tile.openstreetmap.org (the refusal arrives as a 403) the map never draws, while
the page, the Leaflet bundle and the polygon data are all same-origin and fine.

Fetch the tiles server-side and serve them from /tiles/{z}/{x}/{y}.png: cached
on disk, so a page opened repeatedly does not re-fetch, and gated on being
signed in, so the route is not an open tile relay. The upstream stays
configurable via LEAFLET_TILE_URL, and LEAFLET_TILE_PROXY_ENABLED=False puts the
browser back on direct fetches.

When a tile still cannot be fetched, answer an empty tile rather than an error
page and say so under the map, instead of leaving a viewer with a blank grid and
no explanation.

apps/compliance/admin.py

@@ -1,9 +1,10 @@
 import json
 
-from django.conf import settings
 from django.contrib import admin
 from django.utils.html import format_html
 
+from apps.core.tiles import browser_tile_url
+
 from .models import (
     ProductSpecification, SpecParameter,
     Facility, OriginZone, CollectorGroup, HarvestIntake, IntakeBatchLineage,
@@ -45,6 +46,10 @@ class OriginZoneAdmin(admin.ModelAdmin):
 
         Display only. It does not validate the geometry and it does not verify
         the zone — those are separate pieces of work.
+
+        The tile URL is the same-origin path from apps/core/tiles.py, so the map
+        keeps working when a viewer's network cannot reach a third-party tile
+        server.
         """
         if obj is None:
             return "Save this zone, then its area can be drawn here."
@@ -60,12 +65,12 @@ class OriginZoneAdmin(admin.ModelAdmin):
         return format_html(
             '<div id="zone-map" data-geometry="{}" data-tile-url="{}" '
             'style="height:420px;max-width:900px;border:1px solid #cbd5e1;'
-            'border-radius:.375rem"></div>'
+            'border-radius:.375rem;background:#eef2f5"></div>'
             '<p style="max-width:900px;margin-top:.5rem;font-size:.8rem;color:#64748b">'
             "Drawn from the stored polygon. A view, not a check: it does not validate the "
             "geometry and it does not verify the zone.</p>",
             json.dumps(obj.geometry),
-            settings.LEAFLET_TILE_URL,
+            browser_tile_url(),
         )
 
 

apps/compliance/static/js/ixg_zone_map.js

@@ -34,10 +34,27 @@
     var map = L.map(el, { scrollWheelZoom: false });
 
     if (el.dataset.tileUrl) {
-      L.tileLayer(el.dataset.tileUrl, {
+      var tiles = L.tileLayer(el.dataset.tileUrl, {
         maxZoom: 19,
         attribution: OSM_ATTRIBUTION,
       }).addTo(map);
+
+      // If the tile source cannot be reached, the polygon still draws but the
+      // background stays blank. Say so, rather than leaving the viewer to wonder
+      // whether the record itself is broken.
+      var note = null;
+      tiles.on("tileerror", function () {
+        if (note) {
+          return;
+        }
+        note = document.createElement("p");
+        note.style.cssText =
+          "max-width:900px;margin-top:.5rem;font-size:.8rem;color:#b45309";
+        note.textContent =
+          "Map imagery could not be loaded. The area below is still drawn from the " +
+          "stored coordinates, so the shape can still be checked.";
+        el.insertAdjacentElement("afterend", note);
+      });
     }
 
     var layer = L.geoJSON(geometry, {

apps/core/tests.py

@@ -1,12 +1,23 @@
-"""Every portal page must actually render.
+"""Cross-cutting checks: every portal page renders, and map tiles are served
+from our own origin.
 
-Nothing in the suite was rendering the ops or portal HTML pages, which is how a
+Neither was covered before. The page sweep exists because a view can lose a
-view can lose a line — a `template_name`, say — and 500 in front of a user while
+line — a `template_name`, say — and 500 in front of a user while the rest of the
-68 tests still pass. These sweeps fetch each read-only page with an empty
+suite stays green; it fetches each read-only page with an empty database, so it
-database, so they cover the empty-state branches as well as the templates.
+covers the empty-state branches too. The tile tests cover the map's only
+external dependency, which used to be fetched by the viewer's browser.
 """
+import shutil
+import tempfile
+import urllib.error
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+from django.conf import settings
 from django.contrib.auth import get_user_model
-from django.test import Client, TestCase
+from django.test import Client, TestCase, override_settings
+
+from .tiles import browser_tile_url, upstream_url
 
 User = get_user_model()
 
@@ -57,3 +68,88 @@ class PortalPagesRenderTests(TestCase):
 
     def test_buyer_pages_render(self):
         self.sweep(self.make_user("buyer-sweep@ixg.local", "buyer"), PORTAL_ROUTES, "portal.ixgltd.com")
+
+
+class TileProxyTests(TestCase):
+    """Tiles come from our origin, so the map does not depend on the viewer's
+    network being able to reach a third-party tile server."""
+
+    def setUp(self):
+        self.cache = Path(tempfile.mkdtemp(prefix="ixg-tile-cache-"))
+        self.addCleanup(shutil.rmtree, self.cache, True)
+        override = override_settings(LEAFLET_TILE_CACHE_DIR=str(self.cache))
+        override.enable()
+        self.addCleanup(override.disable)
+
+        self.user = User.objects.create_user(
+            email="tile-viewer@ixg.local", password="pass1234", role="admin",
+            full_name="Tile Viewer", is_staff=True,
+        )
+        self.anon = Client(SERVER_NAME="app.ixgltd.com")
+        self.viewer = Client(SERVER_NAME="app.ixgltd.com")
+        self.viewer.force_login(self.user)
+
+    @staticmethod
+    def fake_upstream(payload):
+        """Stand in for urllib.request.urlopen returning a tile."""
+        response = MagicMock()
+        response.read.return_value = payload
+        context = MagicMock()
+        context.__enter__.return_value = response
+        return lambda *a, **k: context
+
+    def test_a_viewer_must_be_signed_in(self):
+        self.assertEqual(self.anon.get("/tiles/13/4085/3448.png").status_code, 403)
+
+    def test_tiles_outside_the_zoom_range_are_refused(self):
+        self.assertEqual(self.viewer.get("/tiles/20/1/1.png").status_code, 404)
+        # x beyond the span of that zoom level
+        self.assertEqual(self.viewer.get("/tiles/1/5/1.png").status_code, 404)
+
+    def test_a_tile_is_fetched_from_the_upstream_and_served(self):
+        payload = b"\x89PNG\r\n\x1a\n fake tile"
+        with patch("apps.core.tiles.urllib.request.urlopen", self.fake_upstream(payload)):
+            r = self.viewer.get("/tiles/13/4085/3448.png")
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r["Content-Type"], "image/png")
+        self.assertEqual(r.content, payload)
+        self.assertTrue((self.cache / "13" / "4085" / "3448.png").is_file())
+
+    def test_a_second_request_is_served_from_the_cache(self):
+        payload = b"\x89PNG\r\n\x1a\n cached tile"
+        first = MagicMock(side_effect=self.fake_upstream(payload))
+        with patch("apps.core.tiles.urllib.request.urlopen", first):
+            self.viewer.get("/tiles/13/4085/3448.png")
+            self.assertEqual(first.call_count, 1)
+
+            with patch("apps.core.tiles.urllib.request.urlopen") as second:
+                r = self.viewer.get("/tiles/13/4085/3448.png")
+            self.assertEqual(second.call_count, 0, "cache should have been used")
+
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.content, payload)
+
+    def test_an_unreachable_upstream_answers_an_empty_tile_not_an_error_page(self):
+        with patch(
+            "apps.core.tiles.urllib.request.urlopen",
+            side_effect=urllib.error.HTTPError("u", 403, "Forbidden", None, None),
+        ):
+            r = self.viewer.get("/tiles/13/4085/3448.png")
+        self.assertEqual(r.status_code, 204)
+        self.assertEqual(r.content, b"")
+
+    def test_browser_tile_url_is_same_origin_when_the_proxy_is_on(self):
+        self.assertEqual(browser_tile_url(), "/tiles/{z}/{x}/{y}.png")
+
+    def test_browser_tile_url_is_the_upstream_when_the_proxy_is_off(self):
+        with override_settings(LEAFLET_TILE_PROXY_ENABLED=False):
+            self.assertEqual(browser_tile_url(), settings.LEAFLET_TILE_URL)
+
+    def test_the_upstream_url_gets_the_coordinates_substituted(self):
+        with override_settings(
+            LEAFLET_TILE_URL="https://example.invalid/{s}/{z}/{x}/{y}.png"
+        ):
+            self.assertEqual(
+                upstream_url(13, 4085, 3448),
+                "https://example.invalid/a/13/4085/3448.png",
+            )

apps/core/tiles.py

@@ -0,0 +1,112 @@
+"""Map tiles, served from our own origin.
+
+The origin-zone map is drawn from a stored polygon, but the background imagery
+comes from a public tile server. That makes the map the only part of the admin
+page that depends on a *third-party* domain, and it is the part that breaks:
+a viewer whose network, proxy or browser filter refuses tile.openstreetmap.org
+(the refusal arrives as a 403) gets a map that never draws, even though the page,
+the Leaflet bundle and the polygon data are all fine and same-origin.
+
+So the browser asks us for tiles and we fetch them:
+
+    browser ->  /tiles/{z}/{x}/{y}.png  ->  <upstream tile server>
+
+Two things this buys. The map stops depending on the viewer's network path to a
+third party, and tiles are cached on this server, so a page that is opened
+repeatedly does not re-fetch them.
+
+The upstream stays configurable (``LEAFLET_TILE_URL``) because OpenStreetMap's
+tile policy deliberately does not cover heavy commercial use — the same setting
+points the map at a paid provider or a self-hosted tile server without further
+code changes.
+"""
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+from django.conf import settings
+from django.http import HttpResponse, HttpResponseForbidden, HttpResponseNotFound
+
+# How long a browser may reuse a tile. Tiles are effectively static.
+BROWSER_CACHE_SECONDS = 60 * 60 * 24 * 7
+
+MAX_ZOOM = 19
+FETCH_TIMEOUT_SECONDS = 5
+
+# Identify the application to the upstream, as tile providers require.
+UPSTREAM_USER_AGENT = (
+    "IXG-Platform/1.0 (internal origin-zone map; contact the platform administrator)"
+)
+
+
+def browser_tile_url():
+    """The tile URL template the *browser* should use.
+
+    With the proxy on (the default) this is a same-origin path; with it off, the
+    browser talks to the upstream directly, as it did before.
+    """
+    if settings.LEAFLET_TILE_PROXY_ENABLED:
+        return "/tiles/{z}/{x}/{y}.png"
+    return settings.LEAFLET_TILE_URL
+
+
+def cache_dir():
+    return Path(settings.LEAFLET_TILE_CACHE_DIR)
+
+
+def upstream_url(z, x, y):
+    """Format the configured upstream template for one tile."""
+    url = settings.LEAFLET_TILE_URL
+    # Some providers use a {s} subdomain placeholder; any label works.
+    url = url.replace("{s}", "a")
+    return (
+        url.replace("{z}", str(z)).replace("{x}", str(x)).replace("{y}", str(y))
+    )
+
+
+def valid_tile(z, x, y):
+    """Reject anything that is not a real tile address for the zoom level."""
+    if not 0 <= z <= MAX_ZOOM:
+        return False
+    span = 2 ** z
+    return 0 <= x < span and 0 <= y < span
+
+
+def tile_view(request, z, x, y):
+    """Serve one tile, from cache when possible, otherwise from the upstream."""
+    if not request.user.is_authenticated:
+        return HttpResponseForbidden("Authentication required")
+
+    if not valid_tile(z, x, y):
+        return HttpResponseNotFound("No such tile")
+
+    cached = cache_dir() / str(z) / str(x) / f"{y}.png"
+    if cached.is_file():
+        return serve(cached.read_bytes())
+
+    try:
+        request_obj = urllib.request.Request(
+            upstream_url(z, x, y), headers={"User-Agent": UPSTREAM_USER_AGENT}
+        )
+        with urllib.request.urlopen(request_obj, timeout=FETCH_TIMEOUT_SECONDS) as resp:
+            data = resp.read()
+    except (urllib.error.URLError, urllib.error.HTTPError, OSError, ValueError):
+        # The upstream refused or is unreachable. Answer with an empty tile rather
+        # than an error page: the map keeps its polygon and its controls, and the
+        # viewer sees a blank background instead of a broken grid.
+        return HttpResponse(status=204)
+
+    try:
+        cached.parent.mkdir(parents=True, exist_ok=True)
+        cached.write_bytes(data)
+    except OSError:
+        # A read-only or full cache directory must not break the map.
+        pass
+
+    return serve(data)
+
+
+def serve(data):
+    response = HttpResponse(data, content_type="image/png")
+    response["Cache-Control"] = f"public, max-age={BROWSER_CACHE_SECONDS}"
+    return response

dsp_platform/settings.py

@@ -143,6 +143,18 @@ LEAFLET_TILE_URL = env(
     "LEAFLET_TILE_URL", default="https://tile.openstreetmap.org/{z}/{x}/{y}.png"
 )
 
+# Serve tiles from our own origin instead of letting the browser fetch them from
+# a third party (apps/core/tiles.py). The map is the only part of the admin page
+# that depends on an external domain, and when a viewer's network, proxy or
+# browser filter refuses that domain the map silently never draws — the page,
+# the Leaflet bundle and the polygon are all same-origin and fine. With this on,
+# the browser asks us and we fetch and cache. Set LEAFLET_TILE_PROXY_ENABLED=False
+# to go back to direct browser fetches.
+LEAFLET_TILE_PROXY_ENABLED = env.bool("LEAFLET_TILE_PROXY_ENABLED", default=True)
+LEAFLET_TILE_CACHE_DIR = env(
+    "LEAFLET_TILE_CACHE_DIR", default=str(MEDIA_ROOT / "tile-cache")
+)
+
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 
 AUTH_USER_MODEL = "users.User"

dsp_platform/urls.py

@@ -5,8 +5,11 @@ from django.conf import settings
 from django.conf.urls.static import static
 from rest_framework.authtoken import views as drf_auth
 
+from apps.core.tiles import tile_view
+
 urlpatterns = [
     path("", lambda r: HttpResponseRedirect("/admin/")),
+    path("tiles/<int:z>/<int:x>/<int:y>.png", tile_view, name="tile"),
     path("grappelli/", include("grappelli.urls")),
     path("admin/", admin.site.urls),
     path("oidc/", include("mozilla_django_oidc.urls")),