Reorganize project into scripts/, docs/, data/, output/ directories

Move all Python scripts to scripts/, documentation to docs/, raw input data to data/, and generated HTML/CSV outputs to output/. Update path references in 8 scripts to use Path(__file__).parent.parent as project root so they work correctly from the new location. Update README links and quick-start commands accordingly. Notebooks remain at root. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-27 21:57:22 -07:00
parent a2e295d95b
commit ee5856661a
40 changed files with 31 additions and 30 deletions
--- a/scripts/make_data_center_map.py
+++ b/scripts/make_data_center_map.py
@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+from collections import Counter
+
+import psycopg2
+
+
+DB_NAME = "data_centers"
+POINT_TABLE = "public.master_data_centers"
+
+
+def connect():
+    return psycopg2.connect(
+        host=os.environ["PGWEB_HOST"],
+        port=os.environ["PGWEB_PORT"],
+        user=os.environ["PGWEB_USER"],
+        password=os.environ["PGWEB_PASSWORD"],
+        dbname=DB_NAME,
+    )
+
+
+def load_points(conn):
+    with conn.cursor() as cur:
+        cur.execute(
+            f"""
+            select
+                master_id,
+                source,
+                coalesce(operator, '') as operator,
+                coalesce(name, '') as name,
+                coalesce(city, '') as city,
+                coalesce(state, '') as state,
+                longitude,
+                latitude,
+                coalesce(curated_id, '') as curated_id,
+                coalesce(osm_id, '') as osm_id,
+                coalesce(match_method, '') as match_method,
+                coalesce(geoid, '') as geoid
+            from {POINT_TABLE}
+            where longitude is not null and latitude is not null
+            """
+        )
+        rows = cur.fetchall()
+
+    points = []
+    for row in rows:
+        points.append(
+            {
+                "id": row[0],
+                "source": row[1],
+                "operator": row[2],
+                "name": row[3],
+                "city": row[4],
+                "state": row[5],
+                "lon": float(row[6]),
+                "lat": float(row[7]),
+                "curated_id": row[8],
+                "osm_id": row[9],
+                "match_method": row[10],
+                "geoid": row[11],
+            }
+        )
+    return points
+
+
+def compute_center(points):
+    if not points:
+        return 39.5, -98.35
+    lat = sum(p["lat"] for p in points) / len(points)
+    lon = sum(p["lon"] for p in points) / len(points)
+    return lat, lon
+
+
+def build_stats(points):
+    by_source = Counter(p["source"] or "(blank)" for p in points)
+    by_match = Counter(p["match_method"] or "(none)" for p in points)
+    return {
+        "total": len(points),
+        "by_source": dict(sorted(by_source.items(), key=lambda x: x[0])),
+        "by_match_method": dict(sorted(by_match.items(), key=lambda x: x[0])),
+    }
+
+
+def render_html(points, center_lat, center_lon, output_path):
+    stats = build_stats(points)
+    points_json = json.dumps(points)
+    stats_json = json.dumps(stats)
+
+    html = f"""<!doctype html>
+<html lang=\"en\">
+<head>
+  <meta charset=\"utf-8\" />
+  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />
+  <title>US Data Centers Master Map</title>
+  <link rel=\"stylesheet\" href=\"https://unpkg.com/leaflet@1.9.4/dist/leaflet.css\" />
+  <style>
+    html, body {{ height: 100%; margin: 0; font-family: system-ui, -apple-system, Segoe UI, sans-serif; }}
+    #layout {{ display: grid; grid-template-columns: 320px 1fr; height: 100%; }}
+    #panel {{ padding: 14px; border-right: 1px solid #ddd; overflow: auto; background: #f8fafb; }}
+    #map {{ height: 100%; width: 100%; }}
+    h1 {{ margin: 0 0 8px; font-size: 18px; }}
+    h2 {{ margin: 16px 0 8px; font-size: 14px; }}
+    .stat-row {{ display: flex; justify-content: space-between; padding: 2px 0; font-size: 13px; }}
+    .dot {{ width: 10px; height: 10px; border-radius: 50%; display: inline-block; margin-right: 8px; }}
+    @media (max-width: 900px) {{
+      #layout {{ grid-template-columns: 1fr; grid-template-rows: 220px 1fr; }}
+      #panel {{ border-right: 0; border-bottom: 1px solid #ddd; }}
+    }}
+  </style>
+</head>
+<body>
+  <div id=\"layout\">
+    <div id=\"panel\">
+      <h1>US Data Centers (Master)</h1>
+      <div class=\"stat-row\"><span>Total points</span><strong id=\"total\"></strong></div>
+      <h2>Source</h2>
+      <div id=\"sourceStats\"></div>
+      <h2>Match Method (merged rows)</h2>
+      <div id=\"matchStats\"></div>
+      <h2>Source Colors</h2>
+      <div class=\"stat-row\"><span><span class=\"dot\" style=\"background:#2ca02c\"></span>merged (curated + OSM)</span></div>
+      <div class=\"stat-row\"><span><span class=\"dot\" style=\"background:#1f77b4\"></span>curated only</span></div>
+      <div class=\"stat-row\"><span><span class=\"dot\" style=\"background:#ff7f0e\"></span>osm only</span></div>
+      <div class=\"stat-row\"><span><span class=\"dot\" style=\"background:#7f7f7f\"></span>other</span></div>
+    </div>
+    <div id=\"map\"></div>
+  </div>
+
+  <script src=\"https://unpkg.com/leaflet@1.9.4/dist/leaflet.js\"></script>
+  <script>
+    const points = {points_json};
+    const stats = {stats_json};
+
+    function colorForSource(source) {{
+      if (source === 'merged') return '#2ca02c';
+      if (source === 'curated') return '#1f77b4';
+      if (source === 'osm') return '#ff7f0e';
+      return '#7f7f7f';
+    }}
+
+    function escapeHtml(value) {{
+      return String(value || '')
+        .replaceAll('&', '&amp;')
+        .replaceAll('<', '&lt;')
+        .replaceAll('>', '&gt;')
+        .replaceAll('"', '&quot;')
+        .replaceAll("'", '&#39;');
+    }}
+
+    const map = L.map('map', {{ preferCanvas: true }}).setView([{center_lat}, {center_lon}], 5);
+
+    L.tileLayer('https://tile.openstreetmap.org/{{z}}/{{x}}/{{y}}.png', {{
+      maxZoom: 19,
+      attribution: '&copy; OpenStreetMap contributors'
+    }}).addTo(map);
+
+    const bounds = [];
+    for (const p of points) {{
+      const marker = L.circleMarker([p.lat, p.lon], {{
+        radius: 4,
+        color: colorForSource(p.source),
+        fillColor: colorForSource(p.source),
+        fillOpacity: 0.7,
+        weight: 1
+      }});
+
+      const title = p.name || p.id;
+      const operator = p.operator || '(unknown operator)';
+      const cityState = [p.city, p.state].filter(Boolean).join(', ');
+      const provenance = [
+        p.curated_id ? 'curated_id=' + escapeHtml(p.curated_id) : null,
+        p.osm_id ? 'osm_id=' + escapeHtml(p.osm_id) : null,
+        p.match_method ? 'match=' + escapeHtml(p.match_method) : null,
+      ].filter(Boolean).join('<br>');
+      marker.bindPopup(`
+        <strong>${{escapeHtml(title)}}</strong><br>
+        Operator: ${{escapeHtml(operator)}}<br>
+        Location: ${{escapeHtml(cityState)}}<br>
+        Source: ${{escapeHtml(p.source)}}<br>
+        ${{provenance ? provenance + '<br>' : ''}}
+        GEOID: ${{escapeHtml(p.geoid)}}
+      `);
+
+      marker.addTo(map);
+      bounds.push([p.lat, p.lon]);
+    }}
+
+    if (bounds.length > 0) {{
+      map.fitBounds(bounds, {{ padding: [20, 20] }});
+    }}
+
+    document.getElementById('total').textContent = stats.total;
+
+    const sourceStats = document.getElementById('sourceStats');
+    for (const [k, v] of Object.entries(stats.by_source)) {{
+      const div = document.createElement('div');
+      div.className = 'stat-row';
+      div.innerHTML = `<span>${{escapeHtml(k)}}</span><strong>${{v}}</strong>`;
+      sourceStats.appendChild(div);
+    }}
+
+    const matchStats = document.getElementById('matchStats');
+    for (const [k, v] of Object.entries(stats.by_match_method)) {{
+      const div = document.createElement('div');
+      div.className = 'stat-row';
+      div.innerHTML = `<span>${{escapeHtml(k)}}</span><strong>${{v}}</strong>`;
+      matchStats.appendChild(div);
+    }}
+  </script>
+</body>
+</html>
+"""
+    with open(output_path, "w", encoding="utf-8") as f:
+        f.write(html)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Generate an interactive HTML map from the PostGIS point table."
+    )
+    parser.add_argument(
+        "--output",
+        default="data_center_map.html",
+        help="Output HTML path (default: data_center_map.html)",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    conn = connect()
+    try:
+        points = load_points(conn)
+    finally:
+        conn.close()
+
+    center_lat, center_lon = compute_center(points)
+    render_html(points, center_lat, center_lon, args.output)
+    print(f"wrote {len(points)} points to {args.output}")
+
+
+if __name__ == "__main__":
+    main()