updated map and cluster analysis

ingest_eia_energy_layers.py

@@ -1245,13 +1245,19 @@ def main():
         raise RuntimeError("No datasets selected. Try --category all or --list-only.")
     
     conn = connect_db()
+    failed_datasets: list[tuple[str, str, str]] = []
     try:
         ensure_source_catalog_table(conn)
-        
+
         if not args.skip_ingest:
             for dataset, table_name, category in datasets_to_ingest:
                 print(f"importing {dataset.name} -> public.{table_name} [{category}]")
-                success = import_layer_to_postgis(dataset, table_name, max_records=args.max_records)
+                try:
+                    success = import_layer_to_postgis(dataset, table_name, max_records=args.max_records)
+                except Exception as exc:
+                    print(f"  FAILED: {dataset.name} ({dataset.api_endpoint}): {exc!r}")
+                    failed_datasets.append((dataset.name, dataset.api_endpoint, repr(exc)))
+                    success = False
                 if success:
                     upsert_layer_catalog(conn, table_name, dataset, category)
                     add_geom_index_and_analyze(conn, table_name)
@@ -1293,9 +1299,17 @@ def main():
         print(f"\ndone: catalog_layers={catalog_count}")
         for tbl, n in counts.items():
             print(f"  {tbl}: {n if n is not None else 'missing'} rows")
+
+        if failed_datasets:
+            print(f"\nfailed datasets ({len(failed_datasets)}):")
+            for name, endpoint, err in failed_datasets:
+                print(f"  - {name} ({endpoint}): {err}")
     finally:
         conn.close()
 
+    if failed_datasets:
+        sys.exit(2)
+
 
 if __name__ == "__main__":
     main()