diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2b964dd --- /dev/null +++ b/.gitignore @@ -0,0 +1,76 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache/ +nosetests.xml +coverage.xml +*.cover +*.py,cover +.pytest_cache/ + +# Type checkers / linters +.mypy_cache/ +.pyre/ +.ruff_cache/ + +# Environments +.env +.env.* +.venv/ +venv/ +env/ +ENV/ + +# Jupyter Notebook +.ipynb_checkpoints/ + +# IDEs and editors +.vscode/ +.idea/ +*.swp +*.swo + +# Logs +*.log + +# OS files +.DS_Store +Thumbs.db diff --git a/__pycache__/ingest_eia_energy_layers.cpython-314.pyc b/__pycache__/ingest_eia_energy_layers.cpython-314.pyc index 07a3c0b..abd9810 100644 Binary files a/__pycache__/ingest_eia_energy_layers.cpython-314.pyc and b/__pycache__/ingest_eia_energy_layers.cpython-314.pyc differ diff --git a/ingest_eia_energy_layers.py b/ingest_eia_energy_layers.py index a416875..81ef532 100644 --- a/ingest_eia_energy_layers.py +++ b/ingest_eia_energy_layers.py @@ -80,6 +80,11 @@ EIA_GEOCODABLE_ENDPOINTS = { "electricity/facility-fuel": ["latitude", "longitude"], } +# Endpoints that do not reliably support retry with ad-hoc data[] field requests. +EIA_NO_RETRY_EXTRA_FIELDS = { + "electricity/facility-fuel", +} + # US state abbreviation to FIPS code mapping for state-level GEOID linking STATE_FIPS = { "AL": "01", "AK": "02", "AZ": "04", "AR": "05", "CA": "06", @@ -138,7 +143,7 @@ def query_eia_api(endpoint: str, params: Optional[Dict[str, Any]] = None, extra_ req_params[f"data[{i}]"] = field try: - resp = requests.get(url, params=req_params, timeout=60) + resp = requests.get(url, params=req_params, timeout=(10, 20)) resp.raise_for_status() return resp.json() except requests.RequestException as e: @@ -163,7 +168,11 @@ def fetch_eia_records( data = query_eia_api(endpoint, params=params, extra_data_fields=used_extra_fields) # Some endpoints return 400 when requesting unsupported fields (e.g. lat/lon). - if data is None and used_extra_fields: + if ( + data is None + and used_extra_fields + and endpoint not in EIA_NO_RETRY_EXTRA_FIELDS + ): print(f" retrying {endpoint} without extra data fields") used_extra_fields = None data = query_eia_api(endpoint, params=params, extra_data_fields=None)