diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8c4c5f8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + test: + name: Offline tests (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install package with dev deps + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + - name: Run offline tests + # Network tests download hundreds of MB of real data; they are excluded from CI + # and run manually. Everything else must pass on every supported Python version. + run: pytest -m "not network" + + build: + name: Build & check distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Build sdist and wheel + run: | + python -m pip install --upgrade pip build twine + python -m build + - name: Validate metadata and README rendering + run: twine check dist/* diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..a9f97a5 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,33 @@ +name: deploy + +on: workflow_dispatch + +jobs: + deploy: + environment: Deploy + runs-on: ubuntu-latest + permissions: + # Required for PyPI Trusted Publishing (OIDC); no API token/secret needed. + id-token: write + contents: read + steps: + - run: echo "Starting deployment to PyPi" + - uses: actions/checkout@v5 + - name: Set up Python 3.10 + uses: actions/setup-python@v6 + with: + python-version: "3.10" + - name: Install helpers + run: | + python -m pip install --upgrade pip + pip install build twine + - name: Build + # This project uses a pyproject.toml build backend (no setup.py), so build + # with the PEP 517 frontend rather than `setup.py sdist bdist_wheel`. + run: python -m build + - name: Check distribution metadata + run: twine check dist/* + - name: Deploy to PyPi + # Trusted Publishing: PyPI verifies this repo/workflow/environment via OIDC. + uses: pypa/gh-action-pypi-publish@release/v1 + - run: echo "Successfully deployed to PyPi" diff --git a/.gitignore b/.gitignore index bf2e2f2..cafa253 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,16 @@ .pip/ **/__pycache__/ **/.temp_data/ +test.py +.DS_Store +**/.DS_Store +# packaging build artifacts +build/ +dist/ +*.egg-info/ +src/*.egg-info/ + +# editor / tooling +.vscode/ +CLAUDE.md diff --git a/CHANGELOG.MD b/CHANGELOG.MD index e55e8da..471f7ce 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -4,13 +4,30 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [Unreleased] - yyyy-mm-dd +## [0.1.0] - 2026-06-06 ### Added - Introduce structure for different dataset-specific tools -- ... +- `proxy` and `verify` options on the loaders for downloading behind a proxy / custom TLS +- `Matrix.to_pandas()` to convert a matrix to a labelled DataFrame +- OECD ICIO 2025 edition (`2025-regular`, `2025-extended`) and Eurostat Figaro 2025 edition (`2025`, years 2010-2023) ### Changed +- Renamed the package from `iopy` to `iotables` and adopted a `src/` layout with `pyproject.toml` +- Downloads now use `curl_cffi` (browser TLS impersonation) so Cloudflare-protected OECD files work +- Re-pointed OECD/Figaro download URLs to current hosts (OECD `webfs`, Figaro CIRCABC) +- Cached downloads now live in a user cache dir (`~/.cache/iotables`, override with `IOTABLES_DATA`) +- User-facing argument validation now raises `ValueError` instead of `assert` (so it survives `python -O`) ### Fixed +- OECD `2025-regular` file-id regex and in-zip CSV filenames; cross-edition cache-id collisions +- `remove_downloaded_files(database=...)` deleted the wrong database's files +- `ExioBase` download-failure handler raised `AttributeError` instead of the intended warning +- Loader `db_name` derivation used `str.rstrip('.py')` (a character set, not a suffix) +- Dropped `lru_cache` on `_load_data`, which pinned every loader instance (and its data) in memory +- Downloads are now atomic (written to a `.part` file and moved into place on completion), so an interrupted download no longer leaves a truncated file in the cache +- `download_file` now verifies the downloaded size against `Content-Length` (when advertised and unencoded) and raises on a truncated body +### Other +- Added a `py.typed` marker, `[project.urls]`, Python-version classifiers, and a GitHub Actions CI (offline tests across Python 3.9-3.13 + build/twine check) +- Network tests are marked `network` and moved into fixtures, so the suite collects and the offline tests run without a connection (`pytest -m "not network"`) ___ diff --git a/README.md b/README.md index 1e223ce..c91e530 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# iopy +# iotables ## Description @@ -14,14 +14,15 @@ Currently the following databases are supported: ## Installation ```bash -pip install git+https://github.com/WWakker/iopy.git +pip install iotables ``` ## Structure of input-output data Input-data model the economy in matrix form. Here, we explain input-output data using OECD data as an example. -THE OECD input-output tables contain inputs and outputs in current million USD for 45 sectors and 66 countries as well -as rest-of-world. China and Mexico data are split into CN1, CN2, MX1, and MX2. +The OECD input-output tables contain inputs and outputs in current million USD for a set of sectors and countries (the +exact counts depend on the edition; see `config.py`) as well as rest-of-world. China and Mexico data are split into CN1, +CN2, MX1, and MX2. ![OECD sctructure](res/OECD_structure.png) *Source*: OECD @@ -173,19 +174,20 @@ This is the Ghosh equation, which defines the relation between production output ### Create an instance of the OECD class Creating an instance of the OECD class loads the OECD data and gives access to convenient attributes and methods. -An instance can be create as follows, specifiying a year between 1995 and 2018. +An instance can be created as follows, specifying an available year for the chosen version (e.g. 1995-2018 for the +`2021` version, up to 2022 for the `2025` versions). ```python -import iopy -oecd = iopy.OECD(version='2021', year=2018, refresh=False) +import iotables +oecd = iotables.OECD(version='2021', year=2018, refresh=False) ``` Similary, an instance can be created for other data, for example Figaro and ExioBase as ```python -import iopy -figaro = iopy.Figaro(version='2022', year=2020, kind='industry-by-industry') -exio = iopy.ExioBase(version='3.81', year=2022, kind='industry-by-industry') +import iotables +figaro = iotables.Figaro(version='2025', year=2020, kind='industry-by-industry') +exio = iotables.ExioBase(version='3.81', year=2022, kind='industry-by-industry') ``` Creating an instance of a database class downloads and loads the data into memory, creates standard input-output matrices, and gives access to the following attributes and methods: @@ -223,11 +225,11 @@ All matrices are extended `numpy.ndarray`'s with attributes `info`, `rows` and ` When running a Leontief or Ghosh shock, the percentage shock to final demand/primary inputs in countries and sectors can be specified as ```python -import iopy +import iotables -oecd = iopy.OECD(version='2021', year=2018) +oecd = iotables.OECD(version='2021', year=2018) -df_l = oecd.leontief_demand_shock(shock=-10, regions=['FR', 'DE], sectors=['01T02', '35']) +df_l = oecd.leontief_demand_shock(shock=-10, regions=['FR', 'DE'], sectors=['01T02', '35']) df_g = oecd.ghosh_supply_shock(shock=-10, regions=['FR', 'DE'], sectors=['01T02', '35']) ``` diff --git a/iopy/__about__.py b/iopy/__about__.py deleted file mode 100644 index caf4018..0000000 --- a/iopy/__about__.py +++ /dev/null @@ -1,6 +0,0 @@ -__title__ = "iopy" -__about__ = "A Python package to easily load and work with input-output data" -__version__ = '0.0.1' -__authors__ = 'Wouter Wakker' -__email__ = "wouter.wakker@outlook.com" -__url__ = "" diff --git a/iopy/core/__init__.py b/iopy/core/__init__.py deleted file mode 100644 index 07a74f5..0000000 --- a/iopy/core/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -""" Created on 22/11/2022:: -------------- __init__.py ------------- -**Authors**: W. Wakker - -""" - -from iopy.core.globals import * diff --git a/iopy/core/config.py b/iopy/core/config.py deleted file mode 100644 index ecf9912..0000000 --- a/iopy/core/config.py +++ /dev/null @@ -1,204 +0,0 @@ -""" Created on 08/10/2022:: -------------- config ------------- -**Authors**: W. Wakker - -""" -config = { - 'oecd': - {'2021': - {'links': - {1995: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=91d8e84b-7406-46b9-af5f-ec096242755c', - 1996: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=91d8e84b-7406-46b9-af5f-ec096242755c', - 1997: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=91d8e84b-7406-46b9-af5f-ec096242755c', - 1998: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=91d8e84b-7406-46b9-af5f-ec096242755c', - 1999: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=91d8e84b-7406-46b9-af5f-ec096242755c', - 2000: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8adf89dd-18b4-40fe-bc7f-c822052eb961', - 2001: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8adf89dd-18b4-40fe-bc7f-c822052eb961', - 2002: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8adf89dd-18b4-40fe-bc7f-c822052eb961', - 2003: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8adf89dd-18b4-40fe-bc7f-c822052eb961', - 2004: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8adf89dd-18b4-40fe-bc7f-c822052eb961', - 2005: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=fe218690-0a3b-44aa-a82c-b3e3da6d24db', - 2006: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=fe218690-0a3b-44aa-a82c-b3e3da6d24db', - 2007: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=fe218690-0a3b-44aa-a82c-b3e3da6d24db', - 2008: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=fe218690-0a3b-44aa-a82c-b3e3da6d24db', - 2009: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=fe218690-0a3b-44aa-a82c-b3e3da6d24db', - 2010: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=2c2f499f-5703-4034-9457-2f7518e8f2fc', - 2011: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=2c2f499f-5703-4034-9457-2f7518e8f2fc', - 2012: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=2c2f499f-5703-4034-9457-2f7518e8f2fc', - 2013: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=2c2f499f-5703-4034-9457-2f7518e8f2fc', - 2014: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=2c2f499f-5703-4034-9457-2f7518e8f2fc', - 2015: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=59a3d7f2-3f23-40d5-95ca-48da84c0f861', - 2016: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=59a3d7f2-3f23-40d5-95ca-48da84c0f861', - 2017: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=59a3d7f2-3f23-40d5-95ca-48da84c0f861', - 2018: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=59a3d7f2-3f23-40d5-95ca-48da84c0f861'}, - 'regex_id': r'[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}', - 'num_regions': 71, - 'num_sectors': 45 - }, - '2022-extended': - {'links': - {1995: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=4a2d6739-f717-48ac-a97b-d95f96984c55', - 1996: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=4a2d6739-f717-48ac-a97b-d95f96984c55', - 1997: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=4a2d6739-f717-48ac-a97b-d95f96984c55', - 1998: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=4a2d6739-f717-48ac-a97b-d95f96984c55', - 1999: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=4a2d6739-f717-48ac-a97b-d95f96984c55', - 2000: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8eb6cf87-1899-4547-b337-d76dd4ef608c', - 2001: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8eb6cf87-1899-4547-b337-d76dd4ef608c', - 2002: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8eb6cf87-1899-4547-b337-d76dd4ef608c', - 2003: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8eb6cf87-1899-4547-b337-d76dd4ef608c', - 2004: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8eb6cf87-1899-4547-b337-d76dd4ef608c', - 2005: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=f9b6dd9a-e64b-47fb-832e-bd1628c43b72', - 2006: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=f9b6dd9a-e64b-47fb-832e-bd1628c43b72', - 2007: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=f9b6dd9a-e64b-47fb-832e-bd1628c43b72', - 2008: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=f9b6dd9a-e64b-47fb-832e-bd1628c43b72', - 2009: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=f9b6dd9a-e64b-47fb-832e-bd1628c43b72', - 2010: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=35cfb172-2cc4-4d62-98e5-9e5f1c51d6c9', - 2011: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=35cfb172-2cc4-4d62-98e5-9e5f1c51d6c9', - 2012: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=35cfb172-2cc4-4d62-98e5-9e5f1c51d6c9', - 2013: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=35cfb172-2cc4-4d62-98e5-9e5f1c51d6c9', - 2014: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=35cfb172-2cc4-4d62-98e5-9e5f1c51d6c9', - 2015: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=e5368118-b274-4e42-820e-33dacbfb94ed', - 2016: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=e5368118-b274-4e42-820e-33dacbfb94ed', - 2017: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=e5368118-b274-4e42-820e-33dacbfb94ed', - 2018: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=e5368118-b274-4e42-820e-33dacbfb94ed', - 2019: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=e5368118-b274-4e42-820e-33dacbfb94ed', - 2020: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=92539517-1661-4077-a9df-136abc39b858'}, - 'regex_id': r'[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}', - 'num_regions': 81, - 'num_sectors': 45}, - '2022-small': - {'links': - {1995: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8f69e3c5-8bc0-4c7b-aad2-5ef776c119ea', - 1996: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8f69e3c5-8bc0-4c7b-aad2-5ef776c119ea', - 1997: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8f69e3c5-8bc0-4c7b-aad2-5ef776c119ea', - 1998: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8f69e3c5-8bc0-4c7b-aad2-5ef776c119ea', - 1999: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8f69e3c5-8bc0-4c7b-aad2-5ef776c119ea', - 2000: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=43e7a690-c1ea-4839-b0fb-907e0aa79523', - 2001: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=43e7a690-c1ea-4839-b0fb-907e0aa79523', - 2002: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=43e7a690-c1ea-4839-b0fb-907e0aa79523', - 2003: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=43e7a690-c1ea-4839-b0fb-907e0aa79523', - 2004: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=43e7a690-c1ea-4839-b0fb-907e0aa79523', - 2005: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=9942ca81-9137-4926-adcf-dbe19fa2bcb6', - 2006: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=9942ca81-9137-4926-adcf-dbe19fa2bcb6', - 2007: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=9942ca81-9137-4926-adcf-dbe19fa2bcb6', - 2008: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=9942ca81-9137-4926-adcf-dbe19fa2bcb6', - 2009: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=9942ca81-9137-4926-adcf-dbe19fa2bcb6', - 2010: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=832d60ce-8346-456b-8ab8-cafd81c2f054', - 2011: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=832d60ce-8346-456b-8ab8-cafd81c2f054', - 2012: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=832d60ce-8346-456b-8ab8-cafd81c2f054', - 2013: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=832d60ce-8346-456b-8ab8-cafd81c2f054', - 2014: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=832d60ce-8346-456b-8ab8-cafd81c2f054', - 2015: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=ac018f80-6899-4395-af52-5c21134c51b3', - 2016: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=ac018f80-6899-4395-af52-5c21134c51b3', - 2017: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=ac018f80-6899-4395-af52-5c21134c51b3', - 2018: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=ac018f80-6899-4395-af52-5c21134c51b3', - 2019: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=ac018f80-6899-4395-af52-5c21134c51b3', - 2020: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=60d244bd-5b40-4be3-bd93-0dc35c210ece'}, - 'regex_id': r'[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}', - 'num_regions': 77, - 'num_sectors': 45}, - - }, - 'figaro': - {'2022': - {'links': - {'product-by-product': { - 2010: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2010.csv/bc2b60d9-32f2-1c80-56e4-f839ca2f06fc?t=1655180575504', - 2011: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2011.csv/74e6f603-1740-116c-3469-4db7ebe97aa2?t=1655182400093', - 2012: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2012.csv/17b1bf95-daa9-a506-9d8b-932edf8f3950?t=1655184005469', - 2013: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2013.csv/d32be770-9f71-e28c-e8ac-11a04d8d885c?t=1655185487140', - 2014: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2014.csv/815e7a8e-a1c7-89f4-e621-ce3971e79c4b?t=1655186769271', - 2015: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2015.csv/3c69e27a-8f45-29e9-cdef-59d8a402e927?t=1655186809149', - 2016: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2016.csv/f82150d8-74b3-b0a3-4764-567eab545b5a?t=1655188856261', - 2017: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2017.csv/7ec909a5-0faa-e45d-2173-43f6c1b59689?t=1655188931368', - 2018: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2018.csv/c44abd15-b354-6690-24dc-f186fe147bb2?t=1655196163454', - 2019: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2019.csv/0ab0c866-8dd8-92d6-1ac7-51ffe78fd5dc?t=1655196217023', - 2020: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2020.csv/3b838d62-884e-c1dc-9693-314b6460af0d?t=1655196273194', - }, - 'industry-by-industry': { - 2010: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2010.csv/3db6f05f-8343-a45e-2012-14db4607716b?t=1655180547507', - 2011: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2011.csv/5ce2f4ab-f4c1-e1dd-2aac-68813c714414?t=1655182368997', - 2012: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2012.csv/a1ad1f56-3a20-fac1-ee79-9a80b6d25a46?t=1655183982037', - 2013: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2013.csv/eec78482-56e2-053e-5486-15061be568e6?t=1655185412542', - 2014: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2014.csv/35eb67c4-0550-f2ae-b827-6ac240b5f877?t=1655186676930', - 2015: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2015.csv/af93fe1f-a9fd-c094-dd18-a15d390e0f9c?t=1655186725710', - 2016: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2016.csv/8322aa36-8d5c-5d52-07e5-e3d527875d06?t=1655188541728', - 2017: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2017.csv/30ad9b86-689f-d5a1-c80c-6765ea0f5421?t=1655188598849', - 2018: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2018.csv/a0369f43-bf98-a362-9054-f7075cc01704?t=1655195970852', - 2019: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2019.csv/c1354cbb-b3e6-6da5-780a-5be3319ad6d4?t=1655196037947', - 2020: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2020.csv/6dc6df43-0d95-856c-897f-18ba2ca053f1?t=1655196109971', - } - }, - 'regex_id': r'[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}', - 'num_regions': 46, - 'num_sectors': 64} - }, - 'exiobase': - {'3.81': - {'links': - {'product-by-product': { - 1995: 'https://zenodo.org/record/5589597/files/IOT_1995_pxp.zip?download=1', - 1996: 'https://zenodo.org/record/5589597/files/IOT_1996_pxp.zip?download=1', - 1997: 'https://zenodo.org/record/5589597/files/IOT_1997_pxp.zip?download=1', - 1998: 'https://zenodo.org/record/5589597/files/IOT_1998_pxp.zip?download=1', - 1999: 'https://zenodo.org/record/5589597/files/IOT_1999_pxp.zip?download=1', - 2000: 'https://zenodo.org/record/5589597/files/IOT_2000_pxp.zip?download=1', - 2001: 'https://zenodo.org/record/5589597/files/IOT_2001_pxp.zip?download=1', - 2002: 'https://zenodo.org/record/5589597/files/IOT_2002_pxp.zip?download=1', - 2003: 'https://zenodo.org/record/5589597/files/IOT_2003_pxp.zip?download=1', - 2004: 'https://zenodo.org/record/5589597/files/IOT_2004_pxp.zip?download=1', - 2005: 'https://zenodo.org/record/5589597/files/IOT_2005_pxp.zip?download=1', - 2006: 'https://zenodo.org/record/5589597/files/IOT_2006_pxp.zip?download=1', - 2007: 'https://zenodo.org/record/5589597/files/IOT_2007_pxp.zip?download=1', - 2008: 'https://zenodo.org/record/5589597/files/IOT_2008_pxp.zip?download=1', - 2009: 'https://zenodo.org/record/5589597/files/IOT_2009_pxp.zip?download=1', - 2010: 'https://zenodo.org/record/5589597/files/IOT_2010_pxp.zip?download=1', - 2011: 'https://zenodo.org/record/5589597/files/IOT_2011_pxp.zip?download=1', - 2012: 'https://zenodo.org/record/5589597/files/IOT_2012_pxp.zip?download=1', - 2013: 'https://zenodo.org/record/5589597/files/IOT_2013_pxp.zip?download=1', - 2014: 'https://zenodo.org/record/5589597/files/IOT_2014_pxp.zip?download=1', - 2015: 'https://zenodo.org/record/5589597/files/IOT_2015_pxp.zip?download=1', - 2016: 'https://zenodo.org/record/5589597/files/IOT_2016_pxp.zip?download=1', - 2017: 'https://zenodo.org/record/5589597/files/IOT_2017_pxp.zip?download=1', - 2018: 'https://zenodo.org/record/5589597/files/IOT_2018_pxp.zip?download=1', - 2019: 'https://zenodo.org/record/5589597/files/IOT_2019_pxp.zip?download=1', - 2020: 'https://zenodo.org/record/5589597/files/IOT_2020_pxp.zip?download=1', - 2021: 'https://zenodo.org/record/5589597/files/IOT_2021_pxp.zip?download=1', - 2022: 'https://zenodo.org/record/5589597/files/IOT_2022_pxp.zip?download=1', - }, - 'industry-by-industry': { - 1995: 'https://zenodo.org/record/5589597/files/IOT_1995_ixi.zip?download=1', - 1996: 'https://zenodo.org/record/5589597/files/IOT_1996_ixi.zip?download=1', - 1997: 'https://zenodo.org/record/5589597/files/IOT_1997_ixi.zip?download=1', - 1998: 'https://zenodo.org/record/5589597/files/IOT_1998_ixi.zip?download=1', - 1999: 'https://zenodo.org/record/5589597/files/IOT_1999_ixi.zip?download=1', - 2000: 'https://zenodo.org/record/5589597/files/IOT_2000_ixi.zip?download=1', - 2001: 'https://zenodo.org/record/5589597/files/IOT_2001_ixi.zip?download=1', - 2002: 'https://zenodo.org/record/5589597/files/IOT_2002_ixi.zip?download=1', - 2003: 'https://zenodo.org/record/5589597/files/IOT_2003_ixi.zip?download=1', - 2004: 'https://zenodo.org/record/5589597/files/IOT_2004_ixi.zip?download=1', - 2005: 'https://zenodo.org/record/5589597/files/IOT_2005_ixi.zip?download=1', - 2006: 'https://zenodo.org/record/5589597/files/IOT_2006_ixi.zip?download=1', - 2007: 'https://zenodo.org/record/5589597/files/IOT_2007_ixi.zip?download=1', - 2008: 'https://zenodo.org/record/5589597/files/IOT_2008_ixi.zip?download=1', - 2009: 'https://zenodo.org/record/5589597/files/IOT_2009_ixi.zip?download=1', - 2010: 'https://zenodo.org/record/5589597/files/IOT_2010_ixi.zip?download=1', - 2011: 'https://zenodo.org/record/5589597/files/IOT_2011_ixi.zip?download=1', - 2012: 'https://zenodo.org/record/5589597/files/IOT_2012_ixi.zip?download=1', - 2013: 'https://zenodo.org/record/5589597/files/IOT_2013_ixi.zip?download=1', - 2014: 'https://zenodo.org/record/5589597/files/IOT_2014_ixi.zip?download=1', - 2015: 'https://zenodo.org/record/5589597/files/IOT_2015_ixi.zip?download=1', - 2016: 'https://zenodo.org/record/5589597/files/IOT_2016_ixi.zip?download=1', - 2017: 'https://zenodo.org/record/5589597/files/IOT_2017_ixi.zip?download=1', - 2018: 'https://zenodo.org/record/5589597/files/IOT_2018_ixi.zip?download=1', - 2019: 'https://zenodo.org/record/5589597/files/IOT_2019_ixi.zip?download=1', - 2020: 'https://zenodo.org/record/5589597/files/IOT_2020_ixi.zip?download=1', - 2021: 'https://zenodo.org/record/5589597/files/IOT_2021_ixi.zip?download=1', - 2022: 'https://zenodo.org/record/5589597/files/IOT_2022_ixi.zip?download=1', - } - }, - 'regex_id': r'[A-Z]{3}_[0-9]{4}_[a-z]{3}', - 'num_regions': {'industry-by-industry': 49, 'product-by-product': 49}, - 'num_sectors': {'industry-by-industry': 163, 'product-by-product': 200}} - } -} diff --git a/iopy/core/globals.py b/iopy/core/globals.py deleted file mode 100644 index 1be3caf..0000000 --- a/iopy/core/globals.py +++ /dev/null @@ -1,10 +0,0 @@ -""" Created on 22/11/2022:: -------------- globals ------------- -**Authors**: W. Wakker - -""" -import os - -DATA_FOLDER = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.temp_data') -IS_WINDOWS = os.name == 'nt' -FILES_LOG = os.path.join(DATA_FOLDER, '_files_log.txt') diff --git a/iopy/dev-requirements.txt b/iopy/dev-requirements.txt deleted file mode 100644 index dd8ec32..0000000 --- a/iopy/dev-requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -requests -numpy -pandas>=1.1.5 -matplotlib -tqdm -pytest -pytest-cov diff --git a/iopy/requirements.txt b/iopy/requirements.txt deleted file mode 100644 index 48761f3..0000000 --- a/iopy/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -requests -numpy -pandas>=1.1.5 -matplotlib -tqdm diff --git a/iopy/tests/.coveragerc b/iopy/tests/.coveragerc deleted file mode 100644 index 2a8a768..0000000 --- a/iopy/tests/.coveragerc +++ /dev/null @@ -1,5 +0,0 @@ -[run] -omit = res/* - **/__about__.py - **/adjust_source_in_covxml.py - **/update_badge.py diff --git a/iopy/tests/test_iopy.py b/iopy/tests/test_iopy.py deleted file mode 100644 index 004c5e6..0000000 --- a/iopy/tests/test_iopy.py +++ /dev/null @@ -1,15 +0,0 @@ -""" Created on 22/11/2022:: -------------- test_iopy ------------- -**Authors**: W. Wakker - -""" -import iopy - - -class TestIopy: - - def test1(self): - iopy.remove_downloaded_files(database='figaro') - - def test_folder_size(self): - assert isinstance(iopy.get_size_data_folder(), str) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..612820b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "iotables" +version = "0.1.0" +description = "A Python package to easily load and work with inter-country input-output data" +readme = "README.md" +license = "MIT" +license-files = ["LICENSE"] +authors = [{ name = "Wouter Wakker", email = "wouter.wakker@outlook.com" }] +requires-python = ">=3.9" +dependencies = [ + "curl_cffi>=0.5", + "numpy>=1.21,<3", + "pandas>=1.3", + "matplotlib>=3.4", + "tqdm>=4.0", +] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Operating System :: OS Independent", +] + +[project.urls] +Homepage = "https://github.com/WWakker/iotables" +Repository = "https://github.com/WWakker/iotables" +Changelog = "https://github.com/WWakker/iotables/blob/main/CHANGELOG.MD" + +[project.optional-dependencies] +dev = ["pytest", "pytest-cov"] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +iotables = ["py.typed"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = [ + "network: test downloads real data over the network (deselect with '-m \"not network\"')", +] + +[tool.coverage.run] +source = ["iotables"] +omit = ["**/adjust_source_in_covxml.py"] diff --git a/setup.py b/setup.py deleted file mode 100644 index 2e11119..0000000 --- a/setup.py +++ /dev/null @@ -1,32 +0,0 @@ -import setuptools - -with open("README.md", "r") as f: - long_description = f.read() - -about = {} -with open("iopy/__about__.py") as f: - exec(f.read(), about) - -with open("iopy/requirements.txt") as f: - requirements = f.read().splitlines() - -setuptools.setup( - name="iopy", - version=about['__version__'], - author=about['__authors__'], - author_email=about['__email__'], - description=about['__about__'], - url=about['__url__'], - license='MIT', - long_description=long_description, - long_description_content_type="markdown", - packages=setuptools.find_packages(), - install_requires=requirements, - classifiers=[ - "Programming Language :: Python :: 3", - "Operating System :: OS Independent", - ], - python_requires='>=3.6', - include_package_data=False, - package_data={}, -) diff --git a/iopy/__init__.py b/src/iotables/__init__.py similarity index 55% rename from iopy/__init__.py rename to src/iotables/__init__.py index aa7b0fd..7030d56 100644 --- a/iopy/__init__.py +++ b/src/iotables/__init__.py @@ -1,11 +1,17 @@ -import os -from iopy.core.globals import DATA_FOLDER as __DATA_FOLDER -from iopy.core.globals import IS_WINDOWS as __IS_WINDOWS -from iopy.core.globals import FILES_LOG as __FILES_LOG -from iopy.core.oecd import OECD -from iopy.core.figaro import Figaro -from iopy.core.exiobase import ExioBase -from iopy.core.utils import remove_downloaded_files +import os as _os +from importlib.metadata import version as _version, PackageNotFoundError as _PackageNotFoundError +from iotables.globals import DATA_FOLDER as __DATA_FOLDER +from iotables.globals import IS_WINDOWS as __IS_WINDOWS +from iotables.globals import FILES_LOG as __FILES_LOG +from iotables.oecd import OECD +from iotables.figaro import Figaro +from iotables.exiobase import ExioBase +from iotables.utils import remove_downloaded_files + +try: + __version__ = _version("iotables") +except _PackageNotFoundError: # package not installed (e.g. running from source tree) + __version__ = "0.0.0" def get_size_data_folder(): @@ -34,7 +40,4 @@ def human(size): return get_size(__DATA_FOLDER) -if not os.path.exists(__DATA_FOLDER): - os.mkdir(__DATA_FOLDER) - -del os +_os.makedirs(__DATA_FOLDER, exist_ok=True) diff --git a/iopy/core/base_io.py b/src/iotables/base_io.py similarity index 93% rename from iopy/core/base_io.py rename to src/iotables/base_io.py index 6549b00..2c111ce 100644 --- a/iopy/core/base_io.py +++ b/src/iotables/base_io.py @@ -1,11 +1,7 @@ -""" Created on 18/10/2022:: -------------- io ------------- -**Authors**: W. Wakker - -""" +"""Shared analysis engine: derives the coefficient/inverse matrices and runs shocks.""" from warnings import warn -from iopy.core.matrix import Matrix -from iopy.core.utils import assert_is_subset +from iotables.matrix import Matrix +from iotables.utils import assert_is_subset import matplotlib.pyplot as plt from typing import Union, Iterable, Optional import numpy as np @@ -22,6 +18,7 @@ def __init__(self): 'X', 'V', 'FD', + 'FD_GRAN', 'ADD', 'FD_REGION', 'rs', @@ -29,7 +26,9 @@ def __init__(self): 'regions', 'sectors', 'unit', - 'demand_items'] + 'demand_items', + 'reference', + 'contact'] assert_is_subset(necessary_attrs, dir(self)) # Coefficients matrix, replace 0 with 1 to allow inversion @@ -53,7 +52,7 @@ def __init__(self): self.Z.rows, self.Z.columns) - self.G = Matrix('Output inverse', + self.G = Matrix('Ghosh inverse', (np.eye(self.rs) - self.B).I, self.Z.rows, self.Z.columns) @@ -80,7 +79,8 @@ def _shock(self, if custom_shock_vector is not None: shock_vector = np.array(custom_shock_vector).reshape(self.rs, 1) else: - assert shock and regions and sectors, "Must supply parameters: 'shock', 'regions', 'sectors'" + if shock is None or regions is None or sectors is None: + raise ValueError("Must supply parameters: 'shock', 'regions', 'sectors'") assert_is_subset(regions, self.regions) assert_is_subset(sectors, self.sectors) @@ -171,7 +171,8 @@ def _plot_shock(self, Returns: fig, ax """ - assert by in {'region', 'sector'}, "plot_by must be 'region' or 'sector'" + if by not in {'region', 'sector'}: + raise ValueError("plot_by must be 'region' or 'sector'") assert_is_subset(regions, self.regions) df = self._shock_to_df(x_new) @@ -255,7 +256,7 @@ def ghosh_supply_shock(self, """Executes a Ghosh supply shock Args: - shock: Shock in percentage of original demand + shock: Shock in percentage of original primary inputs (value added) regions: List of regions to be shocked sectors: List of sectors to be shocked custom_shock_vector: Vector of length regions * sectors with percentage shocks, overrides all other shock @@ -297,7 +298,8 @@ def get_imports_exports(self, Returns: float: Sum of trade flow from exporting region-sectors to importing region-sectors """ - assert use_type in {'intermediate', 'final', 'both'}, "use_type must be 'intermediate', 'final' or 'both'" + if use_type not in {'intermediate', 'final', 'both'}: + raise ValueError("use_type must be 'intermediate', 'final' or 'both'") if import_sectors is not None and use_type in {'final', 'both'}: warn('Note that import_sectors only apply to intermediate use, for final use only import_regions is used') diff --git a/src/iotables/config.py b/src/iotables/config.py new file mode 100644 index 0000000..b6686dc --- /dev/null +++ b/src/iotables/config.py @@ -0,0 +1,195 @@ +"""Single source of truth for the data that exists: download links and shapes per database.""" + + +def _chunk_links(base, chunks): + """Expand ``{filename: (start_year, end_year)}`` into ``{year: url}``. + + OECD distributes each ICIO edition as a handful of multi-year zip archives + (e.g. ``2016-2022_SML.zip`` holds one CSV per year), so every year in a range + resolves to the same download URL. + """ + return {year: base + filename + for filename, (start, end) in chunks.items() + for year in range(start, end + 1)} + + +def _circabc_links(kind_tag, ids): + """Build CIRCABC anonymous download URLs for Figaro from ``{year: node_id}``. + + Eurostat distributes the Figaro 2025 edition through CIRCABC; every file has a + stable, auth-free download URL of the form ``/sd/a/{node_id}/{filename}``. + """ + return {year: f'https://circabc.europa.eu/sd/a/{nid}/matrix_eu-ic-io_{kind_tag}_25ed_{year}.csv' + for year, nid in ids.items()} + + +config = { + 'oecd': + {'2021': + {'links': _chunk_links('https://webfs-sti.oecd.org/files/STI-PIE/ICIO/2021/', { + 'ICIO2021_1995-1999.zip': (1995, 1999), + 'ICIO2021_2000-2004.zip': (2000, 2004), + 'ICIO2021_2005-2009.zip': (2005, 2009), + 'ICIO2021_2010-2014.zip': (2010, 2014), + 'ICIO2021_2015-2018.zip': (2015, 2018), + }), + 'regex_id': r'ICIO2021_[0-9]{4}-[0-9]{4}', + 'num_regions': 71, + 'num_sectors': 45 + }, + '2022-extended': + {'links': _chunk_links('https://webfs-sti.oecd.org/files/STI-PIE/ICIO/2023/', { + '1995-2000_EXT.zip': (1995, 2000), + '2001-2005_EXT.zip': (2001, 2005), + '2006-2010_EXT.zip': (2006, 2010), + '2011-2015_EXT.zip': (2011, 2015), + '2016-2020_EXT.zip': (2016, 2020), + }), + 'regex_id': r'2023/[0-9]{4}-[0-9]{4}_EXT', + 'num_regions': 81, + 'num_sectors': 45}, + '2022-small': + {'links': _chunk_links('https://webfs-sti.oecd.org/files/STI-PIE/ICIO/2023/', { + '1995-2000_SML.zip': (1995, 2000), + '2001-2005_SML.zip': (2001, 2005), + '2006-2010_SML.zip': (2006, 2010), + '2011-2015_SML.zip': (2011, 2015), + '2016-2020_SML.zip': (2016, 2020), + }), + 'regex_id': r'2023/[0-9]{4}-[0-9]{4}_SML', + 'num_regions': 77, + 'num_sectors': 45}, + '2025-extended': + {'links': _chunk_links('https://webfs-sti.oecd.org/files/STI-PIE/ICIO/2025/', { + '1995-2000_EXT.zip': (1995, 2000), + '2001-2005_EXT.zip': (2001, 2005), + '2006-2010_EXT.zip': (2006, 2010), + '2011-2015_EXT.zip': (2011, 2015), + '2016-2022_EXT.zip': (2016, 2022), + }), + 'regex_id': r'2025/[0-9]{4}-[0-9]{4}_EXT', + 'num_regions': 85, + 'num_sectors': 50}, + '2025-regular': + {'links': _chunk_links('https://webfs-sti.oecd.org/files/STI-PIE/ICIO/2025/', { + '1995-2000_SML.zip': (1995, 2000), + '2001-2005_SML.zip': (2001, 2005), + '2006-2010_SML.zip': (2006, 2010), + '2011-2015_SML.zip': (2011, 2015), + '2016-2022_SML.zip': (2016, 2022), + }), + 'regex_id': r'2025/[0-9]{4}-[0-9]{4}_SML', + 'num_regions': 81, + 'num_sectors': 50} + + }, + + 'figaro': + { + '2025': + {'links': + {'product-by-product': _circabc_links('prod-by-prod', { + 2010: 'f6d2007e-34c9-420a-a106-8c5ea836c49d', + 2011: 'd27f7d7c-4006-4c36-8820-7d9990679323', + 2012: '7cea8b73-f166-47ff-9c27-f240794b57af', + 2013: 'd56c2ed9-0c4f-40a3-9bbc-1aee49f5bd45', + 2014: 'b5aa0e38-3d1b-496b-b180-bbd9bab3e153', + 2015: 'd037b772-8bf9-424f-97a4-9274d261e6cf', + 2016: '66369d21-8262-47d6-ad8a-dc536a2466d0', + 2017: 'd6848c38-9569-4848-bb68-4144fed38c71', + 2018: '557b9483-023e-497c-96c5-44ce807bd444', + 2019: 'fa0e9127-7a2b-4b74-bbad-1bc9e717197b', + 2020: '46f5665e-d2e1-4270-95a1-970da3d70d32', + 2021: 'a7e2919d-f084-4c35-8b21-e960a854e2bd', + 2022: '31484cee-43fe-45d4-a546-209e5898c6dd', + 2023: 'e213892a-afac-4d34-83e9-45e8a324e7e8', + }), + 'industry-by-industry': _circabc_links('ind-by-ind', { + 2010: 'fc80f855-d144-476e-b4bf-5cfba946819c', + 2011: '1bcb2624-04ed-43e1-8588-df6680ed352a', + 2012: '399671ad-cbb3-493e-ad5f-83e989f1eecc', + 2013: 'a2b4746d-1d11-4a44-ab1c-50ac956f0849', + 2014: 'beba57b2-2696-497a-b92f-2a5beca724c7', + 2015: '1a194b8c-6ea1-4bec-9c73-0cd599febcc3', + 2016: '2cdd74fc-0bce-4ae0-8bf2-34d34546d86d', + 2017: '4a11c796-4186-4cce-a02f-12d353fc5e59', + 2018: '7a57a374-2200-498c-bb5f-cee24202b0b8', + 2019: 'c3467617-8a00-44a0-9b6b-ccad8a2ab58d', + 2020: '4df668e1-2a8a-4e84-ae57-00309d8bc760', + 2021: '6736dea8-da14-450f-b212-a791baf238c8', + 2022: 'b20c339d-984f-413c-a499-54ff76beb90c', + 2023: '21557f49-1e94-431c-8523-d972fec020b8', + })}, + 'regex_id': r'matrix_eu-ic-io_[a-z-]+_25ed_[0-9]{4}', + 'num_regions': 50, + 'num_sectors': 64}, + }, + 'exiobase': + {'3.81': + {'links': + {'product-by-product': { + 1995: 'https://zenodo.org/record/5589597/files/IOT_1995_pxp.zip?download=1', + 1996: 'https://zenodo.org/record/5589597/files/IOT_1996_pxp.zip?download=1', + 1997: 'https://zenodo.org/record/5589597/files/IOT_1997_pxp.zip?download=1', + 1998: 'https://zenodo.org/record/5589597/files/IOT_1998_pxp.zip?download=1', + 1999: 'https://zenodo.org/record/5589597/files/IOT_1999_pxp.zip?download=1', + 2000: 'https://zenodo.org/record/5589597/files/IOT_2000_pxp.zip?download=1', + 2001: 'https://zenodo.org/record/5589597/files/IOT_2001_pxp.zip?download=1', + 2002: 'https://zenodo.org/record/5589597/files/IOT_2002_pxp.zip?download=1', + 2003: 'https://zenodo.org/record/5589597/files/IOT_2003_pxp.zip?download=1', + 2004: 'https://zenodo.org/record/5589597/files/IOT_2004_pxp.zip?download=1', + 2005: 'https://zenodo.org/record/5589597/files/IOT_2005_pxp.zip?download=1', + 2006: 'https://zenodo.org/record/5589597/files/IOT_2006_pxp.zip?download=1', + 2007: 'https://zenodo.org/record/5589597/files/IOT_2007_pxp.zip?download=1', + 2008: 'https://zenodo.org/record/5589597/files/IOT_2008_pxp.zip?download=1', + 2009: 'https://zenodo.org/record/5589597/files/IOT_2009_pxp.zip?download=1', + 2010: 'https://zenodo.org/record/5589597/files/IOT_2010_pxp.zip?download=1', + 2011: 'https://zenodo.org/record/5589597/files/IOT_2011_pxp.zip?download=1', + 2012: 'https://zenodo.org/record/5589597/files/IOT_2012_pxp.zip?download=1', + 2013: 'https://zenodo.org/record/5589597/files/IOT_2013_pxp.zip?download=1', + 2014: 'https://zenodo.org/record/5589597/files/IOT_2014_pxp.zip?download=1', + 2015: 'https://zenodo.org/record/5589597/files/IOT_2015_pxp.zip?download=1', + 2016: 'https://zenodo.org/record/5589597/files/IOT_2016_pxp.zip?download=1', + 2017: 'https://zenodo.org/record/5589597/files/IOT_2017_pxp.zip?download=1', + 2018: 'https://zenodo.org/record/5589597/files/IOT_2018_pxp.zip?download=1', + 2019: 'https://zenodo.org/record/5589597/files/IOT_2019_pxp.zip?download=1', + 2020: 'https://zenodo.org/record/5589597/files/IOT_2020_pxp.zip?download=1', + 2021: 'https://zenodo.org/record/5589597/files/IOT_2021_pxp.zip?download=1', + 2022: 'https://zenodo.org/record/5589597/files/IOT_2022_pxp.zip?download=1', + }, + 'industry-by-industry': { + 1995: 'https://zenodo.org/record/5589597/files/IOT_1995_ixi.zip?download=1', + 1996: 'https://zenodo.org/record/5589597/files/IOT_1996_ixi.zip?download=1', + 1997: 'https://zenodo.org/record/5589597/files/IOT_1997_ixi.zip?download=1', + 1998: 'https://zenodo.org/record/5589597/files/IOT_1998_ixi.zip?download=1', + 1999: 'https://zenodo.org/record/5589597/files/IOT_1999_ixi.zip?download=1', + 2000: 'https://zenodo.org/record/5589597/files/IOT_2000_ixi.zip?download=1', + 2001: 'https://zenodo.org/record/5589597/files/IOT_2001_ixi.zip?download=1', + 2002: 'https://zenodo.org/record/5589597/files/IOT_2002_ixi.zip?download=1', + 2003: 'https://zenodo.org/record/5589597/files/IOT_2003_ixi.zip?download=1', + 2004: 'https://zenodo.org/record/5589597/files/IOT_2004_ixi.zip?download=1', + 2005: 'https://zenodo.org/record/5589597/files/IOT_2005_ixi.zip?download=1', + 2006: 'https://zenodo.org/record/5589597/files/IOT_2006_ixi.zip?download=1', + 2007: 'https://zenodo.org/record/5589597/files/IOT_2007_ixi.zip?download=1', + 2008: 'https://zenodo.org/record/5589597/files/IOT_2008_ixi.zip?download=1', + 2009: 'https://zenodo.org/record/5589597/files/IOT_2009_ixi.zip?download=1', + 2010: 'https://zenodo.org/record/5589597/files/IOT_2010_ixi.zip?download=1', + 2011: 'https://zenodo.org/record/5589597/files/IOT_2011_ixi.zip?download=1', + 2012: 'https://zenodo.org/record/5589597/files/IOT_2012_ixi.zip?download=1', + 2013: 'https://zenodo.org/record/5589597/files/IOT_2013_ixi.zip?download=1', + 2014: 'https://zenodo.org/record/5589597/files/IOT_2014_ixi.zip?download=1', + 2015: 'https://zenodo.org/record/5589597/files/IOT_2015_ixi.zip?download=1', + 2016: 'https://zenodo.org/record/5589597/files/IOT_2016_ixi.zip?download=1', + 2017: 'https://zenodo.org/record/5589597/files/IOT_2017_ixi.zip?download=1', + 2018: 'https://zenodo.org/record/5589597/files/IOT_2018_ixi.zip?download=1', + 2019: 'https://zenodo.org/record/5589597/files/IOT_2019_ixi.zip?download=1', + 2020: 'https://zenodo.org/record/5589597/files/IOT_2020_ixi.zip?download=1', + 2021: 'https://zenodo.org/record/5589597/files/IOT_2021_ixi.zip?download=1', + 2022: 'https://zenodo.org/record/5589597/files/IOT_2022_ixi.zip?download=1', + } + }, + 'regex_id': r'[A-Z]{3}_[0-9]{4}_[a-z]{3}', + 'num_regions': {'industry-by-industry': 49, 'product-by-product': 49}, + 'num_sectors': {'industry-by-industry': 163, 'product-by-product': 200}} + } +} diff --git a/iopy/core/exiobase.py b/src/iotables/exiobase.py similarity index 81% rename from iopy/core/exiobase.py rename to src/iotables/exiobase.py index bec7f33..e043cf0 100644 --- a/iopy/core/exiobase.py +++ b/src/iotables/exiobase.py @@ -1,23 +1,19 @@ -""" Created on 15/11/2022:: -------------- exiobase ------------- -**Authors**: S. Boldrini -""" +"""Loader for EXIOBASE inter-country input-output data.""" -from iopy.core.matrix import Matrix -from functools import lru_cache +from iotables.matrix import Matrix import numpy as np import pandas as pd from tqdm import tqdm from zipfile import ZipFile import re import os -from iopy.core.config import config -from iopy.core.base_io import IO +from iotables.config import config +from iotables.base_io import IO from warnings import warn -from iopy.core.globals import DATA_FOLDER, FILES_LOG -from iopy.core.utils import remove_downloaded_files +from iotables.globals import DATA_FOLDER, FILES_LOG +from iotables.utils import remove_downloaded_files, download_file -db_name = os.path.basename(__file__).rstrip('.py') +db_name = os.path.splitext(os.path.basename(__file__))[0] def process_df(df): @@ -33,7 +29,9 @@ def __init__(self, version: str, year: int, kind: str = 'industry-by-industry', - refresh: bool = False): + refresh: bool = False, + proxy=None, + verify=True): """ Args: @@ -41,9 +39,13 @@ def __init__(self, year: Year from 1995 to 2022 kind: industry-by-industry (default) or product-by-product refresh: Download the data even if it exists on the hard drive + proxy: Optional proxy for downloading; a URL string (applied to http and + https) or a ``{scheme: url}`` dict + verify: Verify the server's TLS certificate (``False`` to skip, or a CA bundle path) """ - assert kind in {'industry-by-industry', 'product-by-product'} + if kind not in {'industry-by-industry', 'product-by-product'}: + raise ValueError("kind must be 'industry-by-industry' or 'product-by-product'") if version not in config['exiobase'].keys(): raise ValueError( @@ -57,6 +59,8 @@ def __init__(self, self.version = version self.year = year self.kind = kind + self._proxy = proxy + self._verify = verify self._url = config['exiobase'][version]['links'][kind][year] self._file_id = re.search(config['exiobase'][version]['regex_id'], self._url).group(0) self._data_file = os.path.join(DATA_FOLDER, self._file_id + '.zip') @@ -72,7 +76,7 @@ def __init__(self, # Load pbar.set_description('Loading data...') self.df = None - self._Z_raw, self._FD_raw, self._X_raw, self._metadata, self._sector_codes, self._FD_codes = self._load_data() + self._Z_raw, self._FD_raw, self._X_raw, self._sector_codes, self._FD_codes = self._load_data() exiobase_sector_name_mapping = self._sector_codes.reset_index().set_index('CodeNr')['Name'].to_dict() exiobase_FD_name_mapping = self._FD_codes.reset_index().set_index('CodeNr')['Name'].to_dict() @@ -93,7 +97,8 @@ def __init__(self, # Create matrices pbar.set_description('Creating matrices...') - assert self._Z_raw.shape[0] == self._Z_raw.shape[1] + if self._Z_raw.shape[0] != self._Z_raw.shape[1]: + raise ValueError('Intermediate-use matrix Z is not square; the downloaded file may be corrupt') self.rs = config['exiobase'][version]['num_regions'][kind] * config['exiobase'][version]['num_sectors'][kind] self.Z = Matrix('Intermediate use', *process_df(self._Z_raw)) @@ -119,10 +124,11 @@ def __init__(self, columns=[r for r, s in self.FD_GRAN.columns]).T fd_region.index.name = 'region' + fd_region = fd_region.groupby('region').sum().T self.FD_REGION = Matrix('Final demand by region', - fd_region.groupby('region').sum(0).T, + fd_region, rows=self.Z.rows, - columns=fd_region.groupby('region').sum(0).T.columns.to_list()) + columns=fd_region.columns.to_list()) self.regions = list(sorted(np.unique([r for r, s in self.Z.rows]))) self.sectors = list(sorted(np.unique([s for r, s in self.Z.rows]))) @@ -136,7 +142,6 @@ def __init__(self, pbar.update() pbar.set_description('Done') - @lru_cache() def _load_data(self): folder = f'IOT_{self.year}_{"ixi" if self.kind == "industry-by-industry" else "pxp"}' with ZipFile(self._data_file, 'r') as zf: @@ -164,24 +169,16 @@ def _load_data(self): with zf.open(f'{folder}/finaldemands.txt', 'r') as csv_file: FD_codes = pd.read_csv(csv_file, sep='\t', index_col=1) - with zf.open(f'{folder}/metadata.json', 'r') as json_file: - metadata = pd.read_json(json_file) - - return z_raw, fd_raw, x_raw, metadata, sector_codes, FD_codes + return z_raw, fd_raw, x_raw, sector_codes, FD_codes def _download_data(self): - import requests - try: - r = requests.get(self._url, stream=True) - with open(self._data_file, "wb") as f: - for chunk in r.iter_content(1024 * 5): - f.write(chunk) + download_file(self._url, self._data_file, proxy=self._proxy, verify=self._verify) with open(FILES_LOG, 'a') as files_log: files_log.write(db_name + ';' + self._data_file + '\n') except Exception as e: warn(f"Couldn't download the data. Try downloading manually from {self._url} " - f"and save the csv file as {self._file_id}.csv in {self._data_folder}") + f"and save the zip file as {self._file_id}.zip in {DATA_FOLDER}") raise e @staticmethod diff --git a/iopy/core/figaro.py b/src/iotables/figaro.py similarity index 81% rename from iopy/core/figaro.py rename to src/iotables/figaro.py index 19ae9e3..1868f7b 100644 --- a/iopy/core/figaro.py +++ b/src/iotables/figaro.py @@ -1,23 +1,18 @@ -""" Created on 14/10/2022:: -------------- figaro ------------- -**Authors**: W. Wakker - -""" -from iopy.core.mappings import figaro_sector_name_mapping_pxp_2022, figaro_sector_name_mapping_ixi_2022, figaro_demand_items -from iopy.core.matrix import Matrix -from functools import lru_cache +"""Loader for Eurostat Figaro inter-country input-output data.""" +from iotables.mappings import figaro_sector_name_mapping_pxp_2022, figaro_sector_name_mapping_ixi_2022, figaro_demand_items +from iotables.matrix import Matrix import numpy as np import pandas as pd from tqdm import tqdm import re import os -from iopy.core.config import config +from iotables.config import config from warnings import warn -from iopy.core.base_io import IO -from iopy.core.globals import DATA_FOLDER, FILES_LOG -from iopy.core.utils import remove_downloaded_files +from iotables.base_io import IO +from iotables.globals import DATA_FOLDER, FILES_LOG +from iotables.utils import remove_downloaded_files, download_file -db_name = os.path.basename(__file__).rstrip('.py') +db_name = os.path.splitext(os.path.basename(__file__))[0] def process_df(df): @@ -42,16 +37,22 @@ def __init__(self, version: str, year: int, kind='industry-by-industry', - refresh: bool = False): + refresh: bool = False, + proxy=None, + verify=True): """ Args: - version: Edition (year of publication), e.g. '2022' - year: Year from 2010 to 2020 + version: Edition (year of publication), e.g. '2025' + year: Year; availability depends on the edition (e.g. 2010-2023 for '2025') kind: industry-by-industry (default) or product-by-product refresh: Download the data even if it exists on the hard drive + proxy: Optional proxy for downloading; a URL string (applied to http and + https) or a ``{scheme: url}`` dict + verify: Verify the server's TLS certificate (``False`` to skip, or a CA bundle path) """ - assert kind in {'industry-by-industry', 'product-by-product'} + if kind not in {'industry-by-industry', 'product-by-product'}: + raise ValueError("kind must be 'industry-by-industry' or 'product-by-product'") if version not in config['figaro'].keys(): raise ValueError( @@ -65,6 +66,8 @@ def __init__(self, self.version = version self.year = year self.kind = kind + self._proxy = proxy + self._verify = verify self._url = config['figaro'][version]['links'][kind][year] self._file_id = re.search(config['figaro'][version]['regex_id'], self._url).group(0) self._data_file = os.path.join(DATA_FOLDER, self._file_id + '.csv') @@ -114,7 +117,7 @@ def __init__(self, # Create region level FD fd_region = pd.DataFrame(self.FD_GRAN, columns=[r for r, s in self.FD_GRAN.columns]).T fd_region.index.name = 'region' - fd_region = fd_region.groupby('region').sum(0).T + fd_region = fd_region.groupby('region').sum().T self.FD_REGION = Matrix('Final demand by region', fd_region, rows=self.Z.rows, @@ -133,19 +136,13 @@ def __init__(self, pbar.update() pbar.set_description('Done') - @lru_cache() def _load_data(self): df = pd.read_csv(self._data_file, index_col=0) return df def _download_data(self): - import requests - try: - r = requests.get(self._url, stream=True) - with open(self._data_file, "wb") as f: - for chunk in r.iter_content(1024 * 5): - f.write(chunk) + download_file(self._url, self._data_file, proxy=self._proxy, verify=self._verify) with open(FILES_LOG, 'a') as files_log: files_log.write(db_name + ';' + self._data_file + '\n') except Exception as e: diff --git a/src/iotables/globals.py b/src/iotables/globals.py new file mode 100644 index 0000000..347aa47 --- /dev/null +++ b/src/iotables/globals.py @@ -0,0 +1,10 @@ +"""Module-level constants: the download cache location and bookkeeping paths.""" +import os + +# Downloaded source files are cached in a user-level cache directory rather than inside +# the installed package. Override with the IOTABLES_DATA environment variable; otherwise +# fall back to XDG_CACHE_HOME (or ~/.cache) per platform convention. +_cache_root = os.environ.get('XDG_CACHE_HOME') or os.path.join(os.path.expanduser('~'), '.cache') +DATA_FOLDER = os.environ.get('IOTABLES_DATA', os.path.join(_cache_root, 'iotables')) +IS_WINDOWS = os.name == 'nt' +FILES_LOG = os.path.join(DATA_FOLDER, '_files_log.txt') diff --git a/iopy/core/mappings.py b/src/iotables/mappings.py similarity index 83% rename from iopy/core/mappings.py rename to src/iotables/mappings.py index 30b2c54..b70c7d8 100644 --- a/iopy/core/mappings.py +++ b/src/iotables/mappings.py @@ -1,8 +1,4 @@ -""" Created on 06/09/2022:: -------------- mappings ------------- -**Authors**: W. Wakker - -""" +"""Per-database sector-code → human-name maps, demand-item lists, and the OECD 2022→2021 remap.""" oecd_sector_2022_2021_mapping = {'A01_02': '01T02', 'A03': '03', @@ -143,6 +139,60 @@ '94T96': 'Other service activities', '97T98': 'Households'} +# OECD ICIO 2025 edition uses a 50-industry ISIC Rev.4 breakdown (codes differ from the +# 45-sector classification above). Source: ReadMe_ICIO_*.xlsx "Area_Activities" sheet. +oecd_sector_name_mapping_2025 = {'A01': 'Agriculture and hunting', + 'A02': 'Forestry and logging', + 'A03': 'Fishing and aquaculture', + 'B05': 'Mining of coal and lignite', + 'B06': 'Extraction of crude petroleum and natural gas', + 'B07': 'Mining of metal ores', + 'B08': 'Other mining and quarrying', + 'B09': 'Mining support service activities', + 'C10T12': 'Manufacture of food products; beverages and tobacco products', + 'C13T15': 'Manufacture of textiles, wearing apparel, leather and related products', + 'C16': 'Manufacture of wood and of products of wood and cork', + 'C17_18': 'Manufacture of paper and paper products; Printing and reproduction of recorded media', + 'C19': 'Manufacture of coke and refined petroleum products', + 'C20': 'Manufacture of chemicals and chemical products', + 'C21': 'Manufacture of basic pharmaceutical products and pharmaceutical preparations', + 'C22': 'Manufacture of rubber and plastic products', + 'C23': 'Manufacture of other non-metallic mineral products', + 'C24A': 'Manufacture of basic iron and steel', + 'C24B': 'Manufacture of basic precious and other non-ferrous metals', + 'C25': 'Manufacture of fabricated metal products', + 'C26': 'Manufacture of computer, electronic and optical products', + 'C27': 'Manufacture of electrical equipment', + 'C28': 'Manufacture of machinery and equipment n.e.c.', + 'C29': 'Manufacture of motor vehicles, trailers and semi-trailers', + 'C301': 'Building of ships and boats', + 'C302T309': 'Manufacture of other transport equipment', + 'C31T33': 'Manufacture of furniture; other manufacturing; repair and installation of machinery and equipment', + 'D': 'Electricity, gas, steam and air conditioning supply', + 'E': 'Water supply; sewerage, waste management and remediation activities', + 'F': 'Construction', + 'G': 'Wholesale and retail trade; repair of motor vehicles and motorcycles', + 'H49': 'Land transport and transport via pipelines', + 'H50': 'Water transport', + 'H51': 'Air transport', + 'H52': 'Warehousing and support activities for transportation', + 'H53': 'Postal and courier activities', + 'I': 'Accommodation and food service activities', + 'J58T60': 'Publishing, Motion picture, video, television programme production and broadcasting activities', + 'J61': 'Telecommunications', + 'J62_63': 'Computer programming and information service activities', + 'K': 'Financial and insurance activities', + 'L': 'Real estate activities', + 'M': 'Professional, scientific and technical activities', + 'N': 'Administrative and support service activities', + 'O': 'Public administration and defence; compulsory social security', + 'P': 'Education', + 'Q': 'Human health and social work activities', + 'R': 'Arts, entertainment and recreation activities', + 'S': 'Other service activities', + 'T': 'Activities of households as employers; undifferentiated goods- and services-producing ' + 'activities of households for own use'} + oecd_demand_items = {'HFCE': 'Household Final Consumption Expenditure', 'NPISH': 'Non-Profit Institutions Serving Households', 'GGFC': 'General Government Final Consumption', diff --git a/iopy/core/matrix.py b/src/iotables/matrix.py similarity index 61% rename from iopy/core/matrix.py rename to src/iotables/matrix.py index c818b74..b9c48cb 100644 --- a/iopy/core/matrix.py +++ b/src/iotables/matrix.py @@ -1,17 +1,16 @@ -""" Created on 03/10/2022:: -------------- matrix ------------- -**Authors**: W. Wakker - -""" +"""A labelled 2-D ``numpy.ndarray`` subclass carrying region/sector row and column metadata.""" import numpy as np +import pandas as pd class Matrix(np.ndarray): def __new__(cls, info, input_array, rows, columns): obj = np.asarray(input_array).view(cls) - assert len(obj.shape) == 2, "Array must be 2-dimensional" - assert obj.shape == (len(rows), len(columns)), "Rows and columns do not have the shape of the array" + if len(obj.shape) != 2: + raise ValueError("Array must be 2-dimensional") + if obj.shape != (len(rows), len(columns)): + raise ValueError("Rows and columns do not have the shape of the array") obj.info = info obj.rows = rows obj.columns = columns @@ -68,3 +67,18 @@ def to_numpy(self): numpy array """ return np.asarray(self) + + def to_pandas(self): + """Convert to pandas DataFrame + + Returns: + pandas DataFrame + """ + def _to_index(labels): + if len(labels) and isinstance(labels[0], tuple): + return pd.MultiIndex.from_tuples(labels, names=['region', 'sector']) + return pd.Index(labels) + + return pd.DataFrame(np.asarray(self), + index=_to_index(self.rows), + columns=_to_index(self.columns)) diff --git a/iopy/core/oecd.py b/src/iotables/oecd.py similarity index 78% rename from iopy/core/oecd.py rename to src/iotables/oecd.py index d723dfd..44ad4a0 100644 --- a/iopy/core/oecd.py +++ b/src/iotables/oecd.py @@ -1,40 +1,36 @@ -""" Created on 06/09/2022:: -------------- oecd ------------- -**Authors**: W. Wakker - -""" -from iopy.core.mappings import oecd_sector_name_mapping, oecd_demand_items, oecd_sector_2022_2021_mapping -from iopy.core.matrix import Matrix -from functools import lru_cache +"""Loader for OECD ICIO inter-country input-output data.""" +from iotables.mappings import oecd_sector_name_mapping, oecd_sector_name_mapping_2025, oecd_demand_items, \ + oecd_sector_2022_2021_mapping +from iotables.matrix import Matrix import numpy as np import pandas as pd -from iopy.core.utils import ALPHA3_TO_ALPHA2 +from iotables.utils import ALPHA3_TO_ALPHA2 from tqdm import tqdm from zipfile import ZipFile import re import os -from iopy.core.config import config -from iopy.core.base_io import IO -from iopy.core.utils import replace_if_exists, remove_downloaded_files -from iopy.core.globals import DATA_FOLDER, FILES_LOG +from iotables.config import config +from iotables.base_io import IO +from iotables.utils import replace_if_exists, remove_downloaded_files, download_file +from iotables.globals import DATA_FOLDER, FILES_LOG from warnings import warn from functools import partial -db_name = os.path.basename(__file__).rstrip('.py') +db_name = os.path.splitext(os.path.basename(__file__))[0] def process_df(df): if df.shape[0] > 1 and df.shape[1] > 1: return (df, - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_')], - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_')]) + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_', n=1)], + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_', n=1)]) elif df.shape[0] == 1: return (df, df.index.to_list(), - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_')]) + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_', n=1)]) else: return (df, - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_')], + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_', n=1)], df.columns.to_list()) @@ -44,13 +40,18 @@ class OECD(IO): def __init__(self, version: str, year: int, - refresh: bool = False): + refresh: bool = False, + proxy=None, + verify=True): """ Args: version: Publication version of the data; '2021', '2022-small' or '2022-extended' year: Year refresh: Download the data even if it exists on the hard drive + proxy: Optional proxy for downloading; a URL string (applied to http and + https) or a ``{scheme: url}`` dict + verify: Verify the server's TLS certificate (``False`` to skip, or a CA bundle path) """ if version not in config['oecd'].keys(): @@ -64,8 +65,10 @@ def __init__(self, self.year = year self.version = version + self._proxy = proxy + self._verify = verify self._url = config['oecd'][version]['links'][year] - self._file_id = re.search(config['oecd'][version]['regex_id'], self._url).group(0) + self._file_id = re.search(config['oecd'][version]['regex_id'], self._url).group(0).replace('/', '_') self._data_file = os.path.join(DATA_FOLDER, self._file_id + '.zip') file_exists = os.path.exists(self._data_file) download = not file_exists or refresh @@ -119,7 +122,7 @@ def __init__(self, # Create region level FD fd_region = pd.DataFrame(self.FD_GRAN, columns=[r for r, s in self.FD_GRAN.columns]).T fd_region.index.name = 'region' - fd_region = fd_region.groupby('region').sum(0).T + fd_region = fd_region.groupby('region').sum().T self.FD_REGION = Matrix('Final demand by region', fd_region, rows=self.Z.rows, @@ -128,7 +131,8 @@ def __init__(self, self.regions = list(sorted(np.unique([r for r, s in self.Z.rows]))) self.sectors = list(sorted(np.unique([s for r, s in self.Z.rows]))) self.unit = 'Million USD' - self.sector_name_mapping = oecd_sector_name_mapping + self.sector_name_mapping = oecd_sector_name_mapping_2025 if version in ('2025-extended', '2025-regular') \ + else oecd_sector_name_mapping self.demand_items = oecd_demand_items self.reference = f'OECD ({self.version[:4]}), OECD Inter-Country Input-Output Database, http://oe.cd/icio' self.contact = 'ICIO-TiVA.Contact@oecd.org, mentioning ICIO' @@ -138,27 +142,25 @@ def __init__(self, pbar.update() pbar.set_description('Done') - @lru_cache() def _load_data(self): if self.version == '2021': filename = f'ICIO2021_{self.year}.csv' elif self.version == '2022-extended': - filename = f'{self.year}.CSV' + filename = f'{self.year}.csv' elif self.version == '2022-small': - filename = f'{self.year}SML.CSV' + filename = f'{self.year}_SML.csv' + elif self.version == '2025-extended': + filename = f'{self.year}.csv' + elif self.version == '2025-regular': + filename = f'{self.year}_SML.csv' with ZipFile(self._data_file, 'r') as zf: with zf.open(filename, 'r') as csv_file: df = pd.read_csv(csv_file, index_col=0) return df def _download_data(self): - import requests - try: - r = requests.get(self._url, stream=True) - with open(self._data_file, "wb") as f: - for chunk in r.iter_content(1024 * 5): - f.write(chunk) + download_file(self._url, self._data_file, proxy=self._proxy, verify=self._verify) with open(FILES_LOG, 'a') as files_log: files_log.write(db_name + ';' + self._data_file + '\n') except Exception as e: diff --git a/src/iotables/py.typed b/src/iotables/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/iopy/core/utils.py b/src/iotables/utils.py similarity index 70% rename from iopy/core/utils.py rename to src/iotables/utils.py index 26fca1b..2b45d22 100644 --- a/iopy/core/utils.py +++ b/src/iotables/utils.py @@ -1,9 +1,5 @@ -""" Created on 11/10/2022:: -------------- utils ------------- -**Authors**: W. Wakker - -""" -from iopy.core.globals import FILES_LOG +"""Shared helpers: validation, the downloader, cache cleanup, and country-code maps.""" +from iotables.globals import FILES_LOG from collections import defaultdict import os @@ -13,6 +9,57 @@ def assert_is_subset(subset, superset): raise ValueError(f'Not found: {set(subset).difference(superset)}') +def download_file(url, dest, proxy=None, verify=True): + """Stream-download ``url`` to the local path ``dest``. + + Args: + url: Source URL. + dest: Local file path to write to. + proxy: Optional proxy. Either a single URL string (e.g. + ``'http://user:pass@host:port'``) applied to both http and https, + or a ``{scheme: url}`` dict passed straight through. + verify: Verify the server's TLS certificate. Set to ``False`` to skip + verification (e.g. behind a TLS-intercepting proxy), or pass a + path to a CA bundle. + """ + from curl_cffi import requests + + kwargs = {} + if proxy is not None: + kwargs['proxies'] = {'http': proxy, 'https': proxy} if isinstance(proxy, str) else proxy + + r = requests.get(url, stream=True, impersonate='chrome', verify=verify, **kwargs) + if not r.ok: + raise ConnectionError(r.reason or f'HTTP {r.status_code}') + + # Download to a temporary file and atomically move it into place only once the + # stream completes, so an interrupted download never leaves a truncated file in + # the cache (which would otherwise load as a wrong-shaped, silently corrupt matrix). + tmp = dest + '.part' + try: + written = 0 + with open(tmp, 'wb') as f: + for chunk in r.iter_content(): + f.write(chunk) + written += len(chunk) + + # Guard against a silently truncated body (server returns 200 then closes the + # stream early). Only enforce when the server advertised a length and did not + # transform the bytes -- e.g. CIRCABC serves gzip/chunked with no Content-Length, + # where the written size legitimately differs from any advertised length. + content_encoding = (r.headers.get('Content-Encoding') or '').lower() + expected = r.headers.get('Content-Length') + if expected is not None and content_encoding in ('', 'identity') and written != int(expected): + raise ConnectionError( + f'Incomplete download from {url}: got {written} bytes, expected {expected}') + + os.replace(tmp, dest) + except BaseException: + if os.path.exists(tmp): + os.remove(tmp) + raise + + def replace_if_exists(x, mapping): """Replace if x exists in mapping, otherwise return x @@ -53,7 +100,7 @@ def remove_downloaded_files(database: str = 'all', print(f'no files found for {database}, only for {list(files.keys())}') return other_files = {k: v for k, v in files.items() if k != database} - files = files[db] + files = files[database] else: files = {item for sublist in files.values() for item in sublist} for path in files: diff --git a/iopy/tests/__init__.py b/tests/__init__.py similarity index 100% rename from iopy/tests/__init__.py rename to tests/__init__.py diff --git a/iopy/tests/results/__init__.py b/tests/results/__init__.py similarity index 100% rename from iopy/tests/results/__init__.py rename to tests/results/__init__.py diff --git a/iopy/tests/results/adjust_source_in_covxml.py b/tests/results/adjust_source_in_covxml.py similarity index 83% rename from iopy/tests/results/adjust_source_in_covxml.py rename to tests/results/adjust_source_in_covxml.py index 153a10f..5d6ea26 100644 --- a/iopy/tests/results/adjust_source_in_covxml.py +++ b/tests/results/adjust_source_in_covxml.py @@ -10,7 +10,7 @@ with open(f'{parent}/cov.xml', 'r+') as f: covxml = f.read() covxml = re.sub(r'.*', - r'/builds/.../.../iopy/iopy', + r'/builds/.../.../src/iotables', covxml) f.seek(0) f.write(covxml) diff --git a/iopy/tests/test_exiobase.py b/tests/test_exiobase.py similarity index 67% rename from iopy/tests/test_exiobase.py rename to tests/test_exiobase.py index 0815b05..a285c61 100644 --- a/iopy/tests/test_exiobase.py +++ b/tests/test_exiobase.py @@ -1,11 +1,9 @@ -""" Created on 17/11/2022:: -------------- test_exiobase ------------- -**Authors**: W. Wakker - -""" -from iopy import ExioBase +"""Network tests for the ExioBase loader (download real data; run with the 'network' marker).""" +import pytest +from iotables import ExioBase +@pytest.mark.network class TestExioBase: def test_load(self): diff --git a/iopy/tests/test_figaro.py b/tests/test_figaro.py similarity index 85% rename from iopy/tests/test_figaro.py rename to tests/test_figaro.py index 03e2ae9..2c212cf 100644 --- a/iopy/tests/test_figaro.py +++ b/tests/test_figaro.py @@ -1,50 +1,53 @@ -""" Created on 19/10/2022:: -------------- test_figaro ------------- -**Authors**: W. Wakker - -""" +"""Network tests for the Figaro loader (download real data; run with the 'network' marker).""" import pytest -from iopy import Figaro +from iotables import Figaro import numpy as np import matplotlib import matplotlib.pyplot as plt -f = Figaro(version='2022', year=2018, kind='industry-by-industry') +EA = ['AT', 'BE', 'CY', 'DE', 'EE', 'ES', 'FI', 'FR', 'GR', 'HR', 'IE', 'IT', 'LT', 'LU', 'LV', 'MT', 'NL', 'PT', 'SI', 'SK'] -custom_shock_vector = np.random.uniform(size=f.rs, low=-10, high=10).reshape(-1, 1) -EA = ['AT', 'BE', 'CY', 'DE', 'EE', 'ES', 'FI', 'FR', 'GR', 'HR', 'IE', 'IT', 'LT', 'LU', 'LV', 'MT', 'NL', 'PT', 'SI', 'SK'] +@pytest.fixture(scope="module") +def f(): + return Figaro(version='2025', year=2018, kind='industry-by-industry') +@pytest.fixture(scope="module") +def custom_shock_vector(f): + return np.random.uniform(size=f.rs, low=-10, high=10).reshape(-1, 1) + + +@pytest.mark.network class TestFigaro: def test_download(self): - Figaro(version='2022', year=2018, refresh=True) + Figaro(version='2025', year=2018, refresh=True) def test_load(self): - fi = Figaro(version='2022', year=2018, kind='industry-by-industry') + fi = Figaro(version='2025', year=2018, kind='industry-by-industry') assert set(fi.sectors).issubset(fi.sector_name_mapping) - fi = Figaro(version='2022', year=2018, kind='product-by-product') + fi = Figaro(version='2025', year=2018, kind='product-by-product') assert set(fi.sectors).issubset(fi.sector_name_mapping) - def test_matrices(self): + def test_matrices(self, f): for attr in ['Z', 'A', 'B', 'L', 'G', 'V', 'FD', 'X']: assert hasattr(f, attr) for attr_attr in ['info', 'rows', 'columns', 'I']: assert attr_attr in dir(getattr(f, attr)) - def test_leontief(self): + def test_leontief(self, f): fd = (np.eye(f.rs) - f.A) @ f.X assert np.isclose(f.FD, fd, atol=.001).all() x = (np.eye(f.rs) - f.A).I @ f.FD assert np.isclose(f.X, x, atol=.001).all() - def test_ghosh(self): + def test_ghosh(self, f): x = f.G.T @ f.V.T assert np.isclose(f.X, x, atol=.001).all() - def test_shock(self): + def test_shock(self, f, custom_shock_vector): assert np.array_equal(f._shock(model='ghosh', custom_shock_vector=custom_shock_vector), (f.G.T @ (f.V.T * (custom_shock_vector / 100))) + f.X) @@ -52,7 +55,7 @@ def test_shock(self): ((np.eye(f.rs) - f.A).I @ ( f.FD * (custom_shock_vector / 100))) + f.X) - with pytest.raises(AssertionError): + with pytest.raises(ValueError): f._shock(model='leontief') shock_vector = np.array([-.1 if r in EA and s == 'A01' else 0 for r, s in f.X.rows]).reshape(-1, 1) @@ -72,17 +75,17 @@ def test_shock(self): with pytest.raises(ValueError): f._shock(model='leontief', shock=-10, regions=EA + ['something'], sectors=['A01']) - def test_leontief_shock(self): + def test_leontief_shock(self, f): assert np.array_equal( f.leontief_demand_shock(shock=-10, regions=EA, sectors=['A01']).x_new.values.reshape(-1, 1), f._shock(model='leontief', shock=-10, regions=EA, sectors=['A01'])) - def test_ghosh_shock(self): + def test_ghosh_shock(self, f): assert np.array_equal( f.ghosh_supply_shock(shock=-10, regions=EA, sectors=['A01']).x_new.values.reshape(-1, 1), f._shock(model='ghosh', shock=-10, regions=EA, sectors=['A01'])) - def test_plot(self): + def test_plot(self, f): fig, ax = f.ghosh_supply_shock(shock=-10, regions=EA, sectors=['A01'], plot_regions=EA, plot=True, show=False) assert isinstance(fig, matplotlib.figure.Figure) assert isinstance(ax, plt.Axes) @@ -101,25 +104,25 @@ def test_plot(self): f.ghosh_supply_shock(shock=-10, regions=EA, sectors=['35'], plot=True, show=True, plot_by='region') - def test_get_imports_exports(self): + def test_get_imports_exports(self, f): assert np.isclose(f.get_imports_exports(import_regions=['CN'], export_regions='AU', import_sectors=None, export_sectors=None, - use_type='intermediate'), 86280.8749) + use_type='intermediate'), 72202.248) assert np.isclose(f.get_imports_exports(import_regions='CN', export_regions='AU', import_sectors=None, export_sectors=None, - use_type='final'), 13770.405) + use_type='final'), 8212.555) assert np.isclose(f.get_imports_exports(import_regions=['CN'], export_regions='AU', import_sectors=None, export_sectors=None, - use_type='both'), 86280.8749 + 13770.405) + use_type='both'), 72202.248 + 8212.555) f.get_imports_exports(import_regions=['CN'], export_regions='AU', @@ -142,5 +145,5 @@ def test_get_imports_exports(self): import_sectors='A01', use_type='both') - def test_remove_local_files(self): + def test_remove_local_files(self, f): f.remove_downloaded_files() diff --git a/tests/test_iotables.py b/tests/test_iotables.py new file mode 100644 index 0000000..2107f62 --- /dev/null +++ b/tests/test_iotables.py @@ -0,0 +1,135 @@ +"""Offline tests for top-level helpers and the shared downloader.""" +import pytest +import iotables +from iotables import utils + + +class TestIotables: + + def test1(self): + iotables.remove_downloaded_files(database='figaro') + + def test_folder_size(self): + assert isinstance(iotables.get_size_data_folder(), str) + + +class TestRemoveDownloadedFiles: + """Offline tests for remove_downloaded_files (no network).""" + + def _setup_log(self, tmp_path, monkeypatch): + oecd_file = tmp_path / 'oecd.zip' + figaro_file = tmp_path / 'figaro.csv' + oecd_file.write_text('x') + figaro_file.write_text('y') + log = tmp_path / '_files_log.txt' + # Order matters: oecd is written first so the last line is figaro. The + # earlier wrong-database bug used the leftover loop variable (last line), + # which would have made remove(database='oecd') target figaro instead. + log.write_text(f'oecd;{oecd_file}\nfigaro;{figaro_file}\n') + monkeypatch.setattr(utils, 'FILES_LOG', str(log)) + return oecd_file, figaro_file, log + + def test_remove_single_database_keeps_others(self, tmp_path, monkeypatch): + oecd_file, figaro_file, log = self._setup_log(tmp_path, monkeypatch) + + utils.remove_downloaded_files(database='oecd', verbose=False) + + assert not oecd_file.exists() # requested database removed + assert figaro_file.exists() # other database untouched + assert log.exists() # log rewritten, not deleted + assert log.read_text().strip() == f'figaro;{figaro_file}' + + def test_remove_all(self, tmp_path, monkeypatch): + oecd_file, figaro_file, log = self._setup_log(tmp_path, monkeypatch) + + utils.remove_downloaded_files(database='all', verbose=False) + + assert not oecd_file.exists() + assert not figaro_file.exists() + assert not log.exists() + + def test_remove_unknown_database_is_noop(self, tmp_path, monkeypatch): + oecd_file, figaro_file, log = self._setup_log(tmp_path, monkeypatch) + + utils.remove_downloaded_files(database='nope', verbose=False) + + assert oecd_file.exists() + assert figaro_file.exists() + assert log.exists() + + def test_no_log_is_noop(self, tmp_path, monkeypatch): + monkeypatch.setattr(utils, 'FILES_LOG', str(tmp_path / 'missing.txt')) + utils.remove_downloaded_files(database='all', verbose=False) # must not raise + + +class _FakeResponse: + def __init__(self, chunks, headers, ok=True, status_code=200): + self._chunks = chunks + self.headers = headers + self.ok = ok + self.status_code = status_code + self.reason = 'OK' + + def iter_content(self): + return iter(self._chunks) + + +class TestDownloadFile: + """Offline tests for download_file's atomic + integrity behaviour (no network).""" + + def _patch(self, monkeypatch, response): + import curl_cffi + monkeypatch.setattr(curl_cffi.requests, 'get', lambda *a, **k: response) + + def test_truncated_body_raises_and_leaves_no_files(self, tmp_path, monkeypatch): + # Server advertises 100 bytes but only streams 50: a silent truncation. + resp = _FakeResponse([b'x' * 50], {'Content-Length': '100'}) + self._patch(monkeypatch, resp) + dest = tmp_path / 'data.zip' + + with pytest.raises(ConnectionError): + utils.download_file('http://example/data.zip', str(dest)) + + assert not dest.exists() + assert not (tmp_path / 'data.zip.part').exists() + + def test_complete_body_succeeds(self, tmp_path, monkeypatch): + resp = _FakeResponse([b'ab', b'cd'], {'Content-Length': '4'}) + self._patch(monkeypatch, resp) + dest = tmp_path / 'data.zip' + + utils.download_file('http://example/data.zip', str(dest)) + + assert dest.read_bytes() == b'abcd' + assert not (tmp_path / 'data.zip.part').exists() + + def test_length_check_skipped_for_encoded_body(self, tmp_path, monkeypatch): + # gzip/chunked (e.g. CIRCABC): advertised length differs from decoded bytes, + # so the check must be skipped rather than false-positive. + resp = _FakeResponse([b'x' * 50], {'Content-Length': '100', 'Content-Encoding': 'gzip'}) + self._patch(monkeypatch, resp) + dest = tmp_path / 'data.csv' + + utils.download_file('http://example/data.csv', str(dest)) + + assert dest.read_bytes() == b'x' * 50 + + def test_no_content_length_succeeds(self, tmp_path, monkeypatch): + resp = _FakeResponse([b'hello'], {}) + self._patch(monkeypatch, resp) + dest = tmp_path / 'data.csv' + + utils.download_file('http://example/data.csv', str(dest)) + + assert dest.read_bytes() == b'hello' + + def test_http_error_raises(self, tmp_path, monkeypatch): + resp = _FakeResponse([], {}, ok=False, status_code=504) + resp.reason = 'Gateway Timeout' + self._patch(monkeypatch, resp) + dest = tmp_path / 'data.zip' + + with pytest.raises(ConnectionError): + utils.download_file('http://example/data.zip', str(dest)) + + assert not dest.exists() diff --git a/iopy/tests/test_matrix.py b/tests/test_matrix.py similarity index 79% rename from iopy/tests/test_matrix.py rename to tests/test_matrix.py index 3dd379a..c23a7f9 100644 --- a/iopy/tests/test_matrix.py +++ b/tests/test_matrix.py @@ -1,27 +1,23 @@ -""" Created on 03/10/2022:: -------------- test_matrix ------------- -**Authors**: W. Wakker - -""" -from iopy.core.matrix import Matrix +"""Offline tests for the labelled Matrix subclass.""" +from iotables.matrix import Matrix import pandas as pd import numpy as np import pytest -from iopy.core.utils import ALPHA3_TO_ALPHA2 +from iotables.utils import ALPHA3_TO_ALPHA2 def process_df(df): if df.shape[0] > 1 and df.shape[1] > 1: return (df, - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_')], - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_')]) + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_', n=1)], + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_', n=1)]) elif df.shape[0] == 1: return (df, df.index.to_list(), - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_')]) + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_', n=1)]) else: return (df, - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_')], + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_', n=1)], df.columns.to_list()) @@ -39,7 +35,7 @@ def test1(self): Matrix('something', *process_df(df)) def test2(self): - with pytest.raises(AssertionError): + with pytest.raises(ValueError): Matrix('something', [1, 2, 3], ['something'], ['something']) def testI(self): diff --git a/iopy/tests/test_oecd.py b/tests/test_oecd.py similarity index 86% rename from iopy/tests/test_oecd.py rename to tests/test_oecd.py index 51afe0a..fc75462 100644 --- a/iopy/tests/test_oecd.py +++ b/tests/test_oecd.py @@ -1,25 +1,28 @@ -""" Created on 03/10/2022:: -------------- test ------------- -**Authors**: W. Wakker - -""" +"""Network tests for the OECD loader (download real data; run with the 'network' marker).""" import pytest -from iopy import OECD +from iotables import OECD import numpy as np import matplotlib import matplotlib.pyplot as plt -oecd = OECD(version='2021', year=2018) +EA = ['AT', 'BE', 'CY', 'DE', 'EE', 'ES', 'FI', 'FR', 'GR', 'HR', 'IE', 'IT', 'LT', 'LU', 'LV', 'MT', 'NL', 'PT', 'SI', 'SK'] + -custom_shock_vector = np.random.uniform(size=oecd.rs, low=-10, high=10).reshape(-1, 1) +@pytest.fixture(scope="module") +def oecd(): + return OECD(version='2021', year=2018) -EA = ['AT', 'BE', 'CY', 'DE', 'EE', 'ES', 'FI', 'FR', 'GR', 'HR', 'IE', 'IT', 'LT', 'LU', 'LV', 'MT', 'NL', 'PT', 'SI', 'SK'] +@pytest.fixture(scope="module") +def custom_shock_vector(oecd): + return np.random.uniform(size=oecd.rs, low=-10, high=10).reshape(-1, 1) + +@pytest.mark.network class TestOECD: def test_download(self): - o = OECD(version='2021', year=2018, refresh=True) + OECD(version='2021', year=2018, refresh=True) def test_load(self): o = OECD(version='2021', year=2018) @@ -28,25 +31,29 @@ def test_load(self): assert set(o.sectors).issubset(o.sector_name_mapping) o = OECD(version='2022-small', year=2018) assert set(o.sectors).issubset(o.sector_name_mapping) + o = OECD(version='2025-extended', year=2022) + assert set(o.sectors).issubset(o.sector_name_mapping) + o = OECD(version='2025-regular', year=2022) + assert set(o.sectors).issubset(o.sector_name_mapping) - def test_matrices(self): + def test_matrices(self, oecd): for attr in ['Z', 'A', 'B', 'L', 'G', 'V', 'FD', 'X']: assert hasattr(oecd, attr) for attr_attr in ['info', 'rows', 'columns', 'I']: assert attr_attr in dir(getattr(oecd, attr)) - def test_leontief(self): + def test_leontief(self, oecd): fd = (np.eye(oecd.rs) - oecd.A) @ oecd.X assert np.isclose(oecd.FD, fd, atol=.001).all() x = (np.eye(oecd.rs) - oecd.A).I @ oecd.FD assert np.isclose(oecd.X, x, atol=.001).all() - def test_ghosh(self): + def test_ghosh(self, oecd): x = oecd.G.T @ oecd.V.T assert np.isclose(oecd.X, x, atol=.001).all() - def test_shock(self): + def test_shock(self, oecd, custom_shock_vector): assert np.array_equal(oecd._shock(model='ghosh', custom_shock_vector=custom_shock_vector), (oecd.G.T @ (oecd.V.T * (custom_shock_vector / 100))) + oecd.X) @@ -54,7 +61,7 @@ def test_shock(self): ((np.eye(oecd.rs) - oecd.A).I @ ( oecd.FD * (custom_shock_vector / 100))) + oecd.X) - with pytest.raises(AssertionError): + with pytest.raises(ValueError): oecd._shock(model='leontief') shock_vector = np.array([-.1 if r in EA and s == '35' else 0 for r, s in oecd.X.rows]).reshape(-1, 1) @@ -74,17 +81,17 @@ def test_shock(self): with pytest.raises(ValueError): oecd._shock(model='leontief', shock=-10, regions=EA + ['something'], sectors=['35']) - def test_leontief_shock(self): + def test_leontief_shock(self, oecd): assert np.array_equal( oecd.leontief_demand_shock(shock=-10, regions=EA, sectors=['35']).x_new.values.reshape(-1, 1), oecd._shock(model='leontief', shock=-10, regions=EA, sectors=['35'])) - def test_ghosh_shock(self): + def test_ghosh_shock(self, oecd): assert np.array_equal( oecd.ghosh_supply_shock(shock=-10, regions=EA, sectors=['35']).x_new.values.reshape(-1, 1), oecd._shock(model='ghosh', shock=-10, regions=EA, sectors=['35'])) - def test_plot(self): + def test_plot(self, oecd): fig, ax = oecd.ghosh_supply_shock(shock=-10, regions=EA, sectors=['35'], plot_regions=EA, plot=True, show=False) assert isinstance(fig, matplotlib.figure.Figure) assert isinstance(ax, plt.Axes) @@ -103,7 +110,7 @@ def test_plot(self): oecd.ghosh_supply_shock(shock=-10, regions=EA, sectors=['35'], plot=True, show=True, plot_by='region') - def test_get_imports_exports(self): + def test_get_imports_exports(self, oecd): assert np.isclose(oecd.get_imports_exports(import_regions=['CN1', 'CN2'], export_regions='AU',