From d6e8f17a55d3de7f210789c8331843f04f5e665f Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Wed, 3 Jun 2026 22:40:36 +0200 Subject: [PATCH 01/15] Update OECD links --- test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test.py diff --git a/test.py b/test.py new file mode 100644 index 0000000..e69de29 From a5eeb172a85cda5b49ffd758a9ebfe8f4907a3fb Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Wed, 3 Jun 2026 22:41:00 +0200 Subject: [PATCH 02/15] Update OECD links --- .DS_Store | Bin 0 -> 6148 bytes .gitignore | 2 +- .vscode/settings.json | 4 ++ CLAUDE.md | 100 ++++++++++++++++++++++++++ iopy/.DS_Store | Bin 0 -> 6148 bytes iopy/core/config.py | 143 ++++++++++++++++---------------------- iopy/core/exiobase.py | 9 +-- iopy/core/figaro.py | 9 +-- iopy/core/oecd.py | 19 +++-- iopy/core/utils.py | 19 +++++ iopy/dev-requirements.txt | 1 + iopy/requirements.txt | 1 + test.py | 3 + 13 files changed, 203 insertions(+), 107 deletions(-) create mode 100644 .DS_Store create mode 100644 .vscode/settings.json create mode 100644 CLAUDE.md create mode 100644 iopy/.DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..e393e44e60e428aaa0bb5c3151f1dbb932e86b57 GIT binary patch literal 6148 zcmeHKL2uJA6n<_4O&ODVVA3u~k+_c1G7TYd>AF^95(8;LZ~#=ArEC#dTs7$`R8`6u z{sVu3D}Mz33n%!V?T)71u7HqVWq;1|_ni21@>~;N%dP2`=pXw`Y>#zz~1^zV!_}jfnd-Rxg=_$V3-{0>cbq->A9K5xD|o>q9h&jJ2E#Z?vqs~)D3zUS*DG$tU31^~r*h(F{cN1J z`=eKU^<2t0n78}Ei)b+E)o$&{B?r^JawO8Am zPVet*Z+Q=!vn_AhY&3Szey}~8Ro#s{_Z}Vf-p6N&{HSI|0{KkYy1_ZH1{d=V)8jb6 z^Ks{e^V}aLu}n_j<3ZNvA|WLX=mfvJ@Fet(e5drRic}eCih2;>QxZ zN=IPsQil$KbvmQ47*XOUS{%PjBRo-BWQp5LESFNSG8$u{>#K`h`nJTcfQwO4U&2{kI1^{j3U(IU@^{GRJ#bHdb*nf3$)RtzVjJ_A1AWW^-Oii9@& z%G^1z|E|Iw&{fMBa?HN)dTDsIG2uQ7OzY8c)v!C|=Qzb07h;P?{<4|&<{ahRuhgSI zVOfvk-WqXw-}7$?E8C|$RDw&{vAz>t8X8(z=i?2 zA0nz?9I*1JR|gv{0f=o*tFSM%g!n`O;f47m1iG#v0q;ceYH woV>LW^_D6kc9qAC3p@H0(^j71GpY*bBk2(1fR#tiQ0$L@(qN4-@T&}b0BA{aT>t<8 literal 0 HcmV?d00001 diff --git a/iopy/core/config.py b/iopy/core/config.py index ecf9912..22f67f8 100644 --- a/iopy/core/config.py +++ b/iopy/core/config.py @@ -3,102 +3,81 @@ **Authors**: W. Wakker """ + + +def _chunk_links(base, chunks): + """Expand ``{filename: (start_year, end_year)}`` into ``{year: url}``. + + OECD distributes each ICIO edition as a handful of multi-year zip archives + (e.g. ``2016-2022_SML.zip`` holds one CSV per year), so every year in a range + resolves to the same download URL. + """ + return {year: base + filename + for filename, (start, end) in chunks.items() + for year in range(start, end + 1)} + + config = { 'oecd': {'2021': - {'links': - {1995: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=91d8e84b-7406-46b9-af5f-ec096242755c', - 1996: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=91d8e84b-7406-46b9-af5f-ec096242755c', - 1997: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=91d8e84b-7406-46b9-af5f-ec096242755c', - 1998: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=91d8e84b-7406-46b9-af5f-ec096242755c', - 1999: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=91d8e84b-7406-46b9-af5f-ec096242755c', - 2000: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8adf89dd-18b4-40fe-bc7f-c822052eb961', - 2001: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8adf89dd-18b4-40fe-bc7f-c822052eb961', - 2002: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8adf89dd-18b4-40fe-bc7f-c822052eb961', - 2003: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8adf89dd-18b4-40fe-bc7f-c822052eb961', - 2004: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8adf89dd-18b4-40fe-bc7f-c822052eb961', - 2005: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=fe218690-0a3b-44aa-a82c-b3e3da6d24db', - 2006: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=fe218690-0a3b-44aa-a82c-b3e3da6d24db', - 2007: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=fe218690-0a3b-44aa-a82c-b3e3da6d24db', - 2008: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=fe218690-0a3b-44aa-a82c-b3e3da6d24db', - 2009: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=fe218690-0a3b-44aa-a82c-b3e3da6d24db', - 2010: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=2c2f499f-5703-4034-9457-2f7518e8f2fc', - 2011: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=2c2f499f-5703-4034-9457-2f7518e8f2fc', - 2012: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=2c2f499f-5703-4034-9457-2f7518e8f2fc', - 2013: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=2c2f499f-5703-4034-9457-2f7518e8f2fc', - 2014: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=2c2f499f-5703-4034-9457-2f7518e8f2fc', - 2015: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=59a3d7f2-3f23-40d5-95ca-48da84c0f861', - 2016: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=59a3d7f2-3f23-40d5-95ca-48da84c0f861', - 2017: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=59a3d7f2-3f23-40d5-95ca-48da84c0f861', - 2018: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=59a3d7f2-3f23-40d5-95ca-48da84c0f861'}, - 'regex_id': r'[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}', + {'links': _chunk_links('https://webfs-sti.oecd.org/files/STI-PIE/ICIO/2021/', { + 'ICIO2021_1995-1999.zip': (1995, 1999), + 'ICIO2021_2000-2004.zip': (2000, 2004), + 'ICIO2021_2005-2009.zip': (2005, 2009), + 'ICIO2021_2010-2014.zip': (2010, 2014), + 'ICIO2021_2015-2018.zip': (2015, 2018), + }), + 'regex_id': r'ICIO2021_[0-9]{4}-[0-9]{4}', 'num_regions': 71, 'num_sectors': 45 }, '2022-extended': - {'links': - {1995: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=4a2d6739-f717-48ac-a97b-d95f96984c55', - 1996: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=4a2d6739-f717-48ac-a97b-d95f96984c55', - 1997: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=4a2d6739-f717-48ac-a97b-d95f96984c55', - 1998: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=4a2d6739-f717-48ac-a97b-d95f96984c55', - 1999: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=4a2d6739-f717-48ac-a97b-d95f96984c55', - 2000: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8eb6cf87-1899-4547-b337-d76dd4ef608c', - 2001: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8eb6cf87-1899-4547-b337-d76dd4ef608c', - 2002: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8eb6cf87-1899-4547-b337-d76dd4ef608c', - 2003: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8eb6cf87-1899-4547-b337-d76dd4ef608c', - 2004: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8eb6cf87-1899-4547-b337-d76dd4ef608c', - 2005: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=f9b6dd9a-e64b-47fb-832e-bd1628c43b72', - 2006: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=f9b6dd9a-e64b-47fb-832e-bd1628c43b72', - 2007: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=f9b6dd9a-e64b-47fb-832e-bd1628c43b72', - 2008: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=f9b6dd9a-e64b-47fb-832e-bd1628c43b72', - 2009: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=f9b6dd9a-e64b-47fb-832e-bd1628c43b72', - 2010: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=35cfb172-2cc4-4d62-98e5-9e5f1c51d6c9', - 2011: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=35cfb172-2cc4-4d62-98e5-9e5f1c51d6c9', - 2012: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=35cfb172-2cc4-4d62-98e5-9e5f1c51d6c9', - 2013: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=35cfb172-2cc4-4d62-98e5-9e5f1c51d6c9', - 2014: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=35cfb172-2cc4-4d62-98e5-9e5f1c51d6c9', - 2015: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=e5368118-b274-4e42-820e-33dacbfb94ed', - 2016: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=e5368118-b274-4e42-820e-33dacbfb94ed', - 2017: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=e5368118-b274-4e42-820e-33dacbfb94ed', - 2018: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=e5368118-b274-4e42-820e-33dacbfb94ed', - 2019: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=e5368118-b274-4e42-820e-33dacbfb94ed', - 2020: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=92539517-1661-4077-a9df-136abc39b858'}, - 'regex_id': r'[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}', + {'links': _chunk_links('https://webfs-sti.oecd.org/files/STI-PIE/ICIO/2023/', { + '1995-2000_EXT.zip': (1995, 2000), + '2001-2005_EXT.zip': (2001, 2005), + '2006-2010_EXT.zip': (2006, 2010), + '2011-2015_EXT.zip': (2011, 2015), + '2016-2020_EXT.zip': (2016, 2020), + }), + 'regex_id': r'2023/[0-9]{4}-[0-9]{4}_EXT', 'num_regions': 81, 'num_sectors': 45}, '2022-small': - {'links': - {1995: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8f69e3c5-8bc0-4c7b-aad2-5ef776c119ea', - 1996: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8f69e3c5-8bc0-4c7b-aad2-5ef776c119ea', - 1997: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8f69e3c5-8bc0-4c7b-aad2-5ef776c119ea', - 1998: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8f69e3c5-8bc0-4c7b-aad2-5ef776c119ea', - 1999: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=8f69e3c5-8bc0-4c7b-aad2-5ef776c119ea', - 2000: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=43e7a690-c1ea-4839-b0fb-907e0aa79523', - 2001: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=43e7a690-c1ea-4839-b0fb-907e0aa79523', - 2002: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=43e7a690-c1ea-4839-b0fb-907e0aa79523', - 2003: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=43e7a690-c1ea-4839-b0fb-907e0aa79523', - 2004: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=43e7a690-c1ea-4839-b0fb-907e0aa79523', - 2005: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=9942ca81-9137-4926-adcf-dbe19fa2bcb6', - 2006: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=9942ca81-9137-4926-adcf-dbe19fa2bcb6', - 2007: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=9942ca81-9137-4926-adcf-dbe19fa2bcb6', - 2008: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=9942ca81-9137-4926-adcf-dbe19fa2bcb6', - 2009: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=9942ca81-9137-4926-adcf-dbe19fa2bcb6', - 2010: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=832d60ce-8346-456b-8ab8-cafd81c2f054', - 2011: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=832d60ce-8346-456b-8ab8-cafd81c2f054', - 2012: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=832d60ce-8346-456b-8ab8-cafd81c2f054', - 2013: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=832d60ce-8346-456b-8ab8-cafd81c2f054', - 2014: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=832d60ce-8346-456b-8ab8-cafd81c2f054', - 2015: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=ac018f80-6899-4395-af52-5c21134c51b3', - 2016: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=ac018f80-6899-4395-af52-5c21134c51b3', - 2017: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=ac018f80-6899-4395-af52-5c21134c51b3', - 2018: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=ac018f80-6899-4395-af52-5c21134c51b3', - 2019: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=ac018f80-6899-4395-af52-5c21134c51b3', - 2020: 'https://stats.oecd.org/wbos/fileview2.aspx?IDFile=60d244bd-5b40-4be3-bd93-0dc35c210ece'}, - 'regex_id': r'[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}', + {'links': _chunk_links('https://webfs-sti.oecd.org/files/STI-PIE/ICIO/2023/', { + '1995-2000_SML.zip': (1995, 2000), + '2001-2005_SML.zip': (2001, 2005), + '2006-2010_SML.zip': (2006, 2010), + '2011-2015_SML.zip': (2011, 2015), + '2016-2020_SML.zip': (2016, 2020), + }), + 'regex_id': r'2023/[0-9]{4}-[0-9]{4}_SML', 'num_regions': 77, 'num_sectors': 45}, + '2025-extended': + {'links': _chunk_links('https://webfs-sti.oecd.org/files/STI-PIE/ICIO/2025/', { + '1995-2000_EXT.zip': (1995, 2000), + '2001-2005_EXT.zip': (2001, 2005), + '2006-2010_EXT.zip': (2006, 2010), + '2011-2015_EXT.zip': (2011, 2015), + '2016-2022_EXT.zip': (2016, 2022), + }), + 'regex_id': r'2025/[0-9]{4}-[0-9]{4}_EXT', + 'num_regions': 85, + 'num_sectors': 50}, + '2025-regular': + {'links': _chunk_links('https://webfs-sti.oecd.org/files/STI-PIE/ICIO/2025/', { + '1995-2000_SML.zip': (1995, 2000), + '2001-2005_SML.zip': (2001, 2005), + '2006-2010_SML.zip': (2006, 2010), + '2011-2015_SML.zip': (2011, 2015), + '2016-2022_SML.zip': (2016, 2022), + }), + 'regex_id': r'2025/[0-9]{4}-[0-9]{4}_SML', + 'num_regions': 81, + 'num_sectors': 50} }, + 'figaro': {'2022': {'links': diff --git a/iopy/core/exiobase.py b/iopy/core/exiobase.py index bec7f33..8314655 100644 --- a/iopy/core/exiobase.py +++ b/iopy/core/exiobase.py @@ -15,7 +15,7 @@ from iopy.core.base_io import IO from warnings import warn from iopy.core.globals import DATA_FOLDER, FILES_LOG -from iopy.core.utils import remove_downloaded_files +from iopy.core.utils import remove_downloaded_files, download_file db_name = os.path.basename(__file__).rstrip('.py') @@ -170,13 +170,8 @@ def _load_data(self): return z_raw, fd_raw, x_raw, metadata, sector_codes, FD_codes def _download_data(self): - import requests - try: - r = requests.get(self._url, stream=True) - with open(self._data_file, "wb") as f: - for chunk in r.iter_content(1024 * 5): - f.write(chunk) + download_file(self._url, self._data_file) with open(FILES_LOG, 'a') as files_log: files_log.write(db_name + ';' + self._data_file + '\n') except Exception as e: diff --git a/iopy/core/figaro.py b/iopy/core/figaro.py index 19ae9e3..2d0f840 100644 --- a/iopy/core/figaro.py +++ b/iopy/core/figaro.py @@ -15,7 +15,7 @@ from warnings import warn from iopy.core.base_io import IO from iopy.core.globals import DATA_FOLDER, FILES_LOG -from iopy.core.utils import remove_downloaded_files +from iopy.core.utils import remove_downloaded_files, download_file db_name = os.path.basename(__file__).rstrip('.py') @@ -139,13 +139,8 @@ def _load_data(self): return df def _download_data(self): - import requests - try: - r = requests.get(self._url, stream=True) - with open(self._data_file, "wb") as f: - for chunk in r.iter_content(1024 * 5): - f.write(chunk) + download_file(self._url, self._data_file) with open(FILES_LOG, 'a') as files_log: files_log.write(db_name + ';' + self._data_file + '\n') except Exception as e: diff --git a/iopy/core/oecd.py b/iopy/core/oecd.py index d723dfd..a821490 100644 --- a/iopy/core/oecd.py +++ b/iopy/core/oecd.py @@ -15,7 +15,7 @@ import os from iopy.core.config import config from iopy.core.base_io import IO -from iopy.core.utils import replace_if_exists, remove_downloaded_files +from iopy.core.utils import replace_if_exists, remove_downloaded_files, download_file from iopy.core.globals import DATA_FOLDER, FILES_LOG from warnings import warn from functools import partial @@ -65,7 +65,7 @@ def __init__(self, self.year = year self.version = version self._url = config['oecd'][version]['links'][year] - self._file_id = re.search(config['oecd'][version]['regex_id'], self._url).group(0) + self._file_id = re.search(config['oecd'][version]['regex_id'], self._url).group(0).replace('/', '_') self._data_file = os.path.join(DATA_FOLDER, self._file_id + '.zip') file_exists = os.path.exists(self._data_file) download = not file_exists or refresh @@ -143,22 +143,21 @@ def _load_data(self): if self.version == '2021': filename = f'ICIO2021_{self.year}.csv' elif self.version == '2022-extended': - filename = f'{self.year}.CSV' + filename = f'{self.year}.csv' elif self.version == '2022-small': - filename = f'{self.year}SML.CSV' + filename = f'{self.year}_SML.csv' + elif self.version == '2025-extended': + filename = f'{self.year}.csv' + elif self.version == '2025-regular': + filename = f'{self.year}_SML.csv' with ZipFile(self._data_file, 'r') as zf: with zf.open(filename, 'r') as csv_file: df = pd.read_csv(csv_file, index_col=0) return df def _download_data(self): - import requests - try: - r = requests.get(self._url, stream=True) - with open(self._data_file, "wb") as f: - for chunk in r.iter_content(1024 * 5): - f.write(chunk) + download_file(self._url, self._data_file) with open(FILES_LOG, 'a') as files_log: files_log.write(db_name + ';' + self._data_file + '\n') except Exception as e: diff --git a/iopy/core/utils.py b/iopy/core/utils.py index 26fca1b..3cd103b 100644 --- a/iopy/core/utils.py +++ b/iopy/core/utils.py @@ -13,6 +13,25 @@ def assert_is_subset(subset, superset): raise ValueError(f'Not found: {set(subset).difference(superset)}') +def download_file(url, dest): + """Stream-download ``url`` to the local path ``dest``. + + Uses ``curl_cffi`` with browser (TLS) impersonation so downloads succeed even + when the host sits behind Cloudflare's bot challenge. The OECD file server + (``webfs-sti.oecd.org``) rejects plain ``requests``/``urllib`` traffic with + HTTP 403; impersonating a real browser passes the challenge and is harmless + for the non-protected Eurostat/EXIOBASE hosts. + """ + from curl_cffi import requests + + r = requests.get(url, stream=True, impersonate='chrome') + if not r.ok: + raise ConnectionError(r.reason or f'HTTP {r.status_code}') + with open(dest, 'wb') as f: + for chunk in r.iter_content(): + f.write(chunk) + + def replace_if_exists(x, mapping): """Replace if x exists in mapping, otherwise return x diff --git a/iopy/dev-requirements.txt b/iopy/dev-requirements.txt index dd8ec32..25c21b6 100644 --- a/iopy/dev-requirements.txt +++ b/iopy/dev-requirements.txt @@ -1,4 +1,5 @@ requests +curl_cffi numpy pandas>=1.1.5 matplotlib diff --git a/iopy/requirements.txt b/iopy/requirements.txt index 48761f3..3c710c3 100644 --- a/iopy/requirements.txt +++ b/iopy/requirements.txt @@ -1,4 +1,5 @@ requests +curl_cffi numpy pandas>=1.1.5 matplotlib diff --git a/test.py b/test.py index e69de29..dd633bc 100644 --- a/test.py +++ b/test.py @@ -0,0 +1,3 @@ +from iopy import OECD + +io = OECD(version='2025-extended', year=2022, refresh=True) From 93d4f88d5bde9df298d395424bab44969d605c4b Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Wed, 3 Jun 2026 22:53:55 +0200 Subject: [PATCH 03/15] oecd working --- iopy/core/mappings.py | 54 +++++++++++++++++++++++++++++++++++++++++ iopy/core/oecd.py | 14 ++++++----- iopy/tests/test_oecd.py | 4 +++ 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/iopy/core/mappings.py b/iopy/core/mappings.py index 30b2c54..c482d57 100644 --- a/iopy/core/mappings.py +++ b/iopy/core/mappings.py @@ -143,6 +143,60 @@ '94T96': 'Other service activities', '97T98': 'Households'} +# OECD ICIO 2025 edition uses a 50-industry ISIC Rev.4 breakdown (codes differ from the +# 45-sector classification above). Source: ReadMe_ICIO_*.xlsx "Area_Activities" sheet. +oecd_sector_name_mapping_2025 = {'A01': 'Agriculture and hunting', + 'A02': 'Forestry and logging', + 'A03': 'Fishing and aquaculture', + 'B05': 'Mining of coal and lignite', + 'B06': 'Extraction of crude petroleum and natural gas', + 'B07': 'Mining of metal ores', + 'B08': 'Other mining and quarrying', + 'B09': 'Mining support service activities', + 'C10T12': 'Manufacture of food products; beverages and tobacco products', + 'C13T15': 'Manufacture of textiles, wearing apparel, leather and related products', + 'C16': 'Manufacture of wood and of products of wood and cork', + 'C17_18': 'Manufacture of paper and paper products; Printing and reproduction of recorded media', + 'C19': 'Manufacture of coke and refined petroleum products', + 'C20': 'Manufacture of chemicals and chemical products', + 'C21': 'Manufacture of basic pharmaceutical products and pharmaceutical preparations', + 'C22': 'Manufacture of rubber and plastic products', + 'C23': 'Manufacture of other non-metallic mineral products', + 'C24A': 'Manufacture of basic iron and steel', + 'C24B': 'Manufacture of basic precious and other non-ferrous metals', + 'C25': 'Manufacture of fabricated metal products', + 'C26': 'Manufacture of computer, electronic and optical products', + 'C27': 'Manufacture of electrical equipment', + 'C28': 'Manufacture of machinery and equipment n.e.c.', + 'C29': 'Manufacture of motor vehicles, trailers and semi-trailers', + 'C301': 'Building of ships and boats', + 'C302T309': 'Manufacture of other transport equipment', + 'C31T33': 'Manufacture of furniture; other manufacturing; repair and installation of machinery and equipment', + 'D': 'Electricity, gas, steam and air conditioning supply', + 'E': 'Water supply; sewerage, waste management and remediation activities', + 'F': 'Construction', + 'G': 'Wholesale and retail trade; repair of motor vehicles and motorcycles', + 'H49': 'Land transport and transport via pipelines', + 'H50': 'Water transport', + 'H51': 'Air transport', + 'H52': 'Warehousing and support activities for transportation', + 'H53': 'Postal and courier activities', + 'I': 'Accommodation and food service activities', + 'J58T60': 'Publishing, Motion picture, video, television programme production and broadcasting activities', + 'J61': 'Telecommunications', + 'J62_63': 'Computer programming and information service activities', + 'K': 'Financial and insurance activities', + 'L': 'Real estate activities', + 'M': 'Professional, scientific and technical activities', + 'N': 'Administrative and support service activities', + 'O': 'Public administration and defence; compulsory social security', + 'P': 'Education', + 'Q': 'Human health and social work activities', + 'R': 'Arts, entertainment and recreation activities', + 'S': 'Other service activities', + 'T': 'Activities of households as employers; undifferentiated goods- and services-producing ' + 'activities of households for own use'} + oecd_demand_items = {'HFCE': 'Household Final Consumption Expenditure', 'NPISH': 'Non-Profit Institutions Serving Households', 'GGFC': 'General Government Final Consumption', diff --git a/iopy/core/oecd.py b/iopy/core/oecd.py index a821490..36cddf1 100644 --- a/iopy/core/oecd.py +++ b/iopy/core/oecd.py @@ -3,7 +3,8 @@ **Authors**: W. Wakker """ -from iopy.core.mappings import oecd_sector_name_mapping, oecd_demand_items, oecd_sector_2022_2021_mapping +from iopy.core.mappings import oecd_sector_name_mapping, oecd_sector_name_mapping_2025, oecd_demand_items, \ + oecd_sector_2022_2021_mapping from iopy.core.matrix import Matrix from functools import lru_cache import numpy as np @@ -26,15 +27,15 @@ def process_df(df): if df.shape[0] > 1 and df.shape[1] > 1: return (df, - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_')], - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_')]) + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_', n=1)], + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_', n=1)]) elif df.shape[0] == 1: return (df, df.index.to_list(), - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_')]) + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_', n=1)]) else: return (df, - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_')], + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_', n=1)], df.columns.to_list()) @@ -128,7 +129,8 @@ def __init__(self, self.regions = list(sorted(np.unique([r for r, s in self.Z.rows]))) self.sectors = list(sorted(np.unique([s for r, s in self.Z.rows]))) self.unit = 'Million USD' - self.sector_name_mapping = oecd_sector_name_mapping + self.sector_name_mapping = oecd_sector_name_mapping_2025 if version in ('2025-extended', '2025-regular') \ + else oecd_sector_name_mapping self.demand_items = oecd_demand_items self.reference = f'OECD ({self.version[:4]}), OECD Inter-Country Input-Output Database, http://oe.cd/icio' self.contact = 'ICIO-TiVA.Contact@oecd.org, mentioning ICIO' diff --git a/iopy/tests/test_oecd.py b/iopy/tests/test_oecd.py index 51afe0a..927bdfb 100644 --- a/iopy/tests/test_oecd.py +++ b/iopy/tests/test_oecd.py @@ -28,6 +28,10 @@ def test_load(self): assert set(o.sectors).issubset(o.sector_name_mapping) o = OECD(version='2022-small', year=2018) assert set(o.sectors).issubset(o.sector_name_mapping) + o = OECD(version='2025-extended', year=2022) + assert set(o.sectors).issubset(o.sector_name_mapping) + o = OECD(version='2025-regular', year=2022) + assert set(o.sectors).issubset(o.sector_name_mapping) def test_matrices(self): for attr in ['Z', 'A', 'B', 'L', 'G', 'V', 'FD', 'X']: From dac796e6413720095c8a1cfd0d00e9adbb0cc5ab Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Wed, 3 Jun 2026 23:17:18 +0200 Subject: [PATCH 04/15] figaro 2025 working --- iopy/core/config.py | 49 ++++++++++++++++++++++++++++++++++++++- iopy/core/figaro.py | 4 ++-- iopy/tests/test_figaro.py | 14 +++++------ 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/iopy/core/config.py b/iopy/core/config.py index 22f67f8..4f685dd 100644 --- a/iopy/core/config.py +++ b/iopy/core/config.py @@ -17,6 +17,16 @@ def _chunk_links(base, chunks): for year in range(start, end + 1)} +def _circabc_links(kind_tag, ids): + """Build CIRCABC anonymous download URLs for Figaro from ``{year: node_id}``. + + Eurostat distributes the Figaro 2025 edition through CIRCABC; every file has a + stable, auth-free download URL of the form ``/sd/a/{node_id}/{filename}``. + """ + return {year: f'https://circabc.europa.eu/sd/a/{nid}/matrix_eu-ic-io_{kind_tag}_25ed_{year}.csv' + for year, nid in ids.items()} + + config = { 'oecd': {'2021': @@ -110,7 +120,44 @@ def _chunk_links(base, chunks): }, 'regex_id': r'[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}', 'num_regions': 46, - 'num_sectors': 64} + 'num_sectors': 64}, + '2025': + {'links': + {'product-by-product': _circabc_links('prod-by-prod', { + 2010: 'f6d2007e-34c9-420a-a106-8c5ea836c49d', + 2011: 'd27f7d7c-4006-4c36-8820-7d9990679323', + 2012: '7cea8b73-f166-47ff-9c27-f240794b57af', + 2013: 'd56c2ed9-0c4f-40a3-9bbc-1aee49f5bd45', + 2014: 'b5aa0e38-3d1b-496b-b180-bbd9bab3e153', + 2015: 'd037b772-8bf9-424f-97a4-9274d261e6cf', + 2016: '66369d21-8262-47d6-ad8a-dc536a2466d0', + 2017: 'd6848c38-9569-4848-bb68-4144fed38c71', + 2018: '557b9483-023e-497c-96c5-44ce807bd444', + 2019: 'fa0e9127-7a2b-4b74-bbad-1bc9e717197b', + 2020: '46f5665e-d2e1-4270-95a1-970da3d70d32', + 2021: 'a7e2919d-f084-4c35-8b21-e960a854e2bd', + 2022: '31484cee-43fe-45d4-a546-209e5898c6dd', + 2023: 'e213892a-afac-4d34-83e9-45e8a324e7e8', + }), + 'industry-by-industry': _circabc_links('ind-by-ind', { + 2010: 'fc80f855-d144-476e-b4bf-5cfba946819c', + 2011: '1bcb2624-04ed-43e1-8588-df6680ed352a', + 2012: '399671ad-cbb3-493e-ad5f-83e989f1eecc', + 2013: 'a2b4746d-1d11-4a44-ab1c-50ac956f0849', + 2014: 'beba57b2-2696-497a-b92f-2a5beca724c7', + 2015: '1a194b8c-6ea1-4bec-9c73-0cd599febcc3', + 2016: '2cdd74fc-0bce-4ae0-8bf2-34d34546d86d', + 2017: '4a11c796-4186-4cce-a02f-12d353fc5e59', + 2018: '7a57a374-2200-498c-bb5f-cee24202b0b8', + 2019: 'c3467617-8a00-44a0-9b6b-ccad8a2ab58d', + 2020: '4df668e1-2a8a-4e84-ae57-00309d8bc760', + 2021: '6736dea8-da14-450f-b212-a791baf238c8', + 2022: 'b20c339d-984f-413c-a499-54ff76beb90c', + 2023: '21557f49-1e94-431c-8523-d972fec020b8', + })}, + 'regex_id': r'matrix_eu-ic-io_[a-z-]+_25ed_[0-9]{4}', + 'num_regions': 50, + 'num_sectors': 64}, }, 'exiobase': {'3.81': diff --git a/iopy/core/figaro.py b/iopy/core/figaro.py index 2d0f840..3613082 100644 --- a/iopy/core/figaro.py +++ b/iopy/core/figaro.py @@ -46,8 +46,8 @@ def __init__(self, """ Args: - version: Edition (year of publication), e.g. '2022' - year: Year from 2010 to 2020 + version: Edition (year of publication), e.g. '2025' + year: Year; availability depends on the edition (e.g. 2010-2023 for '2025') kind: industry-by-industry (default) or product-by-product refresh: Download the data even if it exists on the hard drive """ diff --git a/iopy/tests/test_figaro.py b/iopy/tests/test_figaro.py index 03e2ae9..021db1c 100644 --- a/iopy/tests/test_figaro.py +++ b/iopy/tests/test_figaro.py @@ -9,7 +9,7 @@ import matplotlib import matplotlib.pyplot as plt -f = Figaro(version='2022', year=2018, kind='industry-by-industry') +f = Figaro(version='2025', year=2018, kind='industry-by-industry') custom_shock_vector = np.random.uniform(size=f.rs, low=-10, high=10).reshape(-1, 1) @@ -19,12 +19,12 @@ class TestFigaro: def test_download(self): - Figaro(version='2022', year=2018, refresh=True) + Figaro(version='2025', year=2018, refresh=True) def test_load(self): - fi = Figaro(version='2022', year=2018, kind='industry-by-industry') + fi = Figaro(version='2025', year=2018, kind='industry-by-industry') assert set(fi.sectors).issubset(fi.sector_name_mapping) - fi = Figaro(version='2022', year=2018, kind='product-by-product') + fi = Figaro(version='2025', year=2018, kind='product-by-product') assert set(fi.sectors).issubset(fi.sector_name_mapping) def test_matrices(self): @@ -107,19 +107,19 @@ def test_get_imports_exports(self): export_regions='AU', import_sectors=None, export_sectors=None, - use_type='intermediate'), 86280.8749) + use_type='intermediate'), 72202.248) assert np.isclose(f.get_imports_exports(import_regions='CN', export_regions='AU', import_sectors=None, export_sectors=None, - use_type='final'), 13770.405) + use_type='final'), 8212.555) assert np.isclose(f.get_imports_exports(import_regions=['CN'], export_regions='AU', import_sectors=None, export_sectors=None, - use_type='both'), 86280.8749 + 13770.405) + use_type='both'), 72202.248 + 8212.555) f.get_imports_exports(import_regions=['CN'], export_regions='AU', From 6b66ee52b35b601189e7a88b042b854d15aac0b0 Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Fri, 5 Jun 2026 21:58:58 +0200 Subject: [PATCH 05/15] Removed figaro 2022 --- iopy/core/config.py | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/iopy/core/config.py b/iopy/core/config.py index 4f685dd..1867450 100644 --- a/iopy/core/config.py +++ b/iopy/core/config.py @@ -89,38 +89,7 @@ def _circabc_links(kind_tag, ids): }, 'figaro': - {'2022': - {'links': - {'product-by-product': { - 2010: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2010.csv/bc2b60d9-32f2-1c80-56e4-f839ca2f06fc?t=1655180575504', - 2011: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2011.csv/74e6f603-1740-116c-3469-4db7ebe97aa2?t=1655182400093', - 2012: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2012.csv/17b1bf95-daa9-a506-9d8b-932edf8f3950?t=1655184005469', - 2013: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2013.csv/d32be770-9f71-e28c-e8ac-11a04d8d885c?t=1655185487140', - 2014: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2014.csv/815e7a8e-a1c7-89f4-e621-ce3971e79c4b?t=1655186769271', - 2015: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2015.csv/3c69e27a-8f45-29e9-cdef-59d8a402e927?t=1655186809149', - 2016: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2016.csv/f82150d8-74b3-b0a3-4764-567eab545b5a?t=1655188856261', - 2017: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2017.csv/7ec909a5-0faa-e45d-2173-43f6c1b59689?t=1655188931368', - 2018: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2018.csv/c44abd15-b354-6690-24dc-f186fe147bb2?t=1655196163454', - 2019: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2019.csv/0ab0c866-8dd8-92d6-1ac7-51ffe78fd5dc?t=1655196217023', - 2020: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_prod-by-prod_2020.csv/3b838d62-884e-c1dc-9693-314b6460af0d?t=1655196273194', - }, - 'industry-by-industry': { - 2010: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2010.csv/3db6f05f-8343-a45e-2012-14db4607716b?t=1655180547507', - 2011: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2011.csv/5ce2f4ab-f4c1-e1dd-2aac-68813c714414?t=1655182368997', - 2012: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2012.csv/a1ad1f56-3a20-fac1-ee79-9a80b6d25a46?t=1655183982037', - 2013: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2013.csv/eec78482-56e2-053e-5486-15061be568e6?t=1655185412542', - 2014: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2014.csv/35eb67c4-0550-f2ae-b827-6ac240b5f877?t=1655186676930', - 2015: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2015.csv/af93fe1f-a9fd-c094-dd18-a15d390e0f9c?t=1655186725710', - 2016: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2016.csv/8322aa36-8d5c-5d52-07e5-e3d527875d06?t=1655188541728', - 2017: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2017.csv/30ad9b86-689f-d5a1-c80c-6765ea0f5421?t=1655188598849', - 2018: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2018.csv/a0369f43-bf98-a362-9054-f7075cc01704?t=1655195970852', - 2019: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2019.csv/c1354cbb-b3e6-6da5-780a-5be3319ad6d4?t=1655196037947', - 2020: 'https://ec.europa.eu/eurostat/documents/51957/12789261/matrix_eu-ic-io_ind-by-ind_2020.csv/6dc6df43-0d95-856c-897f-18ba2ca053f1?t=1655196109971', - } - }, - 'regex_id': r'[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}', - 'num_regions': 46, - 'num_sectors': 64}, + { '2025': {'links': {'product-by-product': _circabc_links('prod-by-prod', { From ba0a542c4c8bf72015e3414d8707dfa3ba76caa4 Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Fri, 5 Jun 2026 22:08:32 +0200 Subject: [PATCH 06/15] Added proxy and verify --- iopy/core/exiobase.py | 11 +++++++++-- iopy/core/figaro.py | 11 +++++++++-- iopy/core/matrix.py | 16 ++++++++++++++++ iopy/core/oecd.py | 11 +++++++++-- iopy/core/utils.py | 18 ++++++++++++++++-- 5 files changed, 59 insertions(+), 8 deletions(-) diff --git a/iopy/core/exiobase.py b/iopy/core/exiobase.py index 8314655..6631fb1 100644 --- a/iopy/core/exiobase.py +++ b/iopy/core/exiobase.py @@ -33,7 +33,9 @@ def __init__(self, version: str, year: int, kind: str = 'industry-by-industry', - refresh: bool = False): + refresh: bool = False, + proxy=None, + verify=True): """ Args: @@ -41,6 +43,9 @@ def __init__(self, year: Year from 1995 to 2022 kind: industry-by-industry (default) or product-by-product refresh: Download the data even if it exists on the hard drive + proxy: Optional proxy for downloading; a URL string (applied to http and + https) or a ``{scheme: url}`` dict + verify: Verify the server's TLS certificate (``False`` to skip, or a CA bundle path) """ assert kind in {'industry-by-industry', 'product-by-product'} @@ -57,6 +62,8 @@ def __init__(self, self.version = version self.year = year self.kind = kind + self._proxy = proxy + self._verify = verify self._url = config['exiobase'][version]['links'][kind][year] self._file_id = re.search(config['exiobase'][version]['regex_id'], self._url).group(0) self._data_file = os.path.join(DATA_FOLDER, self._file_id + '.zip') @@ -171,7 +178,7 @@ def _load_data(self): def _download_data(self): try: - download_file(self._url, self._data_file) + download_file(self._url, self._data_file, proxy=self._proxy, verify=self._verify) with open(FILES_LOG, 'a') as files_log: files_log.write(db_name + ';' + self._data_file + '\n') except Exception as e: diff --git a/iopy/core/figaro.py b/iopy/core/figaro.py index 3613082..6f4f870 100644 --- a/iopy/core/figaro.py +++ b/iopy/core/figaro.py @@ -42,7 +42,9 @@ def __init__(self, version: str, year: int, kind='industry-by-industry', - refresh: bool = False): + refresh: bool = False, + proxy=None, + verify=True): """ Args: @@ -50,6 +52,9 @@ def __init__(self, year: Year; availability depends on the edition (e.g. 2010-2023 for '2025') kind: industry-by-industry (default) or product-by-product refresh: Download the data even if it exists on the hard drive + proxy: Optional proxy for downloading; a URL string (applied to http and + https) or a ``{scheme: url}`` dict + verify: Verify the server's TLS certificate (``False`` to skip, or a CA bundle path) """ assert kind in {'industry-by-industry', 'product-by-product'} @@ -65,6 +70,8 @@ def __init__(self, self.version = version self.year = year self.kind = kind + self._proxy = proxy + self._verify = verify self._url = config['figaro'][version]['links'][kind][year] self._file_id = re.search(config['figaro'][version]['regex_id'], self._url).group(0) self._data_file = os.path.join(DATA_FOLDER, self._file_id + '.csv') @@ -140,7 +147,7 @@ def _load_data(self): def _download_data(self): try: - download_file(self._url, self._data_file) + download_file(self._url, self._data_file, proxy=self._proxy, verify=self._verify) with open(FILES_LOG, 'a') as files_log: files_log.write(db_name + ';' + self._data_file + '\n') except Exception as e: diff --git a/iopy/core/matrix.py b/iopy/core/matrix.py index c818b74..5dbd21e 100644 --- a/iopy/core/matrix.py +++ b/iopy/core/matrix.py @@ -4,6 +4,7 @@ """ import numpy as np +import pandas as pd class Matrix(np.ndarray): @@ -68,3 +69,18 @@ def to_numpy(self): numpy array """ return np.asarray(self) + + def to_pandas(self): + """Convert to pandas DataFrame + + Returns: + pandas DataFrame + """ + def _to_index(labels): + if len(labels) and isinstance(labels[0], tuple): + return pd.MultiIndex.from_tuples(labels, names=['region', 'sector']) + return pd.Index(labels) + + return pd.DataFrame(np.asarray(self), + index=_to_index(self.rows), + columns=_to_index(self.columns)) diff --git a/iopy/core/oecd.py b/iopy/core/oecd.py index 36cddf1..f6c8077 100644 --- a/iopy/core/oecd.py +++ b/iopy/core/oecd.py @@ -45,13 +45,18 @@ class OECD(IO): def __init__(self, version: str, year: int, - refresh: bool = False): + refresh: bool = False, + proxy=None, + verify=True): """ Args: version: Publication version of the data; '2021', '2022-small' or '2022-extended' year: Year refresh: Download the data even if it exists on the hard drive + proxy: Optional proxy for downloading; a URL string (applied to http and + https) or a ``{scheme: url}`` dict + verify: Verify the server's TLS certificate (``False`` to skip, or a CA bundle path) """ if version not in config['oecd'].keys(): @@ -65,6 +70,8 @@ def __init__(self, self.year = year self.version = version + self._proxy = proxy + self._verify = verify self._url = config['oecd'][version]['links'][year] self._file_id = re.search(config['oecd'][version]['regex_id'], self._url).group(0).replace('/', '_') self._data_file = os.path.join(DATA_FOLDER, self._file_id + '.zip') @@ -159,7 +166,7 @@ def _load_data(self): def _download_data(self): try: - download_file(self._url, self._data_file) + download_file(self._url, self._data_file, proxy=self._proxy, verify=self._verify) with open(FILES_LOG, 'a') as files_log: files_log.write(db_name + ';' + self._data_file + '\n') except Exception as e: diff --git a/iopy/core/utils.py b/iopy/core/utils.py index 3cd103b..8a470ba 100644 --- a/iopy/core/utils.py +++ b/iopy/core/utils.py @@ -13,7 +13,7 @@ def assert_is_subset(subset, superset): raise ValueError(f'Not found: {set(subset).difference(superset)}') -def download_file(url, dest): +def download_file(url, dest, proxy=None, verify=True): """Stream-download ``url`` to the local path ``dest``. Uses ``curl_cffi`` with browser (TLS) impersonation so downloads succeed even @@ -21,10 +21,24 @@ def download_file(url, dest): (``webfs-sti.oecd.org``) rejects plain ``requests``/``urllib`` traffic with HTTP 403; impersonating a real browser passes the challenge and is harmless for the non-protected Eurostat/EXIOBASE hosts. + + Args: + url: Source URL. + dest: Local file path to write to. + proxy: Optional proxy. Either a single URL string (e.g. + ``'http://user:pass@host:port'``) applied to both http and https, + or a ``{scheme: url}`` dict passed straight through. + verify: Verify the server's TLS certificate. Set to ``False`` to skip + verification (e.g. behind a TLS-intercepting proxy), or pass a + path to a CA bundle. """ from curl_cffi import requests - r = requests.get(url, stream=True, impersonate='chrome') + kwargs = {} + if proxy is not None: + kwargs['proxies'] = {'http': proxy, 'https': proxy} if isinstance(proxy, str) else proxy + + r = requests.get(url, stream=True, impersonate='chrome', verify=verify, **kwargs) if not r.ok: raise ConnectionError(r.reason or f'HTTP {r.status_code}') with open(dest, 'wb') as f: From 057dc615dd9636364a6926dc62fc5116fbae55a1 Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Fri, 5 Jun 2026 22:38:59 +0200 Subject: [PATCH 07/15] Renamed iopy to iotables --- .gitignore | 6 ++ CHANGELOG.MD | 9 ++- CLAUDE.md | 58 +++++++++++------- README.md | 18 +++--- iopy/.DS_Store | Bin 6148 -> 0 bytes iopy/__about__.py | 6 -- iopy/core/__init__.py | 7 --- iopy/core/globals.py | 10 --- iopy/dev-requirements.txt | 8 --- iopy/requirements.txt | 6 -- iopy/tests/.coveragerc | 5 -- iopy/tests/test_iopy.py | 15 ----- pyproject.toml | 37 +++++++++++ setup.py | 32 ---------- {iopy => src/iotables}/__init__.py | 23 ++++--- {iopy/core => src/iotables}/base_io.py | 4 +- {iopy/core => src/iotables}/config.py | 0 {iopy/core => src/iotables}/exiobase.py | 10 +-- {iopy/core => src/iotables}/figaro.py | 12 ++-- src/iotables/globals.py | 14 +++++ {iopy/core => src/iotables}/mappings.py | 0 {iopy/core => src/iotables}/matrix.py | 0 {iopy/core => src/iotables}/oecd.py | 14 ++--- {iopy/core => src/iotables}/utils.py | 2 +- test.py | 2 +- {iopy/tests => tests}/__init__.py | 0 {iopy/tests => tests}/results/__init__.py | 0 .../results/adjust_source_in_covxml.py | 2 +- {iopy/tests => tests}/test_exiobase.py | 2 +- {iopy/tests => tests}/test_figaro.py | 2 +- tests/test_iotables.py | 15 +++++ {iopy/tests => tests}/test_matrix.py | 4 +- {iopy/tests => tests}/test_oecd.py | 2 +- 33 files changed, 166 insertions(+), 159 deletions(-) delete mode 100644 iopy/.DS_Store delete mode 100644 iopy/__about__.py delete mode 100644 iopy/core/__init__.py delete mode 100644 iopy/core/globals.py delete mode 100644 iopy/dev-requirements.txt delete mode 100644 iopy/requirements.txt delete mode 100644 iopy/tests/.coveragerc delete mode 100644 iopy/tests/test_iopy.py create mode 100644 pyproject.toml delete mode 100644 setup.py rename {iopy => src/iotables}/__init__.py (56%) rename {iopy/core => src/iotables}/base_io.py (99%) rename {iopy/core => src/iotables}/config.py (100%) rename {iopy/core => src/iotables}/exiobase.py (97%) rename {iopy/core => src/iotables}/figaro.py (95%) create mode 100644 src/iotables/globals.py rename {iopy/core => src/iotables}/mappings.py (100%) rename {iopy/core => src/iotables}/matrix.py (100%) rename {iopy/core => src/iotables}/oecd.py (95%) rename {iopy/core => src/iotables}/utils.py (99%) rename {iopy/tests => tests}/__init__.py (100%) rename {iopy/tests => tests}/results/__init__.py (100%) rename {iopy/tests => tests}/results/adjust_source_in_covxml.py (83%) rename {iopy/tests => tests}/test_exiobase.py (93%) rename {iopy/tests => tests}/test_figaro.py (99%) create mode 100644 tests/test_iotables.py rename {iopy/tests => tests}/test_matrix.py (96%) rename {iopy/tests => tests}/test_oecd.py (99%) diff --git a/.gitignore b/.gitignore index f58a6e3..8492abb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,9 @@ **/__pycache__/ **/.temp_data/ test.py + +# packaging build artifacts +build/ +dist/ +*.egg-info/ +src/*.egg-info/ diff --git a/CHANGELOG.MD b/CHANGELOG.MD index e55e8da..f582eed 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -8,9 +8,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added - Introduce structure for different dataset-specific tools -- ... +- `proxy` and `verify` options on the loaders for downloading behind a proxy / custom TLS +- `Matrix.to_pandas()` to convert a matrix to a labelled DataFrame +- OECD ICIO 2025 edition (`2025-regular`, `2025-extended`) and Eurostat Figaro 2025 edition (`2025`, years 2010-2023) ### Changed +- Renamed the package from `iopy` to `iotables` and adopted a `src/` layout with `pyproject.toml` +- Downloads now use `curl_cffi` (browser TLS impersonation) so Cloudflare-protected OECD files work +- Re-pointed OECD/Figaro download URLs to current hosts (OECD `webfs`, Figaro CIRCABC) +- Cached downloads now live in a user cache dir (`~/.cache/iotables`, override with `IOTABLES_DATA`) ### Fixed +- OECD `2025-regular` file-id regex and in-zip CSV filenames; cross-edition cache-id collisions ___ diff --git a/CLAUDE.md b/CLAUDE.md index 3b047e2..091818f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,39 +4,42 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What this is -`iopy` is a Python package for loading inter-country input-output (IO) data from three +`iotables` is a Python package for loading inter-country input-output (IO) data from three public databases (OECD ICIO, Eurostat Figaro, EXIOBASE) and running economic analyses on them — primarily Leontief demand shocks and Ghosh supply shocks. See [README.md](README.md) for the economic theory (Leontief/Ghosh derivations) and the full public API table. +The package uses a `src/` layout (`src/iotables/`) with packaging configured in +[pyproject.toml](pyproject.toml); tests live in the top-level [tests/](tests/) directory. + ## Commands ```bash -# Install for development (editable) -pip install -e . -pip install -r iopy/dev-requirements.txt # runtime deps + pytest, pytest-cov +# Install for development (editable, with test deps) +pip install -e ".[dev]" # Run the full test suite -pytest iopy/tests +pytest # Run a single test file / test -pytest iopy/tests/test_oecd.py -pytest iopy/tests/test_oecd.py::TestOECD::test_leontief +pytest tests/test_oecd.py +pytest tests/test_oecd.py::TestOECD::test_leontief -# Coverage (config in iopy/tests/.coveragerc) -pytest --cov=iopy iopy/tests +# Coverage (config in pyproject.toml [tool.coverage]) +pytest --cov=iotables ``` Note: the tests **download real data over the network** (tens to hundreds of MB) and instantiate the loader classes, so they are slow and require internet. Downloaded files are -cached under `iopy/.temp_data/`, so reruns are faster. `test_download` forces a re-download -with `refresh=True`. +cached under a user cache dir (`~/.cache/iotables`, override with the `IOTABLES_DATA` +environment variable), so reruns are faster. `test_download` forces a re-download with +`refresh=True`. ## Architecture The core abstraction is a **two-layer design**: a shared analysis engine and per-database loaders. -### `IO` base class — [iopy/core/base_io.py](iopy/core/base_io.py) +### `IO` base class — [src/iotables/base_io.py](src/iotables/base_io.py) The engine. Its `__init__` asserts that the subclass has already set a fixed contract of attributes (`Z`, `X`, `V`, `FD`, `FD_REGION`, `ADD`, `rs`, `regions`, `sectors`, `sector_name_mapping`, `unit`, `demand_items`), then derives the coefficient and inverse @@ -49,8 +52,8 @@ It provides the public methods `leontief_demand_shock`, `ghosh_supply_shock`, an Zeros in `X` are replaced with 1 before division so the matrices stay invertible. ### Loader classes — `OECD`, `Figaro`, `ExioBase` -[oecd.py](iopy/core/oecd.py), [figaro.py](iopy/core/figaro.py), -[exiobase.py](iopy/core/exiobase.py). Each subclasses `IO` and follows the **same lifecycle** +[oecd.py](src/iotables/oecd.py), [figaro.py](src/iotables/figaro.py), +[exiobase.py](src/iotables/exiobase.py). Each subclasses `IO` and follows the **same lifecycle** inside `__init__`: 1. Validate `version`/`year`/`kind` against `config`. 2. Resolve the download URL from `config`, extract a `_file_id` via the version's `regex_id`, @@ -61,36 +64,45 @@ inside `__init__`: When adding a new data version or year, the change is almost always **data-only**: add the download links and `num_regions`/`num_sectors`/`regex_id` to -[iopy/core/config.py](iopy/core/config.py). Each loader has a module-level `process_df` +[src/iotables/config.py](src/iotables/config.py). Each loader has a module-level `process_df` helper that converts the raw dataframe index/columns into `(region, sector)` tuples — the three differ because each source formats its labels differently (OECD maps ISO alpha-3 → alpha-2 via `ALPHA3_TO_ALPHA2`; EXIOBASE uses MultiIndex columns). -### `Matrix` — [iopy/core/matrix.py](iopy/core/matrix.py) +### `Matrix` — [src/iotables/matrix.py](src/iotables/matrix.py) A thin `numpy.ndarray` subclass carrying `info` (label), `rows`, and `columns` metadata (lists of `(region, sector)` tuples), plus `.I` (inverse) and `.T` (transpose that also swaps row/column labels). All IO matrices are `Matrix` instances; row/column selection in `get_imports_exports` relies on these label lists. ### Supporting modules -- [config.py](iopy/core/config.py) — nested dict of `{database: {version: {links, regex_id, +- [config.py](src/iotables/config.py) — nested dict of `{database: {version: {links, regex_id, num_regions, num_sectors}}}`. The single source of truth for what data exists. -- [mappings.py](iopy/core/mappings.py) — sector-code → human-name dicts and demand-item lists +- [mappings.py](src/iotables/mappings.py) — sector-code → human-name dicts and demand-item lists per database; also the OECD 2022→2021 sector-code remapping. -- [globals.py](iopy/core/globals.py) — `DATA_FOLDER` (`iopy/.temp_data`), `FILES_LOG`, +- [globals.py](src/iotables/globals.py) — `DATA_FOLDER` (`~/.cache/iotables`, `IOTABLES_DATA` override), `FILES_LOG`, `IS_WINDOWS`. -- [utils.py](iopy/core/utils.py) — `assert_is_subset` (the standard validation used - throughout), `remove_downloaded_files`, and the `ALPHA3_TO_ALPHA2` country-code map. +- [utils.py](src/iotables/utils.py) — `assert_is_subset` (the standard validation used + throughout), `remove_downloaded_files`, the `ALPHA3_TO_ALPHA2` country-code map, and + `download_file` (the single shared downloader used by all three loaders). + +### Downloading +All three loaders download via `utils.download_file`, which uses **`curl_cffi`** with browser +TLS impersonation (`impersonate='chrome'`). This is required because the OECD file server +(`webfs-sti.oecd.org`) sits behind Cloudflare and rejects plain `requests`/`urllib` with HTTP +403; impersonation is harmless for the Eurostat (CIRCABC) and EXIOBASE (Zenodo) hosts. Each +loader's `__init__` accepts `proxy=` (URL string or `{scheme: url}` dict) and `verify=` +(TLS verification toggle / CA-bundle path), threaded through to `download_file`. ### Download cache and cleanup Downloaded archives live in `DATA_FOLDER` and every download appends a `db_name;path` line to `FILES_LOG` (`_files_log.txt`). `remove_downloaded_files(database=...)` reads that log to -delete files for one database or `'all'`. This is exposed as `iopy.remove_downloaded_files` +delete files for one database or `'all'`. This is exposed as `iotables.remove_downloaded_files` and as a `remove_downloaded_files` static method on each loader. ## Conventions -- The public surface is re-exported from [iopy/\_\_init\_\_.py](iopy/__init__.py): `OECD`, +- The public surface is re-exported from [src/iotables/\_\_init\_\_.py](src/iotables/__init__.py): `OECD`, `Figaro`, `ExioBase`, `remove_downloaded_files`, `get_size_data_folder`. Keep new user-facing entry points exported there. - Validation is done with `assert_is_subset` (raises `ValueError` listing what's missing) and diff --git a/README.md b/README.md index 1e223ce..b3e9a25 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# iopy +# iotables ## Description @@ -14,7 +14,7 @@ Currently the following databases are supported: ## Installation ```bash -pip install git+https://github.com/WWakker/iopy.git +pip install iotables ``` ## Structure of input-output data @@ -176,16 +176,16 @@ Creating an instance of the OECD class loads the OECD data and gives access to c An instance can be create as follows, specifiying a year between 1995 and 2018. ```python -import iopy -oecd = iopy.OECD(version='2021', year=2018, refresh=False) +import iotables +oecd = iotables.OECD(version='2021', year=2018, refresh=False) ``` Similary, an instance can be created for other data, for example Figaro and ExioBase as ```python -import iopy -figaro = iopy.Figaro(version='2022', year=2020, kind='industry-by-industry') -exio = iopy.ExioBase(version='3.81', year=2022, kind='industry-by-industry') +import iotables +figaro = iotables.Figaro(version='2025', year=2020, kind='industry-by-industry') +exio = iotables.ExioBase(version='3.81', year=2022, kind='industry-by-industry') ``` Creating an instance of a database class downloads and loads the data into memory, creates standard input-output matrices, and gives access to the following attributes and methods: @@ -223,9 +223,9 @@ All matrices are extended `numpy.ndarray`'s with attributes `info`, `rows` and ` When running a Leontief or Ghosh shock, the percentage shock to final demand/primary inputs in countries and sectors can be specified as ```python -import iopy +import iotables -oecd = iopy.OECD(version='2021', year=2018) +oecd = iotables.OECD(version='2021', year=2018) df_l = oecd.leontief_demand_shock(shock=-10, regions=['FR', 'DE], sectors=['01T02', '35']) df_g = oecd.ghosh_supply_shock(shock=-10, regions=['FR', 'DE'], sectors=['01T02', '35']) diff --git a/iopy/.DS_Store b/iopy/.DS_Store deleted file mode 100644 index c71720f927fd2fd7fe1ab504465f460efd4e9fca..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}&BV5dIc1LGi%BgBO#&fF!;_C3x`61E8hEkd#&_5WV&8!}xrj^qbiYp#@Dm zXrh@(X21QJnPtCeb{PO}uu3n19)J#2uyw>{kI1^{j3U(IU@^{GRJ#bHdb*nf3$)RtzVjJ_A1AWW^-Oii9@& z%G^1z|E|Iw&{fMBa?HN)dTDsIG2uQ7OzY8c)v!C|=Qzb07h;P?{<4|&<{ahRuhgSI zVOfvk-WqXw-}7$?E8C|$RDw&{vAz>t8X8(z=i?2 zA0nz?9I*1JR|gv{0f=o*tFSM%g!n`O;f47m1iG#v0q;ceYH woV>LW^_D6kc9qAC3p@H0(^j71GpY*bBk2(1fR#tiQ0$L@(qN4-@T&}b0BA{aT>t<8 diff --git a/iopy/__about__.py b/iopy/__about__.py deleted file mode 100644 index caf4018..0000000 --- a/iopy/__about__.py +++ /dev/null @@ -1,6 +0,0 @@ -__title__ = "iopy" -__about__ = "A Python package to easily load and work with input-output data" -__version__ = '0.0.1' -__authors__ = 'Wouter Wakker' -__email__ = "wouter.wakker@outlook.com" -__url__ = "" diff --git a/iopy/core/__init__.py b/iopy/core/__init__.py deleted file mode 100644 index 07a74f5..0000000 --- a/iopy/core/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -""" Created on 22/11/2022:: -------------- __init__.py ------------- -**Authors**: W. Wakker - -""" - -from iopy.core.globals import * diff --git a/iopy/core/globals.py b/iopy/core/globals.py deleted file mode 100644 index 1be3caf..0000000 --- a/iopy/core/globals.py +++ /dev/null @@ -1,10 +0,0 @@ -""" Created on 22/11/2022:: -------------- globals ------------- -**Authors**: W. Wakker - -""" -import os - -DATA_FOLDER = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.temp_data') -IS_WINDOWS = os.name == 'nt' -FILES_LOG = os.path.join(DATA_FOLDER, '_files_log.txt') diff --git a/iopy/dev-requirements.txt b/iopy/dev-requirements.txt deleted file mode 100644 index 25c21b6..0000000 --- a/iopy/dev-requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -requests -curl_cffi -numpy -pandas>=1.1.5 -matplotlib -tqdm -pytest -pytest-cov diff --git a/iopy/requirements.txt b/iopy/requirements.txt deleted file mode 100644 index 3c710c3..0000000 --- a/iopy/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -requests -curl_cffi -numpy -pandas>=1.1.5 -matplotlib -tqdm diff --git a/iopy/tests/.coveragerc b/iopy/tests/.coveragerc deleted file mode 100644 index 2a8a768..0000000 --- a/iopy/tests/.coveragerc +++ /dev/null @@ -1,5 +0,0 @@ -[run] -omit = res/* - **/__about__.py - **/adjust_source_in_covxml.py - **/update_badge.py diff --git a/iopy/tests/test_iopy.py b/iopy/tests/test_iopy.py deleted file mode 100644 index 004c5e6..0000000 --- a/iopy/tests/test_iopy.py +++ /dev/null @@ -1,15 +0,0 @@ -""" Created on 22/11/2022:: -------------- test_iopy ------------- -**Authors**: W. Wakker - -""" -import iopy - - -class TestIopy: - - def test1(self): - iopy.remove_downloaded_files(database='figaro') - - def test_folder_size(self): - assert isinstance(iopy.get_size_data_folder(), str) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7e462b9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,37 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "iotables" +version = "0.0.1" +description = "A Python package to easily load and work with inter-country input-output data" +readme = "README.md" +license = { file = "LICENSE" } +authors = [{ name = "Wouter Wakker", email = "wouter.wakker@outlook.com" }] +requires-python = ">=3.8" +dependencies = [ + "curl_cffi", + "numpy", + "pandas>=1.1.5", + "matplotlib", + "tqdm", +] +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + "License :: OSI Approved :: MIT License", +] + +[project.optional-dependencies] +dev = ["pytest", "pytest-cov"] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.coverage.run] +source = ["iotables"] +omit = ["**/adjust_source_in_covxml.py", "**/update_badge.py"] diff --git a/setup.py b/setup.py deleted file mode 100644 index 2e11119..0000000 --- a/setup.py +++ /dev/null @@ -1,32 +0,0 @@ -import setuptools - -with open("README.md", "r") as f: - long_description = f.read() - -about = {} -with open("iopy/__about__.py") as f: - exec(f.read(), about) - -with open("iopy/requirements.txt") as f: - requirements = f.read().splitlines() - -setuptools.setup( - name="iopy", - version=about['__version__'], - author=about['__authors__'], - author_email=about['__email__'], - description=about['__about__'], - url=about['__url__'], - license='MIT', - long_description=long_description, - long_description_content_type="markdown", - packages=setuptools.find_packages(), - install_requires=requirements, - classifiers=[ - "Programming Language :: Python :: 3", - "Operating System :: OS Independent", - ], - python_requires='>=3.6', - include_package_data=False, - package_data={}, -) diff --git a/iopy/__init__.py b/src/iotables/__init__.py similarity index 56% rename from iopy/__init__.py rename to src/iotables/__init__.py index aa7b0fd..1e7c8b2 100644 --- a/iopy/__init__.py +++ b/src/iotables/__init__.py @@ -1,11 +1,17 @@ import os -from iopy.core.globals import DATA_FOLDER as __DATA_FOLDER -from iopy.core.globals import IS_WINDOWS as __IS_WINDOWS -from iopy.core.globals import FILES_LOG as __FILES_LOG -from iopy.core.oecd import OECD -from iopy.core.figaro import Figaro -from iopy.core.exiobase import ExioBase -from iopy.core.utils import remove_downloaded_files +from importlib.metadata import version as _version, PackageNotFoundError as _PackageNotFoundError +from iotables.globals import DATA_FOLDER as __DATA_FOLDER +from iotables.globals import IS_WINDOWS as __IS_WINDOWS +from iotables.globals import FILES_LOG as __FILES_LOG +from iotables.oecd import OECD +from iotables.figaro import Figaro +from iotables.exiobase import ExioBase +from iotables.utils import remove_downloaded_files + +try: + __version__ = _version("iotables") +except _PackageNotFoundError: # package not installed (e.g. running from source tree) + __version__ = "0.0.0" def get_size_data_folder(): @@ -34,7 +40,6 @@ def human(size): return get_size(__DATA_FOLDER) -if not os.path.exists(__DATA_FOLDER): - os.mkdir(__DATA_FOLDER) +os.makedirs(__DATA_FOLDER, exist_ok=True) del os diff --git a/iopy/core/base_io.py b/src/iotables/base_io.py similarity index 99% rename from iopy/core/base_io.py rename to src/iotables/base_io.py index 6549b00..de177a8 100644 --- a/iopy/core/base_io.py +++ b/src/iotables/base_io.py @@ -4,8 +4,8 @@ """ from warnings import warn -from iopy.core.matrix import Matrix -from iopy.core.utils import assert_is_subset +from iotables.matrix import Matrix +from iotables.utils import assert_is_subset import matplotlib.pyplot as plt from typing import Union, Iterable, Optional import numpy as np diff --git a/iopy/core/config.py b/src/iotables/config.py similarity index 100% rename from iopy/core/config.py rename to src/iotables/config.py diff --git a/iopy/core/exiobase.py b/src/iotables/exiobase.py similarity index 97% rename from iopy/core/exiobase.py rename to src/iotables/exiobase.py index 6631fb1..5ac2435 100644 --- a/iopy/core/exiobase.py +++ b/src/iotables/exiobase.py @@ -3,7 +3,7 @@ **Authors**: S. Boldrini """ -from iopy.core.matrix import Matrix +from iotables.matrix import Matrix from functools import lru_cache import numpy as np import pandas as pd @@ -11,11 +11,11 @@ from zipfile import ZipFile import re import os -from iopy.core.config import config -from iopy.core.base_io import IO +from iotables.config import config +from iotables.base_io import IO from warnings import warn -from iopy.core.globals import DATA_FOLDER, FILES_LOG -from iopy.core.utils import remove_downloaded_files, download_file +from iotables.globals import DATA_FOLDER, FILES_LOG +from iotables.utils import remove_downloaded_files, download_file db_name = os.path.basename(__file__).rstrip('.py') diff --git a/iopy/core/figaro.py b/src/iotables/figaro.py similarity index 95% rename from iopy/core/figaro.py rename to src/iotables/figaro.py index 6f4f870..d63df0a 100644 --- a/iopy/core/figaro.py +++ b/src/iotables/figaro.py @@ -3,19 +3,19 @@ **Authors**: W. Wakker """ -from iopy.core.mappings import figaro_sector_name_mapping_pxp_2022, figaro_sector_name_mapping_ixi_2022, figaro_demand_items -from iopy.core.matrix import Matrix +from iotables.mappings import figaro_sector_name_mapping_pxp_2022, figaro_sector_name_mapping_ixi_2022, figaro_demand_items +from iotables.matrix import Matrix from functools import lru_cache import numpy as np import pandas as pd from tqdm import tqdm import re import os -from iopy.core.config import config +from iotables.config import config from warnings import warn -from iopy.core.base_io import IO -from iopy.core.globals import DATA_FOLDER, FILES_LOG -from iopy.core.utils import remove_downloaded_files, download_file +from iotables.base_io import IO +from iotables.globals import DATA_FOLDER, FILES_LOG +from iotables.utils import remove_downloaded_files, download_file db_name = os.path.basename(__file__).rstrip('.py') diff --git a/src/iotables/globals.py b/src/iotables/globals.py new file mode 100644 index 0000000..6f06481 --- /dev/null +++ b/src/iotables/globals.py @@ -0,0 +1,14 @@ +""" Created on 22/11/2022:: +------------- globals ------------- +**Authors**: W. Wakker + +""" +import os + +# Downloaded source files are cached in a user-level cache directory rather than inside +# the installed package. Override with the IOTABLES_DATA environment variable; otherwise +# fall back to XDG_CACHE_HOME (or ~/.cache) per platform convention. +_cache_root = os.environ.get('XDG_CACHE_HOME') or os.path.join(os.path.expanduser('~'), '.cache') +DATA_FOLDER = os.environ.get('IOTABLES_DATA', os.path.join(_cache_root, 'iotables')) +IS_WINDOWS = os.name == 'nt' +FILES_LOG = os.path.join(DATA_FOLDER, '_files_log.txt') diff --git a/iopy/core/mappings.py b/src/iotables/mappings.py similarity index 100% rename from iopy/core/mappings.py rename to src/iotables/mappings.py diff --git a/iopy/core/matrix.py b/src/iotables/matrix.py similarity index 100% rename from iopy/core/matrix.py rename to src/iotables/matrix.py diff --git a/iopy/core/oecd.py b/src/iotables/oecd.py similarity index 95% rename from iopy/core/oecd.py rename to src/iotables/oecd.py index f6c8077..5991410 100644 --- a/iopy/core/oecd.py +++ b/src/iotables/oecd.py @@ -3,21 +3,21 @@ **Authors**: W. Wakker """ -from iopy.core.mappings import oecd_sector_name_mapping, oecd_sector_name_mapping_2025, oecd_demand_items, \ +from iotables.mappings import oecd_sector_name_mapping, oecd_sector_name_mapping_2025, oecd_demand_items, \ oecd_sector_2022_2021_mapping -from iopy.core.matrix import Matrix +from iotables.matrix import Matrix from functools import lru_cache import numpy as np import pandas as pd -from iopy.core.utils import ALPHA3_TO_ALPHA2 +from iotables.utils import ALPHA3_TO_ALPHA2 from tqdm import tqdm from zipfile import ZipFile import re import os -from iopy.core.config import config -from iopy.core.base_io import IO -from iopy.core.utils import replace_if_exists, remove_downloaded_files, download_file -from iopy.core.globals import DATA_FOLDER, FILES_LOG +from iotables.config import config +from iotables.base_io import IO +from iotables.utils import replace_if_exists, remove_downloaded_files, download_file +from iotables.globals import DATA_FOLDER, FILES_LOG from warnings import warn from functools import partial diff --git a/iopy/core/utils.py b/src/iotables/utils.py similarity index 99% rename from iopy/core/utils.py rename to src/iotables/utils.py index 8a470ba..915382d 100644 --- a/iopy/core/utils.py +++ b/src/iotables/utils.py @@ -3,7 +3,7 @@ **Authors**: W. Wakker """ -from iopy.core.globals import FILES_LOG +from iotables.globals import FILES_LOG from collections import defaultdict import os diff --git a/test.py b/test.py index dd633bc..06c3584 100644 --- a/test.py +++ b/test.py @@ -1,3 +1,3 @@ -from iopy import OECD +from iotables import OECD io = OECD(version='2025-extended', year=2022, refresh=True) diff --git a/iopy/tests/__init__.py b/tests/__init__.py similarity index 100% rename from iopy/tests/__init__.py rename to tests/__init__.py diff --git a/iopy/tests/results/__init__.py b/tests/results/__init__.py similarity index 100% rename from iopy/tests/results/__init__.py rename to tests/results/__init__.py diff --git a/iopy/tests/results/adjust_source_in_covxml.py b/tests/results/adjust_source_in_covxml.py similarity index 83% rename from iopy/tests/results/adjust_source_in_covxml.py rename to tests/results/adjust_source_in_covxml.py index 153a10f..5d6ea26 100644 --- a/iopy/tests/results/adjust_source_in_covxml.py +++ b/tests/results/adjust_source_in_covxml.py @@ -10,7 +10,7 @@ with open(f'{parent}/cov.xml', 'r+') as f: covxml = f.read() covxml = re.sub(r'.*', - r'/builds/.../.../iopy/iopy', + r'/builds/.../.../src/iotables', covxml) f.seek(0) f.write(covxml) diff --git a/iopy/tests/test_exiobase.py b/tests/test_exiobase.py similarity index 93% rename from iopy/tests/test_exiobase.py rename to tests/test_exiobase.py index 0815b05..d7841fd 100644 --- a/iopy/tests/test_exiobase.py +++ b/tests/test_exiobase.py @@ -3,7 +3,7 @@ **Authors**: W. Wakker """ -from iopy import ExioBase +from iotables import ExioBase class TestExioBase: diff --git a/iopy/tests/test_figaro.py b/tests/test_figaro.py similarity index 99% rename from iopy/tests/test_figaro.py rename to tests/test_figaro.py index 021db1c..6371edf 100644 --- a/iopy/tests/test_figaro.py +++ b/tests/test_figaro.py @@ -4,7 +4,7 @@ """ import pytest -from iopy import Figaro +from iotables import Figaro import numpy as np import matplotlib import matplotlib.pyplot as plt diff --git a/tests/test_iotables.py b/tests/test_iotables.py new file mode 100644 index 0000000..c0110eb --- /dev/null +++ b/tests/test_iotables.py @@ -0,0 +1,15 @@ +""" Created on 22/11/2022:: +------------- test_iotables ------------- +**Authors**: W. Wakker + +""" +import iotables + + +class TestIotables: + + def test1(self): + iotables.remove_downloaded_files(database='figaro') + + def test_folder_size(self): + assert isinstance(iotables.get_size_data_folder(), str) diff --git a/iopy/tests/test_matrix.py b/tests/test_matrix.py similarity index 96% rename from iopy/tests/test_matrix.py rename to tests/test_matrix.py index 3dd379a..9f3df9e 100644 --- a/iopy/tests/test_matrix.py +++ b/tests/test_matrix.py @@ -3,11 +3,11 @@ **Authors**: W. Wakker """ -from iopy.core.matrix import Matrix +from iotables.matrix import Matrix import pandas as pd import numpy as np import pytest -from iopy.core.utils import ALPHA3_TO_ALPHA2 +from iotables.utils import ALPHA3_TO_ALPHA2 def process_df(df): diff --git a/iopy/tests/test_oecd.py b/tests/test_oecd.py similarity index 99% rename from iopy/tests/test_oecd.py rename to tests/test_oecd.py index 927bdfb..8762427 100644 --- a/iopy/tests/test_oecd.py +++ b/tests/test_oecd.py @@ -4,7 +4,7 @@ """ import pytest -from iopy import OECD +from iotables import OECD import numpy as np import matplotlib import matplotlib.pyplot as plt From c2a2dff2244d823823215fa96150fa90085f87a8 Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Sat, 6 Jun 2026 13:44:06 +0200 Subject: [PATCH 08/15] Stop tracking .vscode and CLAUDE.md --- .vscode/settings.json | 4 -- CLAUDE.md | 112 ------------------------------------------ 2 files changed, 116 deletions(-) delete mode 100644 .vscode/settings.json delete mode 100644 CLAUDE.md diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 4b5a294..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "python-envs.defaultEnvManager": "ms-python.python:conda", - "python-envs.defaultPackageManager": "ms-python.python:conda" -} \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 091818f..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,112 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## What this is - -`iotables` is a Python package for loading inter-country input-output (IO) data from three -public databases (OECD ICIO, Eurostat Figaro, EXIOBASE) and running economic analyses on -them — primarily Leontief demand shocks and Ghosh supply shocks. See [README.md](README.md) -for the economic theory (Leontief/Ghosh derivations) and the full public API table. - -The package uses a `src/` layout (`src/iotables/`) with packaging configured in -[pyproject.toml](pyproject.toml); tests live in the top-level [tests/](tests/) directory. - -## Commands - -```bash -# Install for development (editable, with test deps) -pip install -e ".[dev]" - -# Run the full test suite -pytest - -# Run a single test file / test -pytest tests/test_oecd.py -pytest tests/test_oecd.py::TestOECD::test_leontief - -# Coverage (config in pyproject.toml [tool.coverage]) -pytest --cov=iotables -``` - -Note: the tests **download real data over the network** (tens to hundreds of MB) and -instantiate the loader classes, so they are slow and require internet. Downloaded files are -cached under a user cache dir (`~/.cache/iotables`, override with the `IOTABLES_DATA` -environment variable), so reruns are faster. `test_download` forces a re-download with -`refresh=True`. - -## Architecture - -The core abstraction is a **two-layer design**: a shared analysis engine and per-database loaders. - -### `IO` base class — [src/iotables/base_io.py](src/iotables/base_io.py) -The engine. Its `__init__` asserts that the subclass has already set a fixed contract of -attributes (`Z`, `X`, `V`, `FD`, `FD_REGION`, `ADD`, `rs`, `regions`, `sectors`, -`sector_name_mapping`, `unit`, `demand_items`), then derives the coefficient and inverse -matrices from them: -- `A` = technical coefficients (`Z / X`), `L` = Leontief inverse `(I − A)⁻¹` -- `B` = allocation coefficients, `G` = Ghosh inverse `(I − B)⁻¹` - -It provides the public methods `leontief_demand_shock`, `ghosh_supply_shock`, and -`get_imports_exports`, plus the private `_shock` / `_shock_to_df` / `_plot_shock` helpers. -Zeros in `X` are replaced with 1 before division so the matrices stay invertible. - -### Loader classes — `OECD`, `Figaro`, `ExioBase` -[oecd.py](src/iotables/oecd.py), [figaro.py](src/iotables/figaro.py), -[exiobase.py](src/iotables/exiobase.py). Each subclasses `IO` and follows the **same lifecycle** -inside `__init__`: -1. Validate `version`/`year`/`kind` against `config`. -2. Resolve the download URL from `config`, extract a `_file_id` via the version's `regex_id`, - download to `DATA_FOLDER` if not already cached (or `refresh=True`). -3. Load the raw file into `self.df`, slice it into the contract matrices (`Z`, `FD`, `X`, - `V`, `FD_REGION`, plus database-specific extras in the `ADD` dict). -4. Call `super().__init__()` **last** to build `A`/`L`/`B`/`G`. - -When adding a new data version or year, the change is almost always **data-only**: add the -download links and `num_regions`/`num_sectors`/`regex_id` to -[src/iotables/config.py](src/iotables/config.py). Each loader has a module-level `process_df` -helper that converts the raw dataframe index/columns into `(region, sector)` tuples — the -three differ because each source formats its labels differently (OECD maps ISO alpha-3 → -alpha-2 via `ALPHA3_TO_ALPHA2`; EXIOBASE uses MultiIndex columns). - -### `Matrix` — [src/iotables/matrix.py](src/iotables/matrix.py) -A thin `numpy.ndarray` subclass carrying `info` (label), `rows`, and `columns` metadata -(lists of `(region, sector)` tuples), plus `.I` (inverse) and `.T` (transpose that also swaps -row/column labels). All IO matrices are `Matrix` instances; row/column selection in -`get_imports_exports` relies on these label lists. - -### Supporting modules -- [config.py](src/iotables/config.py) — nested dict of `{database: {version: {links, regex_id, - num_regions, num_sectors}}}`. The single source of truth for what data exists. -- [mappings.py](src/iotables/mappings.py) — sector-code → human-name dicts and demand-item lists - per database; also the OECD 2022→2021 sector-code remapping. -- [globals.py](src/iotables/globals.py) — `DATA_FOLDER` (`~/.cache/iotables`, `IOTABLES_DATA` override), `FILES_LOG`, - `IS_WINDOWS`. -- [utils.py](src/iotables/utils.py) — `assert_is_subset` (the standard validation used - throughout), `remove_downloaded_files`, the `ALPHA3_TO_ALPHA2` country-code map, and - `download_file` (the single shared downloader used by all three loaders). - -### Downloading -All three loaders download via `utils.download_file`, which uses **`curl_cffi`** with browser -TLS impersonation (`impersonate='chrome'`). This is required because the OECD file server -(`webfs-sti.oecd.org`) sits behind Cloudflare and rejects plain `requests`/`urllib` with HTTP -403; impersonation is harmless for the Eurostat (CIRCABC) and EXIOBASE (Zenodo) hosts. Each -loader's `__init__` accepts `proxy=` (URL string or `{scheme: url}` dict) and `verify=` -(TLS verification toggle / CA-bundle path), threaded through to `download_file`. - -### Download cache and cleanup -Downloaded archives live in `DATA_FOLDER` and every download appends a `db_name;path` line to -`FILES_LOG` (`_files_log.txt`). `remove_downloaded_files(database=...)` reads that log to -delete files for one database or `'all'`. This is exposed as `iotables.remove_downloaded_files` -and as a `remove_downloaded_files` static method on each loader. - -## Conventions - -- The public surface is re-exported from [src/iotables/\_\_init\_\_.py](src/iotables/__init__.py): `OECD`, - `Figaro`, `ExioBase`, `remove_downloaded_files`, `get_size_data_folder`. Keep new - user-facing entry points exported there. -- Validation is done with `assert_is_subset` (raises `ValueError` listing what's missing) and - plain `assert` for argument-shape checks — match this style rather than introducing custom - exception types. -- Version strings are dict keys (e.g. `'2021'`, `'2022-extended'`, `'2025-regular'`), not - numbers; `kind` is `'industry-by-industry'` or `'product-by-product'` (OECD has no `kind`). From 8ff5473bc151a5786a6ee35f792ca5ef1b3d434c Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Sat, 6 Jun 2026 13:44:26 +0200 Subject: [PATCH 09/15] Add .vscode and CLAUDE.md to gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 8492abb..5e78aaa 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,7 @@ build/ dist/ *.egg-info/ src/*.egg-info/ + +# editor / tooling +.vscode/ +CLAUDE.md From 81c7cadef8a8da892eaf4f6c16ba47bb3ca8a003 Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Sat, 6 Jun 2026 13:46:38 +0200 Subject: [PATCH 10/15] Stop tracking test.py and .DS_Store --- .DS_Store | Bin 6148 -> 0 bytes .gitignore | 2 ++ test.py | 3 --- 3 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 .DS_Store delete mode 100644 test.py diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index e393e44e60e428aaa0bb5c3151f1dbb932e86b57..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKL2uJA6n<_4O&ODVVA3u~k+_c1G7TYd>AF^95(8;LZ~#=ArEC#dTs7$`R8`6u z{sVu3D}Mz33n%!V?T)71u7HqVWq;1|_ni21@>~;N%dP2`=pXw`Y>#zz~1^zV!_}jfnd-Rxg=_$V3-{0>cbq->A9K5xD|o>q9h&jJ2E#Z?vqs~)D3zUS*DG$tU31^~r*h(F{cN1J z`=eKU^<2t0n78}Ei)b+E)o$&{B?r^JawO8Am zPVet*Z+Q=!vn_AhY&3Szey}~8Ro#s{_Z}Vf-p6N&{HSI|0{KkYy1_ZH1{d=V)8jb6 z^Ks{e^V}aLu}n_j<3ZNvA|WLX=mfvJ@Fet(e5drRic}eCih2;>QxZ zN=IPsQil$KbvmQ47*XOUS{%PjBRo-BWQp5LESFNSG8$u{>#K`h`nJTcfQwO4U&2 Date: Sat, 6 Jun 2026 14:27:32 +0200 Subject: [PATCH 11/15] Bug fixes --- CHANGELOG.MD | 8 ++++++- README.md | 10 ++++---- pyproject.toml | 14 +++++------ src/iotables/base_io.py | 13 ++++++---- src/iotables/exiobase.py | 12 +++++----- src/iotables/figaro.py | 7 +++--- src/iotables/matrix.py | 6 +++-- src/iotables/oecd.py | 4 +--- src/iotables/utils.py | 19 +++++++++++---- tests/test_figaro.py | 2 +- tests/test_iotables.py | 51 ++++++++++++++++++++++++++++++++++++++++ tests/test_matrix.py | 2 +- tests/test_oecd.py | 2 +- 13 files changed, 111 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index f582eed..1901d03 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [Unreleased] - yyyy-mm-dd +## [0.1.0] - 2026-06-06 ### Added - Introduce structure for different dataset-specific tools @@ -16,8 +16,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Downloads now use `curl_cffi` (browser TLS impersonation) so Cloudflare-protected OECD files work - Re-pointed OECD/Figaro download URLs to current hosts (OECD `webfs`, Figaro CIRCABC) - Cached downloads now live in a user cache dir (`~/.cache/iotables`, override with `IOTABLES_DATA`) +- User-facing argument validation now raises `ValueError` instead of `assert` (so it survives `python -O`) ### Fixed - OECD `2025-regular` file-id regex and in-zip CSV filenames; cross-edition cache-id collisions +- `remove_downloaded_files(database=...)` deleted the wrong database's files +- `ExioBase` download-failure handler raised `AttributeError` instead of the intended warning +- Loader `db_name` derivation used `str.rstrip('.py')` (a character set, not a suffix) +- Dropped `lru_cache` on `_load_data`, which pinned every loader instance (and its data) in memory +- Downloads are now atomic (written to a `.part` file and moved into place on completion), so an interrupted download no longer leaves a truncated file in the cache ___ diff --git a/README.md b/README.md index b3e9a25..c91e530 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,9 @@ pip install iotables ## Structure of input-output data Input-data model the economy in matrix form. Here, we explain input-output data using OECD data as an example. -THE OECD input-output tables contain inputs and outputs in current million USD for 45 sectors and 66 countries as well -as rest-of-world. China and Mexico data are split into CN1, CN2, MX1, and MX2. +The OECD input-output tables contain inputs and outputs in current million USD for a set of sectors and countries (the +exact counts depend on the edition; see `config.py`) as well as rest-of-world. China and Mexico data are split into CN1, +CN2, MX1, and MX2. ![OECD sctructure](res/OECD_structure.png) *Source*: OECD @@ -173,7 +174,8 @@ This is the Ghosh equation, which defines the relation between production output ### Create an instance of the OECD class Creating an instance of the OECD class loads the OECD data and gives access to convenient attributes and methods. -An instance can be create as follows, specifiying a year between 1995 and 2018. +An instance can be created as follows, specifying an available year for the chosen version (e.g. 1995-2018 for the +`2021` version, up to 2022 for the `2025` versions). ```python import iotables @@ -227,7 +229,7 @@ import iotables oecd = iotables.OECD(version='2021', year=2018) -df_l = oecd.leontief_demand_shock(shock=-10, regions=['FR', 'DE], sectors=['01T02', '35']) +df_l = oecd.leontief_demand_shock(shock=-10, regions=['FR', 'DE'], sectors=['01T02', '35']) df_g = oecd.ghosh_supply_shock(shock=-10, regions=['FR', 'DE'], sectors=['01T02', '35']) ``` diff --git a/pyproject.toml b/pyproject.toml index 7e462b9..858c3be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,18 +4,18 @@ build-backend = "setuptools.build_meta" [project] name = "iotables" -version = "0.0.1" +version = "0.1.0" description = "A Python package to easily load and work with inter-country input-output data" readme = "README.md" license = { file = "LICENSE" } authors = [{ name = "Wouter Wakker", email = "wouter.wakker@outlook.com" }] -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ - "curl_cffi", - "numpy", - "pandas>=1.1.5", - "matplotlib", - "tqdm", + "curl_cffi>=0.5", + "numpy>=1.21,<3", + "pandas>=1.3", + "matplotlib>=3.4", + "tqdm>=4.0", ] classifiers = [ "Programming Language :: Python :: 3", diff --git a/src/iotables/base_io.py b/src/iotables/base_io.py index de177a8..4dda7b6 100644 --- a/src/iotables/base_io.py +++ b/src/iotables/base_io.py @@ -53,7 +53,7 @@ def __init__(self): self.Z.rows, self.Z.columns) - self.G = Matrix('Output inverse', + self.G = Matrix('Ghosh inverse', (np.eye(self.rs) - self.B).I, self.Z.rows, self.Z.columns) @@ -80,7 +80,8 @@ def _shock(self, if custom_shock_vector is not None: shock_vector = np.array(custom_shock_vector).reshape(self.rs, 1) else: - assert shock and regions and sectors, "Must supply parameters: 'shock', 'regions', 'sectors'" + if shock is None or regions is None or sectors is None: + raise ValueError("Must supply parameters: 'shock', 'regions', 'sectors'") assert_is_subset(regions, self.regions) assert_is_subset(sectors, self.sectors) @@ -171,7 +172,8 @@ def _plot_shock(self, Returns: fig, ax """ - assert by in {'region', 'sector'}, "plot_by must be 'region' or 'sector'" + if by not in {'region', 'sector'}: + raise ValueError("plot_by must be 'region' or 'sector'") assert_is_subset(regions, self.regions) df = self._shock_to_df(x_new) @@ -255,7 +257,7 @@ def ghosh_supply_shock(self, """Executes a Ghosh supply shock Args: - shock: Shock in percentage of original demand + shock: Shock in percentage of original primary inputs (value added) regions: List of regions to be shocked sectors: List of sectors to be shocked custom_shock_vector: Vector of length regions * sectors with percentage shocks, overrides all other shock @@ -297,7 +299,8 @@ def get_imports_exports(self, Returns: float: Sum of trade flow from exporting region-sectors to importing region-sectors """ - assert use_type in {'intermediate', 'final', 'both'}, "use_type must be 'intermediate', 'final' or 'both'" + if use_type not in {'intermediate', 'final', 'both'}: + raise ValueError("use_type must be 'intermediate', 'final' or 'both'") if import_sectors is not None and use_type in {'final', 'both'}: warn('Note that import_sectors only apply to intermediate use, for final use only import_regions is used') diff --git a/src/iotables/exiobase.py b/src/iotables/exiobase.py index 5ac2435..7f1ded3 100644 --- a/src/iotables/exiobase.py +++ b/src/iotables/exiobase.py @@ -4,7 +4,6 @@ """ from iotables.matrix import Matrix -from functools import lru_cache import numpy as np import pandas as pd from tqdm import tqdm @@ -17,7 +16,7 @@ from iotables.globals import DATA_FOLDER, FILES_LOG from iotables.utils import remove_downloaded_files, download_file -db_name = os.path.basename(__file__).rstrip('.py') +db_name = os.path.splitext(os.path.basename(__file__))[0] def process_df(df): @@ -48,7 +47,8 @@ def __init__(self, verify: Verify the server's TLS certificate (``False`` to skip, or a CA bundle path) """ - assert kind in {'industry-by-industry', 'product-by-product'} + if kind not in {'industry-by-industry', 'product-by-product'}: + raise ValueError("kind must be 'industry-by-industry' or 'product-by-product'") if version not in config['exiobase'].keys(): raise ValueError( @@ -100,7 +100,8 @@ def __init__(self, # Create matrices pbar.set_description('Creating matrices...') - assert self._Z_raw.shape[0] == self._Z_raw.shape[1] + if self._Z_raw.shape[0] != self._Z_raw.shape[1]: + raise ValueError('Intermediate-use matrix Z is not square; the downloaded file may be corrupt') self.rs = config['exiobase'][version]['num_regions'][kind] * config['exiobase'][version]['num_sectors'][kind] self.Z = Matrix('Intermediate use', *process_df(self._Z_raw)) @@ -143,7 +144,6 @@ def __init__(self, pbar.update() pbar.set_description('Done') - @lru_cache() def _load_data(self): folder = f'IOT_{self.year}_{"ixi" if self.kind == "industry-by-industry" else "pxp"}' with ZipFile(self._data_file, 'r') as zf: @@ -183,7 +183,7 @@ def _download_data(self): files_log.write(db_name + ';' + self._data_file + '\n') except Exception as e: warn(f"Couldn't download the data. Try downloading manually from {self._url} " - f"and save the csv file as {self._file_id}.csv in {self._data_folder}") + f"and save the zip file as {self._file_id}.zip in {DATA_FOLDER}") raise e @staticmethod diff --git a/src/iotables/figaro.py b/src/iotables/figaro.py index d63df0a..cfe25a7 100644 --- a/src/iotables/figaro.py +++ b/src/iotables/figaro.py @@ -5,7 +5,6 @@ """ from iotables.mappings import figaro_sector_name_mapping_pxp_2022, figaro_sector_name_mapping_ixi_2022, figaro_demand_items from iotables.matrix import Matrix -from functools import lru_cache import numpy as np import pandas as pd from tqdm import tqdm @@ -17,7 +16,7 @@ from iotables.globals import DATA_FOLDER, FILES_LOG from iotables.utils import remove_downloaded_files, download_file -db_name = os.path.basename(__file__).rstrip('.py') +db_name = os.path.splitext(os.path.basename(__file__))[0] def process_df(df): @@ -56,7 +55,8 @@ def __init__(self, https) or a ``{scheme: url}`` dict verify: Verify the server's TLS certificate (``False`` to skip, or a CA bundle path) """ - assert kind in {'industry-by-industry', 'product-by-product'} + if kind not in {'industry-by-industry', 'product-by-product'}: + raise ValueError("kind must be 'industry-by-industry' or 'product-by-product'") if version not in config['figaro'].keys(): raise ValueError( @@ -140,7 +140,6 @@ def __init__(self, pbar.update() pbar.set_description('Done') - @lru_cache() def _load_data(self): df = pd.read_csv(self._data_file, index_col=0) return df diff --git a/src/iotables/matrix.py b/src/iotables/matrix.py index 5dbd21e..cc10f67 100644 --- a/src/iotables/matrix.py +++ b/src/iotables/matrix.py @@ -11,8 +11,10 @@ class Matrix(np.ndarray): def __new__(cls, info, input_array, rows, columns): obj = np.asarray(input_array).view(cls) - assert len(obj.shape) == 2, "Array must be 2-dimensional" - assert obj.shape == (len(rows), len(columns)), "Rows and columns do not have the shape of the array" + if len(obj.shape) != 2: + raise ValueError("Array must be 2-dimensional") + if obj.shape != (len(rows), len(columns)): + raise ValueError("Rows and columns do not have the shape of the array") obj.info = info obj.rows = rows obj.columns = columns diff --git a/src/iotables/oecd.py b/src/iotables/oecd.py index 5991410..f891855 100644 --- a/src/iotables/oecd.py +++ b/src/iotables/oecd.py @@ -6,7 +6,6 @@ from iotables.mappings import oecd_sector_name_mapping, oecd_sector_name_mapping_2025, oecd_demand_items, \ oecd_sector_2022_2021_mapping from iotables.matrix import Matrix -from functools import lru_cache import numpy as np import pandas as pd from iotables.utils import ALPHA3_TO_ALPHA2 @@ -21,7 +20,7 @@ from warnings import warn from functools import partial -db_name = os.path.basename(__file__).rstrip('.py') +db_name = os.path.splitext(os.path.basename(__file__))[0] def process_df(df): @@ -147,7 +146,6 @@ def __init__(self, pbar.update() pbar.set_description('Done') - @lru_cache() def _load_data(self): if self.version == '2021': filename = f'ICIO2021_{self.year}.csv' diff --git a/src/iotables/utils.py b/src/iotables/utils.py index 915382d..88fe8a0 100644 --- a/src/iotables/utils.py +++ b/src/iotables/utils.py @@ -41,9 +41,20 @@ def download_file(url, dest, proxy=None, verify=True): r = requests.get(url, stream=True, impersonate='chrome', verify=verify, **kwargs) if not r.ok: raise ConnectionError(r.reason or f'HTTP {r.status_code}') - with open(dest, 'wb') as f: - for chunk in r.iter_content(): - f.write(chunk) + + # Download to a temporary file and atomically move it into place only once the + # stream completes, so an interrupted download never leaves a truncated file in + # the cache (which would otherwise load as a wrong-shaped, silently corrupt matrix). + tmp = dest + '.part' + try: + with open(tmp, 'wb') as f: + for chunk in r.iter_content(): + f.write(chunk) + os.replace(tmp, dest) + except BaseException: + if os.path.exists(tmp): + os.remove(tmp) + raise def replace_if_exists(x, mapping): @@ -86,7 +97,7 @@ def remove_downloaded_files(database: str = 'all', print(f'no files found for {database}, only for {list(files.keys())}') return other_files = {k: v for k, v in files.items() if k != database} - files = files[db] + files = files[database] else: files = {item for sublist in files.values() for item in sublist} for path in files: diff --git a/tests/test_figaro.py b/tests/test_figaro.py index 6371edf..76222c6 100644 --- a/tests/test_figaro.py +++ b/tests/test_figaro.py @@ -52,7 +52,7 @@ def test_shock(self): ((np.eye(f.rs) - f.A).I @ ( f.FD * (custom_shock_vector / 100))) + f.X) - with pytest.raises(AssertionError): + with pytest.raises(ValueError): f._shock(model='leontief') shock_vector = np.array([-.1 if r in EA and s == 'A01' else 0 for r, s in f.X.rows]).reshape(-1, 1) diff --git a/tests/test_iotables.py b/tests/test_iotables.py index c0110eb..a4cfc36 100644 --- a/tests/test_iotables.py +++ b/tests/test_iotables.py @@ -3,7 +3,9 @@ **Authors**: W. Wakker """ +import os import iotables +from iotables import utils class TestIotables: @@ -13,3 +15,52 @@ def test1(self): def test_folder_size(self): assert isinstance(iotables.get_size_data_folder(), str) + + +class TestRemoveDownloadedFiles: + """Offline tests for remove_downloaded_files (no network).""" + + def _setup_log(self, tmp_path, monkeypatch): + oecd_file = tmp_path / 'oecd.zip' + figaro_file = tmp_path / 'figaro.csv' + oecd_file.write_text('x') + figaro_file.write_text('y') + log = tmp_path / '_files_log.txt' + # Order matters: oecd is written first so the last line is figaro. The + # earlier wrong-database bug used the leftover loop variable (last line), + # which would have made remove(database='oecd') target figaro instead. + log.write_text(f'oecd;{oecd_file}\nfigaro;{figaro_file}\n') + monkeypatch.setattr(utils, 'FILES_LOG', str(log)) + return oecd_file, figaro_file, log + + def test_remove_single_database_keeps_others(self, tmp_path, monkeypatch): + oecd_file, figaro_file, log = self._setup_log(tmp_path, monkeypatch) + + utils.remove_downloaded_files(database='oecd', verbose=False) + + assert not oecd_file.exists() # requested database removed + assert figaro_file.exists() # other database untouched + assert log.exists() # log rewritten, not deleted + assert log.read_text().strip() == f'figaro;{figaro_file}' + + def test_remove_all(self, tmp_path, monkeypatch): + oecd_file, figaro_file, log = self._setup_log(tmp_path, monkeypatch) + + utils.remove_downloaded_files(database='all', verbose=False) + + assert not oecd_file.exists() + assert not figaro_file.exists() + assert not log.exists() + + def test_remove_unknown_database_is_noop(self, tmp_path, monkeypatch): + oecd_file, figaro_file, log = self._setup_log(tmp_path, monkeypatch) + + utils.remove_downloaded_files(database='nope', verbose=False) + + assert oecd_file.exists() + assert figaro_file.exists() + assert log.exists() + + def test_no_log_is_noop(self, tmp_path, monkeypatch): + monkeypatch.setattr(utils, 'FILES_LOG', str(tmp_path / 'missing.txt')) + utils.remove_downloaded_files(database='all', verbose=False) # must not raise diff --git a/tests/test_matrix.py b/tests/test_matrix.py index 9f3df9e..b2971af 100644 --- a/tests/test_matrix.py +++ b/tests/test_matrix.py @@ -39,7 +39,7 @@ def test1(self): Matrix('something', *process_df(df)) def test2(self): - with pytest.raises(AssertionError): + with pytest.raises(ValueError): Matrix('something', [1, 2, 3], ['something'], ['something']) def testI(self): diff --git a/tests/test_oecd.py b/tests/test_oecd.py index 8762427..ad12d85 100644 --- a/tests/test_oecd.py +++ b/tests/test_oecd.py @@ -58,7 +58,7 @@ def test_shock(self): ((np.eye(oecd.rs) - oecd.A).I @ ( oecd.FD * (custom_shock_vector / 100))) + oecd.X) - with pytest.raises(AssertionError): + with pytest.raises(ValueError): oecd._shock(model='leontief') shock_vector = np.array([-.1 if r in EA and s == '35' else 0 for r, s in oecd.X.rows]).reshape(-1, 1) From 5cc31cd4778ce874fda9ecae3023e8c7cd1d2548 Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Sat, 6 Jun 2026 15:05:20 +0200 Subject: [PATCH 12/15] Harden download integrity, add CI, and polish packaging - download_file: verify body size against Content-Length (when present and unencoded) and raise on a truncated download; offline tests added - tests: move loaders into fixtures and mark network tests so the suite collects and the offline tier runs without a connection (pytest -m "not network") - CI: GitHub Actions running offline tests on Python 3.9-3.13 + build/twine check - packaging: PEP 639 SPDX license (clean License-Expression: MIT), py.typed, [project.urls], Python-version classifiers; drop stale coverage omit - cleanups: real module docstrings, IO attribute contract includes FD_GRAN/reference/contact, .sum() instead of .sum(0), drop unused EXIOBASE metadata load and the del os half-measure --- .github/workflows/ci.yml | 44 ++++++++++++++++++++++ CHANGELOG.MD | 4 ++ pyproject.toml | 24 ++++++++++-- src/iotables/__init__.py | 6 +-- src/iotables/base_io.py | 11 +++--- src/iotables/config.py | 6 +-- src/iotables/exiobase.py | 17 +++------ src/iotables/figaro.py | 8 +--- src/iotables/globals.py | 6 +-- src/iotables/mappings.py | 6 +-- src/iotables/matrix.py | 6 +-- src/iotables/oecd.py | 8 +--- src/iotables/py.typed | 0 src/iotables/utils.py | 19 +++++++--- tests/test_exiobase.py | 8 ++-- tests/test_figaro.py | 37 +++++++++--------- tests/test_iotables.py | 81 +++++++++++++++++++++++++++++++++++++--- tests/test_matrix.py | 14 +++---- tests/test_oecd.py | 37 +++++++++--------- 19 files changed, 226 insertions(+), 116 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 src/iotables/py.typed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..20a2e1d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + test: + name: Offline tests (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install package with dev deps + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + - name: Run offline tests + # Network tests download hundreds of MB of real data; they are excluded from CI + # and run manually. Everything else must pass on every supported Python version. + run: pytest -m "not network" + + build: + name: Build & check distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Build sdist and wheel + run: | + python -m pip install --upgrade pip build twine + python -m build + - name: Validate metadata and README rendering + run: twine check dist/* diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 1901d03..471f7ce 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -24,6 +24,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Loader `db_name` derivation used `str.rstrip('.py')` (a character set, not a suffix) - Dropped `lru_cache` on `_load_data`, which pinned every loader instance (and its data) in memory - Downloads are now atomic (written to a `.part` file and moved into place on completion), so an interrupted download no longer leaves a truncated file in the cache +- `download_file` now verifies the downloaded size against `Content-Length` (when advertised and unencoded) and raises on a truncated body +### Other +- Added a `py.typed` marker, `[project.urls]`, Python-version classifiers, and a GitHub Actions CI (offline tests across Python 3.9-3.13 + build/twine check) +- Network tests are marked `network` and moved into fixtures, so the suite collects and the offline tests run without a connection (`pytest -m "not network"`) ___ diff --git a/pyproject.toml b/pyproject.toml index 858c3be..612820b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=64", "wheel"] +requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] @@ -7,7 +7,8 @@ name = "iotables" version = "0.1.0" description = "A Python package to easily load and work with inter-country input-output data" readme = "README.md" -license = { file = "LICENSE" } +license = "MIT" +license-files = ["LICENSE"] authors = [{ name = "Wouter Wakker", email = "wouter.wakker@outlook.com" }] requires-python = ">=3.9" dependencies = [ @@ -19,19 +20,34 @@ dependencies = [ ] classifiers = [ "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Operating System :: OS Independent", - "License :: OSI Approved :: MIT License", ] +[project.urls] +Homepage = "https://github.com/WWakker/iotables" +Repository = "https://github.com/WWakker/iotables" +Changelog = "https://github.com/WWakker/iotables/blob/main/CHANGELOG.MD" + [project.optional-dependencies] dev = ["pytest", "pytest-cov"] [tool.setuptools.packages.find] where = ["src"] +[tool.setuptools.package-data] +iotables = ["py.typed"] + [tool.pytest.ini_options] testpaths = ["tests"] +markers = [ + "network: test downloads real data over the network (deselect with '-m \"not network\"')", +] [tool.coverage.run] source = ["iotables"] -omit = ["**/adjust_source_in_covxml.py", "**/update_badge.py"] +omit = ["**/adjust_source_in_covxml.py"] diff --git a/src/iotables/__init__.py b/src/iotables/__init__.py index 1e7c8b2..7030d56 100644 --- a/src/iotables/__init__.py +++ b/src/iotables/__init__.py @@ -1,4 +1,4 @@ -import os +import os as _os from importlib.metadata import version as _version, PackageNotFoundError as _PackageNotFoundError from iotables.globals import DATA_FOLDER as __DATA_FOLDER from iotables.globals import IS_WINDOWS as __IS_WINDOWS @@ -40,6 +40,4 @@ def human(size): return get_size(__DATA_FOLDER) -os.makedirs(__DATA_FOLDER, exist_ok=True) - -del os +_os.makedirs(__DATA_FOLDER, exist_ok=True) diff --git a/src/iotables/base_io.py b/src/iotables/base_io.py index 4dda7b6..2c111ce 100644 --- a/src/iotables/base_io.py +++ b/src/iotables/base_io.py @@ -1,8 +1,4 @@ -""" Created on 18/10/2022:: -------------- io ------------- -**Authors**: W. Wakker - -""" +"""Shared analysis engine: derives the coefficient/inverse matrices and runs shocks.""" from warnings import warn from iotables.matrix import Matrix from iotables.utils import assert_is_subset @@ -22,6 +18,7 @@ def __init__(self): 'X', 'V', 'FD', + 'FD_GRAN', 'ADD', 'FD_REGION', 'rs', @@ -29,7 +26,9 @@ def __init__(self): 'regions', 'sectors', 'unit', - 'demand_items'] + 'demand_items', + 'reference', + 'contact'] assert_is_subset(necessary_attrs, dir(self)) # Coefficients matrix, replace 0 with 1 to allow inversion diff --git a/src/iotables/config.py b/src/iotables/config.py index 1867450..b6686dc 100644 --- a/src/iotables/config.py +++ b/src/iotables/config.py @@ -1,8 +1,4 @@ -""" Created on 08/10/2022:: -------------- config ------------- -**Authors**: W. Wakker - -""" +"""Single source of truth for the data that exists: download links and shapes per database.""" def _chunk_links(base, chunks): diff --git a/src/iotables/exiobase.py b/src/iotables/exiobase.py index 7f1ded3..e043cf0 100644 --- a/src/iotables/exiobase.py +++ b/src/iotables/exiobase.py @@ -1,7 +1,4 @@ -""" Created on 15/11/2022:: -------------- exiobase ------------- -**Authors**: S. Boldrini -""" +"""Loader for EXIOBASE inter-country input-output data.""" from iotables.matrix import Matrix import numpy as np @@ -79,7 +76,7 @@ def __init__(self, # Load pbar.set_description('Loading data...') self.df = None - self._Z_raw, self._FD_raw, self._X_raw, self._metadata, self._sector_codes, self._FD_codes = self._load_data() + self._Z_raw, self._FD_raw, self._X_raw, self._sector_codes, self._FD_codes = self._load_data() exiobase_sector_name_mapping = self._sector_codes.reset_index().set_index('CodeNr')['Name'].to_dict() exiobase_FD_name_mapping = self._FD_codes.reset_index().set_index('CodeNr')['Name'].to_dict() @@ -127,10 +124,11 @@ def __init__(self, columns=[r for r, s in self.FD_GRAN.columns]).T fd_region.index.name = 'region' + fd_region = fd_region.groupby('region').sum().T self.FD_REGION = Matrix('Final demand by region', - fd_region.groupby('region').sum(0).T, + fd_region, rows=self.Z.rows, - columns=fd_region.groupby('region').sum(0).T.columns.to_list()) + columns=fd_region.columns.to_list()) self.regions = list(sorted(np.unique([r for r, s in self.Z.rows]))) self.sectors = list(sorted(np.unique([s for r, s in self.Z.rows]))) @@ -171,10 +169,7 @@ def _load_data(self): with zf.open(f'{folder}/finaldemands.txt', 'r') as csv_file: FD_codes = pd.read_csv(csv_file, sep='\t', index_col=1) - with zf.open(f'{folder}/metadata.json', 'r') as json_file: - metadata = pd.read_json(json_file) - - return z_raw, fd_raw, x_raw, metadata, sector_codes, FD_codes + return z_raw, fd_raw, x_raw, sector_codes, FD_codes def _download_data(self): try: diff --git a/src/iotables/figaro.py b/src/iotables/figaro.py index cfe25a7..1868f7b 100644 --- a/src/iotables/figaro.py +++ b/src/iotables/figaro.py @@ -1,8 +1,4 @@ -""" Created on 14/10/2022:: -------------- figaro ------------- -**Authors**: W. Wakker - -""" +"""Loader for Eurostat Figaro inter-country input-output data.""" from iotables.mappings import figaro_sector_name_mapping_pxp_2022, figaro_sector_name_mapping_ixi_2022, figaro_demand_items from iotables.matrix import Matrix import numpy as np @@ -121,7 +117,7 @@ def __init__(self, # Create region level FD fd_region = pd.DataFrame(self.FD_GRAN, columns=[r for r, s in self.FD_GRAN.columns]).T fd_region.index.name = 'region' - fd_region = fd_region.groupby('region').sum(0).T + fd_region = fd_region.groupby('region').sum().T self.FD_REGION = Matrix('Final demand by region', fd_region, rows=self.Z.rows, diff --git a/src/iotables/globals.py b/src/iotables/globals.py index 6f06481..347aa47 100644 --- a/src/iotables/globals.py +++ b/src/iotables/globals.py @@ -1,8 +1,4 @@ -""" Created on 22/11/2022:: -------------- globals ------------- -**Authors**: W. Wakker - -""" +"""Module-level constants: the download cache location and bookkeeping paths.""" import os # Downloaded source files are cached in a user-level cache directory rather than inside diff --git a/src/iotables/mappings.py b/src/iotables/mappings.py index c482d57..b70c7d8 100644 --- a/src/iotables/mappings.py +++ b/src/iotables/mappings.py @@ -1,8 +1,4 @@ -""" Created on 06/09/2022:: -------------- mappings ------------- -**Authors**: W. Wakker - -""" +"""Per-database sector-code → human-name maps, demand-item lists, and the OECD 2022→2021 remap.""" oecd_sector_2022_2021_mapping = {'A01_02': '01T02', 'A03': '03', diff --git a/src/iotables/matrix.py b/src/iotables/matrix.py index cc10f67..b9c48cb 100644 --- a/src/iotables/matrix.py +++ b/src/iotables/matrix.py @@ -1,8 +1,4 @@ -""" Created on 03/10/2022:: -------------- matrix ------------- -**Authors**: W. Wakker - -""" +"""A labelled 2-D ``numpy.ndarray`` subclass carrying region/sector row and column metadata.""" import numpy as np import pandas as pd diff --git a/src/iotables/oecd.py b/src/iotables/oecd.py index f891855..44ad4a0 100644 --- a/src/iotables/oecd.py +++ b/src/iotables/oecd.py @@ -1,8 +1,4 @@ -""" Created on 06/09/2022:: -------------- oecd ------------- -**Authors**: W. Wakker - -""" +"""Loader for OECD ICIO inter-country input-output data.""" from iotables.mappings import oecd_sector_name_mapping, oecd_sector_name_mapping_2025, oecd_demand_items, \ oecd_sector_2022_2021_mapping from iotables.matrix import Matrix @@ -126,7 +122,7 @@ def __init__(self, # Create region level FD fd_region = pd.DataFrame(self.FD_GRAN, columns=[r for r, s in self.FD_GRAN.columns]).T fd_region.index.name = 'region' - fd_region = fd_region.groupby('region').sum(0).T + fd_region = fd_region.groupby('region').sum().T self.FD_REGION = Matrix('Final demand by region', fd_region, rows=self.Z.rows, diff --git a/src/iotables/py.typed b/src/iotables/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/iotables/utils.py b/src/iotables/utils.py index 88fe8a0..a596715 100644 --- a/src/iotables/utils.py +++ b/src/iotables/utils.py @@ -1,8 +1,4 @@ -""" Created on 11/10/2022:: -------------- utils ------------- -**Authors**: W. Wakker - -""" +"""Shared helpers: validation, the downloader, cache cleanup, and country-code maps.""" from iotables.globals import FILES_LOG from collections import defaultdict import os @@ -47,9 +43,22 @@ def download_file(url, dest, proxy=None, verify=True): # the cache (which would otherwise load as a wrong-shaped, silently corrupt matrix). tmp = dest + '.part' try: + written = 0 with open(tmp, 'wb') as f: for chunk in r.iter_content(): f.write(chunk) + written += len(chunk) + + # Guard against a silently truncated body (server returns 200 then closes the + # stream early). Only enforce when the server advertised a length and did not + # transform the bytes -- e.g. CIRCABC serves gzip/chunked with no Content-Length, + # where the written size legitimately differs from any advertised length. + content_encoding = (r.headers.get('Content-Encoding') or '').lower() + expected = r.headers.get('Content-Length') + if expected is not None and content_encoding in ('', 'identity') and written != int(expected): + raise ConnectionError( + f'Incomplete download from {url}: got {written} bytes, expected {expected}') + os.replace(tmp, dest) except BaseException: if os.path.exists(tmp): diff --git a/tests/test_exiobase.py b/tests/test_exiobase.py index d7841fd..a285c61 100644 --- a/tests/test_exiobase.py +++ b/tests/test_exiobase.py @@ -1,11 +1,9 @@ -""" Created on 17/11/2022:: -------------- test_exiobase ------------- -**Authors**: W. Wakker - -""" +"""Network tests for the ExioBase loader (download real data; run with the 'network' marker).""" +import pytest from iotables import ExioBase +@pytest.mark.network class TestExioBase: def test_load(self): diff --git a/tests/test_figaro.py b/tests/test_figaro.py index 76222c6..2c212cf 100644 --- a/tests/test_figaro.py +++ b/tests/test_figaro.py @@ -1,21 +1,24 @@ -""" Created on 19/10/2022:: -------------- test_figaro ------------- -**Authors**: W. Wakker - -""" +"""Network tests for the Figaro loader (download real data; run with the 'network' marker).""" import pytest from iotables import Figaro import numpy as np import matplotlib import matplotlib.pyplot as plt -f = Figaro(version='2025', year=2018, kind='industry-by-industry') +EA = ['AT', 'BE', 'CY', 'DE', 'EE', 'ES', 'FI', 'FR', 'GR', 'HR', 'IE', 'IT', 'LT', 'LU', 'LV', 'MT', 'NL', 'PT', 'SI', 'SK'] -custom_shock_vector = np.random.uniform(size=f.rs, low=-10, high=10).reshape(-1, 1) -EA = ['AT', 'BE', 'CY', 'DE', 'EE', 'ES', 'FI', 'FR', 'GR', 'HR', 'IE', 'IT', 'LT', 'LU', 'LV', 'MT', 'NL', 'PT', 'SI', 'SK'] +@pytest.fixture(scope="module") +def f(): + return Figaro(version='2025', year=2018, kind='industry-by-industry') + + +@pytest.fixture(scope="module") +def custom_shock_vector(f): + return np.random.uniform(size=f.rs, low=-10, high=10).reshape(-1, 1) +@pytest.mark.network class TestFigaro: def test_download(self): @@ -27,24 +30,24 @@ def test_load(self): fi = Figaro(version='2025', year=2018, kind='product-by-product') assert set(fi.sectors).issubset(fi.sector_name_mapping) - def test_matrices(self): + def test_matrices(self, f): for attr in ['Z', 'A', 'B', 'L', 'G', 'V', 'FD', 'X']: assert hasattr(f, attr) for attr_attr in ['info', 'rows', 'columns', 'I']: assert attr_attr in dir(getattr(f, attr)) - def test_leontief(self): + def test_leontief(self, f): fd = (np.eye(f.rs) - f.A) @ f.X assert np.isclose(f.FD, fd, atol=.001).all() x = (np.eye(f.rs) - f.A).I @ f.FD assert np.isclose(f.X, x, atol=.001).all() - def test_ghosh(self): + def test_ghosh(self, f): x = f.G.T @ f.V.T assert np.isclose(f.X, x, atol=.001).all() - def test_shock(self): + def test_shock(self, f, custom_shock_vector): assert np.array_equal(f._shock(model='ghosh', custom_shock_vector=custom_shock_vector), (f.G.T @ (f.V.T * (custom_shock_vector / 100))) + f.X) @@ -72,17 +75,17 @@ def test_shock(self): with pytest.raises(ValueError): f._shock(model='leontief', shock=-10, regions=EA + ['something'], sectors=['A01']) - def test_leontief_shock(self): + def test_leontief_shock(self, f): assert np.array_equal( f.leontief_demand_shock(shock=-10, regions=EA, sectors=['A01']).x_new.values.reshape(-1, 1), f._shock(model='leontief', shock=-10, regions=EA, sectors=['A01'])) - def test_ghosh_shock(self): + def test_ghosh_shock(self, f): assert np.array_equal( f.ghosh_supply_shock(shock=-10, regions=EA, sectors=['A01']).x_new.values.reshape(-1, 1), f._shock(model='ghosh', shock=-10, regions=EA, sectors=['A01'])) - def test_plot(self): + def test_plot(self, f): fig, ax = f.ghosh_supply_shock(shock=-10, regions=EA, sectors=['A01'], plot_regions=EA, plot=True, show=False) assert isinstance(fig, matplotlib.figure.Figure) assert isinstance(ax, plt.Axes) @@ -101,7 +104,7 @@ def test_plot(self): f.ghosh_supply_shock(shock=-10, regions=EA, sectors=['35'], plot=True, show=True, plot_by='region') - def test_get_imports_exports(self): + def test_get_imports_exports(self, f): assert np.isclose(f.get_imports_exports(import_regions=['CN'], export_regions='AU', @@ -142,5 +145,5 @@ def test_get_imports_exports(self): import_sectors='A01', use_type='both') - def test_remove_local_files(self): + def test_remove_local_files(self, f): f.remove_downloaded_files() diff --git a/tests/test_iotables.py b/tests/test_iotables.py index a4cfc36..2107f62 100644 --- a/tests/test_iotables.py +++ b/tests/test_iotables.py @@ -1,9 +1,5 @@ -""" Created on 22/11/2022:: -------------- test_iotables ------------- -**Authors**: W. Wakker - -""" -import os +"""Offline tests for top-level helpers and the shared downloader.""" +import pytest import iotables from iotables import utils @@ -64,3 +60,76 @@ def test_remove_unknown_database_is_noop(self, tmp_path, monkeypatch): def test_no_log_is_noop(self, tmp_path, monkeypatch): monkeypatch.setattr(utils, 'FILES_LOG', str(tmp_path / 'missing.txt')) utils.remove_downloaded_files(database='all', verbose=False) # must not raise + + +class _FakeResponse: + def __init__(self, chunks, headers, ok=True, status_code=200): + self._chunks = chunks + self.headers = headers + self.ok = ok + self.status_code = status_code + self.reason = 'OK' + + def iter_content(self): + return iter(self._chunks) + + +class TestDownloadFile: + """Offline tests for download_file's atomic + integrity behaviour (no network).""" + + def _patch(self, monkeypatch, response): + import curl_cffi + monkeypatch.setattr(curl_cffi.requests, 'get', lambda *a, **k: response) + + def test_truncated_body_raises_and_leaves_no_files(self, tmp_path, monkeypatch): + # Server advertises 100 bytes but only streams 50: a silent truncation. + resp = _FakeResponse([b'x' * 50], {'Content-Length': '100'}) + self._patch(monkeypatch, resp) + dest = tmp_path / 'data.zip' + + with pytest.raises(ConnectionError): + utils.download_file('http://example/data.zip', str(dest)) + + assert not dest.exists() + assert not (tmp_path / 'data.zip.part').exists() + + def test_complete_body_succeeds(self, tmp_path, monkeypatch): + resp = _FakeResponse([b'ab', b'cd'], {'Content-Length': '4'}) + self._patch(monkeypatch, resp) + dest = tmp_path / 'data.zip' + + utils.download_file('http://example/data.zip', str(dest)) + + assert dest.read_bytes() == b'abcd' + assert not (tmp_path / 'data.zip.part').exists() + + def test_length_check_skipped_for_encoded_body(self, tmp_path, monkeypatch): + # gzip/chunked (e.g. CIRCABC): advertised length differs from decoded bytes, + # so the check must be skipped rather than false-positive. + resp = _FakeResponse([b'x' * 50], {'Content-Length': '100', 'Content-Encoding': 'gzip'}) + self._patch(monkeypatch, resp) + dest = tmp_path / 'data.csv' + + utils.download_file('http://example/data.csv', str(dest)) + + assert dest.read_bytes() == b'x' * 50 + + def test_no_content_length_succeeds(self, tmp_path, monkeypatch): + resp = _FakeResponse([b'hello'], {}) + self._patch(monkeypatch, resp) + dest = tmp_path / 'data.csv' + + utils.download_file('http://example/data.csv', str(dest)) + + assert dest.read_bytes() == b'hello' + + def test_http_error_raises(self, tmp_path, monkeypatch): + resp = _FakeResponse([], {}, ok=False, status_code=504) + resp.reason = 'Gateway Timeout' + self._patch(monkeypatch, resp) + dest = tmp_path / 'data.zip' + + with pytest.raises(ConnectionError): + utils.download_file('http://example/data.zip', str(dest)) + + assert not dest.exists() diff --git a/tests/test_matrix.py b/tests/test_matrix.py index b2971af..c23a7f9 100644 --- a/tests/test_matrix.py +++ b/tests/test_matrix.py @@ -1,8 +1,4 @@ -""" Created on 03/10/2022:: -------------- test_matrix ------------- -**Authors**: W. Wakker - -""" +"""Offline tests for the labelled Matrix subclass.""" from iotables.matrix import Matrix import pandas as pd import numpy as np @@ -13,15 +9,15 @@ def process_df(df): if df.shape[0] > 1 and df.shape[1] > 1: return (df, - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_')], - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_')]) + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_', n=1)], + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_', n=1)]) elif df.shape[0] == 1: return (df, df.index.to_list(), - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_')]) + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.columns.str.split('_', n=1)]) else: return (df, - [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_')], + [(ALPHA3_TO_ALPHA2[r] if r in ALPHA3_TO_ALPHA2 else r, s) for r, s in df.index.str.split('_', n=1)], df.columns.to_list()) diff --git a/tests/test_oecd.py b/tests/test_oecd.py index ad12d85..fc75462 100644 --- a/tests/test_oecd.py +++ b/tests/test_oecd.py @@ -1,25 +1,28 @@ -""" Created on 03/10/2022:: -------------- test ------------- -**Authors**: W. Wakker - -""" +"""Network tests for the OECD loader (download real data; run with the 'network' marker).""" import pytest from iotables import OECD import numpy as np import matplotlib import matplotlib.pyplot as plt -oecd = OECD(version='2021', year=2018) +EA = ['AT', 'BE', 'CY', 'DE', 'EE', 'ES', 'FI', 'FR', 'GR', 'HR', 'IE', 'IT', 'LT', 'LU', 'LV', 'MT', 'NL', 'PT', 'SI', 'SK'] -custom_shock_vector = np.random.uniform(size=oecd.rs, low=-10, high=10).reshape(-1, 1) -EA = ['AT', 'BE', 'CY', 'DE', 'EE', 'ES', 'FI', 'FR', 'GR', 'HR', 'IE', 'IT', 'LT', 'LU', 'LV', 'MT', 'NL', 'PT', 'SI', 'SK'] +@pytest.fixture(scope="module") +def oecd(): + return OECD(version='2021', year=2018) + + +@pytest.fixture(scope="module") +def custom_shock_vector(oecd): + return np.random.uniform(size=oecd.rs, low=-10, high=10).reshape(-1, 1) +@pytest.mark.network class TestOECD: def test_download(self): - o = OECD(version='2021', year=2018, refresh=True) + OECD(version='2021', year=2018, refresh=True) def test_load(self): o = OECD(version='2021', year=2018) @@ -33,24 +36,24 @@ def test_load(self): o = OECD(version='2025-regular', year=2022) assert set(o.sectors).issubset(o.sector_name_mapping) - def test_matrices(self): + def test_matrices(self, oecd): for attr in ['Z', 'A', 'B', 'L', 'G', 'V', 'FD', 'X']: assert hasattr(oecd, attr) for attr_attr in ['info', 'rows', 'columns', 'I']: assert attr_attr in dir(getattr(oecd, attr)) - def test_leontief(self): + def test_leontief(self, oecd): fd = (np.eye(oecd.rs) - oecd.A) @ oecd.X assert np.isclose(oecd.FD, fd, atol=.001).all() x = (np.eye(oecd.rs) - oecd.A).I @ oecd.FD assert np.isclose(oecd.X, x, atol=.001).all() - def test_ghosh(self): + def test_ghosh(self, oecd): x = oecd.G.T @ oecd.V.T assert np.isclose(oecd.X, x, atol=.001).all() - def test_shock(self): + def test_shock(self, oecd, custom_shock_vector): assert np.array_equal(oecd._shock(model='ghosh', custom_shock_vector=custom_shock_vector), (oecd.G.T @ (oecd.V.T * (custom_shock_vector / 100))) + oecd.X) @@ -78,17 +81,17 @@ def test_shock(self): with pytest.raises(ValueError): oecd._shock(model='leontief', shock=-10, regions=EA + ['something'], sectors=['35']) - def test_leontief_shock(self): + def test_leontief_shock(self, oecd): assert np.array_equal( oecd.leontief_demand_shock(shock=-10, regions=EA, sectors=['35']).x_new.values.reshape(-1, 1), oecd._shock(model='leontief', shock=-10, regions=EA, sectors=['35'])) - def test_ghosh_shock(self): + def test_ghosh_shock(self, oecd): assert np.array_equal( oecd.ghosh_supply_shock(shock=-10, regions=EA, sectors=['35']).x_new.values.reshape(-1, 1), oecd._shock(model='ghosh', shock=-10, regions=EA, sectors=['35'])) - def test_plot(self): + def test_plot(self, oecd): fig, ax = oecd.ghosh_supply_shock(shock=-10, regions=EA, sectors=['35'], plot_regions=EA, plot=True, show=False) assert isinstance(fig, matplotlib.figure.Figure) assert isinstance(ax, plt.Axes) @@ -107,7 +110,7 @@ def test_plot(self): oecd.ghosh_supply_shock(shock=-10, regions=EA, sectors=['35'], plot=True, show=True, plot_by='region') - def test_get_imports_exports(self): + def test_get_imports_exports(self, oecd): assert np.isclose(oecd.get_imports_exports(import_regions=['CN1', 'CN2'], export_regions='AU', From 57fa156d5744facbebab84fb5ea6444e1f9ac0f4 Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Sat, 6 Jun 2026 15:12:02 +0200 Subject: [PATCH 13/15] Add PyPI deploy workflow (manual dispatch) --- .github/workflows/deploy.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..a1171fe --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,32 @@ +name: deploy + +on: workflow_dispatch + +jobs: + deploy: + environment: Deploy + runs-on: ubuntu-latest + steps: + - run: echo "Starting deployment to PyPi" + - uses: actions/checkout@v5 + - name: Set up Python 3.10 + uses: actions/setup-python@v6 + with: + python-version: "3.10" + - name: Install helpers + run: | + python -m pip install --upgrade pip + pip install build twine + - name: Build + # This project uses a pyproject.toml build backend (no setup.py), so build + # with the PEP 517 frontend rather than `setup.py sdist bdist_wheel`. + run: python -m build + - name: Check distribution metadata + run: twine check dist/* + - name: Deploy to PyPi + env: + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + run: | + twine upload dist/* + - run: echo "Successfully deployed to PyPi" From ed3252e01e2547eb85279a527eafc905d4a18518 Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Sat, 6 Jun 2026 15:13:53 +0200 Subject: [PATCH 14/15] Switch deploy workflow to PyPI Trusted Publishing (OIDC) --- .github/workflows/deploy.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a1171fe..a9f97a5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -6,6 +6,10 @@ jobs: deploy: environment: Deploy runs-on: ubuntu-latest + permissions: + # Required for PyPI Trusted Publishing (OIDC); no API token/secret needed. + id-token: write + contents: read steps: - run: echo "Starting deployment to PyPi" - uses: actions/checkout@v5 @@ -24,9 +28,6 @@ jobs: - name: Check distribution metadata run: twine check dist/* - name: Deploy to PyPi - env: - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - run: | - twine upload dist/* + # Trusted Publishing: PyPI verifies this repo/workflow/environment via OIDC. + uses: pypa/gh-action-pypi-publish@release/v1 - run: echo "Successfully deployed to PyPi" From 2db2ed7c7262889bd0a1ea3558c96cd703bac0ef Mon Sep 17 00:00:00 2001 From: Wouter Wakker Date: Sat, 6 Jun 2026 22:27:14 +0200 Subject: [PATCH 15/15] Minor fixes --- .github/workflows/ci.yml | 2 +- src/iotables/utils.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20a2e1d..8c4c5f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.10" - name: Build sdist and wheel run: | python -m pip install --upgrade pip build twine diff --git a/src/iotables/utils.py b/src/iotables/utils.py index a596715..2b45d22 100644 --- a/src/iotables/utils.py +++ b/src/iotables/utils.py @@ -12,12 +12,6 @@ def assert_is_subset(subset, superset): def download_file(url, dest, proxy=None, verify=True): """Stream-download ``url`` to the local path ``dest``. - Uses ``curl_cffi`` with browser (TLS) impersonation so downloads succeed even - when the host sits behind Cloudflare's bot challenge. The OECD file server - (``webfs-sti.oecd.org``) rejects plain ``requests``/``urllib`` traffic with - HTTP 403; impersonating a real browser passes the challenge and is harmless - for the non-protected Eurostat/EXIOBASE hosts. - Args: url: Source URL. dest: Local file path to write to.