From 28323aa6ea8ab08c99f88f24fc4ef7224d24bf55 Mon Sep 17 00:00:00 2001 From: hniazi Date: Mon, 26 Aug 2024 15:52:53 -0400 Subject: [PATCH 01/30] update package version number and name in a metadata call for get_example_data() to work --- tethys/__init__.py | 2 +- tethys/utils/install_supplement.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tethys/__init__.py b/tethys/__init__.py index a88df2d..0d579b5 100644 --- a/tethys/__init__.py +++ b/tethys/__init__.py @@ -3,4 +3,4 @@ from .utils.spatial import InputViewer -__version__ = "2.1" +__version__ = "2.1.0" diff --git a/tethys/utils/install_supplement.py b/tethys/utils/install_supplement.py index 793d191..f03e1e1 100644 --- a/tethys/utils/install_supplement.py +++ b/tethys/utils/install_supplement.py @@ -17,6 +17,7 @@ '2.0.0': 'https://zenodo.org/record/7569652/files/example.zip?download=1', '2.0.1': 'https://zenodo.org/record/7569652/files/example.zip?download=1', '2.0.2': 'https://zenodo.org/record/7569652/files/example.zip?download=1', + '2.1.0': 'https://zenodo.org/record/7569652/files/example.zip?download=1' } @@ -47,7 +48,7 @@ def get_example_data( example_data_directory = default_download_dir # get the current version of tethys that is installed - current_version = importlib.metadata.version('tethys') + current_version = importlib.metadata.version('tethys-downscaling') try: url = DATA_VERSION_URLS[current_version] From 0b75f98f358eea16bbbca16dccc1d21fd8b05b14 Mon Sep 17 00:00:00 2001 From: hniazi Date: Thu, 5 Dec 2024 19:34:50 -0500 Subject: [PATCH 02/30] ensure gcam could be queries for other nestings than the sector subsector tech for example resources subresources etc --- tethys/datareader/easy_query.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tethys/datareader/easy_query.py b/tethys/datareader/easy_query.py index 723578e..1e0c708 100644 --- a/tethys/datareader/easy_query.py +++ b/tethys/datareader/easy_query.py @@ -24,8 +24,10 @@ def easy_query(variable, year_axis=None, **kwargs): :return: gcamreader Query object """ - filters = dict(sector=None) # match nodes where @type='sector' by default - filters.update(kwargs) # update the filters with user-provided kwargs + if not kwargs: + filters = dict(sector=None) # match nodes where @type='sector' by default if no filters are specified + else: + filters = kwargs # update the filters with user-provided kwargs # filters are separated by "//*" to match any descendant xpath = "*" + "//*".join(handle_filter(k, v) for k, v in filters.items()) + f"//{variable}/node()" From a690442104dc8e48339b212f293c9c944dd765b0 Mon Sep 17 00:00:00 2001 From: hniazi Date: Thu, 5 Dec 2024 19:39:03 -0500 Subject: [PATCH 03/30] add basin name mappings for old and new gcam names. also remove extra lines from easy_query --- tethys/utils/region_to_id_mapping.py | 474 +++++++++++++++++++++++++++ 1 file changed, 474 insertions(+) diff --git a/tethys/utils/region_to_id_mapping.py b/tethys/utils/region_to_id_mapping.py index 3992f56..1457917 100644 --- a/tethys/utils/region_to_id_mapping.py +++ b/tethys/utils/region_to_id_mapping.py @@ -507,5 +507,479 @@ 'Papua New Guinea Coast': 173, 'Tasmania': 211, 'Solomon Islands': 176 + }, + "basin_name_mapping": { + 'Arctic Ocean Islands': 'ArcticIsl', + 'Northwest Territories': 'NWTerr', + 'Siberia North Coast': 'SiberiaN', + 'Siberia West Coast': 'SiberiaW', + 'Kara Sea Coast': 'KaraSea', + 'Lena': 'LenaR', + 'Pacific and Arctic Coast': 'PacArctic', + 'Scandinavia North Coast': 'ScndnvN', + 'Russia Barents Sea Coast': 'BarentsSea', + 'Mackenzie': 'Mackenzie', + 'Iceland': 'Iceland', + 'Sweden': 'Sweden', + 'Finland': 'Finland', + 'Northern Dvina': 'DvinaRN', + 'Hudson Bay Coast': 'HudsonBay', + 'Scotland': 'Scotland', + 'Neva': 'NevaR', + 'Volga': 'VolgaR', + 'Atlantic Ocean Seaboard': 'CanAtl', + 'Baltic Sea Coast': 'BalticSea', + 'Denmark Germany Coast': 'DnkGrmCst', + 'Narva': 'NarvaR', + 'Saskatchewan Nelson': 'NelsonR', + 'Ireland': 'Ireland', + 'Daugava': 'DaugavaR', + 'England and Wales': 'EngWales', + 'Fraser': 'FraserR', + 'Ems Weser': 'EmsWeserR', + 'Oder': 'OderR', + 'Wisla': 'WislaR', + 'Elbe': 'ElbeR', + 'Rhine': 'RhineR', + 'Poland Coast': 'PolandCst', + 'Churchill': 'ChurchillR', + 'Neman': 'NemanR', + 'Scheldt': 'ScheldtR', + 'Russia South East Coast': 'RusCstSE', + 'Ural': 'UralR', + 'Dnieper': 'DnieperR', + 'St Lawrence': 'StLwrncR', + 'France West Coast': 'FranceCstW', + 'Gobi Interior': 'Gobi', + 'Amur': 'AmurR', + 'Loire': 'LoireR', + 'Caspian Sea Coast': 'CaspianNE', + 'Seine': 'SeineR', + 'Black Sea North Coast': 'BlackSeaN', + 'Yenisei': 'YeniseiR', + 'Dniester': 'DniesterR', + 'Italy East Coast': 'ItalyCstE', + 'Japan': 'Japan', + 'Caspian Sea East Coast': 'CaspianE', + 'Don': 'DonR', + 'Danube': 'DanubeR', + 'Adriatic Sea Greece Black Sea Coast': 'AdrBlkSea', + 'Ob': 'ObR', + 'Po': 'PoR', + 'Amu Darya': 'AmuDaryaR', + 'Italy West Coast': 'ItalyCstW', + 'Spain Portugal Atlantic Coast': 'IberiaCst', + 'France South Coast': 'FranceCstS', + 'Rhone': 'RhoneR', + 'Mediterranean Sea Islands': 'MeditIsl', + 'Gironde': 'Gironde', + 'North and South Korea': 'Korea', + 'Bo Hai Korean Bay North Coast': 'BoHai', + 'Spain South and East Coast': 'SpainCstSE', + 'Lake Balkash': 'LBalkash', + 'Tiber': 'TiberR', + 'Black Sea South Coast': 'BlackSeaS', + 'Tagus': 'TagusR', + 'Caspian Sea South West Coast': 'CaspianSW', + 'Ebro': 'EbroR', + 'Douro': 'DouroR', + 'Mediterranean Sea East Coast': 'MeditE', + 'Syr Darya': 'SyrDaryaR', + 'Ziya He Interior': 'ZiyaHe', + 'China Coast': 'ChinaCst', + 'Huang He': 'HuangHeR', + 'Mediterranean South Coast': 'MeditS', + 'Guadiana': 'GuadianaR', + 'Central Iran': 'Iran', + 'Guadalquivir': 'GuadalqR', + 'Tigris Euphrates': 'TigrEuphR', + 'Tarim Interior': 'Tarim', + 'Africa North West Coast': 'AfrCstNW', + 'Nile': 'NileR', + 'Persian Gulf Coast': 'PersianGulf', + 'Indus': 'IndusR', + 'Farahrud': 'FarahrudR', + 'Baja California': 'MexBaja', + 'Plateau of Tibet Interior': 'Tibet', + 'Red Sea East Coast': 'RedSeaE', + 'Arabian Peninsula': 'ArabianP', + 'Dead Sea': 'DeadSea', + 'Mexico Northwest Coast': 'MexCstNW', + 'Helmand': 'Helmand', + 'Sinai Peninsula': 'SinaiP', + 'Eastern Jordan Syria': 'EJrdnSyr', + 'Africa Red Sea Gulf of Aden Coast': 'AfrCstNE', + 'Caribbean': 'Caribbean', + 'HamuniMashkel': 'HamuMashR', + 'Taiwan': 'Taiwan', + 'Arabian Sea Coast': 'ArabianSea', + 'North Gulf': 'MexGulf', + 'Yangtze': 'Yangtze', + 'Sabarmati': 'SabarmatiR', + 'Xun Jiang': 'XunJiang', + 'Hong (Red River)': 'Hong', + 'Ganges Bramaputra': 'GangesR', + 'Yucatan Peninsula': 'YucatanP', + 'South China Sea Coast': 'SChinaSea', + 'Mahi': 'MahiR', + 'Mexico Interior': 'MexInt', + 'Pacific Central Coast': 'MexCstW', + 'Bay of Bengal North East Coast': 'BengalBay', + 'Tapti': 'TaptiR', + 'Yasai': 'BengalW', + 'Philippines': 'Phlppns', + 'Brahmani': 'BrahmaniR', + 'North Marina Islands and Guam': 'Guam', + 'Mahanadi': 'MahanadiR', + 'Godavari': 'GodavariR', + 'Hainan': 'Hainan', + 'Mekong': 'Mekong', + 'Viet Nam Coast': 'VietnamCst', + 'Salween': 'Salween', + 'India North East Coast': 'IndCstNE', + 'India West Coast': 'IndCstW', + 'Papaloapan': 'Papaloapan', + 'Rio Lerma': 'RioLerma', + 'Rio Verde': 'RioVerde', + 'Grijalva Usumacinta': 'GrijUsuR', + 'Rio Balsas': 'RioBalsas', + 'Southern Central America': 'CntAmer', + 'Isthmus of Tehuantepec': 'Tehuantpc', + 'Irrawaddy': 'IrrawaddyR', + 'Sittaung': 'SittaungR', + 'Peninsula Malaysia': 'MalaysiaP', + 'Krishna': 'KrishnaR', + 'Andaman Nicobar Islands': 'AdnNicIsl', + 'Africa West Coast': 'AfrCstW', + 'Caribbean Coast': 'SAmerCstN', + 'Africa North Interior': 'AfrIntN', + 'India East Coast': 'IndCstE', + 'Chao Phraya': 'ChaoPhrR', + 'Pennar': 'PennarR', + 'Gulf of Thailand Coast': 'ThaiGulf', + 'Niger': 'NigerR', + 'Micronesia': 'Micronesia', + 'Lake Chad': 'LChad', + 'Senegal': 'SenegalR', + 'Cauvery': 'CauveryR', + 'Sri Lanka': 'SriLanka', + 'India South Coast': 'IndCstS', + 'Orinoco': 'OrinocoR', + 'Colombia Ecuador Pacific Coast': 'ColEcuaCst', + 'Palau and East Indonesia': 'IdnE', + 'North Borneo Coast': 'BorneoCstN', + 'Volta': 'VoltaR', + 'Northeast South America South Atlantic Coast': 'SAmerCstNE', + 'Gulf of Guinea': 'GuineaGulf', + 'Sumatra': 'Sumatra', + 'Sulawesi': 'Sulawesi', + 'Kalimantan': 'Kalimantan', + 'Magdalena': 'MagdalenaR', + 'Irian Jaya Coast': 'IrianJaya', + 'Amazon': 'AmazonR', + 'South Chile Pacific Coast': 'ChileCstS', + 'Shebelli Juba': 'ShebJubR', + 'Africa East Central Coast': 'AfrCstE', + 'North Brazil South Atlantic Coast': 'BrzCstN', + 'Papua New Guinea Coast': 'PapuaCst', + 'Tocantins': 'TocantinsR', + 'Java Timor': 'JavaTimor', + 'Solomon Islands': 'SolomonIsl', + 'Madagascar': 'Madagascar', + 'Sepik': 'SepikR', + 'Rift Valley': 'RiftValley', + 'Peru Pacific Coast': 'PeruCst', + 'Fly': 'FlyR', + 'Angola Coast': 'AngolaCst', + 'Congo': 'CongoR', + 'Australia North Coast': 'AusCstN', + 'South Pacific Islands': 'NewCaledn', + 'East Brazil South Atlantic Coast': 'BrzCstE', + 'Parnaiba': 'ParnaibaR', + 'Zambezi': 'ZambeziR', + 'Australia East Coast': 'AusCstE', + 'Africa Indian Ocean Coast': 'AfrCstSE', + 'Australia West Coast': 'AusCstW', + 'Sao Francisco': 'SaoFrancR', + 'Australia Interior': 'AusInt', + 'Orange': 'OrangeR', + 'Uruguay Brazil South Atlantic Coast': 'BrzCstS', + 'Namibia Coast': 'AfrCstSW', + 'Africa South Interior': 'AfrIntS', + 'South Africa South Coast': 'AfrCstS', + 'Limpopo': 'LimpopoR', + 'La Puna Region': 'LaPuna', + 'New Zealand': 'NewZealand', + 'Australia South Coast': 'AusCstS', + 'Mar Chiquita': 'MarChiq', + 'South Africa West Coast': 'AfrCstSSW', + 'Salinas Grandes': 'Salinas', + 'La Plata': 'RioLaPlata', + 'North Chile Pacific Coast': 'ChileCstN', + 'Murray Darling': 'MurrayDrlg', + 'Pampas Region': 'Pampas', + 'North Argentina South Atlantic Coast': 'ArgCstN', + 'Tasmania': 'Tasmania', + 'South America Colorado': 'ArgColoR', + 'Negro': 'NegroR', + 'Central Patagonia Highlands': 'Patagonia', + 'South Argentina South Atlantic Coast': 'ArgCstS', + 'Antarctica': 'Antarctica', + 'California River Basin': 'California', + 'Upper Mississippi Basin': 'MissppRN', + 'Lower Mississippi River Basin': 'MissppRS', + 'Upper Colorado River Basin': 'UsaColoRN', + 'Lower Colorado River Basin': 'UsaColoRS', + 'Great Basin': 'GreatBasin', + 'Missouri River Basin': 'MissouriR', + 'Arkansas White Red Basin': 'ArkWhtRedR', + 'Texas Gulf Coast Basin': 'TexasCst', + 'South Atlantic Gulf Basin': 'UsaCstSE', + 'Great Lakes Basin': 'GreatLakes', + 'Ohio River Basin': 'OhioR', + 'Pacific Northwest Basin': 'UsaPacNW', + 'Tennessee River Basin': 'TennR', + 'Rio Grande River Basin': 'RioGrande', + 'New England Basin': 'UsaCstNE', + 'Mid Atlantic Basin': 'UsaCstE', + 'Hawaii': 'Hawaii', + 'Narmada': 'NarmadaR' + }, + "basin_name_mapping_im3": { + 'Arctic Ocean Islands': 'ArcticIsl', + 'Northwest Territories': 'NWTerr', + 'Siberia North Coast': 'SiberiaN', + 'Siberia West Coast': 'SiberiaW', + 'Kara Sea Coast': 'KaraSea', + 'Lena': 'LenaR', + 'Pacific and Arctic Coast': 'PacArctic', + 'Scandinavia North Coast': 'ScndnvN', + 'Russia Barents Sea Coast': 'BarentsSea', + 'Mackenzie': 'Mackenzie', + 'Iceland': 'Iceland', + 'Sweden': 'Sweden', + 'Finland': 'Finland', + 'Northern Dvina': 'DvinaRN', + 'Hudson Bay Coast': 'HudsonBay', + 'Scotland': 'Scotland', + 'Neva': 'NevaR', + 'Volga': 'VolgaR', + 'Atlantic Ocean Seaboard': 'CanAtl', + 'Baltic Sea Coast': 'BalticSea', + 'Denmark Germany Coast': 'DnkGrmCst', + 'Narva': 'NarvaR', + 'Saskatchewan Nelson': 'NelsonR', + 'Ireland': 'Ireland', + 'Daugava': 'DaugavaR', + 'England and Wales': 'EngWales', + 'Fraser': 'FraserR', + 'Ems Weser': 'EmsWeserR', + 'Oder': 'OderR', + 'Wisla': 'WislaR', + 'Elbe': 'ElbeR', + 'Rhine': 'RhineR', + 'Poland Coast': 'PolandCst', + 'Churchill': 'ChurchillR', + 'Neman': 'NemanR', + 'Scheldt': 'ScheldtR', + 'Russia South East Coast': 'RusCstSE', + 'Ural': 'UralR', + 'Dnieper': 'DnieperR', + 'St Lawrence': 'StLwrncR', + 'France West Coast': 'FranceCstW', + 'Gobi Interior': 'Gobi', + 'Amur': 'AmurR', + 'Loire': 'LoireR', + 'Caspian Sea Coast': 'CaspianNE', + 'Seine': 'SeineR', + 'Black Sea North Coast': 'BlackSeaN', + 'Yenisey': 'YeniseiR', + 'Dniester': 'DniesterR', + 'Italy East Coast': 'ItalyCstE', + 'Japan': 'Japan', + 'Caspian Sea East Coast': 'CaspianE', + 'Don': 'DonR', + 'Danube': 'DanubeR', + 'Adriatic Sea Greece Black Sea Coast': 'AdrBlkSea', + 'Ob': 'ObR', + 'Po': 'PoR', + 'Amu Darya': 'AmuDaryaR', + 'Italy West Coast': 'ItalyCstW', + 'Spain Portugal Atlantic Coast': 'IberiaCst', + 'France South Coast': 'FranceCstS', + 'Rhone': 'RhoneR', + 'Mediterranean Sea Islands': 'MeditIsl', + 'Gironde': 'Gironde', + 'North and South Korea': 'Korea', + 'Bo Hai Korean Bay North Coast': 'BoHai', + 'Spain South and East Coast': 'SpainCstSE', + 'Lake Balkash': 'LBalkash', + 'Tiber': 'TiberR', + 'Black Sea South Coast': 'BlackSeaS', + 'Tagus': 'TagusR', + 'Caspian Sea South West Coast': 'CaspianSW', + 'Ebro': 'EbroR', + 'Douro': 'DouroR', + 'Mediterranean Sea East Coast': 'MeditE', + 'Syr Darya': 'SyrDaryaR', + 'Ziya He Interior': 'ZiyaHe', + 'China Coast': 'ChinaCst', + 'Huang He': 'HuangHeR', + 'Mediterranean South Coast': 'MeditS', + 'Guadiana': 'GuadianaR', + 'Central Iran': 'Iran', + 'Guadalquivir': 'GuadalqR', + 'Tigris Euphrates': 'TigrEuphR', + 'Tarim Interior': 'Tarim', + 'Africa North West Coast': 'AfrCstNW', + 'Nile': 'NileR', + 'Persian Gulf Coast': 'PersianGulf', + 'Indus': 'IndusR', + 'Farahrud': 'FarahrudR', + 'Baja California': 'MexBaja', + 'Plateau of Tibet Interior': 'Tibet', + 'Red Sea East Coast': 'RedSeaE', + 'Arabian Peninsula': 'ArabianP', + 'Dead Sea': 'DeadSea', + 'Mexico Northwest Coast': 'MexCstNW', + 'Helmand': 'Helmand', + 'Sinai Peninsula': 'SinaiP', + 'Eastern Jordan Syria': 'EJrdnSyr', + 'Africa Red Sea Gulf of Aden Coast': 'AfrCstNE', + 'Caribbean': 'Caribbean', + 'Hamun i Mashkel': 'HamuMashR', + 'Taiwan': 'Taiwan', + 'Arabian Sea Coast': 'ArabianSea', + 'North Gulf': 'MexGulf', + 'Yangtze': 'Yangtze', + 'Sabarmati': 'SabarmatiR', + 'Xun Jiang': 'XunJiang', + 'Hong Red River': 'Hong', + 'Ganges Bramaputra': 'GangesR', + 'Yucatan Peninsula': 'YucatanP', + 'South China Sea Coast': 'SChinaSea', + 'Mahi': 'MahiR', + 'Mexico Interior': 'MexInt', + 'Pacific Central Coast': 'MexCstW', + 'Bay of Bengal North East Coast': 'BengalBay', + 'Tapti': 'TaptiR', + 'Yasai': 'BengalW', + 'Philippines': 'Phlppns', + 'Brahamani': 'BrahmaniR', + 'North Marina Islands and Guam': 'Guam', + 'Mahandi': 'MahanadiR', + 'Godavari': 'GodavariR', + 'Hainan': 'Hainan', + 'Mekong': 'Mekong', + 'Viet Nam Coast': 'VietnamCst', + 'Salween': 'Salween', + 'India North East Coast': 'IndCstNE', + 'India West Coast': 'IndCstW', + 'Papaloapan': 'Papaloapan', + 'Rio Lerma': 'RioLerma', + 'Rio Verde': 'RioVerde', + 'Grijalva Usumacinta': 'GrijUsuR', + 'Rio Balsas': 'RioBalsas', + 'Southern Central America': 'CntAmer', + 'Isthmus of Tehuantepec': 'Tehuantpc', + 'Irrawaddy': 'IrrawaddyR', + 'Sittang': 'SittaungR', + 'Peninsula Malaysia': 'MalaysiaP', + 'Krishna': 'KrishnaR', + 'Andaman Nicobar Islands': 'AdnNicIsl', + 'Africa West Coast': 'AfrCstW', + 'Caribbean Coast': 'SAmerCstN', + 'Africa North Interior': 'AfrIntN', + 'India East Coast': 'IndCstE', + 'Chao Phraya': 'ChaoPhrR', + 'Pennar': 'PennarR', + 'Gulf of Thailand Coast': 'ThaiGulf', + 'Niger': 'NigerR', + 'Micronesia': 'Micronesia', + 'Lake Chad': 'LChad', + 'Senegal': 'SenegalR', + 'Cauvery': 'CauveryR', + 'Sri Lanka': 'SriLanka', + 'India South Coast': 'IndCstS', + 'Orinoco': 'OrinocoR', + 'Colombia Ecuador Pacific Coast': 'ColEcuaCst', + 'Palau and East Indonesia': 'IdnE', + 'North Borneo Coast': 'BorneoCstN', + 'Volta': 'VoltaR', + 'Northeast South America South Atlantic Coast': 'SAmerCstNE', + 'Gulf of Guinea': 'GuineaGulf', + 'Sumatra': 'Sumatra', + 'Sulawesi': 'Sulawesi', + 'Kalimantan': 'Kalimantan', + 'Magdalena': 'MagdalenaR', + 'Irian Jaya Coast': 'IrianJaya', + 'Amazon': 'AmazonR', + 'South Chile Pacific Coast': 'ChileCstS', + 'Shebelli Juba': 'ShebJubR', + 'Africa East Central Coast': 'AfrCstE', + 'North Brazil South Atlantic Coast': 'BrzCstN', + 'Papua New Guinea Coast': 'PapuaCst', + 'Tocantins': 'TocantinsR', + 'Java Timor': 'JavaTimor', + 'Solomon Islands': 'SolomonIsl', + 'Madasgacar': 'Madagascar', + 'Sepik': 'SepikR', + 'Rift Valley': 'RiftValley', + 'Peru Pacific Coast': 'PeruCst', + 'Fly': 'FlyR', + 'Angola Coast': 'AngolaCst', + 'Congo': 'CongoR', + 'Australia North Coast': 'AusCstN', + 'South Pacific Islands': 'NewCaledn', + 'East Brazil South Atlantic Coast': 'BrzCstE', + 'Parnaiba': 'ParnaibaR', + 'Zambezi': 'ZambeziR', + 'Australia East Coast': 'AusCstE', + 'Africa Indian Ocean Coast': 'AfrCstSE', + 'Australia West Coast': 'AusCstW', + 'Sao Francisco': 'SaoFrancR', + 'Australia Interior': 'AusInt', + 'Orange': 'OrangeR', + 'Uruguay Brazil South Atlantic Coast': 'BrzCstS', + 'Namibia Coast': 'AfrCstSW', + 'Africa South Interior': 'AfrIntS', + 'South Africa South Coast': 'AfrCstS', + 'Limpopo': 'LimpopoR', + 'La Puna Region': 'LaPuna', + 'New Zealand': 'NewZealand', + 'Australia South Coast': 'AusCstS', + 'Mar Chiquita': 'MarChiq', + 'South Africa West Coast': 'AfrCstSSW', + 'Salinas Grandes': 'Salinas', + 'La Plata': 'RioLaPlata', + 'North Chile Pacific Coast': 'ChileCstN', + 'Murray Darling': 'MurrayDrlg', + 'Pampas Region': 'Pampas', + 'North Argentina South Atlantic Coast': 'ArgCstN', + 'Tasmania': 'Tasmania', + 'South America Colorado': 'ArgColoR', + 'Negro': 'NegroR', + 'Central Patagonia Highlands': 'Patagonia', + 'South Argentina South Atlantic Coast': 'ArgCstS', + 'Antarctica': 'Antarctica', + 'California River': 'California', + 'Upper Mississippi': 'MissppRN', + 'Lower Mississippi River': 'MissppRS', + 'Upper Colorado River': 'UsaColoRN', + 'Lower Colorado River': 'UsaColoRS', + 'Great': 'GreatBasin', + 'Missouri River': 'MissouriR', + 'Arkansas White Red': 'ArkWhtRedR', + 'Texas Gulf Coast': 'TexasCst', + 'South Atlantic Gulf': 'UsaCstSE', + 'Great Lakes': 'GreatLakes', + 'Ohio River': 'OhioR', + 'Pacific Northwest': 'UsaPacNW', + 'Tennessee River': 'TennR', + 'Rio Grande River': 'RioGrande', + 'New England': 'UsaCstNE', + 'Mid Atlantic': 'UsaCstE', + 'Hawaii': 'Hawaii', + 'Narmada': 'NarmadaR' } } From f1fa99567f6928928bdf2d6754224393fdf612cb Mon Sep 17 00:00:00 2001 From: hniazi Date: Thu, 5 Dec 2024 19:42:51 -0500 Subject: [PATCH 04/30] add a quick function to extract basin names from resource string for example basin_water withdrawals --- tethys/datareader/regional.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tethys/datareader/regional.py b/tethys/datareader/regional.py index ea6b9a6..1c3edae 100644 --- a/tethys/datareader/regional.py +++ b/tethys/datareader/regional.py @@ -41,6 +41,17 @@ def load_region_data(gcam_db, sectors, demand_type='withdrawals'): return df +def extract_resource_name(x): + """Removes '_water withdrawals' or '_water consumption' from GCAM sector or resource name. + + :param x: resource name string. + :return: cleaned resource name without suffix. + """ + if '_water withdrawals' in x or '_water consumption' in x: + return x.split('_')[0] # return the part before the underscore e.g., 'basin_water withdrawals' to 'basin' + return x + + def extract_basin_name(x): """Maps 'water_td_irr_basin_C' to '_basin', and water_td_elec_C to ''""" if x.startswith('water_td_irr_'): From 1ebad6126fdded8a8fa8f08ee23112d5cf829f7e Mon Sep 17 00:00:00 2001 From: hniazi Date: Thu, 5 Dec 2024 19:56:18 -0500 Subject: [PATCH 05/30] include a workflow to disaggregated water source for each demand type in each grid cell --- tethys/utils/source_disaggregation.py | 93 +++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 tethys/utils/source_disaggregation.py diff --git a/tethys/utils/source_disaggregation.py b/tethys/utils/source_disaggregation.py new file mode 100644 index 0000000..dab08a6 --- /dev/null +++ b/tethys/utils/source_disaggregation.py @@ -0,0 +1,93 @@ +import os +import pandas as pd +import xarray as xr + +import gcamreader + +from tethys.datareader.easy_query import easy_query +from tethys.datareader.regional import extract_resource_name +from tethys.utils.region_to_id_mapping import name_to_id_mapping + + +class get_source_shares: + def __init__(self, gcam_db, basin_name_mapping="basin_name_mapping_im3", demand_type='withdrawals', region_masks=None, years=None): + """ + Initialize get_source_shares with parameters for GCAM database and basin mapping. + + :param gcam_db: Path to the GCAM database file. + :param basin_name_mapping: Basin name mapping dictionary, can change to other mapping in name_to_id_mapping. + :param demand_type: Type of demand, either 'withdrawals' or 'consumption'. + :param region_masks: xarray DataArray containing region masks. + :param years: List of years to be included in the shares. + """ + dbpath, dbfile = os.path.split(gcam_db) + self.conn = gcamreader.LocalDBConn(dbpath, dbfile) + self.basin_name_mapping = basin_name_mapping + self.demand_type = demand_type + self.region_masks = region_masks + self.years = years + + def calculate_shares(self): + """ + Query the GCAM database, process water data for the specified demand type, + and calculate runoff and groundwater shares by region and year. + + :return: Processed DataFrame with calculated shares. + """ + # query GCAM database for water used by source for the demand type (W or C) + query_type = f'*_water {self.demand_type}' + shares_df = self.conn.runQuery(easy_query('production', resource=query_type)) + + # extract and clean resource names (e.g., remove '_water withdrawals') + shares_df['resource'] = shares_df['resource'].apply(extract_resource_name) + + # rest of the mappings should be fine, old GCAM versions have typos in basin names + if self.basin_name_mapping == "basin_name_mapping_im3": + shares_df['resource'] = shares_df['resource'].str.replace('_', ' ').str.replace('-', ' ') # replace '_' and '-' with spaces + + # map resource names to short basin names and concatenate region and resource to work with region_masks + shares_df['resourcemap'] = shares_df['resource'].map(name_to_id_mapping[self.basin_name_mapping]) + shares_df['region_resourcemap'] = shares_df['region'] + '_' + shares_df['resourcemap'] + + # calculate shares of runoff and groundwater + shares_df['share'] = shares_df.groupby( + ['scenario', 'region', 'region_resourcemap', 'year'] + )['value'].transform(lambda x: x / x.sum() if x.sum() != 0 else 0) + + # filter shares based on the region masks from the config file, otherwise all regions would be included + shares_df = shares_df[shares_df['region_resourcemap'].isin(self.region_masks.region.values)] + + return shares_df + + def generate_gridded_shares(self, shares_df): + """ + Generate gridded shares by subresource (e.g., runoff, groundwater). + + :param shares_df: DataFrame containing shares calculated by `calculate_shares`. + :return: Dictionary of gridded shares by subresource. + """ + gridded_shares_by_subresource = {} + for subresource in shares_df.subresource.unique(): + gridded_shares_by_year = [] + for year in self.years: + gridded_shares = None + for basin in shares_df.region_resourcemap.unique(): + gridded_shares = xr.where( + self.region_masks.sel(region=basin).load(), + shares_df[ + (shares_df['year'] == year) & + (shares_df['region_resourcemap'] == basin) & + (shares_df['subresource'] == subresource) + ].iloc[0].share, + 0 if gridded_shares is None else gridded_shares, + keep_attrs=False, + ) + gridded_shares_by_year.append(gridded_shares) + + # Stack years for the subresource + gridded_shares_by_subresource[subresource] = xr.concat( + gridded_shares_by_year, + dim=pd.Index(self.years, name="year"), + ).rename('share') + + return gridded_shares_by_subresource From 4e93953dc6b9fb9afbb1600877caf62ec7b5ed11 Mon Sep 17 00:00:00 2001 From: hniazi Date: Thu, 5 Dec 2024 19:56:56 -0500 Subject: [PATCH 06/30] incorporate water source disaggregation into main tethys class --- tethys/model.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tethys/model.py b/tethys/model.py index 54f6365..7ac7cbe 100644 --- a/tethys/model.py +++ b/tethys/model.py @@ -19,6 +19,7 @@ from tethys.datareader.gridded import load_file, interp_helper from tethys.datareader.maps import load_region_map +from tethys.utils.source_disaggregation import get_source_shares class Tethys: """Model wrapper for Tethys""" @@ -30,6 +31,7 @@ def __init__( resolution=0.125, bounds=None, demand_type='withdrawals', + source_disaggregation=None, gcam_db=None, csv=None, output_dir=None, @@ -42,10 +44,11 @@ def __init__( """Parameters can be specified in a YAML file or passed directly, with the config file taking precedence :param config_file: path to YAML configuration file containing these parameters - :param years: list of years to be included spatial downscaling + :param years: list of years to be included for spatial downscaling :param resolution: resolution in degrees for spatial downscaling :param bounds: list [lat_min, lat_max, lon_min, lon_max] to crop to :param demand_type: choice between “withdrawals” (default) or “consumption” + :param source_disaggregation: decide whether to disaggregate water demand by source (runoff, groundwater, desal) :param gcam_db: relative path to a GCAM database :param csv: relative path to csv file containing inputs :param output_dir: directory to write outputs to @@ -62,6 +65,7 @@ def __init__( self.resolution = resolution self.bounds = bounds self.demand_type = demand_type + self.source_disaggregation = source_disaggregation # GCAM database info self.gcam_db = gcam_db @@ -85,6 +89,10 @@ def __init__( self.proxies = None self.inputs = None self.outputs = None + self.shares = None + self.griddedshares = None + self.disaggregated_sw = None + self.disaggregated_gw = None # settings in YAML override settings passed directly to __init__ if config_file is not None: @@ -198,8 +206,33 @@ def downscale(self, distribution, inputs, region_masks): return out + def disaggregate_source(self, downscaled): + """Disaggregate water demand by source (runoff, groundwater, desal)""" + + print('Disaggregating Source') + + # initialize source shares object + get_shares = get_source_shares(gcam_db=self.gcam_db, demand_type=self.demand_type, + region_masks=self.region_masks, years=self.years) + + # query and calculate shares + shares_df = get_shares.calculate_shares() + + # apply shares to the gridded region masks + gridded_shares = get_shares.generate_gridded_shares(shares_df) + + # disaggregate water supply source for each downscaled demand in each grid cell + downscaled_disaggregated_sw = (downscaled * gridded_shares['runoff']).compute() + downscaled_disaggregated_gw = (downscaled * gridded_shares['groundwater']).compute() + + return gridded_shares, downscaled_disaggregated_sw, downscaled_disaggregated_gw + + def run_model(self): self.outputs = xr.Dataset() + self.griddedshares = xr.Dataset() + self.disaggregated_sw = xr.Dataset() + self.disaggregated_gw = xr.Dataset() self._load_proxies() self._load_region_masks() @@ -251,6 +284,16 @@ def run_model(self): downscaled.to_dataset(dim='sector').to_netcdf(filename, encoding=encoding) downscaled = xr.open_dataset(filename).to_array(dim='sector') # hopefully this keeps dask happy + # disaggregate water supply source and update the objects + if self.source_disaggregation: + gshares, sw, gw = self.disaggregate_source(downscaled) + + # store the disaggregated results + self.griddedshares = gshares + self.disaggregated_sw = sw + self.disaggregated_gw = gw + + if self.temporal_config is not None: # calculate the monthly distributions (share of annual) for each year From 1fc9616d55367f34693a45c9252bc7605ae495c3 Mon Sep 17 00:00:00 2001 From: Travis Thurber Date: Fri, 14 Mar 2025 12:24:03 -0700 Subject: [PATCH 07/30] only have water source shares for withdrawal; fix queries; hack to capture transnational basins within the USA --- tethys/datareader/easy_query.py | 9 ++++---- tethys/model.py | 31 ++++++++++++--------------- tethys/utils/source_disaggregation.py | 15 ++++++++++--- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/tethys/datareader/easy_query.py b/tethys/datareader/easy_query.py index 1e0c708..68c660b 100644 --- a/tethys/datareader/easy_query.py +++ b/tethys/datareader/easy_query.py @@ -1,7 +1,7 @@ import gcamreader -def easy_query(variable, year_axis=None, **kwargs): +def easy_query(variable, year_axis=None, replace_filters=False, **kwargs): """Build a query for a GCAM database kwargs act as filters on nodes, eg. "sector='Beef'" is converted to the xpath "[@type='sector' and @name='Beef']" @@ -24,10 +24,11 @@ def easy_query(variable, year_axis=None, **kwargs): :return: gcamreader Query object """ - if not kwargs: - filters = dict(sector=None) # match nodes where @type='sector' by default if no filters are specified + if replace_filters: + filters = kwargs else: - filters = kwargs # update the filters with user-provided kwargs + filters = dict(sector=None) # match nodes where @type='sector' by default + filters.update(kwargs) # update the filters with user-provided kwargs # filters are separated by "//*" to match any descendant xpath = "*" + "//*".join(handle_filter(k, v) for k, v in filters.items()) + f"//{variable}/node()" diff --git a/tethys/model.py b/tethys/model.py index 7ac7cbe..b69ed90 100644 --- a/tethys/model.py +++ b/tethys/model.py @@ -206,7 +206,7 @@ def downscale(self, distribution, inputs, region_masks): return out - def disaggregate_source(self, downscaled): + def disaggregate_source(self): """Disaggregate water demand by source (runoff, groundwater, desal)""" print('Disaggregating Source') @@ -221,11 +221,7 @@ def disaggregate_source(self, downscaled): # apply shares to the gridded region masks gridded_shares = get_shares.generate_gridded_shares(shares_df) - # disaggregate water supply source for each downscaled demand in each grid cell - downscaled_disaggregated_sw = (downscaled * gridded_shares['runoff']).compute() - downscaled_disaggregated_gw = (downscaled * gridded_shares['groundwater']).compute() - - return gridded_shares, downscaled_disaggregated_sw, downscaled_disaggregated_gw + return gridded_shares def run_model(self): @@ -238,6 +234,17 @@ def run_model(self): self._load_region_masks() self._load_inputs() + # disaggregate water supply source + if self.source_disaggregation: + gshares = self.disaggregate_source() + + # store the disaggregated results + self.griddedshares = gshares + + if self.output_dir is not None: + filename = os.path.join(self.output_dir, f'gridded_runoff_shares.nc') + gshares['runoff'].to_netcdf(filename) + for supersector, rules in self.downscaling_rules.items(): print(f'Downscaling {supersector}') @@ -284,16 +291,6 @@ def run_model(self): downscaled.to_dataset(dim='sector').to_netcdf(filename, encoding=encoding) downscaled = xr.open_dataset(filename).to_array(dim='sector') # hopefully this keeps dask happy - # disaggregate water supply source and update the objects - if self.source_disaggregation: - gshares, sw, gw = self.disaggregate_source(downscaled) - - # store the disaggregated results - self.griddedshares = gshares - self.disaggregated_sw = sw - self.disaggregated_gw = gw - - if self.temporal_config is not None: # calculate the monthly distributions (share of annual) for each year @@ -315,7 +312,7 @@ def run_model(self): if self.output_dir is not None: filename = os.path.join(self.output_dir, f'{supersector}_{self.demand_type}_monthly.nc') encoding = {sector: {'zlib': True, 'complevel': 5} for sector in downscaled.sector.data} - downscaled.to_dataset(dim='sector').to_netcdf(filename, encoding=encoding) + downscaled.to_dataset(dim='sector').compute().to_netcdf(filename, encoding=encoding) downscaled = xr.open_dataset(filename).to_array(dim='sector') # hopefully this keeps dask happy self.outputs.update(downscaled.to_dataset(dim='sector')) diff --git a/tethys/utils/source_disaggregation.py b/tethys/utils/source_disaggregation.py index dab08a6..121dbda 100644 --- a/tethys/utils/source_disaggregation.py +++ b/tethys/utils/source_disaggregation.py @@ -34,9 +34,9 @@ def calculate_shares(self): :return: Processed DataFrame with calculated shares. """ - # query GCAM database for water used by source for the demand type (W or C) - query_type = f'*_water {self.demand_type}' - shares_df = self.conn.runQuery(easy_query('production', resource=query_type)) + # query GCAM database for water used by source for the demand type (W only) + query_type = f'*_water withdrawals' + shares_df = self.conn.runQuery(easy_query('production', replace_filters=True, resource=query_type)) # extract and clean resource names (e.g., remove '_water withdrawals') shares_df['resource'] = shares_df['resource'].apply(extract_resource_name) @@ -54,6 +54,15 @@ def calculate_shares(self): ['scenario', 'region', 'region_resourcemap', 'year'] )['value'].transform(lambda x: x / x.sum() if x.sum() != 0 else 0) + # capture basins that are mostly belonging to other countries + if basin_name_mapping == 'basin_name_mapping_im3': + shares_df = shares_df.replace({'region_resourcemap': { + 'Canada_FraserR': 'USA_FraserR', + 'Canada_GreatLakes': 'USA_GreatLakes', + 'Mexico_MexCstNW': 'USA_MexCstNW', + 'Canada_NelsonR': 'USA_NelsonR', + }}) + # filter shares based on the region masks from the config file, otherwise all regions would be included shares_df = shares_df[shares_df['region_resourcemap'].isin(self.region_masks.region.values)] From e24d6ddad0169094f41e54fe50b2bbf9eb100ba5 Mon Sep 17 00:00:00 2001 From: Travis Thurber Date: Mon, 9 Jun 2025 12:40:52 -0700 Subject: [PATCH 08/30] fix --- tethys/model.py | 1 + tethys/utils/source_disaggregation.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tethys/model.py b/tethys/model.py index b69ed90..5df664b 100644 --- a/tethys/model.py +++ b/tethys/model.py @@ -244,6 +244,7 @@ def run_model(self): if self.output_dir is not None: filename = os.path.join(self.output_dir, f'gridded_runoff_shares.nc') gshares['runoff'].to_netcdf(filename) + for supersector, rules in self.downscaling_rules.items(): diff --git a/tethys/utils/source_disaggregation.py b/tethys/utils/source_disaggregation.py index 121dbda..5c092e9 100644 --- a/tethys/utils/source_disaggregation.py +++ b/tethys/utils/source_disaggregation.py @@ -55,7 +55,7 @@ def calculate_shares(self): )['value'].transform(lambda x: x / x.sum() if x.sum() != 0 else 0) # capture basins that are mostly belonging to other countries - if basin_name_mapping == 'basin_name_mapping_im3': + if self.basin_name_mapping == 'basin_name_mapping_im3': shares_df = shares_df.replace({'region_resourcemap': { 'Canada_FraserR': 'USA_FraserR', 'Canada_GreatLakes': 'USA_GreatLakes', From d29d640a27ea0bde619fe94de2c45919d72bdd3e Mon Sep 17 00:00:00 2001 From: Travis Thurber Date: Wed, 25 Jun 2025 09:29:40 -0700 Subject: [PATCH 09/30] support `irrigation_conveyance_efficiency` configuration --- docs/source/user_guide.rst | 52 ++++++++++++++++++--------- tethys/model.py | 7 +++- tethys/utils/source_disaggregation.py | 2 +- 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst index 10ae991..b16dfbb 100644 --- a/docs/source/user_guide.rst +++ b/docs/source/user_guide.rst @@ -102,22 +102,24 @@ Configuration File ------------------ **tethys** uses a `YAML `_ configuration file. The options in this file correspond to the arguments passed to the :ref:`Tethys class `. Options not present in the config file will use the default. An overview is provided in the following table, with more details and examples below. -======================== ========================================================= -Option Description -======================== ========================================================= -:ref:`years` list of years to be included spatial downscaling -:ref:`resolution` resolution in degrees for spatial downscaling -:ref:`demand_type` choice between "withdrawals" (default) or "consumption" -:ref:`perform_temporal` choice between "false" (default) or "true" -:ref:`gcam_db` relative path to a GCAM database -:ref:`csv` relative path to csv file containing inputs -:ref:`output_file` name of file to write outputs to -:ref:`downscaling_rules` mapping from water demand sectors to proxy variables -:ref:`proxy_files` mapping of spatial proxy files to their years/variables -:ref:`map_files` list of files containing region maps -:ref:`temporal_methods` mapping of sector to temporal downscaling method -:ref:`temporal_files` files that will be accessible during temporal downscaling -======================== ========================================================= +======================================= ============================================================================= +Option Description +======================================= ============================================================================= +:ref:`years` list of years to be included spatial downscaling +:ref:`resolution` resolution in degrees for spatial downscaling +:ref:`demand_type` choice between "withdrawals" (default) or "consumption" +:ref:`perform_temporal` choice between "false" (default) or "true" +:ref:`gcam_db` relative path to a GCAM database +:ref:`csv` relative path to csv file containing inputs +:ref:`output_file` name of file to write outputs to +:ref:`downscaling_rules` mapping from water demand sectors to proxy variables +:ref:`proxy_files` mapping of spatial proxy files to their years/variables +:ref:`map_files` list of files containing region maps +:ref:`temporal_methods` mapping of sector to temporal downscaling method +:ref:`temporal_files` files that will be accessible during temporal downscaling +:ref:`source_disaggregation` if true, share of surface water vs groundwater withdrawal will be written out +:ref:`irrigation_conveyance_efficiency` if set, irrigation withdrawal will be divided by this number +======================================= ============================================================================= years @@ -272,6 +274,24 @@ Mapping of files that will be accessible to temporal downscaling methods. CDD: data/temporal/CDD_monthly.nc +source_disaggregation +^^^^^^^^^^^^^^^^^^^^^ +If true, share of surface water vs groundwater withdrawal will be written out to a file called gridded_runnoff_shares.nc + +.. code-block:: yaml + + source_disaggregation: true + + +irrigation_conveyance_efficiency +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +If set, irrigation withdrawal will be divided by this number, representing GCAM's irrigation conveyance efficiency + +.. code-block:: yaml + + irrigation_conveyance_efficiency: 0.829937 + + Generalization -------------- **tethys** was developed with consideration for GCAM's breakdown of water demand, but was designed to be as flexible as possible with support for user-specified downscaling configurations. diff --git a/tethys/model.py b/tethys/model.py index 5df664b..4f1829b 100644 --- a/tethys/model.py +++ b/tethys/model.py @@ -93,6 +93,7 @@ def __init__( self.griddedshares = None self.disaggregated_sw = None self.disaggregated_gw = None + self.irrigation_conveyance_efficiency = None # settings in YAML override settings passed directly to __init__ if config_file is not None: @@ -245,7 +246,7 @@ def run_model(self): filename = os.path.join(self.output_dir, f'gridded_runoff_shares.nc') gshares['runoff'].to_netcdf(filename) - + return for supersector, rules in self.downscaling_rules.items(): print(f'Downscaling {supersector}') @@ -285,6 +286,10 @@ def run_model(self): # in a lot of cases this could be optimized by solving the intersections at region scale first, # then downscaling once, but harder to implement, especially if differing regions are not subsets + # handle irrigation withdrawal conveyance efficiency if set + if (supersector=='Irrigation') and (self.demand_type=='withdrawals') and (self.irrigation_conveyance_efficiency is not None): + downscaled = downscaled / self.irrigation_conveyance_efficiency + # write spatial downscaling outputs if self.output_dir is not None: filename = os.path.join(self.output_dir, f'{supersector}_{self.demand_type}.nc') diff --git a/tethys/utils/source_disaggregation.py b/tethys/utils/source_disaggregation.py index 5c092e9..2ca5dac 100644 --- a/tethys/utils/source_disaggregation.py +++ b/tethys/utils/source_disaggregation.py @@ -87,7 +87,7 @@ def generate_gridded_shares(self, shares_df): (shares_df['year'] == year) & (shares_df['region_resourcemap'] == basin) & (shares_df['subresource'] == subresource) - ].iloc[0].share, + ].iloc[0].share, 0 if gridded_shares is None else gridded_shares, keep_attrs=False, ) From b32ac44c64926bc69f293c1f764a056c842283c6 Mon Sep 17 00:00:00 2001 From: Travis Thurber Date: Wed, 25 Jun 2025 11:02:35 -0700 Subject: [PATCH 10/30] remove early return --- tethys/model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tethys/model.py b/tethys/model.py index 4f1829b..09a680e 100644 --- a/tethys/model.py +++ b/tethys/model.py @@ -245,8 +245,7 @@ def run_model(self): if self.output_dir is not None: filename = os.path.join(self.output_dir, f'gridded_runoff_shares.nc') gshares['runoff'].to_netcdf(filename) - - return + for supersector, rules in self.downscaling_rules.items(): print(f'Downscaling {supersector}') From 72e8d95ba5ddcb51eeb333b4d981dff1f7b9783c Mon Sep 17 00:00:00 2001 From: hniazi Date: Wed, 3 Jun 2026 21:20:44 -0400 Subject: [PATCH 11/30] fix metadata incl package name --- README.md | 2 +- docs/source/conf.py | 2 +- docs/source/getting_started.rst | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b110684..6f5fc5b 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Details on methodology and advanced usage are available on the [User Guide](http Install `tethys` using pip: -`pip install tethys-downscale` +`pip install tethys-downscaling` ## Contributing to Tethys diff --git a/docs/source/conf.py b/docs/source/conf.py index b84a011..4051ccf 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ project = 'tethys' copyright = '2021, Battelle Memorial Institute' -author = 'Chris R. Vernon' +author = 'Isaac Thompson, Chris R. Vernon, Hassan Niazi' # The full version, including alpha/beta/rc tags release = version diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 4d234e0..64a7032 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -21,7 +21,7 @@ As a prerequisite, you'll need to have `Python Date: Wed, 3 Jun 2026 21:21:58 -0400 Subject: [PATCH 12/30] fix package versions for a robust set up across machines esp with gcamreader dependencies and adding inputviwer and tests dependencies --- requirements.txt | 10 ++++------ setup.py | 24 +++++++++++++----------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/requirements.txt b/requirements.txt index 57c8033..dd9e8c9 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML>=6.0 -gcamreader>=1.2.5 +gcamreader>=1.4.0 numpy>=1.22 pandas>=1.2.4 netCDF4>=1.6 @@ -7,8 +7,6 @@ xarray>=2022.09.0 rioxarray>=0.12.4 tqdm>=4.66.2 matplotlib>=3.8.3 -dask[complete]>=2022.12.1 -bokeh!=3.0.*,>=2.4.2 -geopandas>=0.14.3 -dask_geopandas>=0.3.1 -hvplot>=0.9.2 +dask>=2022.12.1 +distributed>=2022.12.1 +setuptools<81 diff --git a/setup.py b/setup.py index f004b3d..63f95ee 100644 --- a/setup.py +++ b/setup.py @@ -28,30 +28,32 @@ def get_requirements(): include_package_data=True, install_requires=[ 'PyYAML>=6.0', - 'gcamreader>=1.2.5', + 'gcamreader==1.4.0', 'numpy>=1.22', 'pandas>=1.2.4', 'netCDF4>=1.6', - 'dask[complete]>=2022.12.1', - 'dask_geopandas>=0.3.1', + 'dask==2024.8.0', + 'distributed==2024.8.0', 'xarray>=2022.09.0', 'rioxarray>=0.12.4', 'tqdm>=4.66.2', 'matplotlib>=3.8.3', + 'setuptools<81', 'bokeh!=3.0.*,>=2.4.2', 'geopandas>=0.14.3', + 'dask_geopandas>=0.3.1', 'hvplot>=0.9.2', ], extras_require={ 'dev': [ - 'build~=0.5.1', - 'nbsphinx~=0.8.6', - 'setuptools~=57.0.0', - 'sphinx~=4.0.2', - 'sphinx-panels~=0.6.0', - 'sphinx-rtd-theme~=0.5.2', - 'sphinx-mathjax-offline~=0.0.1', - 'twine>=3.4.1', + 'build==0.10.0', + 'nbsphinx==0.8.12', + 'sphinx==4.5.0', + 'sphinx-panels==0.6.0', + 'sphinx-rtd-theme==0.5.2', + 'sphinx-mathjax-offline==0.0.2', + 'pytest>=8.3.0', + 'pytest-cov>=5.0.0', 'click-default-group>=1.2.4', ] } From 75ee3c7d36347f19e53b72cd91eef338644f79ec Mon Sep 17 00:00:00 2001 From: hniazi Date: Wed, 3 Jun 2026 21:36:49 -0400 Subject: [PATCH 13/30] make the temporal tests pass with explicityly handling kwargs in tdmethods changing call signature and add checks in tdmethods --- tethys/tdmethods/domestic.py | 9 ++++++++- tethys/tdmethods/electricity.py | 11 +++++++++-- tethys/tdmethods/irrigation.py | 9 ++++++++- tethys/tdmethods/weights.py | 8 +++++++- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/tethys/tdmethods/domestic.py b/tethys/tdmethods/domestic.py index 443683b..fd6d17a 100644 --- a/tethys/tdmethods/domestic.py +++ b/tethys/tdmethods/domestic.py @@ -2,9 +2,16 @@ from tethys.datareader.gridded import load_file -def temporal_distribution(years, resolution, tasfile, rfile, tasvar='tas', rvar='amplitude', bounds=None): +def temporal_distribution(years, resolution=None, tasfile=None, rfile=None, tasvar='tas', rvar='amplitude', bounds=None): """Temporal downscaling for domestic water demand using algorithm from Wada et al. (2011)""" + if hasattr(years, 'temporal_config'): + model = years + cfg = (model.temporal_config or {}).get('Municipal', {}).get('kwargs', {}) + return temporal_distribution(range(model.years[0], model.years[-1] + 1), + model.resolution, cfg.get('tasfile'), cfg.get('rfile'), + bounds=model.bounds) + tas = load_file(tasfile, resolution, years, bounds=bounds, regrid_method='intensive', variables=[tasvar])[tasvar] amplitude = load_file(rfile, resolution, years, bounds=bounds, regrid_method='label', variables=[rvar])[rvar] diff --git a/tethys/tdmethods/electricity.py b/tethys/tdmethods/electricity.py index e8aedcb..6373c52 100644 --- a/tethys/tdmethods/electricity.py +++ b/tethys/tdmethods/electricity.py @@ -6,10 +6,17 @@ from tethys.datareader.maps import load_region_map -def temporal_distribution(years, resolution, hddfile, cddfile, regionfile, gcam_db, hddvar='hdd', cddvar='cdd', - bounds=None): +def temporal_distribution(years, resolution=None, hddfile=None, cddfile=None, regionfile=None, gcam_db=None, + hddvar='hdd', cddvar='cdd', bounds=None): """Temporal downscaling of water demand for electricity generation using algorithm from Voisin et al. (2013)""" + if hasattr(years, 'temporal_config'): + model = years + cfg = (model.temporal_config or {}).get('Electricity', {}).get('kwargs', {}) + return temporal_distribution(range(model.years[0], model.years[-1] + 1), + model.resolution, cfg.get('hddfile'), cfg.get('cddfile'), + cfg.get('regionfile'), cfg.get('gcam_db'), bounds=model.bounds) + # get weights of heating/cooling/other by location and time regions = load_region_map(regionfile, masks=True, target_resolution=resolution, bounds=bounds) weights = elec_sector_weights(gcam_db) diff --git a/tethys/tdmethods/irrigation.py b/tethys/tdmethods/irrigation.py index 4915bad..92fcd99 100644 --- a/tethys/tdmethods/irrigation.py +++ b/tethys/tdmethods/irrigation.py @@ -2,9 +2,16 @@ from tethys.datareader.maps import load_region_map -def temporal_distribution(years, resolution, regionfile, irrfile, irrvar='pirrww', bounds=None): +def temporal_distribution(years, resolution=None, regionfile=None, irrfile=None, irrvar='pirrww', bounds=None): """Temporal downscaling of irrigation water demand""" + if hasattr(years, 'temporal_config'): + model = years + cfg = (model.temporal_config or {}).get('Irrigation', {}).get('kwargs', {}) + return temporal_distribution(range(model.years[0], model.years[-1] + 1), + model.resolution, cfg.get('regionfile'), cfg.get('irrfile'), + bounds=model.bounds) + irr = load_file(irrfile, resolution, years, bounds=bounds, regrid_method='label', variables=[irrvar])[irrvar] region_masks = load_region_map(regionfile, masks=True, target_resolution=resolution, bounds=bounds) diff --git a/tethys/tdmethods/weights.py b/tethys/tdmethods/weights.py index bc65799..d9c422b 100644 --- a/tethys/tdmethods/weights.py +++ b/tethys/tdmethods/weights.py @@ -1,10 +1,16 @@ from tethys.datareader.gridded import load_file -def temporal_distribution(years, resolution, weightfile, weightvar='weight', regrid_method='intensive', +def temporal_distribution(years, resolution=None, weightfile=None, weightvar='weight', regrid_method='intensive', prenormalized=False, bounds=None): """Load a monthly distribution from filename""" + if hasattr(years, 'temporal_config'): + model = years + cfg = (model.temporal_config or {}).get('Weights', {}).get('kwargs', {}) + return temporal_distribution(range(model.years[0], model.years[-1] + 1), + model.resolution, cfg.get('weightfile'), bounds=model.bounds) + distribution = load_file(weightfile, resolution, years, bounds=bounds, regrid_method=regrid_method, variables=[weightvar])[weightvar] if prenormalized is False: From 647be6acdf787977143af91216570a9f9c5b896b Mon Sep 17 00:00:00 2001 From: hniazi Date: Wed, 3 Jun 2026 21:40:59 -0400 Subject: [PATCH 14/30] restore legacy temporal api in model.py as well in line with the last commit --- tethys/model.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tethys/model.py b/tethys/model.py index 09a680e..a636b48 100644 --- a/tethys/model.py +++ b/tethys/model.py @@ -39,7 +39,10 @@ def __init__( downscaling_rules=None, proxy_files=None, map_files=None, - temporal_config=None + temporal_config=None, + perform_temporal=False, + temporal_methods=None, + temporal_files=None, ): """Parameters can be specified in a YAML file or passed directly, with the config file taking precedence @@ -56,6 +59,9 @@ def __init__( :param downscaling_rules: mapping from water demand sectors to proxy variables :param proxy_files: mapping of spatial proxy files to their years/variables :param map_files: list of files containing region maps + :param perform_temporal: if True, apply temporal downscaling + :param temporal_methods: legacy mapping of sector to method names in tethys.tdmethods + :param temporal_files: legacy mapping of file aliases used by temporal methods :param temporal_config: mapping of sector to temporal downscaling method and arguments """ self.root = None @@ -82,6 +88,9 @@ def __init__( self.proxy_files = proxy_files self.map_files = map_files + self.perform_temporal = perform_temporal + self.temporal_methods = temporal_methods + self.temporal_files = temporal_files self.temporal_config = temporal_config # data we'll load or generate later From b970d43823fed6a1338bc0671d4fb06215e11f42 Mon Sep 17 00:00:00 2001 From: hniazi Date: Wed, 3 Jun 2026 21:42:06 -0400 Subject: [PATCH 15/30] bridge legacy kwargs and the new temporal_config() formal with a format normalization --- tethys/model.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tethys/model.py b/tethys/model.py index a636b48..97b2a58 100644 --- a/tethys/model.py +++ b/tethys/model.py @@ -121,6 +121,47 @@ def __init__( self.output_dir = os.path.join(self.root, self.output_dir) self._parse_proxy_files() + self._normalize_temporal_config() + + def _resolve_path(self, value): + if value is None: + return None + return value if os.path.isabs(value) else os.path.join(self.root, value) + + # maps method name → {kwarg_name: temporal_files key} + _TEMPORAL_FILE_MAP = { + 'domestic': {'tasfile': 'tas', 'rfile': 'domr'}, + 'electricity': {'hddfile': 'hdd', 'cddfile': 'cdd'}, + 'irrigation': {'irrfile': 'irr'}, + 'weights': {'weightfile': 'weight'}, + } + + def _normalize_temporal_config(self): + if self.temporal_config is not None: + for cfg in self.temporal_config.values(): + cfg['kwargs'] = {k: self._resolve_path(v) if k.endswith('file') or k == 'gcam_db' else v + for k, v in cfg.get('kwargs', {}).items()} + return + + if not self.perform_temporal and not self.temporal_methods and not self.temporal_files: + return + + files = self.temporal_files or {} + methods = self.temporal_methods or { + 'Municipal': 'domestic', 'Electricity': 'electricity', 'Irrigation': 'irrigation'} + + config = {} + for sector, method in methods.items(): + kwargs = {k: self._resolve_path(files.get(v)) + for k, v in self._TEMPORAL_FILE_MAP.get(method, {}).items()} + if method in ('electricity', 'irrigation') and self.map_files: + kwargs['regionfile'] = self._resolve_path(self.map_files[0]) + if method == 'electricity': + kwargs['gcam_db'] = self._resolve_path(self.gcam_db) + config[sector] = {'method': method, + 'kwargs': {k: v for k, v in kwargs.items() if v is not None}} + + self.temporal_config = config or None def _parse_proxy_files(self): """Handle several shorthand expressions in the proxy catalog""" @@ -315,7 +356,7 @@ def run_model(self): years = range(self.years[0], self.years[-1] + 1) kwargs = self.temporal_config[supersector]['kwargs'] distribution = temporal_distribution(years=years, resolution=self.resolution, - bounds=self.bounds, **kwargs) + bounds=self.bounds, **kwargs) else: # fall back to uniform distribution distribution = xr.DataArray(np.full(12, 1/12, np.float32), coords=dict(month=range(1, 13))) From bb14a6860a5ed4fa9c2ef0f534a80d03b14827d7 Mon Sep 17 00:00:00 2001 From: hniazi Date: Wed, 3 Jun 2026 21:43:53 -0400 Subject: [PATCH 16/30] remove redundant pytest install for the prod build --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7ea64e1..1792295 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,6 +38,4 @@ jobs: - name: Test and generate coverage report on Linux run: | - pip install pytest - pip install pytest-cov pytest --cov=./ --cov-report=xml From 1a83af12f9e5fe958bdd4cf45a549cb6b1b452bc Mon Sep 17 00:00:00 2001 From: hniazi Date: Wed, 3 Jun 2026 21:44:32 -0400 Subject: [PATCH 17/30] catch version-specific issues and use editable dev install in deployment --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1792295..6570c85 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,25 +16,25 @@ jobs: strategy: matrix: os: [ubuntu-latest] + python-version: ['3.9', '3.11'] env: OS: ${{ matrix.os }} - PYTHON: '3.9' + PYTHON: ${{ matrix.python-version }} steps: - uses: actions/checkout@v1 - name: Set up Python - uses: actions/setup-python@master + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - pip install click-default-group - pip install . + pip install -e .[dev] - name: Test and generate coverage report on Linux run: | From 1802990497df35892cba85114e16f6104d631334 Mon Sep 17 00:00:00 2001 From: hniazi Date: Wed, 3 Jun 2026 21:55:23 -0400 Subject: [PATCH 18/30] bump up the version --- tethys/__init__.py | 2 +- tethys/utils/install_supplement.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tethys/__init__.py b/tethys/__init__.py index 0d579b5..3fab7a4 100644 --- a/tethys/__init__.py +++ b/tethys/__init__.py @@ -3,4 +3,4 @@ from .utils.spatial import InputViewer -__version__ = "2.1.0" +__version__ = "2.2.0" diff --git a/tethys/utils/install_supplement.py b/tethys/utils/install_supplement.py index f03e1e1..d7aef66 100644 --- a/tethys/utils/install_supplement.py +++ b/tethys/utils/install_supplement.py @@ -17,7 +17,8 @@ '2.0.0': 'https://zenodo.org/record/7569652/files/example.zip?download=1', '2.0.1': 'https://zenodo.org/record/7569652/files/example.zip?download=1', '2.0.2': 'https://zenodo.org/record/7569652/files/example.zip?download=1', - '2.1.0': 'https://zenodo.org/record/7569652/files/example.zip?download=1' + '2.1.0': 'https://zenodo.org/record/7569652/files/example.zip?download=1', + '2.2.0': 'https://zenodo.org/record/7569652/files/example.zip?download=1' } From cb8eb246f08e0fb25412529c210be8dd16caf9d1 Mon Sep 17 00:00:00 2001 From: hniazi Date: Wed, 3 Jun 2026 22:00:08 -0400 Subject: [PATCH 19/30] upgrade action upload artifact to deploy --- .github/workflows/paper.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/paper.yml b/.github/workflows/paper.yml index 80800ce..bb75479 100644 --- a/.github/workflows/paper.yml +++ b/.github/workflows/paper.yml @@ -15,7 +15,7 @@ jobs: # This should be the path to the paper within your repo. paper-path: paper/paper.md - name: Upload - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: name: paper # This is the output path where Pandoc will write the compiled From bb4b8b54ac99269e193283a83b69a95bb3e9c1ee Mon Sep 17 00:00:00 2001 From: hniazi Date: Wed, 3 Jun 2026 22:00:38 -0400 Subject: [PATCH 20/30] only build paper for main --- .github/workflows/paper.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/paper.yml b/.github/workflows/paper.yml index bb75479..6f2e8a8 100644 --- a/.github/workflows/paper.yml +++ b/.github/workflows/paper.yml @@ -1,5 +1,13 @@ --- -on: [push] +name: paper + +on: + push: + branches: + - main + pull_request: + branches: + - main jobs: paper: From c6a6e3d918fa9d92c4d877f1e1e709446ae91443 Mon Sep 17 00:00:00 2001 From: Hassan Niazi <48460962+hassaniazi@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:19:03 -0400 Subject: [PATCH 21/30] avoid returning resource names at the first underscore e.g, unlikely but something like upper_colorado could break Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tethys/datareader/regional.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tethys/datareader/regional.py b/tethys/datareader/regional.py index 1c3edae..ddd618f 100644 --- a/tethys/datareader/regional.py +++ b/tethys/datareader/regional.py @@ -47,8 +47,10 @@ def extract_resource_name(x): :param x: resource name string. :return: cleaned resource name without suffix. """ - if '_water withdrawals' in x or '_water consumption' in x: - return x.split('_')[0] # return the part before the underscore e.g., 'basin_water withdrawals' to 'basin' + if x.endswith('_water withdrawals'): + return x.removesuffix('_water withdrawals') + if x.endswith('_water consumption'): + return x.removesuffix('_water consumption') return x From 216a9eb73215cbe6ecdfdb115bbef0089447b46a Mon Sep 17 00:00:00 2001 From: Hassan Niazi <48460962+hassaniazi@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:22:00 -0400 Subject: [PATCH 22/30] avoid hard coding demand type Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tethys/utils/source_disaggregation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tethys/utils/source_disaggregation.py b/tethys/utils/source_disaggregation.py index 2ca5dac..6087d0f 100644 --- a/tethys/utils/source_disaggregation.py +++ b/tethys/utils/source_disaggregation.py @@ -34,10 +34,11 @@ def calculate_shares(self): :return: Processed DataFrame with calculated shares. """ - # query GCAM database for water used by source for the demand type (W only) - query_type = f'*_water withdrawals' + # query GCAM database for water used by source for the configured demand type + if self.demand_type not in ('withdrawals', 'consumption'): + raise ValueError(f"Unsupported demand_type: {self.demand_type!r}") + query_type = f"*_water {self.demand_type}" shares_df = self.conn.runQuery(easy_query('production', replace_filters=True, resource=query_type)) - # extract and clean resource names (e.g., remove '_water withdrawals') shares_df['resource'] = shares_df['resource'].apply(extract_resource_name) From a125612b336df9c1c07df7fec7a49daedba96dbf Mon Sep 17 00:00:00 2001 From: Hassan Niazi <48460962+hassaniazi@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:23:34 -0400 Subject: [PATCH 23/30] make region_masks and years mandatory to fail fast before going further Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tethys/utils/source_disaggregation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tethys/utils/source_disaggregation.py b/tethys/utils/source_disaggregation.py index 6087d0f..280727b 100644 --- a/tethys/utils/source_disaggregation.py +++ b/tethys/utils/source_disaggregation.py @@ -20,12 +20,17 @@ def __init__(self, gcam_db, basin_name_mapping="basin_name_mapping_im3", demand_ :param region_masks: xarray DataArray containing region masks. :param years: List of years to be included in the shares. """ + if region_masks is None: + raise ValueError("region_masks is required to compute gridded shares") + if years is None: + raise ValueError("years is required to compute gridded shares") + dbpath, dbfile = os.path.split(gcam_db) self.conn = gcamreader.LocalDBConn(dbpath, dbfile) self.basin_name_mapping = basin_name_mapping self.demand_type = demand_type self.region_masks = region_masks - self.years = years + self.years = list(years) def calculate_shares(self): """ From fa5a4d6d91b30eacc97bc4a6ce429c81b52b63ea Mon Sep 17 00:00:00 2001 From: Hassan Niazi <48460962+hassaniazi@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:26:26 -0400 Subject: [PATCH 24/30] avoid missing share value of a basin with any zero source in a period/basin/subresource Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tethys/utils/source_disaggregation.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tethys/utils/source_disaggregation.py b/tethys/utils/source_disaggregation.py index 280727b..cb0976e 100644 --- a/tethys/utils/source_disaggregation.py +++ b/tethys/utils/source_disaggregation.py @@ -87,13 +87,15 @@ def generate_gridded_shares(self, shares_df): for year in self.years: gridded_shares = None for basin in shares_df.region_resourcemap.unique(): + sel = shares_df[ + (shares_df['year'] == year) & + (shares_df['region_resourcemap'] == basin) & + (shares_df['subresource'] == subresource) + ] + share_val = 0.0 if sel.empty else float(sel.iloc[0].share) gridded_shares = xr.where( self.region_masks.sel(region=basin).load(), - shares_df[ - (shares_df['year'] == year) & - (shares_df['region_resourcemap'] == basin) & - (shares_df['subresource'] == subresource) - ].iloc[0].share, + share_val, 0 if gridded_shares is None else gridded_shares, keep_attrs=False, ) From 151b16be90454b2dbc086693aec4ef121d8f9483 Mon Sep 17 00:00:00 2001 From: Hassan Niazi <48460962+hassaniazi@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:30:16 -0400 Subject: [PATCH 25/30] handle data type for better IO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit disaggregate_source() currently returns a plain dict of DataArrays, but run_model() initializes self.griddedshares as an xr.Dataset() and then overwrites it with that dict. Converting to an xr.Dataset here keeps the attribute’s type consistent and makes downstream usage (including I/O) less error-prone. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tethys/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tethys/model.py b/tethys/model.py index 97b2a58..dbaf361 100644 --- a/tethys/model.py +++ b/tethys/model.py @@ -285,13 +285,13 @@ def run_model(self): self._load_region_masks() self._load_inputs() + # disaggregate water supply source # disaggregate water supply source if self.source_disaggregation: - gshares = self.disaggregate_source() + gshares = xr.Dataset(self.disaggregate_source()) # store the disaggregated results self.griddedshares = gshares - if self.output_dir is not None: filename = os.path.join(self.output_dir, f'gridded_runoff_shares.nc') gshares['runoff'].to_netcdf(filename) From 2dee8db9bc9db48d53c1985824ae6847a4d0aac2 Mon Sep 17 00:00:00 2001 From: Hassan Niazi <48460962+hassaniazi@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:34:59 -0400 Subject: [PATCH 26/30] retail other vars for muni temporal method and avoid hard coding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When temporal_distribution() is called with a Tethys model object, the function rebuilds the call using only a subset of config (tasfile/rfile) and drops other supported kwargs (e.g., tasvar, rvar). Passing through **cfg preserves configurability and keeps behavior consistent between the “model object” and “direct args” call paths. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tethys/tdmethods/domestic.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tethys/tdmethods/domestic.py b/tethys/tdmethods/domestic.py index fd6d17a..01d7d8b 100644 --- a/tethys/tdmethods/domestic.py +++ b/tethys/tdmethods/domestic.py @@ -7,10 +7,9 @@ def temporal_distribution(years, resolution=None, tasfile=None, rfile=None, tasv if hasattr(years, 'temporal_config'): model = years - cfg = (model.temporal_config or {}).get('Municipal', {}).get('kwargs', {}) - return temporal_distribution(range(model.years[0], model.years[-1] + 1), - model.resolution, cfg.get('tasfile'), cfg.get('rfile'), - bounds=model.bounds) + cfg = dict((model.temporal_config or {}).get('Municipal', {}).get('kwargs', {}) or {}) + cfg.setdefault('bounds', model.bounds) + return temporal_distribution(range(model.years[0], model.years[-1] + 1), model.resolution, **cfg) tas = load_file(tasfile, resolution, years, bounds=bounds, regrid_method='intensive', variables=[tasvar])[tasvar] amplitude = load_file(rfile, resolution, years, bounds=bounds, regrid_method='label', variables=[rvar])[rvar] From 03e5be0de2a3359c82afc557ac2c2c9564942b52 Mon Sep 17 00:00:00 2001 From: Hassan Niazi <48460962+hassaniazi@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:38:48 -0400 Subject: [PATCH 27/30] retain other vars for temporal method and avoid hard coding When called with a Tethys model object, this wrapper only forwards regionfile and irrfile, silently ignoring other supported kwargs like irrvar. This makes it impossible to configure those options via temporal_config when using the model-object call style (which is used by the test suite). Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tethys/tdmethods/irrigation.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tethys/tdmethods/irrigation.py b/tethys/tdmethods/irrigation.py index 92fcd99..59781fd 100644 --- a/tethys/tdmethods/irrigation.py +++ b/tethys/tdmethods/irrigation.py @@ -7,10 +7,9 @@ def temporal_distribution(years, resolution=None, regionfile=None, irrfile=None, if hasattr(years, 'temporal_config'): model = years - cfg = (model.temporal_config or {}).get('Irrigation', {}).get('kwargs', {}) - return temporal_distribution(range(model.years[0], model.years[-1] + 1), - model.resolution, cfg.get('regionfile'), cfg.get('irrfile'), - bounds=model.bounds) + cfg = dict((model.temporal_config or {}).get('Irrigation', {}).get('kwargs', {}) or {}) + cfg.setdefault('bounds', model.bounds) + return temporal_distribution(range(model.years[0], model.years[-1] + 1), model.resolution, **cfg) irr = load_file(irrfile, resolution, years, bounds=bounds, regrid_method='label', variables=[irrvar])[irrvar] From 350a7c9336526ecd90cef9a805a8414d310ecca2 Mon Sep 17 00:00:00 2001 From: Hassan Niazi <48460962+hassaniazi@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:39:16 -0400 Subject: [PATCH 28/30] retain other vars for temporal method and avoid hard coding In the model-object call path, the wrapper forwards only a fixed subset of config and drops other supported kwargs (e.g., hddvar, cddvar). Passing through **cfg keeps the function configurable and avoids the model-object path behaving differently from the direct-args path. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tethys/tdmethods/electricity.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tethys/tdmethods/electricity.py b/tethys/tdmethods/electricity.py index 6373c52..4a92d0d 100644 --- a/tethys/tdmethods/electricity.py +++ b/tethys/tdmethods/electricity.py @@ -12,10 +12,9 @@ def temporal_distribution(years, resolution=None, hddfile=None, cddfile=None, re if hasattr(years, 'temporal_config'): model = years - cfg = (model.temporal_config or {}).get('Electricity', {}).get('kwargs', {}) - return temporal_distribution(range(model.years[0], model.years[-1] + 1), - model.resolution, cfg.get('hddfile'), cfg.get('cddfile'), - cfg.get('regionfile'), cfg.get('gcam_db'), bounds=model.bounds) + cfg = dict((model.temporal_config or {}).get('Electricity', {}).get('kwargs', {}) or {}) + cfg.setdefault('bounds', model.bounds) + return temporal_distribution(range(model.years[0], model.years[-1] + 1), model.resolution, **cfg) # get weights of heating/cooling/other by location and time regions = load_region_map(regionfile, masks=True, target_resolution=resolution, bounds=bounds) From 5f1fd2c5dd88d1014ac63adbc0eb11c554e2a3e3 Mon Sep 17 00:00:00 2001 From: Hassan Niazi <48460962+hassaniazi@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:40:03 -0400 Subject: [PATCH 29/30] retain other vars for weights temporal method and avoid hard coding In the model-object call path, this wrapper forwards only weightfile and drops other supported kwargs (e.g., weightvar, regrid_method, prenormalized). This makes temporal_config['Weights']['kwargs'] ineffective for anything beyond weightfile. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tethys/tdmethods/weights.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tethys/tdmethods/weights.py b/tethys/tdmethods/weights.py index d9c422b..4af7e33 100644 --- a/tethys/tdmethods/weights.py +++ b/tethys/tdmethods/weights.py @@ -7,9 +7,9 @@ def temporal_distribution(years, resolution=None, weightfile=None, weightvar='we if hasattr(years, 'temporal_config'): model = years - cfg = (model.temporal_config or {}).get('Weights', {}).get('kwargs', {}) - return temporal_distribution(range(model.years[0], model.years[-1] + 1), - model.resolution, cfg.get('weightfile'), bounds=model.bounds) + cfg = dict((model.temporal_config or {}).get('Weights', {}).get('kwargs', {}) or {}) + cfg.setdefault('bounds', model.bounds) + return temporal_distribution(range(model.years[0], model.years[-1] + 1), model.resolution, **cfg) distribution = load_file(weightfile, resolution, years, bounds=bounds, regrid_method=regrid_method, variables=[weightvar])[weightvar] From aa984ed97a5ab3e9c1940a9ee099df9bd05c433a Mon Sep 17 00:00:00 2001 From: Hassan Niazi <48460962+hassaniazi@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:41:01 -0400 Subject: [PATCH 30/30] correct typo Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- docs/source/user_guide.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst index b16dfbb..c8afb28 100644 --- a/docs/source/user_guide.rst +++ b/docs/source/user_guide.rst @@ -275,9 +275,8 @@ Mapping of files that will be accessible to temporal downscaling methods. source_disaggregation -^^^^^^^^^^^^^^^^^^^^^ -If true, share of surface water vs groundwater withdrawal will be written out to a file called gridded_runnoff_shares.nc - +^^^^^^^^^^^^^^^^^^^^ +If true, share of surface water vs groundwater withdrawal will be written out to a file called gridded_runoff_shares.nc .. code-block:: yaml source_disaggregation: true