Skip to content

Commit

Permalink
Merge pull request #730 from PyPSA/protected-datafiles
Browse files Browse the repository at this point in the history
apply protected() to zenodo downloads; only retrieve monthly fuel/CO2…
  • Loading branch information
lisazeyen authored Aug 24, 2023
2 parents 1087b1a + b8dee18 commit 624d240
Show file tree
Hide file tree
Showing 10 changed files with 65 additions and 45 deletions.
14 changes: 7 additions & 7 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -28,18 +28,18 @@ dconf
/data/links_p_nom.csv
/data/*totals.csv
/data/biomass*
/data/emobility/
/data/eea*
/data/jrc*
/data/bundle-sector/emobility/
/data/bundle-sector/eea*
/data/bundle-sector/jrc*
/data/heating/
/data/eurostat*
/data/bundle-sector/eurostat*
/data/odyssee/
/data/transport_data.csv
/data/switzerland*
/data/bundle-sector/switzerland*
/data/.nfs*
/data/Industrial_Database.csv
/data/bundle-sector/Industrial_Database.csv
/data/retro/tabula-calculator-calcsetbuilding.csv
/data/nuts*
/data/bundle-sector/nuts*
data/gas_network/scigrid-gas/
data/costs_*.csv

Expand Down
4 changes: 4 additions & 0 deletions doc/release_notes.rst
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ Upcoming Release

* The minimum capacity for renewable generators when using the myopic option has been fixed.

* Files downloaded from zenodo are now write-protected to prevent accidental re-download.

* Files extracted from sector-coupled data bundle have been moved from ``data/`` to ``data/sector-bundle``.

PyPSA-Eur 0.8.1 (27th July 2023)
================================

Expand Down
6 changes: 4 additions & 2 deletions rules/build_electricity.smk
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,9 @@ rule add_electricity:
hydro_capacities=ancient("data/bundle/hydro_capacities.csv"),
geth_hydro_capacities="data/geth2015_hydro_capacities.csv",
unit_commitment="data/unit_commitment.csv",
fuel_price=RESOURCES + "monthly_fuel_price.csv",
fuel_price=RESOURCES + "monthly_fuel_price.csv"
if config["conventional"]["dynamic_fuel_price"]
else [],
load=RESOURCES + "load.csv",
nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
output:
Expand Down Expand Up @@ -478,7 +480,7 @@ rule prepare_network:
input:
RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
tech_costs=COSTS,
co2_price=RESOURCES + "co2_price.csv",
co2_price=lambda w: RESOURCES + "co2_price.csv" if "Ept" in w.opts else [],
output:
RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
log:
Expand Down
28 changes: 14 additions & 14 deletions rules/build_sector.smk
Original file line number Diff line number Diff line change
Expand Up @@ -242,9 +242,9 @@ rule build_energy_totals:
energy=config["energy"],
input:
nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
co2="data/eea/UNFCCC_v23.csv",
swiss="data/switzerland-sfoe/switzerland-new_format.csv",
idees="data/jrc-idees-2015",
co2="data/bundle-sector/eea/UNFCCC_v23.csv",
swiss="data/bundle-sector/switzerland-sfoe/switzerland-new_format.csv",
idees="data/bundle-sector/jrc-idees-2015",
district_heat_share="data/district_heat_share.csv",
eurostat=input_eurostat,
output:
Expand Down Expand Up @@ -272,7 +272,7 @@ rule build_biomass_potentials:
"https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx",
keep_local=True,
),
nuts2="data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21
nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21
regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"),
swiss_cantons=ancient("data/bundle/ch_cantons.csv"),
Expand Down Expand Up @@ -366,7 +366,7 @@ if not config["sector"]["regional_co2_sequestration_potential"]["enable"]:

rule build_salt_cavern_potentials:
input:
salt_caverns="data/h2_salt_caverns_GWh_per_sqkm.geojson",
salt_caverns="data/bundle-sector/h2_salt_caverns_GWh_per_sqkm.geojson",
regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}_{clusters}.geojson",
output:
Expand All @@ -388,7 +388,7 @@ rule build_ammonia_production:
params:
countries=config["countries"],
input:
usgs="data/myb1-2017-nitro.xls",
usgs="data/bundle-sector/myb1-2017-nitro.xls",
output:
ammonia_production=RESOURCES + "ammonia_production.csv",
threads: 1
Expand All @@ -410,7 +410,7 @@ rule build_industry_sector_ratios:
ammonia=config["sector"].get("ammonia", False),
input:
ammonia_production=RESOURCES + "ammonia_production.csv",
idees="data/jrc-idees-2015",
idees="data/bundle-sector/jrc-idees-2015",
output:
industry_sector_ratios=RESOURCES + "industry_sector_ratios.csv",
threads: 1
Expand All @@ -432,8 +432,8 @@ rule build_industrial_production_per_country:
countries=config["countries"],
input:
ammonia_production=RESOURCES + "ammonia_production.csv",
jrc="data/jrc-idees-2015",
eurostat="data/eurostat-energy_balances-may_2018_edition",
jrc="data/bundle-sector/jrc-idees-2015",
eurostat="data/bundle-sector/eurostat-energy_balances-may_2018_edition",
output:
industrial_production_per_country=RESOURCES
+ "industrial_production_per_country.csv",
Expand Down Expand Up @@ -483,7 +483,7 @@ rule build_industrial_distribution_key:
input:
regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
hotmaps_industrial_database="data/Industrial_Database.csv",
hotmaps_industrial_database="data/bundle-sector/Industrial_Database.csv",
output:
industrial_distribution_key=RESOURCES
+ "industrial_distribution_key_elec_s{simpl}_{clusters}.csv",
Expand Down Expand Up @@ -558,7 +558,7 @@ rule build_industrial_energy_demand_per_country_today:
countries=config["countries"],
industry=config["industry"],
input:
jrc="data/jrc-idees-2015",
jrc="data/bundle-sector/jrc-idees-2015",
ammonia_production=RESOURCES + "ammonia_production.csv",
industrial_production_per_country=RESOURCES
+ "industrial_production_per_country.csv",
Expand Down Expand Up @@ -684,8 +684,8 @@ rule build_transport_demand:
pop_weighted_energy_totals=RESOURCES
+ "pop_weighted_energy_totals_s{simpl}_{clusters}.csv",
transport_data=RESOURCES + "transport_data.csv",
traffic_data_KFZ="data/emobility/KFZ__count",
traffic_data_Pkw="data/emobility/Pkw__count",
traffic_data_KFZ="data/bundle-sector/emobility/KFZ__count",
traffic_data_Pkw="data/bundle-sector/emobility/Pkw__count",
temp_air_total=RESOURCES + "temp_air_total_elec_s{simpl}_{clusters}.nc",
output:
transport_demand=RESOURCES + "transport_demand_s{simpl}_{clusters}.csv",
Expand Down Expand Up @@ -734,7 +734,7 @@ rule prepare_sector_network:
avail_profile=RESOURCES + "avail_profile_s{simpl}_{clusters}.csv",
dsm_profile=RESOURCES + "dsm_profile_s{simpl}_{clusters}.csv",
co2_totals_name=RESOURCES + "co2_totals.csv",
co2="data/eea/UNFCCC_v23.csv",
co2="data/bundle-sector/eea/UNFCCC_v23.csv",
biomass_potentials=RESOURCES + "biomass_potentials_s{simpl}_{clusters}.csv",
heat_profile="data/heat_load_profile_BDEW.csv",
costs="data/costs_{}.csv".format(config["costs"]["year"])
Expand Down
2 changes: 1 addition & 1 deletion rules/common.smk
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def has_internet_access(url="www.zenodo.org") -> bool:
def input_eurostat(w):
# 2016 includes BA, 2017 does not
report_year = config["energy"]["eurostat_report_year"]
return f"data/eurostat-energy_balances-june_{report_year}_edition"
return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition"


def solved_previous_horizon(wildcards):
Expand Down
2 changes: 1 addition & 1 deletion rules/postprocess.smk
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ rule plot_summary:
energy=RESULTS + "csvs/energy.csv",
balances=RESULTS + "csvs/supply_energy.csv",
eurostat=input_eurostat,
co2="data/eea/UNFCCC_v23.csv",
co2="data/bundle-sector/eea/UNFCCC_v23.csv",
output:
costs=RESULTS + "graphs/costs.pdf",
energy=RESULTS + "graphs/energy.pdf",
Expand Down
42 changes: 26 additions & 16 deletions rules/retrieve.smk
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle",

rule retrieve_databundle:
output:
expand("data/bundle/{file}", file=datafiles),
protected(expand("data/bundle/{file}", file=datafiles)),
log:
LOGS + "retrieve_databundle.log",
resources:
Expand Down Expand Up @@ -92,7 +92,7 @@ if config["enable"]["retrieve"] and config["enable"].get(
static=True,
),
output:
RESOURCES + "natura.tiff",
protected(RESOURCES + "natura.tiff"),
log:
LOGS + "retrieve_natura_raster.log",
resources:
Expand All @@ -106,22 +106,30 @@ if config["enable"]["retrieve"] and config["enable"].get(
"retrieve_sector_databundle", True
):
datafiles = [
"data/eea/UNFCCC_v23.csv",
"data/switzerland-sfoe/switzerland-new_format.csv",
"data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
"data/myb1-2017-nitro.xls",
"data/Industrial_Database.csv",
"data/emobility/KFZ__count",
"data/emobility/Pkw__count",
"data/h2_salt_caverns_GWh_per_sqkm.geojson",
directory("data/eurostat-energy_balances-june_2016_edition"),
directory("data/eurostat-energy_balances-may_2018_edition"),
directory("data/jrc-idees-2015"),
"eea/UNFCCC_v23.csv",
"switzerland-sfoe/switzerland-new_format.csv",
"nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
"myb1-2017-nitro.xls",
"Industrial_Database.csv",
"emobility/KFZ__count",
"emobility/Pkw__count",
"h2_salt_caverns_GWh_per_sqkm.geojson",
]

datafolders = [
protected(
directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition")
),
protected(
directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition")
),
protected(directory("data/bundle-sector/jrc-idees-2015")),
]

rule retrieve_sector_databundle:
output:
*datafiles,
protected(expand("data/bundle-sector/{files}", files=datafiles)),
*datafolders,
log:
LOGS + "retrieve_sector_databundle.log",
retries: 2
Expand All @@ -143,7 +151,9 @@ if config["enable"]["retrieve"] and (

rule retrieve_gas_infrastructure_data:
output:
expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles),
protected(
expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles)
),
log:
LOGS + "retrieve_gas_infrastructure_data.log",
retries: 2
Expand Down Expand Up @@ -187,7 +197,7 @@ if config["enable"]["retrieve"]:
static=True,
),
output:
"data/shipdensity_global.zip",
protected("data/shipdensity_global.zip"),
log:
LOGS + "retrieve_ship_raster.log",
resources:
Expand Down
7 changes: 5 additions & 2 deletions scripts/add_existing_baseyear.py
Original file line number Diff line number Diff line change
Expand Up @@ -447,8 +447,11 @@ def add_heating_capacities_installed_before_baseyear(
)
# if rural heating demand for one of the nodes doesn't exist,
# then columns were dropped before and heating demand share should be 0.0
if all(f"{node} {service} rural heat" in p_set_sum.index for service in ["residential", "services"])
else 0.
if all(
f"{node} {service} rural heat" in p_set_sum.index
for service in ["residential", "services"]
)
else 0.0
for node in nodal_df.index
],
index=nodal_df.index,
Expand Down
3 changes: 2 additions & 1 deletion scripts/build_industrial_distribution_key.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,14 @@
import uuid
from itertools import product

import country_converter as coco
import geopandas as gpd
import pandas as pd
from packaging.version import Version, parse

import country_converter as coco
cc = coco.CountryConverter()


def locate_missing_industrial_sites(df):
"""
Locate industrial sites without valid locations based on city and
Expand Down
2 changes: 1 addition & 1 deletion scripts/plot_summary.py
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ def historical_emissions(countries):
"""
# https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16
# downloaded 201228 (modified by EEA last on 201221)
fn = "data/eea/UNFCCC_v23.csv"
fn = "data/bundle-sector/eea/UNFCCC_v23.csv"
df = pd.read_csv(fn, encoding="latin-1")
df.loc[df["Year"] == "1985-1987", "Year"] = 1986
df["Year"] = df["Year"].astype(int)
Expand Down

0 comments on commit 624d240

Please sign in to comment.