Merge pull request #531 from pweigmann/projectPipelines

more project pipelines, CCS subtype change

.buildlibrary

@@ -1,4 +1,4 @@
-ValidationKey: '37558640'
+ValidationKey: '37758420'
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'
 - 'Warning: namespace ''.*'' is not available and has been replaced'

CITATION.cff

@@ -2,7 +2,7 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'mrremind: MadRat REMIND Input Data Package'
-version: 0.188.0
+version: 0.189.0
 date-released: '2024-09-12'
 abstract: The mrremind packages contains data preprocessing for the REMIND model.
 authors:

DESCRIPTION

@@ -1,7 +1,7 @@
 Type: Package
 Package: mrremind
 Title: MadRat REMIND Input Data Package
-Version: 0.188.0
+Version: 0.189.0
 Date: 2024-09-12
 Authors@R: c(
     person("Lavinia", "Baumstark", , "[email protected]", role = c("aut", "cre")),

NAMESPACE

@@ -58,6 +58,7 @@ export(readGlobalCCSinstitute)
 export(readGlobalEnergyMonitor)
 export(readHRE)
 export(readIEA_CCUS)
+export(readIEA_HSMR)
 export(readIEA_WEIO_2014)
 export(readMueller)
 export(readODYM_RECC)

R/calcCCScapacity.R

@@ -9,32 +9,20 @@
 #'
 #' @export
 calcCCScapacity <- function(subtype) {
-  x <- readSource("IEA_CCUS", subtype = subtype)
+  x <- calcOutput("ProjectPipelines", subtype = "CCS", aggregate = F)
 
-  # shift around capacities for EUR for REMIND input data
-  if (subtype == "projections") {
-    # take away 50% of capacities from Norway and UK
-    x[c("NOR", "GBR"), , ] <- x[c("NOR", "GBR"), , ] * 0.5
-
-    mapping <- toolGetMapping("extramapping_EU27.csv", where = "mappingfolder", type = "regional") %>%
-      filter(.data$EU27_map == "EU27")
-    eu27 <- unique(mapping$CountryCode)
-
-    # sum up EU27 capacities and add half of Norway / UK capacities
-    eu27Pool <- dimSums(x[eu27, , ], dim = 1) + dimSums(x[c("NOR", "GBR"), , ], dim = 1)
-    getItems(eu27Pool, dim = 1) <- "EU27"
-
-    # distribute EU27 pool to the countries according to GDP
-    gdp <- calcOutput("GDP", aggregate = FALSE)[eu27, 2020, "gdp_SSP2"]
-    eu27Pool <- toolAggregate(eu27Pool,
-      rel = mapping, weight = gdp,
-      from = "EU27_map", to = "CountryCode"
-    )
-    x[eu27, , ] <- eu27Pool
+  if (subtype == "pipeline") {
+    # used as input-data for CCS bounds
+    x <- x[, c(2020, 2025, 2030), c("operational", "construction", "planned")]
+    # remove "model", "variable" and "unit" dimension
+    x <- collapseDim(x, keepdim = "status")
   }
 
   if (subtype == "historical") {
-    x <- x[, seq(2005, max(getYears(x, as.integer = TRUE))), ]
+    # project pipeline snapshot from beginning of 2024
+    x <- x[, seq(2005, 2023), "operational"]
+    # remove "status" and "unit" dimension
+    x <- collapseDim(x, keepdim = c("model", "variable"))
   }
 
   return(list(

R/calcProjectPipelines.R

@@ -80,6 +80,8 @@ calcProjectPipelines <- function(subtype) {
     # Discussion about Hydro assumptions
     # https://gitlab.pik-potsdam.de/REMIND/committed/-/issues/2
   } else if (subtype == "hydro") {
+    # Source 1: GEM
+    # -> does not include units < 75MW
     # without pumped storage
     x <- readSource("GlobalEnergyMonitor")
     x <- x[, , "Hydro", pmatch = T]
@@ -110,12 +112,48 @@ calcProjectPipelines <- function(subtype) {
                "GlobalEnergyMonitor.Cap|Electricity|Hydro.max_red.GW")
     )
 
+    # Source 2: IEA Hydropower Special Market Report
+    # no access to granular data, only scraping online data explorer,
+    # only 2030 available: scenarios "expected" and "accelerated case"
+    # "operational" corresponds to existing capacities in 2020
+    # without pumped storage (accelerated case has no differentiation,
+    # assume pumped storage equally to expected case)
+    y <- readSource("IEA_HSMR")
+
+    y <- mbind(y,
+               # ASSUMPTION: min_red = operating in 2020
+               setNames(y[, , "operational"],
+                        "IEA_HSMR.Cap|Electricity|Hydro.min_red.GW"),
+
+               # ASSUMPTION: min_yel = expected case
+               setNames(y[, , "expected"],
+                        "IEA_HSMR.Cap|Electricity|Hydro.min_yel.GW"),
+
+               # ASSUMPTION: max_yel = accelerated case
+               setNames(y[, , "accelerated"],
+                        "IEA_HSMR.Cap|Electricity|Hydro.max_yel.GW")
+
+               # ASSUMPTION: max_red = accelerated case + ?
+               # setNames(x[, , "operating"] +
+               #            x[, , "construction"] +
+               #            x[, , "pre-construction"] +
+               #            x[, , "announced"],
+               #          "IEA_HSMR.Cap|Electricity|Hydro.max_red.GW")
+    )
+
+    # add empty 2025 column, so IEA and GEM data can be merged
+    y <- add_columns(y, addnm = "y2025", dim = 2, fill= NA)
+
+    x <- mbind(x, y)
+
+
     # meta data
     unit <- "GW"
-    description <- "Hydro project pipeline from GEM"
+    description <- "Hydro project pipeline from GEM and IEA"
 
   # TODO: coming up next
-  #   # Biomass ####
+
+  #  Biomass ####
   # } else if (subtype == "biomass") {
   #   x <- readSource("GlobalEnergyMonitor")
   #   x <- x[, , "Biomass", pmatch = T]
@@ -157,30 +195,66 @@ calcProjectPipelines <- function(subtype) {
   #   # meta data
   #   unit <- "GW"
   #   description <- "Geothermal project pipeline from GEM"
-  #   # Solar ####
-  # } else if (subtype == "solar") {
-  #   x <- readSource("GlobalEnergyMonitor")
-  #   x <- x[, , "Solar", pmatch = T]
-  #
-  #   # meta data
-  #   unit <- "GW"
-  #   description <- "Solar project pipeline from GEM"
-  #
-  #   # Wind ####
-  # } else if (subtype == "wind") {
-  #   x <- readSource("GlobalEnergyMonitor")
-  #   x <- x[, , "Wind", pmatch = T]
-  #
-  #   #
-  #
-  #
-  #   # meta data
-  #   unit <- "GW"
-  #   description <- "Wind project pipeline from GEM"
-  #
-  }
+  # Solar ####
+  } else if (subtype == "solar") {
+    x <- readSource("GlobalEnergyMonitor")
+    x <- x[, , "Solar", pmatch = T]
+
+    # TODO: PV/CSP differentiation
+    # lower bounds only, as solar can be built quickly
+    x <- mbind(x,
+               # ASSUMPTION: min_red = operating
+               setNames(x[, , "Cap|Electricity|Solar.operating"],
+                        "GlobalEnergyMonitor.Cap|Electricity|Solar.min_red.GW"),
+
+               # ASSUMPTION: min_yel = operating + 0.5*construction + 0.2*pre-construction
+               setNames(x[, , "Cap|Electricity|Solar.operating"] +
+                          x[, , "Cap|Electricity|Solar.construction"]*0.5 +
+                          x[, , "Cap|Electricity|Solar.pre-construction"]*0.2,
+                        "GlobalEnergyMonitor.Cap|Electricity|Solar.min_yel.GW")
+    )
+
+    # meta data
+    unit <- "GW"
+    description <- "Solar project pipeline from GEM"
+
+    # Wind ####
+  } else if (subtype == "wind") {
+    x <- readSource("GlobalEnergyMonitor")
+    x <- x[, , "Wind", pmatch = T]
 
-  x <- x[ ,c(2025, 2030), ]
+    # TODO: On/Offshore differentiation
+    x <- mbind(x,
+               # ASSUMPTION: min_red = operating
+               setNames(x[, , "Cap|Electricity|Wind.operating"],
+                        "GlobalEnergyMonitor.Cap|Electricity|Wind.min_red.GW"),
+
+               # ASSUMPTION: min_yel = operating + 0.5*construction + 0.2*pre-construction
+               setNames(x[, , "Cap|Electricity|Wind.operating"] +
+                          x[, , "Cap|Electricity|Wind.construction"]*0.5 +
+                          x[, , "Cap|Electricity|Wind.pre-construction"]*0.2,
+                        "GlobalEnergyMonitor.Cap|Electricity|Wind.min_yel.GW"),
+
+               # ASSUMPTION: max_yel = operating + construction + 0.8*pre-construction + 0.3*announced
+               setNames(x[, , "Cap|Electricity|Wind.operating"] +
+                          x[, , "Cap|Electricity|Wind.construction"] +
+                          x[, , "Cap|Electricity|Wind.pre-construction"]*0.8 +
+                          x[, , "Cap|Electricity|Wind.announced"]*0.3,
+                        "GlobalEnergyMonitor.Cap|Electricity|Wind.max_yel.GW"),
+
+               # ASSUMPTION: max_red = operating + construction + pre-construction + announced
+               setNames(x[, , "Cap|Electricity|Wind.operating"] +
+                          x[, , "Cap|Electricity|Wind.construction"] +
+                          x[, , "Cap|Electricity|Wind.pre-construction"] +
+                          x[, , "Cap|Electricity|Wind.announced"],
+                        "GlobalEnergyMonitor.Cap|Electricity|Wind.max_red.GW")
+    )
+
+    # meta data
+    unit <- "GW"
+    description <- "Wind project pipeline from GEM"
+
+  }
 
   return(list(
     x = x,

R/convertIEA_HSMR.R

@@ -0,0 +1,14 @@
+#' convert IEA Hydro Special Market Report
+#'
+#' @param x a magclass object returned from `readIEA_HSMR()`
+#' @author Pascal Weigmann
+
+convertIEA_HSMR <- function(x) {
+
+  getItems(x, dim = 1) <- toolCountry2isocode(getItems(x, dim = 1))
+  x <- toolCountryFill(x, fill = 0, verbosity = 2)
+  x[, 2030, "operational"] <- x[, 2020, "operational"]
+  x <- x[, 2030, ]
+
+  return(x)
+}

R/exportThresholds.R

@@ -6,10 +6,14 @@
 #' @author Pascal Weigmann
 #' @param type choose either "config" to export thresholds as used in the
 #'        validationConfig or "full" to export all pipeline data
-#' @param years choose years to include, currenty only 2025 and 2030 are
-#'        available, if NULL uses all available years
+#' @param years choose years to include, currently only 2025 and 2030 are
+#'        available for all sources
 #' @export
-exportThresholds <- function(type = "config", years = NULL) {
+exportThresholds <- function(type = "config", years = c(2025, 2030)) {
+  # extra regions for ELEVATE
+  # setConfig(regionmapping = "regionmappingR10_elevate.csv")
+  # setConfig(forcecache = "GDP")
+  # extraRegions <- c("CHN", "IND", "IDN", "VNM", "PAK")
 
   # get region mappings for aggregation ----
   # Determines all regions data should be aggregated to by examining the columns
@@ -46,8 +50,21 @@ exportThresholds <- function(type = "config", years = NULL) {
                     aggregate = columnsForAggregation, round = 3,
                     warnNA = FALSE, try = FALSE, years = years)
 
+  wind <- calcOutput("ProjectPipelines", subtype = "wind",
+                      aggregate = columnsForAggregation, round = 3,
+                      warnNA = FALSE, try = FALSE, years = years)
+
+  solar <- calcOutput("ProjectPipelines", subtype = "solar",
+                      aggregate = columnsForAggregation, round = 3,
+                      warnNA = FALSE, try = FALSE, years = years)
+
   # combine and export data to madrat output folder
-  out <- mbind(ccs, hydro)
+
+  out <- mbind(ccs, hydro, wind, solar)
+  # remove ROW region, in case it exists
+  out <- out["ROW", , invert = TRUE]
+
+  # export
   if (type == "full") {
     # write report containing all available data, including all statuses and
     # thresholds attached to "variable"
@@ -62,10 +79,15 @@ exportThresholds <- function(type = "config", years = NULL) {
     # write report containing only the "min/max" thresholds in extra columns
     # (as used in a validationConfig)
     outfile <- "thresholds.mif"
-    out[, , c("min_", "max_"), pmatch = TRUE] %>%
+    out <- out[, , c("min_", "max_"), pmatch = TRUE] %>%
       as.quitte() %>%
       pivot_wider(names_from = "status") %>%
-      select(-scenario) %>%
+      select(-scenario)
+    # exclude rows without any threshold
+    out[!(is.na(out$min_red)
+          & is.na(out$min_yel)
+          & is.na(out$max_yel)
+          & is.na(out$max_red)), ] %>%
       write.csv(file = paste0(getConfig("outputfolder"), "/", outfile),
                 row.names = FALSE, quote = FALSE)
 

R/fullREMIND.R

@@ -128,7 +128,7 @@ fullREMIND <- function() {
   calcOutput("FEShares", subtype = "ind_liq",                          round = 5,  file = "p_share_ind_fehos.cs4r")
   calcOutput("Solar",                                                  round = 5,  file = "f_dataRegiSolar.cs3r")
   calcOutput("CapacityNuclear",                                        round = 5,  file = "pm_NuclearConstraint.cs4r")
-  calcOutput("CCScapacity", subtype = "projections",                   round = 8,  file = "p_boundCapCCS.cs4r")
+  calcOutput("CCScapacity", subtype = "pipeline",                      round = 8,  file = "p_boundCapCCS.cs4r")
   calcOutput("CCSbounds",                                              round = 8,  file = "p_boundCapCCSindicator.cs4r")
   calcOutput("LimitCCS",                                               round = 8,  file = "pm_dataccs.cs3r")
   calcOutput('Industry_CCS_limits',

R/fullVALIDATIONREMIND.R

@@ -189,7 +189,7 @@ fullVALIDATIONREMIND <- function(rev = 0) {
     type = "CCScapacity", subtype = "historical", file = valfile,
     aggregate = columnsForAggregation, append = TRUE, warnNA = FALSE,
     try = FALSE, years = years,
-    writeArgs = list(scenario = "historical", model = "IEA CCUS")
+    writeArgs = list(scenario = "historical")
   )
 
   # INNOPATHS ----

R/readIEA_HSMR.R

@@ -0,0 +1,31 @@
+#' read IEA Hydro Special Market Report
+#'
+#' read Hydro capacities and near-term outlook from data scraped from
+#' https://www.iea.org/data-and-statistics/data-tools/hydropower-data-explorer
+#'
+#' @author Pascal Weigmann
+#' @importFrom readxl read_xlsx
+#'
+#' @export
+readIEA_HSMR <- function() {
+
+  x <- read_xlsx("IEA_Hydropower_Special_Market_Report.xlsx") %>%
+    # get capacities in 2030 by adding 2020 values to additions
+    # subtract pumped storage as it is not part of Cap Hydro in REMIND
+    mutate("2020_operational" = capacity_2020,
+           "2030_expected" = capacity_2020 + add_expected_2030 - of_that_pumped,
+           "2030_accelerated" = capacity_2020 + add_accelerated_2030 - of_that_pumped,
+           variable = "Cap|Electricity|Hydro") %>%
+    pivot_longer(cols = c("2020_operational", "2030_expected", "2030_accelerated"),
+                 names_to = "year_scen") %>%
+    select(country, variable, year_scen, unit, value) %>%
+    separate(year_scen, c("year", "status")) %>%
+    as.magpie(spatial = "country")
+
+  x[, 2020, "accelerated"] <- x[, 2020,"operational"]
+  x[, 2020, "expected"]    <- x[, 2020,"operational"]
+
+  x <- add_dimension(x, dim = 3.1, add = "model", nm = "IEA_HSMR")
+
+  return(x)
+}

README.md

@@ -1,6 +1,6 @@
 # MadRat REMIND Input Data Package
 
-R package **mrremind**, version **0.188.0**
+R package **mrremind**, version **0.189.0**
 
 [![CRAN status](https://www.r-pkg.org/badges/version/mrremind)](https://cran.r-project.org/package=mrremind)  [![R build status](https://github.com/pik-piam/mrremind/workflows/check/badge.svg)](https://github.com/pik-piam/mrremind/actions) [![codecov](https://codecov.io/gh/pik-piam/mrremind/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/mrremind) [![r-universe](https://pik-piam.r-universe.dev/badges/mrremind)](https://pik-piam.r-universe.dev/builds)
 
@@ -39,7 +39,7 @@ In case of questions / problems please contact Lavinia Baumstark <lavinia@pik-po
 
 To cite package **mrremind** in publications use:
 
-Baumstark L, Rodrigues R, Levesque A, Oeser J, Bertram C, Mouratiadou I, Malik A, Schreyer F, Soergel B, Rottoli M, Mishra A, Dirnaichner A, Pehl M, Giannousakis A, Klein D, Strefler J, Feldhaus L, Brecha R, Rauner S, Dietrich J, Bi S, Benke F, Weigmann P, Richters O, Hasse R, Fuchs S, Mandaroux R, Koch J (2024). _mrremind: MadRat REMIND Input Data Package_. R package version 0.188.0, <https://github.com/pik-piam/mrremind>.
+Baumstark L, Rodrigues R, Levesque A, Oeser J, Bertram C, Mouratiadou I, Malik A, Schreyer F, Soergel B, Rottoli M, Mishra A, Dirnaichner A, Pehl M, Giannousakis A, Klein D, Strefler J, Feldhaus L, Brecha R, Rauner S, Dietrich J, Bi S, Benke F, Weigmann P, Richters O, Hasse R, Fuchs S, Mandaroux R, Koch J (2024). _mrremind: MadRat REMIND Input Data Package_. R package version 0.189.0, <https://github.com/pik-piam/mrremind>.
 
 A BibTeX entry for LaTeX users is
 
@@ -48,7 +48,7 @@ A BibTeX entry for LaTeX users is
   title = {mrremind: MadRat REMIND Input Data Package},
   author = {Lavinia Baumstark and Renato Rodrigues and Antoine Levesque and Julian Oeser and Christoph Bertram and Ioanna Mouratiadou and Aman Malik and Felix Schreyer and Bjoern Soergel and Marianna Rottoli and Abhijeet Mishra and Alois Dirnaichner and Michaja Pehl and Anastasis Giannousakis and David Klein and Jessica Strefler and Lukas Feldhaus and Regina Brecha and Sebastian Rauner and Jan Philipp Dietrich and Stephen Bi and Falk Benke and Pascal Weigmann and Oliver Richters and Robin Hasse and Sophie Fuchs and Rahel Mandaroux and Johannes Koch},
   year = {2024},
-  note = {R package version 0.188.0},
+  note = {R package version 0.189.0},
   url = {https://github.com/pik-piam/mrremind},
 }
 ```