remove build math addition from backend

src/calliope/backend/backend_model.py

@@ -4,7 +4,6 @@
 
 from __future__ import annotations
 
-import importlib
 import logging
 import time
 import typing
@@ -33,11 +32,9 @@
 from calliope.exceptions import warn as model_warn
 from calliope.io import load_config
 from calliope.util.schema import (
- MATH_SCHEMA,
  MODEL_SCHEMA,
  extract_from_schema,
  update_then_validate_config,
- validate_dict,
 )
 
 if TYPE_CHECKING:
@@ -205,7 +202,6 @@ def _check_inputs(self):
  )
 
  def _build(self) -> None:
- self._add_run_mode_math()
  # The order of adding components matters!
  # 1. Variables, 2. Global Expressions, 3. Constraints, 4. Objectives
  for components in [
@@ -224,26 +220,6 @@ def _build(self) -> None:
  )
  LOGGER.info(f"Optimisation Model | {components} | Generated.")
 
- def _add_run_mode_math(self) -> None:
- """If not given in the add_math list, override model math with run mode math."""
- # FIXME: available modes should not be hardcoded here. They should come from a YAML schema.
- mode = self.inputs.attrs["config"].build.mode
- add_math = self.inputs.attrs["applied_additional_math"]
- not_run_mode = {"plan", "operate", "spores"}.difference([mode])
- run_mode_mismatch = not_run_mode.intersection(add_math)
- if run_mode_mismatch:
- exceptions.warn(
- f"Running in {mode} mode, but run mode(s) {run_mode_mismatch} "
- "math being loaded from file via the model configuration"
- )
-
- if mode != "plan" and mode not in add_math:
- LOGGER.debug(f"Updating math formulation with {mode} mode math.")
- filepath = importlib.resources.files("calliope") / "math" / f"{mode}.yaml"
- self.math.union(AttrDict.from_yaml(filepath), allow_override=True)
-
- validate_dict(self.math, MATH_SCHEMA, "math")
-
  def _add_component(
  self,
  name: str,

src/calliope/model.py

@@ -88,6 +88,7 @@ def __init__(
 
  # new
  self.math: AttrDict = AttrDict()
+ self._math_dir: Path = Path(calliope.__file__).parent / "math"
 
  # try to set logging output format assuming python interactive. Will
  # use CLI logging format if model called from CLI
@@ -312,14 +313,13 @@ def _add_math(self, add_math: list) -> AttrDict:
  Returns:
  AttrDict: Dictionary of math (constraints, variables, objectives, and global expressions).
  """
- math_dir = Path(calliope.__file__).parent / "math"
- base_math = AttrDict.from_yaml(math_dir / "base.yaml")
+ base_math = AttrDict.from_yaml(self._math_dir / "base.yaml")
 
  file_errors = []
 
  for filename in add_math:
  if not f"{filename}".endswith((".yaml", ".yml")):
- yaml_filepath = math_dir / f"{filename}.yaml"
+ yaml_filepath = self._math_dir / f"{filename}.yaml"
  else:
  yaml_filepath = relative_path(self._model_def_path, filename)
 
@@ -371,6 +371,10 @@ def build(self, force: bool = False, **kwargs) -> None:
  )
  else:
  input = self._model_data
+
+ self._add_run_mode_math()
+ validate_dict(self.math, MATH_SCHEMA, "math")
+
  backend_name = updated_build_config["backend"]
  backend = self._BACKENDS[backend_name](input, self.math, **updated_build_config)
  backend._build()
@@ -597,6 +601,24 @@ def validate_math_strings(self, math_dict: dict) -> None:
 
  LOGGER.info("Model: validated math strings")
 
+ def _add_run_mode_math(self) -> AttrDict:
+ """If not given in the add_math list, override model math with run mode math."""
+ # FIXME: available modes should not be hardcoded here. They should come from a YAML schema.
+ mode = self._model_data.attrs["config"].build.mode
+ applied_additional_math = self._model_data.attrs["applied_additional_math"]
+ not_run_mode = {"plan", "operate", "spores"}.difference([mode])
+ run_mode_mismatch = not_run_mode.intersection(applied_additional_math)
+ if run_mode_mismatch:
+ exceptions.warn(
+ f"Running in {mode} mode, but run mode(s) {run_mode_mismatch} "
+ "math being loaded from file via the model configuration"
+ )
+
+ if mode != "plan" and mode not in applied_additional_math:
+ LOGGER.debug(f"Updating math formulation with {mode} mode math.")
+ filepath = self._math_dir / f"{mode}.yaml"
+ self.math.union(AttrDict.from_yaml(filepath), allow_override=True)
+
  def _prepare_operate_mode_inputs(
  self, start_window_idx: int = 0, **config_kwargs
  ) -> xr.Dataset:

tests/test_backend_pyomo.py

@@ -1,6 +1,4 @@
-import importlib
 import logging
-from copy import deepcopy
 from itertools import product
 
 import calliope
@@ -10,8 +8,6 @@
 import pyomo.kernel as pmo
 import pytest # noqa: F401
 import xarray as xr
-from calliope.attrdict import AttrDict
-from calliope.backend.pyomo_backend_model import PyomoBackendModel
 
 from .common.util import build_test_model as build_model
 from .common.util import check_error_or_warning, check_variable_exists
@@ -1625,79 +1621,13 @@ def simple_supply_longnames(self):
  assert m.backend._has_verbose_strings
  return m
 
- @pytest.fixture(scope="class")
- def simple_supply_updated_cost_flow_cap(
- self, simple_supply: calliope.Model
- ) -> calliope.Model:
- simple_supply.backend.verbose_strings()
- simple_supply.backend.update_parameter("cost_flow_cap", DUMMY_INT)
- return simple_supply
-
- @pytest.fixture()
- def temp_path(self, tmpdir_factory):
- return tmpdir_factory.mktemp("custom_math")
-
  def test_new_build_has_backend(self, simple_supply):
  assert hasattr(simple_supply, "backend")
 
  def test_new_build_optimal(self, simple_supply):
  assert hasattr(simple_supply, "results")
  assert simple_supply._model_data.attrs["termination_condition"] == "optimal"
 
- @pytest.mark.parametrize("mode", ["operate", "spores"])
- def test_add_run_mode_custom_math(self, caplog, mode):
- caplog.set_level(logging.DEBUG)
- mode_custom_math = AttrDict.from_yaml(
- importlib.resources.files("calliope") / "math" / f"{mode}.yaml"
- )
- m = build_model({}, "simple_supply,two_hours,investment_costs")
-
- base_math = deepcopy(m.math)
- base_math.union(mode_custom_math, allow_override=True)
-
- backend = PyomoBackendModel(m.inputs, base_math, mode=mode)
- backend._add_run_mode_math()
-
- assert f"Updating math formulation with {mode} mode math." in caplog.text
-
- assert m.math != base_math
- assert backend.math.as_dict() == base_math.as_dict()
-
- def test_add_run_mode_custom_math_before_build(self, caplog, temp_path):
- """A user can override the run mode math by including it directly in the additional math list"""
- caplog.set_level(logging.DEBUG)
- custom_math = AttrDict({"variables": {"flow_cap": {"active": True}}})
- file_path = temp_path.join("custom-math.yaml")
- custom_math.to_yaml(file_path)
-
- m = build_model(
- {"config.init.add_math": ["operate", str(file_path)]},
- "simple_supply,two_hours,investment_costs",
- )
- backend = PyomoBackendModel(m.inputs, m.math, mode="operate")
- backend._add_run_mode_math()
-
- # We set operate mode explicitly in our additional math so it won't be added again
- assert "Updating math formulation with operate mode math." not in caplog.text
-
- # operate mode set it to false, then our math set it back to active
- assert m.math.variables.flow_cap.active
- # operate mode set it to false and our math did not override that
- assert not m.math.variables.storage_cap.active
-
- def test_run_mode_mismatch(self):
- m = build_model(
- {"config.init.add_math": ["operate"]},
- "simple_supply,two_hours,investment_costs",
- )
- backend = PyomoBackendModel(m.inputs, m.math)
- with pytest.warns(exceptions.ModelWarning) as excinfo:
- backend._add_run_mode_math()
-
- assert check_error_or_warning(
- excinfo, "Running in plan mode, but run mode(s) {'operate'}"
- )
-
  @pytest.mark.parametrize(
  "component_type", ["variable", "global_expression", "parameter", "constraint"]
  )

tests/test_core_model.py

@@ -1,9 +1,12 @@
+import importlib
 import logging
+from copy import deepcopy
 
 import calliope
 import numpy as np
 import pandas as pd
 import pytest
+from calliope import Model
 
 from .common.util import build_test_model as build_model
 from .common.util import check_error_or_warning
@@ -205,6 +208,58 @@ def test_override_existing_internal_constraint_merge(
  assert base[i] == new[i]
 
 
+class TestAddRunModeMath:
+ """Tests for the addition of extra math during model build step."""
+
+ @pytest.fixture()
+ def temp_path(self, tmpdir_factory):
+ return tmpdir_factory.mktemp("custom_math")
+
+ @pytest.fixture()
+ def model_w_added_math(self, temp_path) -> Model:
+ """Model with unused additional math."""
+ custom_math = calliope.AttrDict({"variables": {"flow_cap": {"active": True}}})
+ file_path = temp_path.join("custom-math.yaml")
+ custom_math.to_yaml(file_path)
+
+ model = build_model(
+ {"config.init.add_math": ["operate", str(file_path)]},
+ "simple_supply,two_hours,investment_costs",
+ )
+ return model
+
+ @pytest.mark.parametrize("mode", ["operate", "spores", "storage_inter_cluster"])
+ def test_add_run_mode_custom_math(self, caplog, mode):
+ """Mismatches between initialised math and mode trigger math additions."""
+ caplog.set_level(logging.DEBUG)
+ mode_custom_math = calliope.AttrDict.from_yaml(
+ importlib.resources.files("calliope") / "math" / f"{mode}.yaml"
+ )
+ m = build_model({}, "simple_supply,two_hours,investment_costs")
+
+ ini_math = deepcopy(m.math)
+ expected_math = deepcopy(m.math)
+ expected_math.union(mode_custom_math, allow_override=True)
+
+ m.config["build"]["mode"] = mode
+ m._add_run_mode_math()
+
+ assert f"Updating math formulation with {mode}" in caplog.text
+ assert m.math != ini_math
+ assert m.math == expected_math
+
+ def test_no_reload_if_added(self, caplog, model_w_added_math):
+ """Math loaded in init shall not be re-loaded during build."""
+ caplog.set_level(logging.DEBUG)
+ model_w_added_math._add_run_mode_math()
+ assert "Updating math formulation with operate" not in caplog.text
+
+ def test_unused_mode_warning(self, model_w_added_math: Model):
+ """Raise a warning if additional pre-defined math was loaded but not used."""
+ with pytest.warns(calliope.exceptions.ModelWarning):
+ model_w_added_math._add_run_mode_math()
+
+
 class TestValidateMathDict:
  def test_base_math(self, caplog, simple_supply):
  with caplog.at_level(logging.INFO, logger=LOGGER):