Add cgmes v3.0 functionality to converter (e2nIEE#2291)

* add reading inService attribute from cim100 ssh profiles

* adapt handling of inService attribute

* Add std_dev values for all components

* Delete line for current that is already included in the added code

* bugfix inService handling

* fix merging approach for asynchronous machines and generating units

* add handling of op and sc profiles in cgmes converter for v3

* add source, origin_class and origin_id columns to measurement dataframe

* add description to measurement dataframe

* put Analog and AnalogValue ids into measurement dataframe

* adapt eq, ssh and sc profiles of cim converter

* update cim 100 schema

* remove unused elements from cim100 data structure

* fix usage of MeasurementValueSource

* set default value of power transformer end grounded to True

* fix handling of kind/limitType of OperationalLimitType

* add changelog entry for convert cgmes v3.0

* adapt converter documentation for cgmes v3.0

* add basic test for converting cgmes v3.0

* fix type hint error in python 3.8

---------

Co-authored-by: AzureAD\DimitriosZografos <[email protected]>

CHANGELOG.rst

@@ -7,6 +7,7 @@ Change Log
 - [ADDED] PowerFactory converter: option to export lines with sections as a single line with averaged-out impedance, or export line sections as separate individual lines
 - [ADDED] extend plotly function: add zoomlevel-parameter and hvdc lines
 - [ADDED] added support for reading cgmes v3.0 files
+- [ADDED] added support for converting cgmes v3.0
 - [CHANGED] plotting for voltage profile considers also gens that are slacks and only ext_grids and slack gens that are in service
 - [CHANGED] switched from setup.py to pyproject.toml
 - [CHANGED] updated upload_release.py to not call setup.py anymore (see https://packaging.python.org/en/latest/discussions/setup-py-deprecated/)

doc/converter/cgmes.rst

@@ -2,17 +2,17 @@
 CIM CGMES to pandapower
 ===================================================
 
-Converts CIM CGMES 2.4.15 networks to pandapower.
+Converts CIM CGMES 2.4.15 or 3.0 networks to pandapower.
 
-Developed and tested on Python 3.8.
+Developed and tested on Python 3.11.
 
 A `tutorial <https://github.com/e2nIEE/pandapower/blob/develop/tutorials/cim2pp.ipynb>`_ as a Jupyter notebook introduces the converter with an example.
 
 Setup
 -----
 In order to use this converter the following import is all that ist needed. ::
 
-    from pandapower.converter import from_cim
+    from pandapower.converter import from_cim as cim2pp
 
 For a speed increase it is advisable to install numba into the used python environment. ::
 
@@ -43,9 +43,9 @@ Folder of xml or zip files ::
     curr_xml_dir = 'example_cim\\test'
     cgmes_files = [curr_xml_dir + os.sep + x for x in os.listdir(curr_xml_dir)]
 
-To start the converter, the following line is used. As a result it returns a pandapower network. ::
+To start the converter, the following line is used. As cgmes_version also '3.0' can be used for cgmes version 3. As a result it returns a pandapower network. ::
 
-    net = from_cim.from_cim(file_list=cgmes_files)
+    net = cim2pp.from_cim(file_list=cgmes_files, cgmes_version='2.4.15')
 
 In the resulting pandapower-network, the following should be noted:
  - Each component-table (bus, line, trafo, etc.) will get an "origin_id" column which points to the original CIM CGMES UUIDs.

pandapower/converter/cim/cim2pp/build_pp_net.py

@@ -4,7 +4,7 @@
 # and Energy System Technology (IEE), Kassel. All rights reserved.
 import logging
 import traceback
-from typing import Dict
+from typing import Dict, List
 
 import pandas as pd
 
@@ -38,7 +38,19 @@ def __init__(self, cim_parser: cim_classes.CimParser, converter_classes: Dict, *
         self.classes_dict = converter_classes
 
     def merge_eq_ssh_profile(self, cim_type: str, add_cim_type_column: bool = False) -> pd.DataFrame:
-        df = pd.merge(self.cim['eq'][cim_type], self.cim['ssh'][cim_type], how='left', on='rdfId')
+        return self.merge_eq_other_profiles(['ssh'], cim_type, add_cim_type_column)
+
+    def merge_eq_sc_profile(self, cim_type: str, add_cim_type_column: bool = False) -> pd.DataFrame:
+        return self.merge_eq_other_profiles(['sc'], cim_type, add_cim_type_column)
+
+    def merge_eq_other_profiles(self, other_profiles: List[str], cim_type: str,
+                                add_cim_type_column: bool = False) -> pd.DataFrame:
+        df = self.cim['eq'][cim_type]
+        for other_profile in other_profiles:
+            if cim_type not in self.cim[other_profile].keys():
+                self.logger.debug("No entries found in %s profile for cim object %s", other_profile, cim_type)
+                return self.cim['eq'][cim_type].copy()
+            df = pd.merge(df, self.cim[other_profile][cim_type], how='left', on='rdfId')
         if add_cim_type_column:
             df[sc['o_cl']] = cim_type
         return df

pandapower/converter/cim/cim2pp/convert_measurements.py

@@ -39,16 +39,20 @@ def create_measurements_from_analog(self):
         time_start = time.time()
         sc = cim_tools.get_pp_net_special_columns_dict()
         # join the Analogs with the AnalogValues and MeasurementValueSources
+        analogs_prf = 'op' if 'op' in self.cim.keys() else 'eq'
         analogs = pd.merge(
-            self.cim['eq']['Analog'][['rdfId', 'measurementType', 'unitSymbol', 'unitMultiplier', 'Terminal',
-                                      'PowerSystemResource', 'positiveFlowIn']],
-            self.cim['eq']['AnalogValue'][['sensorAccuracy', 'MeasurementValueSource', 'Analog', 'value']],
-            how='inner', left_on='rdfId', right_on='Analog')
-        analogs = analogs.drop(columns=['rdfId', 'Analog'])
-        analogs = pd.merge(analogs, self.cim['eq']['MeasurementValueSource'], how='left',
+            self.cim[analogs_prf]['Analog'][['rdfId', 'name', 'description', 'measurementType', 'unitSymbol',
+                                             'unitMultiplier', 'Terminal', 'PowerSystemResource', 'positiveFlowIn']],
+            self.cim[analogs_prf]['AnalogValue'][['rdfId', 'sensorAccuracy', 'MeasurementValueSource', 'Analog',
+                                                  'value']],
+            how='inner', left_on='rdfId', right_on='Analog', suffixes=("_Analog", "_AnalogValue"))
+        analogs = analogs.drop(columns=['Analog'])
+        analogs = analogs.rename(columns={'name': 'name_analog'})
+        analogs = pd.merge(analogs, self.cim[analogs_prf]['MeasurementValueSource'], how='left',
                            left_on='MeasurementValueSource',
                            right_on='rdfId')
         analogs = analogs.drop(columns=['rdfId', 'MeasurementValueSource'])
+        analogs = analogs.rename(columns={'name': sc['src']})
         # collect all the assets (line, trafo, trafo3w) and its connections
         assets = pd.DataFrame(None, columns=['element_type', 'side'])
         append_dict = dict({'line': {'from_bus': 'from', 'to_bus': 'to'},
@@ -71,17 +75,36 @@ def create_measurements_from_analog(self):
         psr['measurement_type'] = psr.unitSymbol.map({'W': 'p', 'VAr': 'q', 'A': 'i', 'V': 'v'})
         # change the sign if need
         psr['value'].loc[~psr['positiveFlowIn']] = psr.loc[~psr['positiveFlowIn']]['value'] * (-1)
-        # convert all amperes to ka
-        psr['value'].loc[psr['measurement_type'] == 'i'] = psr.loc[psr['measurement_type'] == 'i']['value'] / 1e3
         # move the voltage measurements to the buses
         psr = pd.merge(psr, self.net.bus[['vn_kv']], how='inner', left_on='bus', right_index=True)
-        temp = psr.loc[psr['measurement_type'] == 'v']
-        temp['value'] = temp['value'] / temp['vn_kv']
-        temp['std_dev'] = temp['sensorAccuracy'] / temp['vn_kv']
-        temp['element_type'] = 'bus'
-        temp['element'] = temp['bus']
-        temp['side'] = None
-        psr.loc[psr['measurement_type'] == 'v'] = temp
+        psr_v = psr.loc[psr['measurement_type'] == 'v']     
+        psr_v['value'] = psr_v['value'] / psr_v['vn_kv']
+        psr_v['std_dev'] = psr_v['sensorAccuracy'] / psr_v['vn_kv']
+        psr_v['element_type'] = 'bus'
+        psr_v['element'] = psr_v['bus']
+        psr_v['side'] = None
+        psr.loc[psr['measurement_type'] == 'v', psr_v.columns] = psr_v
+
+        #convert amps to ka and assign std_dev values for 'i' measurements
+        psr_i = psr.loc[psr['measurement_type'] == 'i'] 
+        psr_i['value'] = psr_i['value'] / 1e3
+        psr_i['std_dev'] = psr_i['sensorAccuracy'] / 1e3
+        psr.loc[psr['measurement_type'] == 'i', psr_i.columns] = psr_i
+
+        #assign std_dev values for 'p' measurements
+        psr_p = psr.loc[psr['measurement_type'] == 'p'] 
+        psr_p['std_dev'] = psr_p['sensorAccuracy'] 
+        psr.loc[psr['measurement_type'] == 'p', psr_p.columns] = psr_p
+
+        #assign std_dev values for 'q' measurements
+        psr_q = psr.loc[psr['measurement_type'] == 'q'] 
+        psr_q['std_dev'] = psr_q['sensorAccuracy'] 
+        psr.loc[psr['measurement_type'] == 'q', psr_q.columns] = psr_q
+
+        psr = psr.drop(columns=[sc['o_id']])
+        psr = psr.rename(columns={'rdfId_AnalogValue': sc['o_id'], 'name_analog': sc['name'],
+                                  'rdfId_Analog': sc['a_id']})
+        psr[sc['o_cl']] = 'AnalogValue'
 
         self._copy_to_measurement(psr)
 
@@ -95,14 +118,19 @@ def create_measurements_from_sv(self):
         time_start = time.time()
         sc = cim_tools.get_pp_net_special_columns_dict()
         # get the measurements from the sv profile and set the Terminal as index
-        sv_powerflow = self.cim['sv']['SvPowerFlow'][['Terminal', 'p', 'q']]
+        sv_powerflow = self.cim['sv']['SvPowerFlow'][['rdfId', 'Terminal', 'p', 'q']]
         sv_powerflow = sv_powerflow.set_index('Terminal')
+        sv_powerflow[sc['o_cl']] = 'SvPowerFlow'
+        sv_powerflow[sc['src']] = 'SV'
+        sv_powerflow[sc['desc']] = None
+        sv_powerflow[sc['a_id']] = None
+        sv_powerflow = sv_powerflow.rename(columns={'rdfId': sc['o_id']})
 
         # ---------------------------------------measure: bus v---------------------------------------------------
         busses_temp = self.net.bus[['name', 'vn_kv', sc['ct']]].copy()
         busses_temp = busses_temp.reset_index(level=0)
         busses_temp = busses_temp.rename(columns={'index': 'element', sc['ct']: 'TopologicalNode'})
-        sv_sv_voltages = pd.merge(self.cim['sv']['SvVoltage'][['TopologicalNode', 'v']], busses_temp,
+        sv_sv_voltages = pd.merge(self.cim['sv']['SvVoltage'][['rdfId', 'TopologicalNode', 'v']], busses_temp,
                                   how='left', on='TopologicalNode')
         # drop all the rows mit vn_kv == np.NaN (no measurements available for that bus)
         sv_sv_voltages = sv_sv_voltages.dropna(subset=['vn_kv'])
@@ -121,6 +149,11 @@ def create_measurements_from_sv(self):
         sv_sv_voltages['measurement_type'] = 'v'
         sv_sv_voltages['element_type'] = 'bus'
         sv_sv_voltages['side'] = None
+        sv_sv_voltages[sc['src']] = 'SV'
+        sv_sv_voltages[sc['o_cl']] = 'SvVoltage'
+        sv_sv_voltages[sc['desc']] = None
+        sv_sv_voltages[sc['a_id']] = None
+        sv_sv_voltages = sv_sv_voltages.rename(columns={'rdfId': sc['o_id']})
 
         self._copy_to_measurement(sv_sv_voltages)
 
@@ -150,22 +183,29 @@ def create_measurements_from_sv(self):
         line_temp['value'] = line_temp.p_from
         line_temp['std_dev'] = line_temp.stddev_line_from_p
         line_temp['side'] = line_temp.from_bus
+        line_temp = pd.merge(line_temp, sv_powerflow, left_on=sc['t_from'], right_index=True)
         self._copy_to_measurement(line_temp)
+        line_temp = line_temp.drop(columns=sv_powerflow.columns)
         # ---------------------------------------measure: line p to---------------------------------------------------
         line_temp['value'] = line_temp.p_to
         line_temp['std_dev'] = line_temp.stddev_line_to_p
         line_temp['side'] = line_temp.to_bus
+        line_temp = pd.merge(line_temp, sv_powerflow, left_on=sc['t_to'], right_index=True)
         self._copy_to_measurement(line_temp)
+        line_temp = line_temp.drop(columns=sv_powerflow.columns)
         # ---------------------------------------measure: line q from---------------------------------------------------
         line_temp['measurement_type'] = 'q'
         line_temp['value'] = line_temp.q_from
         line_temp['std_dev'] = line_temp.stddev_line_from_q
         line_temp['side'] = line_temp.from_bus
+        line_temp = pd.merge(line_temp, sv_powerflow, left_on=sc['t_from'], right_index=True)
         self._copy_to_measurement(line_temp)
+        line_temp = line_temp.drop(columns=sv_powerflow.columns)
         # ---------------------------------------measure: line q to---------------------------------------------------
         line_temp['value'] = line_temp.q_to
         line_temp['std_dev'] = line_temp.stddev_line_to_q
         line_temp['side'] = line_temp.to_bus
+        line_temp = pd.merge(line_temp, sv_powerflow, left_on=sc['t_to'], right_index=True)
         self._copy_to_measurement(line_temp)
 
         # ---------------------------------------measure: trafo---------------------------------------------------
@@ -195,22 +235,29 @@ def create_measurements_from_sv(self):
         trafo_temp['value'] = trafo_temp.p_hv
         trafo_temp['std_dev'] = trafo_temp.stddev_trafo_hv_p
         trafo_temp['side'] = trafo_temp.hv_bus
+        trafo_temp = pd.merge(trafo_temp, sv_powerflow, left_on=sc['t_hv'], right_index=True)
         self._copy_to_measurement(trafo_temp)
+        trafo_temp = trafo_temp.drop(columns=sv_powerflow.columns)
         # ---------------------------------------measure: trafo p lv---------------------------------------------------
         trafo_temp['value'] = trafo_temp.p_lv
         trafo_temp['std_dev'] = trafo_temp.stddev_trafo_lv_p
         trafo_temp['side'] = trafo_temp.lv_bus
+        trafo_temp = pd.merge(trafo_temp, sv_powerflow, left_on=sc['t_lv'], right_index=True)
         self._copy_to_measurement(trafo_temp)
+        trafo_temp = trafo_temp.drop(columns=sv_powerflow.columns)
         # ---------------------------------------measure: trafo q hv---------------------------------------------------
         trafo_temp['measurement_type'] = 'q'
         trafo_temp['value'] = trafo_temp.q_hv
         trafo_temp['std_dev'] = trafo_temp.stddev_trafo_hv_q
         trafo_temp['side'] = trafo_temp.hv_bus
+        trafo_temp = pd.merge(trafo_temp, sv_powerflow, left_on=sc['t_hv'], right_index=True)
         self._copy_to_measurement(trafo_temp)
+        trafo_temp = trafo_temp.drop(columns=sv_powerflow.columns)
         # ---------------------------------------measure: trafo q lv---------------------------------------------------
         trafo_temp['value'] = trafo_temp.q_lv
         trafo_temp['std_dev'] = trafo_temp.stddev_trafo_lv_q
         trafo_temp['side'] = trafo_temp.lv_bus
+        trafo_temp = pd.merge(trafo_temp, sv_powerflow, left_on=sc['t_lv'], right_index=True)
         self._copy_to_measurement(trafo_temp)
 
         # ---------------------------------------measure: trafo3w---------------------------------------------------
@@ -246,32 +293,43 @@ def create_measurements_from_sv(self):
         trafo3w_temp['value'] = trafo3w_temp.p_hv
         trafo3w_temp['std_dev'] = trafo3w_temp.stddev_trafo_hv_p
         trafo3w_temp['side'] = trafo3w_temp.hv_bus
+        trafo3w_temp = pd.merge(trafo3w_temp, sv_powerflow, left_on=sc['t_hv'], right_index=True)
         self._copy_to_measurement(trafo3w_temp)
+        trafo3w_temp = trafo3w_temp.drop(columns=sv_powerflow.columns)
         # ---------------------------------------measure: trafo3w p mv---------------------------------------------
         trafo3w_temp['value'] = trafo3w_temp.p_mv
         trafo3w_temp['std_dev'] = trafo3w_temp.stddev_trafo_mv_p
         trafo3w_temp['side'] = trafo3w_temp.mv_bus
+        trafo3w_temp = pd.merge(trafo3w_temp, sv_powerflow, left_on=sc['t_mv'], right_index=True)
         self._copy_to_measurement(trafo3w_temp)
+        trafo3w_temp = trafo3w_temp.drop(columns=sv_powerflow.columns)
         # ---------------------------------------measure: trafo3w p lv---------------------------------------------
         trafo3w_temp['value'] = trafo3w_temp.p_lv
         trafo3w_temp['std_dev'] = trafo3w_temp.stddev_trafo_lv_p
         trafo3w_temp['side'] = trafo3w_temp.lv_bus
+        trafo3w_temp = pd.merge(trafo3w_temp, sv_powerflow, left_on=sc['t_lv'], right_index=True)
         self._copy_to_measurement(trafo3w_temp)
+        trafo3w_temp = trafo3w_temp.drop(columns=sv_powerflow.columns)
         # ---------------------------------------measure: trafo3w q hv---------------------------------------------
         trafo3w_temp['measurement_type'] = 'q'
         trafo3w_temp['value'] = trafo3w_temp.q_hv
         trafo3w_temp['std_dev'] = trafo3w_temp.stddev_trafo_hv_q
         trafo3w_temp['side'] = trafo3w_temp.hv_bus
+        trafo3w_temp = pd.merge(trafo3w_temp, sv_powerflow, left_on=sc['t_hv'], right_index=True)
         self._copy_to_measurement(trafo3w_temp)
+        trafo3w_temp = trafo3w_temp.drop(columns=sv_powerflow.columns)
         # ---------------------------------------measure: trafo3w q mv---------------------------------------------
         trafo3w_temp['value'] = trafo3w_temp.q_mv
         trafo3w_temp['std_dev'] = trafo3w_temp.stddev_trafo_mv_q
         trafo3w_temp['side'] = trafo3w_temp.mv_bus
+        trafo3w_temp = pd.merge(trafo3w_temp, sv_powerflow, left_on=sc['t_mv'], right_index=True)
         self._copy_to_measurement(trafo3w_temp)
+        trafo3w_temp = trafo3w_temp.drop(columns=sv_powerflow.columns)
         # ---------------------------------------measure: trafo3w q lv---------------------------------------------
         trafo3w_temp['value'] = trafo3w_temp.q_lv
         trafo3w_temp['std_dev'] = trafo3w_temp.stddev_trafo_lv_q
         trafo3w_temp['side'] = trafo3w_temp.lv_bus
+        trafo3w_temp = pd.merge(trafo3w_temp, sv_powerflow, left_on=sc['t_lv'], right_index=True)
         self._copy_to_measurement(trafo3w_temp)
 
         # remove NaN values