chore: autoformat sparql through pre-commit

.pre-commit-config.yaml

@@ -19,3 +19,21 @@ repos:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
+  - repo: local
+    hooks:
+      - id: sparql-formatter
+        name: sparql-formatter
+        verbose: true  # To be able to warn on missing npx
+        entry: |
+          bash -c '
+          if ! command -v npx &> /dev/null; then
+            echo "Warning: npx is not installed. Skipping all SPARQL formatting." >&2
+            exit 0
+          fi
+          for file in "$@"; do
+            ./scripts/format-sparql.sh "$file"
+          done
+          ' --
+        language: system
+        types: [file]
+        files: \.sparql$

README.md

@@ -125,3 +125,27 @@ This can be done when initializing the `ServiceCfg` class or by specifying the `
 export SPARQL_REST_API=RDF4J  # To use RDF4J
 export SPARQL_REST_API=BLAZEGRAPH  # To use BlazeGraph
 ```
+
+## Contributing
+
+Contributions are always welcome and encouraged! Whether it's reporting a bug, suggesting an enhancement, or submitting a pull request, your input helps improve the project.
+
+### Development
+
+Dependencies are managed through uv, install with `uv sync`.
+
+It's recommended to install the pre-commit hooks so checks are run automatically on every commit.
+After installing pre-commit itself, install the hooks with `pre-commit install`.
+Checks are normally only run on modified files when committing, but you can run all checks on all files with
+`pre-commit run --all`.
+
+One of the checks is a sparql formatter, this only runs if you have npx installed.
+This is not required, but ensures any new / modified sparql files are nicely formatted.
+npx can for example be installed with npm installed through nvm:
+
+```shell
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash  # Replace the version with the latest release available at https://github.com/nvm-sh/nvm
+nvm install --lts
+npm install -g npx
+npx -v  # To verify the installation
+```

scripts/format-sparql.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+file="$1"
+
+set -e
+
+trap 'rm -f "$file.tmp"' EXIT
+
+sed -i 's/\${/__DOLLAR____LEFTBRACE__/g; s/__DOLLAR____LEFTBRACE__\([^}]*\)}/__DOLLAR____LEFTBRACE__\1__RIGHTBRACE__/g' "$file"
+npx sparql-formatter "$file" > "$file.tmp" && mv "$file.tmp" "$file"
+sed -i 's/__DOLLAR____LEFTBRACE__/\${/g; s/__RIGHTBRACE__/}/g' "$file"

src/cimsparql/sparql/ac_lines.sparql

@@ -1,96 +1,92 @@
 # Name: AC Lines
+
 PREFIX cim: <${cim}>
 PREFIX SN: <${SN}>
 PREFIX xsd: <${xsd}>
-select
+
+SELECT (MAX(?mrid) AS ?mrid) (MAX(?name) AS ?name) (MAX(?node_1) AS ?node_1) (MAX(?node_2) AS ?node_2)
+  (IF(xsd:double(MAX(?p_1)) < xsd:double(MAX(?p_2)), xsd:double(MAX(?p_1)) + xsd:double(MAX(?p_2)), xsd:double(0.0)) AS ?ploss_1)
+  (IF(xsd:double(MAX(?p_1)) > xsd:double(MAX(?p_2)), xsd:double(MAX(?p_1)) + xsd:double(MAX(?p_2)), xsd:double(0.0)) AS ?ploss_2)
+  (MAX(?r) AS ?r) (MAX(?rate) AS ?rate) (COALESCE(MAX(?in_service), MAX(?connected_1) && MAX(?connected_2), true) AS ?status)
+  (MAX(?un) AS ?un) (MAX(?x) AS ?x) (MAX(?b) AS ?b) (MAX(?g) AS ?g) (MAX(?length) AS ?length)
+  (MAX(?connectivity_node_1) AS ?connectivity_node_1) (MAX(?connectivity_node_2) AS ?connectivity_node_2)
 # Below a lot of max aggregation is performed. For variables without any suffix (_1 og _2) all values within the
+WHERE {
 # group are equal and we might as well just pick the maximum value. For variables with the suffix _1 og _2, the
 # value we want to extract either exist as one entry in the group. The other entry is an empty string. Since
 # all strings are greater than the empty string, max-aggregation will pick the value we want. For the boolan
 # value ?connected_1 (or_2), the default value is False. By a similar argument True is larger than False,
 # thus max-aggregation will pick True if it exists in the group
-(max(?mrid) as ?mrid)
-(max(?name) as ?name)
-(max(?node_1) as ?node_1)
-(max(?node_2) as ?node_2)
-
 # If SvPower at node 1 is smaller than SvPower at node 2, we assign the loss (sum of the two) to node_1. In case
 # SvPower at node 1 is larger than on node 2, the loss is assigned to node_2
-(if (xsd:double(max(?p_1)) < xsd:double(max(?p_2)), xsd:double(max(?p_1)) + xsd:double(max(?p_2)), xsd:double(0.0)) as ?ploss_1)
-(if (xsd:double(max(?p_1)) > xsd:double(max(?p_2)), xsd:double(max(?p_1)) + xsd:double(max(?p_2)), xsd:double(0.0)) as ?ploss_2)
-(max(?r) as ?r)
-(max(?rate) as ?rate)
-(coalesce(max(?in_service), max(?connected_1) && max(?connected_2), True) as ?status)
-(max(?un) as ?un)
-(max(?x) as ?x)
-(max(?b) as ?b)
-(max(?g) as ?g)
-(max(?length) as ?length)
-(max(?connectivity_node_1) as ?connectivity_node_1)
-(max(?connectivity_node_2) as ?connectivity_node_2)
-where {
   # TODO: Check if SN:Equipment.networkAnalysisEnable is on it's way into the standard
   # Assume π equivalent with at least r,x and b shunt.
   # Extract properties for ACLineSegments
-  ?_eq_subject <http://entsoe.eu/CIM/EquipmentCore/3/1> ?eq_repo
-  service ?eq_repo {
-  ?acline cim:ACLineSegment.r ?r;
-          cim:ACLineSegment.x ?x;
-          cim:ACLineSegment.bch ?b;
-          cim:IdentifiedObject.mRID ?mrid;
-          cim:IdentifiedObject.name ?name;
-          cim:ConductingEquipment.BaseVoltage/cim:BaseVoltage.nominalVoltage ?un;
-          cim:Conductor.length ?length .
-
-  ?terminal cim:Terminal.ConductingEquipment ?acline;
-            cim:Terminal.ConnectivityNode ?con_node;
-            cim:Terminal.sequenceNumber|cim:ACDCTerminal.sequenceNumber ?nr .
-
+  ?_eq_subject <http://entsoe.eu/CIM/EquipmentCore/3/1> ?eq_repo .
+  SERVICE ?eq_repo {
+    ?acline cim:ACLineSegment.r ?r ;
+            cim:ACLineSegment.x ?x ;
+            cim:ACLineSegment.bch ?b ;
+            cim:IdentifiedObject.mRID ?mrid ;
+            cim:IdentifiedObject.name ?name ;
+            cim:ConductingEquipment.BaseVoltage/cim:BaseVoltage.nominalVoltage ?un ;
+            cim:Conductor.length ?length .
+    ?terminal cim:Terminal.ConductingEquipment ?acline ;
+              cim:Terminal.ConnectivityNode ?con_node ;
+              cim:Terminal.sequenceNumber|cim:ACDCTerminal.sequenceNumber ?nr .
   # Find substation of each connectivity node of the terminals above
-  ?con_node cim:ConnectivityNode.ConnectivityNodeContainer/cim:VoltageLevel.Substation ?substation ;
-            cim:IdentifiedObject.mRID ?connectivity_node .
-
+    ?con_node cim:ConnectivityNode.ConnectivityNodeContainer/cim:VoltageLevel.Substation ?substation ;
+              cim:IdentifiedObject.mRID ?connectivity_node .
   # Find area and optionally bidzone for each substation
-  ?substation cim:Substation.Region/cim:SubGeographicalRegion.Region/cim:IdentifiedObject.name ?area .
-
-  optional {?acline cim:ACLineSegment.gch ?g .}
-  optional {?acline SN:Equipment.networkAnalysisEnable ?analysis_enabled .}
-
+    ?substation cim:Substation.Region/cim:SubGeographicalRegion.Region/cim:IdentifiedObject.name ?area .
+    OPTIONAL {
+      ?acline cim:ACLineSegment.gch ?g .
+    }
+    OPTIONAL {
+      ?acline SN:Equipment.networkAnalysisEnable ?analysis_enabled .
+    }
   # If exists, extract active power limit for acline
-  optional {
-    ?_lim cim:OperationalLimit.OperationalLimitSet/cim:OperationalLimitSet.Equipment ?acline;
-          a cim:ActivePowerLimit;
-          cim:IdentifiedObject.name '${rate}';
-          cim:ActivePowerLimit.value ?rate .
-  }} .
+    OPTIONAL {
+      ?_lim cim:OperationalLimit.OperationalLimitSet/cim:OperationalLimitSet.Equipment ?acline ;
+            a cim:ActivePowerLimit ;
+            cim:IdentifiedObject.name '${rate}' ;
+            cim:ActivePowerLimit.value ?rate .
+    }
+  }
   # Search for acline SV status will be combined with SSH connected
-  optional {?acline ^cim:SvStatus.ConductingEquipment/cim:SvStatus.inService ?in_service . }
-
+  OPTIONAL {
+    ?acline ^cim:SvStatus.ConductingEquipment/cim:SvStatus.inService ?in_service .
+  }
   {
     ?con_node cim:ConnectivityNode.TopologicalNode ?topological_node .
-  } union {
-    filter not exists {?con_node cim:ConnectivityNode.TopologicalNode ?topological_node}
+  }
+  UNION
+  {
+    FILTER NOT EXISTS {
+      ?con_node cim:ConnectivityNode.TopologicalNode ?topological_node .
+    }
     ?terminal cim:Terminal.TopologicalNode ?topological_node .
   }
   ?topological_node cim:IdentifiedObject.mRID ?node .
-
   # Find properties in TP/SV/SSH profile for each terminal.
   ?terminal cim:ACDCTerminal.connected ?connected .
-  optional {?terminal ^cim:SvPowerFlow.Terminal/cim:SvPowerFlow.p ?p .}
-
+  OPTIONAL {
+    ?terminal ^cim:SvPowerFlow.Terminal/cim:SvPowerFlow.p ?p .
+  }
+  BIND (IF(?nr = 1, str(?p), '') AS ?p_1)
+  BIND (IF(?nr = 1, ?node, '') AS ?node_1)
+  BIND (IF(?nr = 1, ?connected, false) AS ?connected_1)
+  BIND (IF(?nr = 2, str(?p), '') AS ?p_2)
+  BIND (IF(?nr = 2, ?node, '') AS ?node_2)
+  BIND (IF(?nr = 2, ?connected, false) AS ?connected_2)
+  BIND (IF(?nr = 1, ?connectivity_node, '') AS ?connectivity_node_1)
+  BIND (IF(?nr = 2, ?connectivity_node, '') AS ?connectivity_node_2)
   # Create variables for node 1 and node 2
-  bind(if(?nr = 1, str(?p), '') as ?p_1)
-  bind(if(?nr = 1, ?node, '') as ?node_1)
-  bind(if(?nr = 1, ?connected, False) as ?connected_1)
-  bind(if(?nr = 2, str(?p), '') as ?p_2)
-  bind(if(?nr = 2, ?node, '') as ?node_2)
-  bind(if(?nr = 2, ?connected, False) as ?connected_2)
-  bind(if(?nr = 1, ?connectivity_node, '') as ?connectivity_node_1)
-  bind(if(?nr = 2, ?connectivity_node, '') as ?connectivity_node_2)
-  filter(regex(?area, '${region}'))
-} group by ?acline
+  FILTER regex(?area, '${region}')
+}
+GROUP BY ?acline
+HAVING ((MAX(?node_1) != MAX(?node_2)) && (MAX(?node_1) != "") && (MAX(?node_2) != "") && (COUNT(*) > 1) && COALESCE(MAX(?analysis_enabled), true))
 # Filtration rules
 # 1) We don't need lines connecting nodes to themselves
 # 2) Only extract lines where at least two nodes exist
 # 3) Only extract lines where SN:Equipment.networkAnalysisEnable is True (if the field exists)
-having((max(?node_1) != max(?node_2)) && (max(?node_1) != "") && (max(?node_2) != "") && (count(*) > 1) && coalesce(max(?analysis_enabled), True))

src/cimsparql/sparql/add_mrid.sparql

@@ -1,8 +1,18 @@
 # Name: Add MRID
+
 PREFIX cim: <${cim}>
-insert {graph ${g} {?s cim:IdentifiedObject.mRID ?mrid}}
-where {
-  graph ${g} {?s a ${rdf_type}}
-  filter (not exists {?s cim:IdentifiedObject.mRID ?mrid})
-  bind(str(?s) as ?mrid)
+
+INSERT {
+  GRAPH ?g {
+    ?s cim:IdentifiedObject.mRID ?mrid .
+  }
+}
+WHERE {
+  GRAPH ?g {
+    ?s a ?rdf_type .
+  }
+  FILTER NOT EXISTS {
+    ?s cim:IdentifiedObject.mRID ?mrid .
+  }
+  BIND (str(?s) AS ?mrid)
 }

src/cimsparql/sparql/associated_switches.sparql

@@ -1,19 +1,23 @@
 # Name: Associated switches
+
 PREFIX cim: <${cim}>
 
-select (SAMPLE(?equipment_mrid) as ?mrid) (SAMPLE(?name) as ?name) (GROUP_CONCAT(?switch_mrid; SEPARATOR=',') as ?switch_mrids) (GROUP_CONCAT(?switch_name; SEPARATOR=',') as ?switch_names) where {
-    {
-    ?equipment ^cim:Terminal.ConductingEquipment/cim:Terminal.ConnectivityNode/^cim:Terminal.ConnectivityNode/cim:Terminal.ConductingEquipment ?switch;
-    	cim:IdentifiedObject.name ?name ;
-		  cim:IdentifiedObject.mRID ?equipment_mrid .
-     ?switch cim:Switch.normalOpen ?_normalOpen;
-		  cim:IdentifiedObject.mRID ?switch_mrid ;
-    	cim:IdentifiedObject.name ?switch_name ;
+SELECT (SAMPLE(?equipment_mrid) AS ?mrid) (SAMPLE(?name) AS ?name) (GROUP_CONCAT(?switch_mrid; SEPARATOR = ',') AS ?switch_mrids)
+  (GROUP_CONCAT(?switch_name; SEPARATOR = ',') AS ?switch_names)
+WHERE {
+  {
+    ?equipment ^cim:Terminal.ConductingEquipment/cim:Terminal.ConnectivityNode/^cim:Terminal.ConnectivityNode/cim:Terminal.ConductingEquipment ?switch ;
+               cim:IdentifiedObject.name ?name ;
+               cim:IdentifiedObject.mRID ?equipment_mrid .
+    ?switch cim:Switch.normalOpen ?_normalOpen ;
+            cim:IdentifiedObject.mRID ?switch_mrid ;
+            cim:IdentifiedObject.name ?switch_name ;
       # Only included switches that has a "second" terminal. There are switches
       # that only have one terminal. These switches can not connect/disconnect anything
-      ^cim:Terminal.ConductingEquipment/cim:ACDCTerminal.sequenceNumber 2 .
-    } minus {
-      ?equipment cim:Switch.normalOpen ?_open
-    }
-
-} group by ?equipment
+            ^cim:Terminal.ConductingEquipment/cim:ACDCTerminal.sequenceNumber 2 .
+  }
+  MINUS {
+    ?equipment cim:Switch.normalOpen ?_open .
+  }
+}
+GROUP BY ?equipment

src/cimsparql/sparql/base_voltage.sparql

@@ -1,20 +1,23 @@
 # Name: Base voltage
+
 PREFIX cim: <${cim}>
 PREFIX xsd: <${xsd}>
 
-select ?mrid ?un ?operating_voltage {
-    {
-        select ?base_voltage (avg(xsd:double(str(?voltage))) as ?operating_voltage) {
-            ?top_node ^cim:SvVoltage.TopologicalNode/cim:SvVoltage.v ?voltage;
-            cim:TopologicalNode.BaseVoltage ?base_voltage .
-            filter(xsd:double(str(?voltage)) > 0.0)
-        } group by ?base_voltage
+SELECT ?mrid ?un ?operating_voltage
+WHERE {
+  {
+    SELECT ?base_voltage (AVG(xsd:double(str(?voltage))) AS ?operating_voltage)
+    WHERE {
+      ?top_node ^cim:SvVoltage.TopologicalNode/cim:SvVoltage.v ?voltage ;
+                cim:TopologicalNode.BaseVoltage ?base_voltage .
+      FILTER (xsd:double(str(?voltage)) > 0.0)
     }
-
-    ?_eq_subject <http://entsoe.eu/CIM/EquipmentCore/3/1> ?eq_repo .
-    service ?eq_repo {
+    GROUP BY ?base_voltage
+  }
+  ?_eq_subject <http://entsoe.eu/CIM/EquipmentCore/3/1> ?eq_repo .
+  SERVICE ?eq_repo {
         # Find the nominal voltage from the base voltage associated with the topological node
-    	?base_voltage cim:BaseVoltage.nominalVoltage ?un ;
-        	cim:IdentifiedObject.mRID ?mrid
-	}
+    ?base_voltage cim:BaseVoltage.nominalVoltage ?un ;
+                  cim:IdentifiedObject.mRID ?mrid .
+  }
 }

src/cimsparql/sparql/borders.sparql

@@ -1,32 +1,35 @@
 # Name: Borders
-PREFIX cim:<${cim}>
-PREFIX SN:<${SN}>
-select  ?mrid ?name ?market_code ?area_1 ?area_2 ?t_mrid_1 ?t_mrid_2
-where {
-  # Extract mRID, name and optionally market_code for aclines
-  ?acline a cim:ACLineSegment;
-         cim:IdentifiedObject.mRID ?mrid;
-         cim:IdentifiedObject.name ?name.
-  optional {?acline cim:Equipment.EquipmentContainer/SN:Line.marketCode ?market_code} .
-  optional {?acline SN:Equipment.networkAnalysisEnable ?_analysis_enabled .}
 
-  # Extract properties for the terminal connected to one end of the acline (sequenceNumber 1)
-  ?terminal1  a cim:Terminal;
-              cim:Terminal.ConductingEquipment ?acline;
-              cim:Terminal.sequenceNumber|cim:ACDCTerminal.sequenceNumber 1;
-              cim:Terminal.ConnectivityNode/cim:ConnectivityNode.ConnectivityNodeContainer/cim:VoltageLevel.Substation/cim:Substation.Region/cim:SubGeographicalRegion.Region/cim:IdentifiedObject.name ?area_1;
-              cim:IdentifiedObject.mRID ?t_mrid_1 .
+PREFIX cim: <${cim}>
+PREFIX SN: <${SN}>
 
+SELECT ?mrid ?name ?market_code ?area_1 ?area_2 ?t_mrid_1 ?t_mrid_2
+WHERE {
+  # Extract mRID, name and optionally market_code for aclines
+  ?acline a cim:ACLineSegment ;
+          cim:IdentifiedObject.mRID ?mrid ;
+          cim:IdentifiedObject.name ?name .
+  OPTIONAL {
+    ?acline cim:Equipment.EquipmentContainer/SN:Line.marketCode ?market_code .
+  }
+  OPTIONAL {
+    ?acline SN:Equipment.networkAnalysisEnable ?_analysis_enabled .
+  }
+  # Extract properties for the terminal connected to one end of the acline (sequenceNumber 1)
+  ?terminal1 a cim:Terminal ;
+             cim:Terminal.ConductingEquipment ?acline ;
+             cim:Terminal.sequenceNumber|cim:ACDCTerminal.sequenceNumber 1 ;
+             cim:Terminal.ConnectivityNode/cim:ConnectivityNode.ConnectivityNodeContainer/cim:VoltageLevel.Substation/cim:Substation.Region/cim:SubGeographicalRegion.Region/cim:IdentifiedObject.name ?area_1 ;
+             cim:IdentifiedObject.mRID ?t_mrid_1 .
   # Extract properties for the terminal connected to the other end of the acline (sequenceNumber 2)
-  ?terminal2  a cim:Terminal;
-              cim:Terminal.ConductingEquipment ?acline;
-              cim:Terminal.sequenceNumber|cim:ACDCTerminal.sequenceNumber 2;
-              cim:Terminal.ConnectivityNode/cim:ConnectivityNode.ConnectivityNodeContainer/cim:VoltageLevel.Substation/cim:Substation.Region/cim:SubGeographicalRegion.Region/cim:IdentifiedObject.name ?area_2 ;
-              cim:IdentifiedObject.mRID ?t_mrid_2 .
-
-  filter (?area_1 != ?area_2)
-  filter (regex(?area_1, '${region}') || regex(?area_2, '${region}'))
-  filter (!regex(?name, 'HVDC'))  # Ignore HVDC
-  bind(coalesce(?_analysis_enabled, True) as ?analysis_enabled)
-  filter(?analysis_enabled)
+  ?terminal2 a cim:Terminal ;
+             cim:Terminal.ConductingEquipment ?acline ;
+             cim:Terminal.sequenceNumber|cim:ACDCTerminal.sequenceNumber 2 ;
+             cim:Terminal.ConnectivityNode/cim:ConnectivityNode.ConnectivityNodeContainer/cim:VoltageLevel.Substation/cim:Substation.Region/cim:SubGeographicalRegion.Region/cim:IdentifiedObject.name ?area_2 ;
+             cim:IdentifiedObject.mRID ?t_mrid_2 .
+  FILTER (?area_1 != ?area_2)
+  FILTER (regex(?area_1, '${region}') || regex(?area_2, '${region}'))
+  FILTER (!regex(?name, 'HVDC'))
+  BIND (COALESCE(?_analysis_enabled, true) AS ?analysis_enabled)  # Ignore HVDC
+  FILTER (?analysis_enabled = true)
 }