when adding a zone, check and mark its underlay IP as allocated

Author: jgallagher

nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs

@@ -117,6 +117,8 @@ pub enum SledEditError {
     ZoneOnNonexistentZpool { zone_id: OmicronZoneUuid, zpool: ZpoolName },
     #[error("ran out of underlay IP addresses")]
     OutOfUnderlayIps,
+    #[error("cannot add zone with invalid underlay IP")]
+    AddZoneBadUnderlayIp(#[source] SledUnderlayIpOutOfRange),
 }
 
 #[derive(Debug)]
@@ -350,18 +352,9 @@ impl ActiveSledEditor {
         // dispositions. If a zone has been fully removed from the blueprint
         // some time after expungement, we may reuse its IP; reconfigurator must
         // know that's safe prior to pruning the expunged zone.
-        let zone_ips = zones.zones(BlueprintZoneFilter::All).filter_map(|z| {
-            // Internal DNS zone's IPs are (intentionally!) outside the sled
-            // subnet, and must be allocated across the rack as a whole. We'll
-            // ignore any internal DNS zones when constructing our underlay IP
-            // allocator (as `SledUnderlayIpAllocator` will fail if we tell it a
-            // zone is using an IP outside of the sled subnet).
-            if z.zone_type.is_internal_dns() {
-                None
-            } else {
-                Some((z.zone_type.kind(), z.underlay_ip()))
-            }
-        });
+        let zone_ips = zones
+            .zones(BlueprintZoneFilter::All)
+            .map(|z| (z.zone_type.kind(), z.underlay_ip()));
 
         Ok(Self {
             underlay_ip_allocator: SledUnderlayIpAllocator::new(
@@ -542,6 +535,13 @@ impl ActiveSledEditor {
         // Ensure we can construct the configs for the datasets for this zone.
         let datasets = ZoneDatasetConfigs::new(&self.disks, &zone)?;
 
+        // Ensure this zone's IP is within our subnet and that future IP
+        // allocations take it into account. (Unless `zone` is an internal DNS
+        // zone, in which case validation is higher-level planner's job.)
+        self.underlay_ip_allocator
+            .mark_as_allocated(zone.zone_type.kind(), zone.underlay_ip())
+            .map_err(SledEditError::AddZoneBadUnderlayIp)?;
+
         // Actually add the zone and its datasets.
         self.zones.add_zone(zone)?;
         datasets.ensure_in_service(&mut self.datasets, rng);

nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/underlay_ip_allocator.rs

@@ -33,6 +33,7 @@ pub struct SledUnderlayIpOutOfRange {
 // general enough to use here, though this one could potentially be used there.
 #[derive(Debug)]
 pub(crate) struct SledUnderlayIpAllocator {
+    minimum: Ipv6Addr,
     last: Ipv6Addr,
     maximum: Ipv6Addr,
 }
@@ -68,22 +69,48 @@ impl SledUnderlayIpAllocator {
         assert!(sled_subnet.net().contains(minimum));
         assert!(sled_subnet.net().contains(maximum));
 
-        let mut last = minimum;
-        for (zone_kind, reserved_ip) in in_use_zone_ips {
-            if reserved_ip < minimum || reserved_ip > maximum {
-                return Err(SledUnderlayIpOutOfRange {
-                    zone_kind,
-                    ip: reserved_ip,
-                    low: minimum,
-                    high: maximum,
-                });
-            }
-            last = Ipv6Addr::max(last, reserved_ip);
+        let mut slf = Self { minimum, last: minimum, maximum };
+        for (zone_kind, ip) in in_use_zone_ips {
+            slf.mark_as_allocated(zone_kind, ip)?;
         }
-        assert!(minimum <= last);
-        assert!(last <= maximum);
+        assert!(slf.minimum <= slf.last);
+        assert!(slf.last <= slf.maximum);
 
-        Ok(Self { last, maximum })
+        Ok(slf)
+    }
+
+    /// Mark an address as used.
+    ///
+    /// Marking an address that has already been handed out by this allocator
+    /// (or could have been handed out by this allocator) is allowed and does
+    /// nothing.
+    ///
+    /// # Errors
+    ///
+    /// Fails if `ip` is outside the subnet of this sled.
+    pub fn mark_as_allocated(
+        &mut self,
+        zone_kind: ZoneKind,
+        ip: Ipv6Addr,
+    ) -> Result<(), SledUnderlayIpOutOfRange> {
+        // We intentionally ignore any internal DNS underlay IPs; they live
+        // outside the sled subnet and are allocated separately.
+        if zone_kind == ZoneKind::InternalDns {
+            return Ok(());
+        }
+
+        if ip < self.minimum || ip > self.maximum {
+            return Err(SledUnderlayIpOutOfRange {
+                zone_kind,
+                ip,
+                low: self.minimum,
+                high: self.maximum,
+            });
+        }
+
+        self.last = Ipv6Addr::max(self.last, ip);
+
+        Ok(())
     }
 
     /// Allocate an unused address from this allocator's range