Added aws-sim provider (#34)

* Added aws-sim provider Signed-off-by: Henry Haase <[email protected]> * Updated model format to suport physical layers found within a CSP Signed-off-by: Henry Haase <[email protected]> * Implemented describe instance topology endpoint in AWS simulation Signed-off-by: Henry Haase <[email protected]> * Updated model tests to match change in model structure Signed-off-by: Henry Haase <[email protected]> * fix typos Signed-off-by: Dmitry Shmulevich <[email protected]> * Changed a few style nits Signed-off-by: Henry Haase <[email protected]> --------- Signed-off-by: Henry Haase <[email protected]> Signed-off-by: Dmitry Shmulevich <[email protected]> Co-authored-by: Dmitry Shmulevich <[email protected]>
NVIDIA · Nov 20, 2024 · 716a491 · 716a491
1 parent 16c5d30
commit 716a491
Show file tree

Hide file tree

Showing 13 changed files with 499 additions and 62 deletions.
diff --git a/README.md b/README.md
@@ -152,7 +152,6 @@ Topograph offers three endpoints for interacting with the service. Below are the
       "secret_access_key": "secret"
     },
     "params": {
-      "use_simulation": "false",
       "model_path": ""
     }
   },

diff --git a/go.mod b/go.mod
@@ -8,12 +8,13 @@ require (
 	cloud.google.com/go/compute v1.28.2
 	cloud.google.com/go/compute/metadata v0.5.2
 	github.com/agrea/ptr v0.2.0
-	github.com/aws/aws-sdk-go-v2 v1.32.3
+	github.com/aws/aws-sdk-go-v2 v1.32.4
 	github.com/aws/aws-sdk-go-v2/config v1.28.1
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.42
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.187.0
 	github.com/go-playground/validator/v10 v10.22.1
+	github.com/golang/protobuf v1.5.4
 	github.com/google/uuid v1.6.0
 	github.com/googleapis/gax-go/v2 v2.13.0
 	github.com/hashicorp/golang-lru v1.0.2
@@ -25,7 +26,6 @@ require (
 	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
 	google.golang.org/api v0.204.0
 	google.golang.org/grpc v1.67.1
-	google.golang.org/protobuf v1.35.1
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.31.2
 	k8s.io/apimachinery v0.31.2
@@ -36,11 +36,11 @@ require (
 require (
 	cloud.google.com/go/auth v0.10.0 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 // indirect
@@ -62,7 +62,6 @@ require (
 	github.com/gofrs/flock v0.12.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
@@ -97,6 +96,7 @@ require (
 	google.golang.org/genproto v0.0.0-20241104194629-dd2ea8efbc28 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20241009091222-67ed5848f094 // indirect
@@ -105,3 +105,11 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+replace (
+	github.com/aws/aws-sdk-go-v2 v1.32.4 => github.com/pkedy/aws-sdk-go-v2 v0.0.0-20241115203348-0198b6c98cd9
+	github.com/aws/aws-sdk-go-v2/config v1.28.4 => github.com/pkedy/aws-sdk-go-v2/config v0.0.0-20241115203348-0198b6c98cd9
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.45 => github.com/pkedy/aws-sdk-go-v2/credentials v0.0.0-20241115203348-0198b6c98cd9
+	github.com/aws/aws-sdk-go-v2/service/autoscaling v1.48.0 => github.com/pkedy/aws-sdk-go-v2/service/autoscaling v0.0.0-20241115203348-0198b6c98cd9
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.187.0 => github.com/pkedy/aws-sdk-go-v2/service/ec2 v0.0.0-20241115203348-0198b6c98cd9
+)
diff --git a/go.sum b/go.sum
@@ -12,26 +12,22 @@ cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/yb
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/agrea/ptr v0.2.0 h1:QSyCkddC52uOrIvkypI8vTqUFw0KAnP71u1JU36EvBk=
 github.com/agrea/ptr v0.2.0/go.mod h1:O85aMmwHY6iqdSLPiaHMVz9AI7qvsZk3JPZ/i13Ec3Y=
-github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk=
-github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
 github.com/aws/aws-sdk-go-v2/config v1.28.1 h1:oxIvOUXy8x0U3fR//0eq+RdCKimWI900+SV+10xsCBw=
 github.com/aws/aws-sdk-go-v2/config v1.28.1/go.mod h1:bRQcttQJiARbd5JZxw6wG0yIK3eLeSCPdg6uqmmlIiI=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.42 h1:sBP0RPjBU4neGpIYyx8mkU2QqLPl5u9cmdTWVzIpHkM=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.42/go.mod h1:FwZBfU530dJ26rv9saAbxa9Ej3eF/AK0OAY86k13n4M=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 h1:68jFVtt3NulEzojFesM/WVarlFpCaXLKaBxDpzkQ9OQ=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18/go.mod h1:Fjnn5jQVIo6VyedMc0/EhPpfNlPl7dHV916O6B+49aE=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 h1:A2w6m6Tmr+BNXjDsr7M90zkWjsu4JXHwrzPg235STs4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23/go.mod h1:35EVp9wyeANdujZruvHiQUAo9E3vbhnIO1mTCAxMlY0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 h1:pgYW9FCabt2M25MoHYCfMrVY2ghiiBKYWUVXfwZs+sU=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23/go.mod h1:c48kLgzO19wAu3CPkDWC28JbaJ+hfQlsdl7I2+oqIbk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.187.0 h1:cA4hWo269CN5RY7Arqt8BfzXF0KIN8DSNo/KcqHKkWk=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.187.0/go.mod h1:ossaD9Z1ugYb6sq9QIqQLEOorCGcqUoxlhud9M9yE70=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 h1:qcxX0JYlgWH3hpPUnd6U0ikcl6LLA9sLkXE2w1fpMvY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3/go.mod h1:cLSNEmI45soc+Ef8K/L+8sEA3A3pYFEYf5B5UI+6bH4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4 h1:tHxQi/XHPK0ctd/wdOw0t7Xrc2OxcRCnVzv8lwWPu0c=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4/go.mod h1:4GQbF1vJzG60poZqWatZlhP31y8PGCCVTvIGPdaaYJ0=
 github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 h1:UTpsIf0loCIWEbrqdLb+0RxnTXfWh2vhw4nQmFi4nPc=
 github.com/aws/aws-sdk-go-v2/service/sso v1.24.3/go.mod h1:FZ9j3PFHHAR+w0BSEjK955w5YD2UwB/l/H0yAK3MJvI=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 h1:2YCmIXv3tmiItw0LlYf6v7gEHebLY45kBEnPezbUKyU=
@@ -167,6 +163,10 @@ github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=
 github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0=
 github.com/oracle/oci-go-sdk/v65 v65.78.0 h1:iM7lFFA7cJkUD4tmrlsAHWgL3HuTuF9mdvTAliMkcFA=
 github.com/oracle/oci-go-sdk/v65 v65.78.0/go.mod h1:IBEV9l1qBzUpo7zgGaRUhbB05BVfcDGYRFBCPlTcPp0=
+github.com/pkedy/aws-sdk-go-v2 v0.0.0-20241115203348-0198b6c98cd9 h1:QhMFD0yJ9nEj4BCX9lREQ7twLM5oEL8y9UwKsRNJamo=
+github.com/pkedy/aws-sdk-go-v2 v0.0.0-20241115203348-0198b6c98cd9/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
+github.com/pkedy/aws-sdk-go-v2/service/ec2 v0.0.0-20241115203348-0198b6c98cd9 h1:wA7yd0OxRH3EWuKaJ7ijRowlWgH2b99nrP+d10+0Sc4=
+github.com/pkedy/aws-sdk-go-v2/service/ec2 v0.0.0-20241115203348-0198b6c98cd9/go.mod h1:0A17IIeys01WfjDKehspGP+Cyo/YH/eNADIbEbRS9yM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

diff --git a/pkg/models/model.go b/pkg/models/model.go
@@ -26,8 +26,9 @@ import (
 )
 
 type Model struct {
-	Switches       []Switch        `yaml:"switches"`
-	CapacityBlocks []CapacityBlock `yaml:"capacity_blocks"`
+	Switches       []Switch         `yaml:"switches"`
+	CapacityBlocks []CapacityBlock  `yaml:"capacity_blocks"`
+	PhysicalLayers []PhysicalLayers `yaml:"physical_layers"`
 
 	// defived
 	Nodes map[string]*Node
@@ -46,11 +47,25 @@ type CapacityBlock struct {
 	Nodes  []string `yaml:"nodes"`
 }
 
+type PhysicalLayers struct {
+	Name           string   `yaml:"name"`
+	Type           string   `yaml:"type"`
+	SubLayers      []string `yaml:"sub_layers"`
+	CapacityBlocks []string `yaml:"capacity_blocks"`
+}
+
+const (
+	PhysicalLayerRegion = "region"
+	PhysicalLayerAZ     = "availability_zone"
+	PhysicalLayerPG     = "placement_group"
+)
+
 type Node struct {
-	Name      string
-	Type      string
-	NVLink    string
-	NetLayers []string
+	Name          string
+	Type          string
+	NVLink        string
+	NetLayers     []string
+	CapacityBlock string
 }
 
 func NewModelFromFile(fname string) (*Model, error) {
@@ -68,6 +83,10 @@ func NewModelFromFile(fname string) (*Model, error) {
 		return nil, err
 	}
 
+	if err = validateLayers(model.PhysicalLayers); err != nil {
+		return nil, err
+	}
+
 	return model, err
 }
 
@@ -110,10 +129,11 @@ func (m *Model) setNodeMap() error {
 				return fmt.Errorf("duplicated node name %q", name)
 			}
 			m.Nodes[name] = &Node{
-				Name:      name,
-				Type:      cb.Type,
-				NVLink:    cb.NVLink,
-				NetLayers: netLayers,
+				Name:          name,
+				Type:          cb.Type,
+				NVLink:        cb.NVLink,
+				NetLayers:     netLayers,
+				CapacityBlock: cb.Name,
 			}
 		}
 	}
@@ -140,6 +160,63 @@ func getNetworkLayers(name string, swmap map[string]string) ([]string, error) {
 	}
 }
 
+// Check to make sure each layer is unique and has only a single parent, if any, and enumerates them into a map of layer name to parent index within the layers list
+// If the layer has no entry within the map, it means that the layer has no parent
+func getLayerParentMap(layers []PhysicalLayers) (map[string]int, error) {
+	parentMap := make(map[string]int)
+	for i, layer := range layers {
+		layerName := layer.Name
+		var parentCount int = 0
+		for j, checkLayer := range layers {
+			if i == j {
+				continue
+			}
+			if layerName == checkLayer.Name {
+				return nil, fmt.Errorf("duplicated physical layer name %q", layerName)
+			}
+			for _, subLayerName := range checkLayer.SubLayers {
+				if layerName == subLayerName {
+					parentMap[layerName] = j
+					parentCount++
+					break
+				}
+			}
+		}
+		if parentCount > 1 {
+			return nil, fmt.Errorf("physical layer with name %q has more than one parent (%d parents)", layerName, parentCount)
+		}
+	}
+	return parentMap, nil
+}
+
+func validateLayers(layers []PhysicalLayers) error {
+	// Validates the parent structure of the layers and gets the map of nodes to parents
+	parentMap, err := getLayerParentMap(layers)
+	if err != nil {
+		return err
+	}
+
+	// Check to make sure there are no loops among the physical layers
+	for _, layer := range layers {
+		layerName := layer.Name
+		currLayerIdx, ok := parentMap[layerName]
+		if ok {
+			currLayerName := layers[currLayerIdx].Name
+			for {
+				currLayerIdx, ok = parentMap[currLayerName]
+				if !ok {
+					break
+				}
+				currLayerName = layers[currLayerIdx].Name
+				if currLayerName == layerName {
+					return fmt.Errorf("circular layer dependencies involving layer with name %q", layerName)
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func (model *Model) ToGraph() (*topology.Vertex, map[string]string) {
 	instance2node := make(map[string]string)
 	nodeVertexMap := make(map[string]*topology.Vertex)
@@ -210,3 +287,42 @@ func (model *Model) ToGraph() (*topology.Vertex, map[string]string) {
 	treeRoot.Metadata = map[string]string{topology.KeyPlugin: topology.TopologyTree}
 	return treeRoot, instance2node
 }
+
+// Get a map that maps from the name of each node in the model to the cloestest physical layer that shares the given type.
+// If no entry exists for the node in the returned map, then there is no layer the node exists in with the given type
+func (model *Model) NodeToLayerMap(layerType string) (map[string]string, error) {
+	// Maps each capacity block to a parent physical layer index
+	cbToLayer := make(map[string]int)
+	for idx, layer := range model.PhysicalLayers {
+		for _, cbName := range layer.CapacityBlocks {
+			cbToLayer[cbName] = idx
+		}
+	}
+
+	// Goes through each node, gets the capacity block, and walks up the parent tree to find the cloest layer of the given type
+	nodeToLayer := make(map[string]string)
+	layerParentMap, err := getLayerParentMap(model.PhysicalLayers)
+	if err != nil {
+		return nil, err
+	}
+	for _, node := range model.Nodes {
+		cb := node.CapacityBlock
+		layerIdx, ok := cbToLayer[cb]
+		if !ok {
+			return nil, fmt.Errorf("capacity block %q not found in any physical layer", cb)
+		}
+		for {
+			layer := model.PhysicalLayers[layerIdx]
+			if layer.Type == layerType {
+				nodeToLayer[node.Name] = layer.Name
+				break
+			}
+			layerIdx, ok = layerParentMap[layer.Name]
+			if !ok {
+				break
+			}
+		}
+	}
+
+	return nodeToLayer, nil
+}