Some bug fix + Scaleway comput instances

boaviztapi/data/archetypes/cloud/aws.csv

@@ -308,12 +308,12 @@ inf2.xlarge,4,16,0,0,1,inf2.48xlarge,
 inf2.8xlarge,32,128,0,0,1,inf2.48xlarge,
 inf2.24xlarge,96,384,0,0,6,inf2.48xlarge,
 inf2.48xlarge,192,768,0,0,12,inf2.48xlarge,
-is4gen.medium,1,6,937,0,0,is4gen.8xlarge,
-is4gen.large,2,12,1875,0,0,is4gen.8xlarge,
-is4gen.xlarge,4,24,3750,0,0,is4gen.8xlarge,
-is4gen.2xlarge,8,48,7500,0,0,is4gen.8xlarge,
-is4gen.4xlarge,16,96,15000,0,0,is4gen.8xlarge,
-is4gen.8xlarge,32,192,30000,0,0,is4gen.8xlarge,
+is4gen.medium,1,6,937,0,0,is4gen.metal,
+is4gen.large,2,12,1875,0,0,is4gen.metal,
+is4gen.xlarge,4,24,3750,0,0,is4gen.metal,
+is4gen.2xlarge,8,48,7500,0,0,is4gen.metal,
+is4gen.4xlarge,16,96,15000,0,0,is4gen.metal,
+is4gen.8xlarge,32,192,30000,0,0,is4gen.metal,
 db.m1.medium,1,3.75,0,0,0,platform_aws_m1,https://aws.amazon.com/ec2/previous-generation/
 m1.medium,1,3.75,410,0,0,platform_aws_m1,https://aws.amazon.com/ec2/previous-generation/
 db.m1.small,1,1.7,0,0,0,platform_aws_m1,https://aws.amazon.com/ec2/previous-generation/

boaviztapi/data/archetypes/cloud/providers.csv

@@ -1,3 +1,4 @@
 provider.name,provider.description
 aws,Amazon Web Services
 azure,Microsoft Azure
+scaleway,Scaleway

boaviztapi/data/archetypes/cloud/scaleway.csv

@@ -0,0 +1,70 @@
+id,vcpu,memory,ssd_storage,hdd_storage,gpu_units,platform,source
+coparm1-16c-64g,16,64,10,0,0,scw_coparm1.base,
+coparm1-2c-8g,2,8,10,0,0,scw_coparm1.base,
+coparm1-32c-128g,32,128,10,0,0,scw_coparm1.base,
+coparm1-4c-16g,4,16,10,0,0,scw_coparm1.base,
+coparm1-8c-32g,8,32,10,0,0,scw_coparm1.base,
+dev1-l,4,8,10,0,0,scw_dev1.base,
+dev1-m,3,4,10,0,0,scw_dev1.base,
+dev1-s,2,2,10,0,0,scw_dev1.base,
+dev1-xl,4,12,10,0,0,scw_dev1.base,
+ent1-2xl,96,384,10,0,0,scw_ent1.base,
+ent1-l,32,128,10,0,0,scw_ent1.base,
+ent1-m,16,64,10,0,0,scw_ent1.base,
+ent1-s,8,32,10,0,0,scw_ent1.base,
+ent1-xl,64,256,10,0,0,scw_ent1.base,
+ent1-xs,4,16,10,0,0,scw_ent1.base,
+ent1-xxs,2,8,10,0,0,scw_ent1.base,
+gp1-l,32,128,10,0,0,scw_gp1.base,
+gp1-m,16,64,10,0,0,scw_gp1.base,
+gp1-s,8,32,10,0,0,scw_gp1.base,
+gp1-viz,8,32,10,0,0,scw_gp1.base,
+gp1-xl,48,256,10,0,0,scw_gp1.base,
+gp1-xs,4,16,10,0,0,scw_gp1.base,
+gpu-3070-s,8,16,10,0,1,scw_renders.base,
+h100-1-80g,24,240,10,0,1,scw_h100.base,
+h100-1-m,24,240,10,0,1,scw_h100.base,
+h100-2-80g,48,480,10,0,2,scw_h100.base,
+h100-2-m,48,480,10,0,2,scw_h100.base,
+l4-1-24g,8,48,10,0,1,scw_l4.base,
+l4-2-24g,16,96,10,0,2,scw_l4.base,
+l4-4-24g,32,192,10,0,4,scw_l4.base,
+l4-8-24g,64,384,10,0,8,scw_l4.base,
+l40s-1-48g,8,96,10,0,1,scw_l4.base,
+l40s-2-48g,16,192,10,0,2,scw_l4.base,
+l40s-4-48g,32,384,10,0,4,scw_l4.base,
+l40s-8-48g,64,768,10,0,8,scw_l4.base,
+play2-micro,4,8,10,0,0,scw_play2.base,
+play2-nano,2,4,10,0,0,scw_play2.base,
+play2-pico,1,2,10,0,0,scw_play2.base,
+pop2-16c-64g,16,64,10,0,0,scw_pop2.base,
+pop2-16c-64g-win,16,64,10,0,0,scw_pop2.base,
+pop2-2c-8g,2,8,10,0,0,scw_pop2.base,
+pop2-2c-8g-win,2,8,10,0,0,scw_pop2.base,
+pop2-32c-128g,32,128,10,0,0,scw_pop2.base,
+pop2-32c-128g-win,32,128,10,0,0,scw_pop2.base,
+pop2-4c-16g,4,16,10,0,0,scw_pop2.base,
+pop2-4c-16g-win,4,16,10,0,0,scw_pop2.base,
+pop2-64c-256g,64,256,10,0,0,scw_pop2.base,
+pop2-8c-32g,8,32,10,0,0,scw_pop2.base,
+pop2-8c-32g-win,8,32,10,0,0,scw_pop2.base,
+pop2-hc-16c-32g,16,32,10,0,0,scw_pop2hc.base,
+pop2-hc-2c-4g,2,4,10,0,0,scw_pop2hc.base,
+pop2-hc-32c-64g,32,64,10,0,0,scw_pop2hc.base,
+pop2-hc-4c-8g,4,8,10,0,0,scw_pop2hc.base,
+pop2-hc-64c-128g,64,128,10,0,0,scw_pop2hc.base,
+pop2-hc-8c-16g,8,16,10,0,0,scw_pop2hc.base,
+pop2-hm-16c-128g,16,128,10,0,0,scw_pop2hm.base,
+pop2-hm-2c-16g,2,16,10,0,0,scw_pop2hm.base,
+pop2-hm-32c-256g,32,256,10,0,0,scw_pop2hm.base,
+pop2-hm-4c-32g,4,32,10,0,0,scw_pop2hm.base,
+pop2-hm-64c-512g,64,512,10,0,0,scw_pop2hm.base,
+pop2-hm-8c-64g,8,64,10,0,0,scw_pop2hm.base,
+pop2-hn-10,4,8,10,0,0,scw_pop2.base,
+pop2-hn-3,2,4,10,0,0,scw_pop2.base,
+pop2-hn-5,4,8,10,0,0,scw_pop2.base,
+pro2-l,32,128,10,0,0,scw_pro2.base,
+pro2-m,16,64,10,0,0,scw_pro2.base,
+pro2-s,8,32,10,0,0,scw_pro2.base,
+pro2-xs,4,16,10,0,0,scw_pro2.base,
+pro2-xxs,2,8,10,0,0,scw_pro2.base,

boaviztapi/data/crowdsourcing/cpu_specs.csv

@@ -34,7 +34,7 @@ AMD EPYC 7252,Rome,EPYC (Zen 2 (Rome)),TSMC,2019-08-07,2.8 GHz,120.0,8.0,16.0,"3
 AMD EPYC 7261,Naples,,,,,,,16.0,,,,,,,,AMD,EPYC,https://github.com/cloud-carbon-footprint/cloud-carbon-coefficients/tree/main/data
 AMD EPYC 7262,Rome,EPYC (Zen 2 (Rome)),TSMC,2019-08-07,3.1 GHz,155.0,8.0,16.0,"3,800 million",7.0,2x 74 mm²,416 mm2,14 nm,564.0,io_die_size (416 mm2) + die_size (2x 74 mm²),AMD,EPYC,https://www.techpowerup.com/cpu-specs/epyc-7262.c2257
 AMD EPYC 7272,Rome,EPYC (Zen 2 (Rome)),TSMC,2019-08-07,2.6 GHz,120.0,12.0,24.0,"3,800 million",7.0,3x 74 mm²,416 mm2,14 nm,638.0,io_die_size (416 mm2) + die_size (3x 74 mm²),AMD,EPYC,https://www.techpowerup.com/cpu-specs/epyc-7272.c2256
-AMD EPYC 7281,Naples,,,,,,,32.0,,,,,,,,AMD,EPYC,https://github.com/cloud-carbon-footprint/cloud-carbon-coefficients/tree/main/data
+AMD EPYC 7281,Naples,,TSMC,,2.1 GHz,170.0,16.0,32.0,,,,,,,,AMD,EPYC,https://www.techpowerup.com/cpu-specs/epyc-7281.c1927
 AMD EPYC 7282,Rome,EPYC (Zen 2 (Rome)),TSMC,2019-08-07,2.8 GHz,120.0,16.0,32.0,"3,800 million",7.0,4x 74 mm²,416 mm2,14 nm,712.0,io_die_size (416 mm2) + die_size (4x 74 mm²),AMD,EPYC,https://www.techpowerup.com/cpu-specs/epyc-7282.c2255
 AMD EPYC 72F3,Milan,EPYC (Zen 3 (Milan)),TSMC,2021-03-15,3.7 GHz,180.0,8.0,16.0,"33,200 million",7.0,8x 81 mm²,416 mm²,12 nm,1064.0,io_die_size (416 mm²) + die_size (8x 81 mm²),AMD,EPYC,https://www.techpowerup.com/cpu-specs/epyc-72f3.c2391
 AMD EPYC 7301,Naples,,,,,,,32.0,,,,,,,,AMD,EPYC,https://github.com/cloud-carbon-footprint/cloud-carbon-coefficients/tree/main/data
@@ -1778,7 +1778,6 @@ Intel Xeon E5-2670 V2,Ivy Bridge,,,,,115.0,10.0,20.0,,,,,,541.0,Max value of cpu
 Intel Xeon E5-2670 V3,Haswell,,,,,,12.0,24.0,,,,,,622.0,Max value of cpu_manufacture https://en.wikichip.org/wiki/intel/microarchitectures/haswell_(client)#Octadeca-core,Intel,Xeon E5,https://github.com/cloud-carbon-footprint/cloud-carbon-coefficients/tree/main/data
 Intel Xeon E5-2676 V3,Haswell,,,,,120.0,12.0,24.0,,,,,,462.4,Linear regression of cpu_manufacture,Intel,Xeon E5,https://docs.google.com/spreadsheets/d/1DqYgQnEDLQVQm5acMAhLgHLD8xXCG9BIrk-_Nv6jF3k/edit#gid=224728652
 Intel Xeon E5-2680 V2,Ivy Bridge,,,,,115.0,10.0,20.0,,,,,,341.0,Value of cpu_manufacture https://en.wikichip.org/wiki/intel/microarchitectures/ivy_bridge_(client)#Deca-core_Ivy_Bridge_Die,Intel,Xeon E5,https://docs.google.com/spreadsheets/d/1DqYgQnEDLQVQm5acMAhLgHLD8xXCG9BIrk-_Nv6jF3k/edit#gid=224728652
-Intel Xeon E5-2686 V4,Broadwell,,,,,145.0,18.0,36.0,,,,,,367.1,Linear regression of cpu_manufacture,Intel,Xeon E5,https://docs.google.com/spreadsheets/d/1DqYgQnEDLQVQm5acMAhLgHLD8xXCG9BIrk-_Nv6jF3k/edit#gid=224728652
 Intel Xeon E7-8880 V3,Haswell,,,,,150.0,18.0,36.0,,,,,,570.1,Linear regression of cpu_manufacture,Intel,Xeon E7,https://docs.google.com/spreadsheets/d/1DqYgQnEDLQVQm5acMAhLgHLD8xXCG9BIrk-_Nv6jF3k/edit#gid=224728652
 Intel Xeon Platinum 8124M,Skylake,,,,,240.0,18.0,36.0,,,,,,485.0,Value of cpu_manufacture https://en.wikichip.org/wiki/intel/microarchitectures/skylake_(server)#High_Core_Count_.28HCC.29,Intel,Xeon Platinum,https://docs.google.com/spreadsheets/d/1DqYgQnEDLQVQm5acMAhLgHLD8xXCG9BIrk-_Nv6jF3k/edit#gid=224728652
 Intel Xeon Platinum 8151,Skylake,,,,,240.0,12.0,24.0,,,,,,336.4,Linear regression of cpu_manufacture,Intel,Xeon Platinum,https://docs.google.com/spreadsheets/d/1DqYgQnEDLQVQm5acMAhLgHLD8xXCG9BIrk-_Nv6jF3k/edit#gid=224728652
@@ -1790,3 +1789,4 @@ Intel Xeon Platinum 8275CL,Cascade Lake,,,,,240.0,24.0,48.0,,,,,,,,Intel,Xeon Pl
 Intel Xeon Platinum 8375C,Ice Lake,,,,,300.0,32.0,64.0,,,,,,529.5,Linear regression of cpu_manufacture,Intel,Xeon Platinum,https://docs.google.com/spreadsheets/d/1DqYgQnEDLQVQm5acMAhLgHLD8xXCG9BIrk-_Nv6jF3k/edit#gid=224728652
 Intel Xeon Platinum 8488C,Sapphire Rapids,Xeon Platinum (Sapphire Rapids-SP),Intel,,2.4 GHz,385.0,48.0,96.0,,,,,,,,Intel,Xeon Platinum,
 Intel Xeon 6455B,Sapphire Rapids,Xeon Scalable (Sapphire Rapids),Intel,2023-10-16,3.9 GHz,350.0,32.0,64.0,,7.0,,,,,,Intel,Xeon, https://docs.google.com/spreadsheets/d/1DqYgQnEDLQVQm5acMAhLgHLD8xXCG9BIrk-_Nv6jF3k/edit?gid=1498560972#gid=1498560972
+Intel Xeon E5-2686 v4,Broadwell,,,,2.3 GHz,145.0,18.0,36.0,,14,,,,456,,,,https://www.x86-guide.net/fr/cpu/Intel-Xeon-E5-2686-v4-cpu-no5808.html

boaviztapi/data/utils/complete_Scaleway_Instances/README.md

@@ -0,0 +1,38 @@
+# Completing and updating Scaleway Instances data
+
+The Scaleway export can be done via the [scw-environmental-footprint](https://github.com/Shillaker/scw-environmental-footprint) project, specifically following [this doc](https://github.com/Shillaker/scw-environmental-footprint/blob/main/docs/boavizta.md).
+
+This does the following:
+
+- Lists all types of Instances using the Scaleway API
+- Maps these to a set of hard-coded base server types (see below)
+- Generates two CSV files: `instances.csv` (the instance types), and `servers.csv` (the base servers)
+
+To add to Boavizta:
+
+- Copy `instances.csv` to `boaviztapi/data/archetypes/cloud/scaleway.csv` (simply overwrite if updating)
+- Add the lines form `servers.csv` to `boaviztapi/data/archetypes/server.csv` (delete all existing servers starting with `scw_` if you are updating)
+
+## Testing
+
+1. Update CSV files
+2. Build the Docker image locally: `docker build -t boaviztapi-dev .`
+3. Run it `docker run -p "5000:5000" -t boaviztapi-dev`
+
+You can then run the script to check the changes using the `check.py` script in this directory:
+
+```
+python3 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+python3 check.py
+```
+
+## Scripting
+
+It would be much better if this code lived in the Boavizta repo. It can be scripted relatively easily:
+
+1. Hard-code the base server types held in the [scw-environmental-footprint](https://github.com/Shillaker/scw-environmental-footprint) repo [here](https://github.com/Shillaker/scw-environmental-footprint/blob/main/model/instances.go)
+2. Use the [scaleway-sdk-python](https://github.com/scaleway/scaleway-sdk-python) to list all the Instance types
+3. Map these the the underlying base server types based on the prefix of the Instance type
+4. Print two CSV files: i) `instances.csv` with the instance types; ii) `servers.csv` with the specs of the base servers

boaviztapi/data/utils/complete_Scaleway_Instances/check.py

@@ -0,0 +1,42 @@
+import requests
+
+CLOUD_URL="http://localhost:5000/v1/cloud/instance"
+PROVIDERS_URL=f"{CLOUD_URL}/all_providers"
+VALID_INSTANCE_TYPE="dev1-l"
+
+def main():
+    resp = requests.get(PROVIDERS_URL)
+
+    if not resp.ok:
+        print(f"Providers request failed: {resp.json()}")
+        exit(1)
+
+    if "scaleway" in resp.json():
+        print("Found Scaleway in list of providers")
+    else:
+        print("Did not find Scaleway in list of providers: {resp.status_code}")
+        exit(1)
+
+    url_params = {
+        "provider": "scaleway",
+        "instance_type": VALID_INSTANCE_TYPE,
+        "verbose": True,
+        "duration": "100",
+        }
+
+    resp = requests.get(CLOUD_URL, params=url_params)
+    if resp.ok:
+        print(f"Got response to impact request for type {VALID_INSTANCE_TYPE}")
+    else:
+        print("Did not find Scaleway in list of providers: {resp.status_code}")
+        exit(1)
+
+    resp_body = resp.json()
+    for impact_name, impact in resp_body["impacts"].items():
+        print(f"\nGot {impact_name}: {impact}")
+
+    cpu = resp_body["verbose"]["CPU-1"]["model_range"]["value"]
+    print(f"\nGot CPU model: {cpu}")
+
+if __name__ == "__main__":
+    main()

boaviztapi/data/utils/complete_Scaleway_Instances/requirements.txt

@@ -0,0 +1 @@
+requests==2.32.3

boaviztapi/dto/device/device.py

@@ -1,5 +1,7 @@
 from typing import Optional, List
 
+from fastapi import HTTPException
+
 from boaviztapi import config
 from boaviztapi.dto.component import CPU, RAM, Disk, PowerSupply
 from boaviztapi.dto.component.cpu import mapper_cpu
@@ -14,7 +16,8 @@
 from boaviztapi.model.device.server import DeviceServer
 from boaviztapi.model.services.cloud_instance import ServiceCloudInstance
 from boaviztapi.model.usage import ModelUsage
-from boaviztapi.service.archetype import get_server_archetype, get_arch_component, get_cloud_instance_archetype
+from boaviztapi.service.archetype import get_server_archetype, get_arch_component, get_cloud_instance_archetype, \
+    get_arch_value
 
 
 class DeviceDTO(BaseDTO):
@@ -75,6 +78,9 @@ class Cloud(Server):
 
 
 def mapper_cloud_instance(cloud_dto: Cloud, archetype=get_cloud_instance_archetype(config["default_cloud_instance"], config["default_cloud_provider"])) -> ServiceCloudInstance:
+    if get_server_archetype(archetype_name=get_arch_value(archetype, 'platform', 'default')) is False:
+        raise HTTPException(status_code=404, detail=f"Cloud platform {get_arch_value(archetype, 'platform', 'default')} not found. Please add it to the server archetypes. For more information, please check the documentation https://doc.api.boavizta.org/contributing/server/.")
+
     model_cloud_instance = ServiceCloudInstance(archetype=archetype)
 
     model_cloud_instance.usage = mapper_usage_cloud(cloud_dto.usage or UsageCloud(), archetype=get_arch_component(model_cloud_instance.archetype, "USAGE"))

boaviztapi/routers/cloud_router.py

@@ -70,7 +70,7 @@ async def instance_cloud_impact(
 
     if not instance_archetype:
         raise HTTPException(status_code=404,
-                            detail=f"{cloud_instance.instance_type} at {cloud_instance.provider} not found")
+                            detail=f"{instance_type} at {provider} not found")
 
     instance_model = mapper_cloud_instance(cloud_instance, archetype=instance_archetype)