Merge pull request #325 from Shillaker/feat/shillaker/scw-instances

feat: scaleway instances

Author: da-ekchajzer

boaviztapi/data/archetypes/cloud/providers.csv

@@ -1,3 +1,4 @@
 provider.name,provider.description
 aws,Amazon Web Services
 azure,Microsoft Azure
+scaleway,Scaleway

boaviztapi/data/archetypes/cloud/scaleway.csv

@@ -0,0 +1,70 @@
+id,vcpu,memory,ssd_storage,hdd_storage,gpu_units,platform,source
+coparm1-16c-64g,16,64,10,0,0,scw_coparm1.base,
+coparm1-2c-8g,2,8,10,0,0,scw_coparm1.base,
+coparm1-32c-128g,32,128,10,0,0,scw_coparm1.base,
+coparm1-4c-16g,4,16,10,0,0,scw_coparm1.base,
+coparm1-8c-32g,8,32,10,0,0,scw_coparm1.base,
+dev1-l,4,8,10,0,0,scw_dev1.base,
+dev1-m,3,4,10,0,0,scw_dev1.base,
+dev1-s,2,2,10,0,0,scw_dev1.base,
+dev1-xl,4,12,10,0,0,scw_dev1.base,
+ent1-2xl,96,384,10,0,0,scw_ent1.base,
+ent1-l,32,128,10,0,0,scw_ent1.base,
+ent1-m,16,64,10,0,0,scw_ent1.base,
+ent1-s,8,32,10,0,0,scw_ent1.base,
+ent1-xl,64,256,10,0,0,scw_ent1.base,
+ent1-xs,4,16,10,0,0,scw_ent1.base,
+ent1-xxs,2,8,10,0,0,scw_ent1.base,
+gp1-l,32,128,10,0,0,scw_gp1.base,
+gp1-m,16,64,10,0,0,scw_gp1.base,
+gp1-s,8,32,10,0,0,scw_gp1.base,
+gp1-viz,8,32,10,0,0,scw_gp1.base,
+gp1-xl,48,256,10,0,0,scw_gp1.base,
+gp1-xs,4,16,10,0,0,scw_gp1.base,
+gpu-3070-s,8,16,10,0,1,scw_renders.base,
+h100-1-80g,24,240,10,0,1,scw_h100.base,
+h100-1-m,24,240,10,0,1,scw_h100.base,
+h100-2-80g,48,480,10,0,2,scw_h100.base,
+h100-2-m,48,480,10,0,2,scw_h100.base,
+l4-1-24g,8,48,10,0,1,scw_l4.base,
+l4-2-24g,16,96,10,0,2,scw_l4.base,
+l4-4-24g,32,192,10,0,4,scw_l4.base,
+l4-8-24g,64,384,10,0,8,scw_l4.base,
+l40s-1-48g,8,96,10,0,1,scw_l4.base,
+l40s-2-48g,16,192,10,0,2,scw_l4.base,
+l40s-4-48g,32,384,10,0,4,scw_l4.base,
+l40s-8-48g,64,768,10,0,8,scw_l4.base,
+play2-micro,4,8,10,0,0,scw_play2.base,
+play2-nano,2,4,10,0,0,scw_play2.base,
+play2-pico,1,2,10,0,0,scw_play2.base,
+pop2-16c-64g,16,64,10,0,0,scw_pop2.base,
+pop2-16c-64g-win,16,64,10,0,0,scw_pop2.base,
+pop2-2c-8g,2,8,10,0,0,scw_pop2.base,
+pop2-2c-8g-win,2,8,10,0,0,scw_pop2.base,
+pop2-32c-128g,32,128,10,0,0,scw_pop2.base,
+pop2-32c-128g-win,32,128,10,0,0,scw_pop2.base,
+pop2-4c-16g,4,16,10,0,0,scw_pop2.base,
+pop2-4c-16g-win,4,16,10,0,0,scw_pop2.base,
+pop2-64c-256g,64,256,10,0,0,scw_pop2.base,
+pop2-8c-32g,8,32,10,0,0,scw_pop2.base,
+pop2-8c-32g-win,8,32,10,0,0,scw_pop2.base,
+pop2-hc-16c-32g,16,32,10,0,0,scw_pop2hc.base,
+pop2-hc-2c-4g,2,4,10,0,0,scw_pop2hc.base,
+pop2-hc-32c-64g,32,64,10,0,0,scw_pop2hc.base,
+pop2-hc-4c-8g,4,8,10,0,0,scw_pop2hc.base,
+pop2-hc-64c-128g,64,128,10,0,0,scw_pop2hc.base,
+pop2-hc-8c-16g,8,16,10,0,0,scw_pop2hc.base,
+pop2-hm-16c-128g,16,128,10,0,0,scw_pop2hm.base,
+pop2-hm-2c-16g,2,16,10,0,0,scw_pop2hm.base,
+pop2-hm-32c-256g,32,256,10,0,0,scw_pop2hm.base,
+pop2-hm-4c-32g,4,32,10,0,0,scw_pop2hm.base,
+pop2-hm-64c-512g,64,512,10,0,0,scw_pop2hm.base,
+pop2-hm-8c-64g,8,64,10,0,0,scw_pop2hm.base,
+pop2-hn-10,4,8,10,0,0,scw_pop2.base,
+pop2-hn-3,2,4,10,0,0,scw_pop2.base,
+pop2-hn-5,4,8,10,0,0,scw_pop2.base,
+pro2-l,32,128,10,0,0,scw_pro2.base,
+pro2-m,16,64,10,0,0,scw_pro2.base,
+pro2-s,8,32,10,0,0,scw_pro2.base,
+pro2-xs,4,16,10,0,0,scw_pro2.base,
+pro2-xxs,2,8,10,0,0,scw_pro2.base,

boaviztapi/data/archetypes/server.csv

@@ -187,4 +187,17 @@ NVasv4-Type1,Azure,rack,1.0,,,AMD EPYC 7V12,,56.0,8.0,2.0,2048.0,0.0,0.0,0.0,AMD
 NVsv3-Type1,Azure,rack,2.0,,,Intel Xeon E5-2690 V4,,56.0,8.0,2.0,2048.0,0.0,0.0,0.0,NVIDIA Tesla M60 ,,2;2;2,2.99;1;5,50;0;100,1,52560.0,0.33;0.2;0.6,RAM units and per unit capacity not verified. RAM capacity from Azure docs was: 448 GiB
 platform_aws_g3,AWS,rack,2.0,,, 	Intel Xeon E5-2686 v4,,8,64,0.0,0.0,0.0,0.0,4.0,NVIDIA Tesla M60,32,2;2;2,2.99;1;5,50;0;100,1,52560.0,0.33;0.2;0.6,RAM units and per unit capacity not verified.
 m7gd.metal,AWS,rack,1.0,,,Annapurna Labs Graviton3,,4,64,0.0,0.0,0.0,0.0,0.0,,0.0,2;2;2,2.99;1;5,50;0;100,1,52560.0,0.33;0.2;0.6,RAM units and per unit capacity not verified.
-platform_aws_m1,AWS,rack,1.0,32,,,64,15,16,4.0,420,0.0,0.0,0.0,,0.0,2;2;2,2.99;1;5,50;0;100,1,52560.0,0.33;0.2;0.6,RAM units and per unit capacity not verified. CPU can be any Intel Xeon Family.
+platform_aws_m1,AWS,rack,1.0,32,,,64,15,16,4.0,420,0.0,0.0,0.0,,0.0,2;2;2,2.99;1;5,50;0;100,1,52560.0,0.33;0.2;0.6,RAM units and per unit capacity not verified. CPU can be any Intel Xeon Family.
+scw_coparm1.base,Scaleway,rack,1,128,,Ampere Altra Max 128,256,16,64,2,937,0,0,0,,0,2,0,50;0;100,1,52560,0.33;0.2;0.6,
+scw_dev1.base,Scaleway,rack,1,16,,AMD EPYC 7281,64,8,32,5,976,0,0,0,,0,2,0,50;0;100,1,52560,0.33;0.2;0.6,
+scw_ent1.base,Scaleway,rack,2,32,,AMD EPYC 7543,128,16,32,2,234,0,0,0,,0,2,0,50;0;100,1,52560,0.33;0.2;0.6,
+scw_gp1.base,Scaleway,rack,2,24,,AMD EPYC 7401P,96,12,32,5,976,0,0,0,,0,2,0,50;0;100,1,52560,0.33;0.2;0.6,
+scw_h100.base,Scaleway,rack,1,32,,AMD EPYC 9334,64,12,64,5,7500,0,0,2,Nvidia H100,80,2,0,50;0;100,1,52560,0.33;0.2;0.6,
+scw_l4.base,Scaleway,rack,2,24,,AMD EPYC 7413,48,16,32,2,234,0,0,8,Nvidia L4,24,4,0,50;0;100,1,52560,0.33;0.2;0.6,
+scw_play2.base,Scaleway,rack,2,32,,AMD EPYC 7543,128,16,64,2,468,0,0,0,,0,2,0,50;0;100,1,52560,0.33;0.2;0.6,
+scw_pop2.base,Scaleway,rack,2,32,,AMD EPYC 7543,128,16,32,2,234,0,0,0,,0,2,0,50;0;100,1,52560,0.33;0.2;0.6,
+scw_pop2hc.base,Scaleway,rack,2,32,,AMD EPYC 7543,128,16,32,2,234,0,0,0,,0,2,0,50;0;100,1,52560,0.33;0.2;0.6,
+scw_pop2hm.base,Scaleway,rack,2,32,,AMD EPYC 7543,128,16,32,2,234,0,0,0,,0,2,0,50;0;100,1,52560,0.33;0.2;0.6,
+scw_pro2.base,Scaleway,rack,2,32,,AMD EPYC 7543,128,24,32,2,468,0,0,0,,0,2,0,50;0;100,1,52560,0.33;0.2;0.6,
+scw_renders.base,Scaleway,rack,2,20,,Intel Xeon Gold 6148,40,12,32,2,3750,0,0,8,Nvidia Tesla P100,24,4,0,50;0;100,1,52560,0.33;0.2;0.6,
+scw_stardust1.base,Scaleway,rack,1,16,,AMD EPYC 7281,64,8,32,5,976,0,0,0,,0,2,0,50;0;100,1,52560,0.33;0.2;0.6,

boaviztapi/data/crowdsourcing/cpu_specs.csv

@@ -34,7 +34,7 @@ AMD EPYC 7252,Rome,EPYC (Zen 2 (Rome)),TSMC,2019-08-07,2.8 GHz,120.0,8.0,16.0,"3
 AMD EPYC 7261,Naples,,,,,,,16.0,,,,,,,,AMD,EPYC,https://github.com/cloud-carbon-footprint/cloud-carbon-coefficients/tree/main/data
 AMD EPYC 7262,Rome,EPYC (Zen 2 (Rome)),TSMC,2019-08-07,3.1 GHz,155.0,8.0,16.0,"3,800 million",7.0,2x 74 mm²,416 mm2,14 nm,564.0,io_die_size (416 mm2) + die_size (2x 74 mm²),AMD,EPYC,https://www.techpowerup.com/cpu-specs/epyc-7262.c2257
 AMD EPYC 7272,Rome,EPYC (Zen 2 (Rome)),TSMC,2019-08-07,2.6 GHz,120.0,12.0,24.0,"3,800 million",7.0,3x 74 mm²,416 mm2,14 nm,638.0,io_die_size (416 mm2) + die_size (3x 74 mm²),AMD,EPYC,https://www.techpowerup.com/cpu-specs/epyc-7272.c2256
-AMD EPYC 7281,Naples,,,,,,,32.0,,,,,,,,AMD,EPYC,https://github.com/cloud-carbon-footprint/cloud-carbon-coefficients/tree/main/data
+AMD EPYC 7281,Naples,,TSMC,,2.1 GHz,170.0,16.0,32.0,,,,,,,,AMD,EPYC,https://www.techpowerup.com/cpu-specs/epyc-7281.c1927
 AMD EPYC 7282,Rome,EPYC (Zen 2 (Rome)),TSMC,2019-08-07,2.8 GHz,120.0,16.0,32.0,"3,800 million",7.0,4x 74 mm²,416 mm2,14 nm,712.0,io_die_size (416 mm2) + die_size (4x 74 mm²),AMD,EPYC,https://www.techpowerup.com/cpu-specs/epyc-7282.c2255
 AMD EPYC 72F3,Milan,EPYC (Zen 3 (Milan)),TSMC,2021-03-15,3.7 GHz,180.0,8.0,16.0,"33,200 million",7.0,8x 81 mm²,416 mm²,12 nm,1064.0,io_die_size (416 mm²) + die_size (8x 81 mm²),AMD,EPYC,https://www.techpowerup.com/cpu-specs/epyc-72f3.c2391
 AMD EPYC 7301,Naples,,,,,,,32.0,,,,,,,,AMD,EPYC,https://github.com/cloud-carbon-footprint/cloud-carbon-coefficients/tree/main/data

boaviztapi/data/utils/complete_Scaleway_Instances/README.md

@@ -0,0 +1,38 @@
+# Completing and updating Scaleway Instances data
+
+The Scaleway export can be done via the [scw-environmental-footprint](https://github.com/Shillaker/scw-environmental-footprint) project, specifically following [this doc](https://github.com/Shillaker/scw-environmental-footprint/blob/main/docs/boavizta.md).
+
+This does the following:
+
+- Lists all types of Instances using the Scaleway API
+- Maps these to a set of hard-coded base server types (see below)
+- Generates two CSV files: `instances.csv` (the instance types), and `servers.csv` (the base servers)
+
+To add to Boavizta:
+
+- Copy `instances.csv` to `boaviztapi/data/archetypes/cloud/scaleway.csv` (simply overwrite if updating)
+- Add the lines form `servers.csv` to `boaviztapi/data/archetypes/server.csv` (delete all existing servers starting with `scw_` if you are updating)
+
+## Testing
+
+1. Update CSV files
+2. Build the Docker image locally: `docker build -t boaviztapi-dev .`
+3. Run it `docker run -p "5000:5000" -t boaviztapi-dev`
+
+You can then run the script to check the changes using the `check.py` script in this directory:
+
+```
+python3 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+python3 check.py
+```
+
+## Scripting
+
+It would be much better if this code lived in the Boavizta repo. It can be scripted relatively easily:
+
+1. Hard-code the base server types held in the [scw-environmental-footprint](https://github.com/Shillaker/scw-environmental-footprint) repo [here](https://github.com/Shillaker/scw-environmental-footprint/blob/main/model/instances.go)
+2. Use the [scaleway-sdk-python](https://github.com/scaleway/scaleway-sdk-python) to list all the Instance types
+3. Map these the the underlying base server types based on the prefix of the Instance type
+4. Print two CSV files: i) `instances.csv` with the instance types; ii) `servers.csv` with the specs of the base servers

boaviztapi/data/utils/complete_Scaleway_Instances/check.py

@@ -0,0 +1,42 @@
+import requests
+
+CLOUD_URL="http://localhost:5000/v1/cloud/instance"
+PROVIDERS_URL=f"{CLOUD_URL}/all_providers"
+VALID_INSTANCE_TYPE="dev1-l"
+
+def main():
+    resp = requests.get(PROVIDERS_URL)
+
+    if not resp.ok:
+        print(f"Providers request failed: {resp.json()}")
+        exit(1)
+
+    if "scaleway" in resp.json():
+        print("Found Scaleway in list of providers")
+    else:
+        print("Did not find Scaleway in list of providers: {resp.status_code}")
+        exit(1)
+
+    url_params = {
+        "provider": "scaleway",
+        "instance_type": VALID_INSTANCE_TYPE,
+        "verbose": True,
+        "duration": "100",
+        }
+
+    resp = requests.get(CLOUD_URL, params=url_params)
+    if resp.ok:
+        print(f"Got response to impact request for type {VALID_INSTANCE_TYPE}")
+    else:
+        print("Did not find Scaleway in list of providers: {resp.status_code}")
+        exit(1)
+
+    resp_body = resp.json()
+    for impact_name, impact in resp_body["impacts"].items():
+        print(f"\nGot {impact_name}: {impact}")
+
+    cpu = resp_body["verbose"]["CPU-1"]["model_range"]["value"]
+    print(f"\nGot CPU model: {cpu}")
+
+if __name__ == "__main__":
+    main()

boaviztapi/data/utils/complete_Scaleway_Instances/requirements.txt

@@ -0,0 +1 @@
+requests==2.32.3

tests/api/test_cloud.py

@@ -967,3 +967,24 @@ async def test_usage_with_complex_time_workload_e8ads_v5():
     )
 
     await test.check_result()
+
+
+@pytest.mark.asyncio
+async def test_empty_usage_scw_dev1_l():
+    test = CloudTest(
+        CloudInstanceRequest("scaleway", "dev1-l"),
+        ADPImpact(
+            ImpactOutput(0.007718, 0.006189, 0.0064, END_OF_LIFE_WARNING),
+            ImpactOutput(0.0004059, 1.513e-05, 8e-05),
+        ),
+        GWPImpact(
+            ImpactOutput(59.5, 30.69, 45.0, END_OF_LIFE_WARNING),
+            ImpactOutput(1375.0, 26.36, 500.0),
+        ),
+        PEImpact(
+            ImpactOutput(789.0, 409.5, 610.0, END_OF_LIFE_WARNING),
+            ImpactOutput(715400.0, 14.9, 20000.0),
+        ),
+    )
+
+    await test.check_result()

tests/data/archetypes/cloud/providers.csv

@@ -1,3 +1,4 @@
 provider.name,provider.description
 aws,Amazon Web Services
-azure,Azure
+azure,Azure
+scaleway,Scaleway

tests/data/archetypes/cloud/scaleway.csv

@@ -0,0 +1,2 @@
+id,vcpu,memory,ssd_storage,hdd_storage,gpu_units,platform,source
+dev1-l,4,8,10,0,0,scw_dev1.base,

tests/data/archetypes/server.csv

@@ -138,3 +138,5 @@ platform_aws_t3,AWS,rack,2,,,Intel Xeon Platinum 8175M,,12,,32,0,0,,0,0,0,,0,2;2
 platform_aws_t3a,AWS,rack,2,,,AMD EPYC 7571,,12,,32,0,0,,0,0,0,,0,2;2;2,2.99;1;5,50;0;100,1,35040,0.33;0.2;0.6,RAM configuration was not verified
 platform_aws_t4g,AWS,rack,1,,,Annapurna Labs Graviton2,,8,,32,0,0,,0,0,0,,0,2;2;2,2.99;1;5,50;0;100,1,35040,0.33;0.2;0.6,RAM configuration was not verified
 Eadsv5-Type1,Azure,rack,1.0,,,AMD EPYC 7763,112.0,48.0,,16.0,2.0,2048.0,,0.0,0.0,0.0,,,2;2;2,2.99;1;5,50;0;100,1,52560.0,0.33;0.2;0.6,RAM units and per unit capacity not verified. RAM capacity from Azure docs was: 768 GiB
+
+scw_dev1.base,Scaleway,rack,1,16,,AMD EPYC 7281,64,8,,32,5,976,,0,0,0,,0,2,0,50;0;100,1,52560,0.33;0.2;0.6,

tests/data/crowdsourcing/cpu_specs.csv

@@ -349,7 +349,8 @@ AMD EPYC 7252,AMD,Rome,EPYC,2.8 GHz,120.0,8.0,,74.0,io_die_size () + die_size (7
 AMD EPYC 7261,AMD,Naples,EPYC,2.5 GHz,170.0,8.0,,213.0,io_die_size () + die_size (213 mm²),https://www.techpowerup.com/cpu-specs/epyc-7261.c2299
 AMD EPYC 7262,AMD,Rome,EPYC,3.1 GHz,155.0,8.0,,74.0,io_die_size () + die_size (74 mm²),https://www.techpowerup.com/cpu-specs/epyc-7262.c2257
 AMD EPYC 7272,AMD,Rome,EPYC,2.6 GHz,120.0,12.0,,74.0,io_die_size () + die_size (74 mm²),https://www.techpowerup.com/cpu-specs/epyc-7272.c2256
-AMD EPYC 7281,AMD,Naples,EPYC,2.1 GHz,170.0,16.0,,213.0,io_die_size () + die_size (213 mm²),https://www.techpowerup.com/cpu-specs/epyc-7281.c1927
+AMD EPYC 7763,AMD,Milan,EPYC,2.45 GHz,280.0,64.0,128.0,1064.0,io_die_size (416 mm²) + die_size (8x 81 mm²),https://www.techpowerup.com/cpu-specs/epyc-7763.c2373
+AMD EPYC 7281,AMD,Naples,EPYC,2.1 GHz,170.0,16.0,32.0,213.0,io_die_size () + die_size (213 mm²),https://www.techpowerup.com/cpu-specs/epyc-7281.c1927
 AMD EPYC 7282,AMD,Rome,EPYC,2.8 GHz,120.0,16.0,,74.0,io_die_size () + die_size (74 mm²),https://www.techpowerup.com/cpu-specs/epyc-7282.c2255
 AMD EPYC 72F3,AMD,Milan,EPYC,3.7 GHz,180.0,8.0,,1064.0,io_die_size (416 mm²) + die_size (8x 81 mm²),https://www.techpowerup.com/cpu-specs/epyc-72f3.c2391
 AMD EPYC 7301,AMD,Naples,EPYC,2.2 GHz,170.0,16.0,,213.0,io_die_size () + die_size (213 mm²),https://www.techpowerup.com/cpu-specs/epyc-7301.c1926
@@ -3255,4 +3256,4 @@ VIA Nano U2350,VIA,CNA,Nano U,1300 MHz,8.0,1.0,,63.0,io_die_size () + die_size (
 VIA Nano U2400,VIA,CNA,Nano U,1300 MHz,8.0,1.0,,63.0,io_die_size () + die_size (63 mm²),https://www.techpowerup.com/cpu-specs/nano-u2400.c2810
 VIA Nano U2500,VIA,CNA,Nano U,1200 MHz,7.0,1.0,,63.0,io_die_size () + die_size (63 mm²),https://www.techpowerup.com/cpu-specs/nano-u2500.c2811
 VIA Nano X2 U4300,VIA,CNC,Nano X2 U,1200 MHz,13.0,2.0,,66.0,io_die_size () + die_size (66 mm²),https://www.techpowerup.com/cpu-specs/nano-x2-u4300.c2814
-,,,,,,
+,,,,,,