[Test] Add EFS and FSxLustre managed storage to cluster created for Multi NICS test.

cli/src/pcluster/cli/commands/configure/easyconfig.py

@@ -177,9 +177,9 @@ def configure(args):  # noqa: C901
     for queue_index in range(number_of_queues):
         while True:
             queue_name = prompt(
-                f"Name of queue {queue_index+1}",
+                f"Name of queue {queue_index + 1}",
                 validator=lambda x: len(NameValidator().execute(x)) == 0,
-                default_value=f"queue{queue_index+1}",
+                default_value=f"queue{queue_index + 1}",
             )
             if queue_name not in queue_names:
                 break
@@ -208,7 +208,7 @@ def configure(args):  # noqa: C901
             if scheduler != "awsbatch":
                 while True:
                     compute_instance_type = prompt(
-                        f"Compute instance type for compute resource {compute_resource_index+1} in {queue_name}",
+                        f"Compute instance type for compute resource {compute_resource_index + 1} in {queue_name}",
                         validator=lambda x: x in AWSApi.instance().ec2.list_instance_types(),
                         default_value=default_instance_type,
                     )

cli/src/pcluster/cli/model.py

@@ -12,6 +12,9 @@
 # limitations under the License.
 import functools
 import importlib
+
+# For importing package resources
+import importlib.resources as importlib_resources
 import json
 
 import jmespath
@@ -20,12 +23,6 @@
 from pcluster.cli.exceptions import APIOperationException
 from pcluster.utils import to_kebab_case, to_snake_case, yaml_load
 
-# For importing package resources
-try:
-    import importlib.resources as pkg_resources  # pylint: disable=ungrouped-imports # nosem
-except ImportError:
-    import importlib_resources as pkg_resources
-
 
 def _param_overrides(operation, param):
     """Provide updates to the model that are specific to the CLI."""
@@ -88,7 +85,7 @@ def _resolve_body(spec, operation):
 
 def package_spec():
     """Load the OpenAPI specification from the package."""
-    with pkg_resources.open_text(openapi, "openapi.yaml") as spec_file:  # pylint: disable=deprecated-method
+    with importlib_resources.open_text(openapi, "openapi.yaml") as spec_file:  # pylint: disable=deprecated-method
         return yaml_load(spec_file.read())
 
 

tests/integration-tests/conftest_resource_bucket.py

@@ -22,7 +22,6 @@
 from pathlib import Path
 
 import boto3
-import pkg_resources
 import pytest
 import urllib3
 from framework.fixture_utils import xdist_session_fixture
@@ -36,7 +35,7 @@
 def install_pc(basepath, pc_version):
     """Install ParallelCluster to a temporary directory"""
     tempdir = Path(basepath) / "python"
-    root = Path(pkg_resources.resource_filename(__name__, "/../.."))
+    root = Path(__file__).parent.parent
     cli_dir = root / "cli"
     try:
         logger.info("installing ParallelCluster packages...")
@@ -105,7 +104,7 @@ def get_resource_map():
 
 @xdist_session_fixture()
 def resource_bucket_shared(request, s3_bucket_factory_shared, lambda_layer_source):
-    root = Path(pkg_resources.resource_filename(__name__, "/../.."))
+    root = Path(__file__).parent.parent
     if request.config.getoption("resource_bucket"):
         return None  # short-circuit this fixture if a resource-bucket is provided
 

tests/integration-tests/tests/common/utils.py

@@ -15,11 +15,12 @@
 import random
 import string
 import time
+from importlib.metadata import version as get_package_version
 
 import boto3
-import pkg_resources
 from assertpy import assert_that
 from botocore.exceptions import ClientError
+from packaging import version as packaging_version
 from remote_command_executor import RemoteCommandExecutionError, RemoteCommandExecutor
 from retrying import retry
 from time_utils import seconds
@@ -270,14 +271,14 @@ def _assert_ami_is_available(region, ami_id):
 def get_installed_parallelcluster_version():
     """Get the version of the installed aws-parallelcluster package."""
     try:
-        return pkg_resources.get_distribution("aws-parallelcluster").version
+        return get_package_version("aws-parallelcluster")
     except Exception:
         logging.info("aws-parallelcluster is not installed through Python. Getting version from `pcluster version`.")
         return json.loads(run_command(["pcluster", "version"]).stdout.strip())["version"]
 
 
 def get_installed_parallelcluster_base_version():
-    return pkg_resources.packaging.version.parse(get_installed_parallelcluster_version()).base_version
+    return packaging_version.parse(get_installed_parallelcluster_version()).base_version
 
 
 CLASSIC_AWS_DOMAIN = "amazonaws.com"

tests/integration-tests/tests/create/test_create.py

@@ -159,7 +159,7 @@ def test_cluster_creation_with_problematic_preinstall_script(
     assert_lines_in_logs(
         remote_command_executor,
         ["/var/log/cfn-init.log"],
-        [f"Failed to execute OnNodeStart script 1 s3://{ bucket_name }/scripts/{script_name}"],
+        [f"Failed to execute OnNodeStart script 1 s3://{bucket_name}/scripts/{script_name}"],
     )
     logging.info("Verifying error in cloudformation failure reason")
     stack_events = cluster.get_stack_events().get("events")
@@ -173,7 +173,7 @@ def test_cluster_creation_with_problematic_preinstall_script(
     )
 
     assert_that(cfn_failure_reason).contains(expected_cfn_failure_reason)
-    assert_that(cfn_failure_reason).does_not_contain(f"s3://{ bucket_name }/scripts/{script_name}")
+    assert_that(cfn_failure_reason).does_not_contain(f"s3://{bucket_name}/scripts/{script_name}")
 
     logging.info("Verifying failures in describe-clusters output")
     expected_failures = [

tests/integration-tests/tests/custom_resource/conftest.py

@@ -15,7 +15,6 @@
 
 import boto3
 import cfn_tools
-import pkg_resources
 import pytest
 import yaml
 from cfn_stacks_factory import CfnStack
@@ -34,7 +33,7 @@ def cfn_fixture(region):
 
 @pytest.fixture(scope="session", name="resources_dir")
 def resources_dir_fixture():
-    return Path(pkg_resources.resource_filename(__name__, "/../../resources"))
+    return Path(__file__).parent.parent.parent / "resources"
 
 
 @pytest.fixture(scope="session", name="cluster_custom_resource_template")

tests/integration-tests/tests/multiple_nics/test_multiple_nics/test_multiple_nics/pcluster.config.yaml

@@ -42,5 +42,14 @@ Scheduling:
         {% endif %}
 SharedStorage:
   - MountDir: /shared
-    Name: name1
+    Name: shared-ebs
     StorageType: Ebs
+  - MountDir: /shared-efs
+    Name: shared-efs
+    StorageType: Efs
+  - MountDir: /shared-fsx-lustre
+    Name: shared-fsx-lustre
+    StorageType: FsxLustre
+    FsxLustreSettings:
+      StorageCapacity: 1200
+      DeletionPolicy: Delete

tests/integration-tests/tests/pcluster_api/test_api.py

@@ -17,7 +17,6 @@
 
 import boto3
 import cfn_tools
-import pkg_resources
 import pytest
 from assertpy import assert_that
 from benchmarks.common.util import get_instance_vcpus
@@ -128,7 +127,7 @@ def _ec2_wait(region, instances, waiter_type):
 
 @pytest.fixture(scope="session", name="resources_dir")
 def resources_dir_fixture():
-    return Path(pkg_resources.resource_filename(__name__, "/../../resources"))
+    return Path(__file__).parent.parent.parent / "resources"
 
 
 @pytest.fixture(scope="session", name="policies_template_path")

tests/integration-tests/tests/schedulers/test_slurm.py

@@ -169,7 +169,7 @@ def test_slurm_ticket_17399(
             "partition": "gpu",
             "test_only": True,
             "other_options": f"--gpus {gpus_per_instance} --nodes 1 --ntasks-per-node 1 "
-            f"--cpus-per-task={cpus_per_instance//gpus_per_instance}",
+            f"--cpus-per-task={cpus_per_instance // gpus_per_instance}",
         }
     )
 
@@ -180,7 +180,7 @@ def test_slurm_ticket_17399(
             "partition": "gpu",
             "test_only": True,
             "other_options": f"--gpus {gpus_per_instance} --nodes 1 --ntasks-per-node 1 "
-            f"--cpus-per-task={cpus_per_instance//gpus_per_instance + 1}",
+            f"--cpus-per-task={cpus_per_instance // gpus_per_instance + 1}",
         }
     )
 

tests/integration-tests/tests/storage/kms_key_factory.py

@@ -3,9 +3,9 @@
 import random
 import string
 import time
+from pathlib import Path
 
 import boto3
-import pkg_resources
 from jinja2 import FileSystemLoader
 from jinja2.sandbox import SandboxedEnvironment
 from utils import get_arn_partition
@@ -112,7 +112,7 @@ def _create_iam_policies(self, iam_policy_name, scheduler):
         # create the iam policy
         # for different scheduler, attach different instance policy
         logging.info("Creating iam policy {0} for iam role...".format(iam_policy_name))
-        file_loader = FileSystemLoader(pkg_resources.resource_filename(__name__, "/../../resources"))
+        file_loader = FileSystemLoader(Path(__file__).parent.parent.parent / "resources")
         env = SandboxedEnvironment(loader=file_loader, trim_blocks=True, lstrip_blocks=True)
         policy_filename = (
             "batch_instance_policy.json" if scheduler == "awsbatch" else "traditional_instance_policy.json"
@@ -155,7 +155,7 @@ def _create_kms_key(self, region):
 
         # create KMS key policy
         logging.info("Attaching key policy...")
-        file_loader = FileSystemLoader(pkg_resources.resource_filename(__name__, "/../../resources"))
+        file_loader = FileSystemLoader(Path(__file__).parent.parent.parent / "resources")
         env = SandboxedEnvironment(loader=file_loader, trim_blocks=True, lstrip_blocks=True)
         key_policy = env.get_template("key_policy.json").render(
             partition=self.partition, account_id=self.account_id, iam_role_name=f"parallelcluster/{self.iam_role_name}"

tests/integration-tests/tests/storage/test_ebs.py

@@ -182,7 +182,7 @@ def test_ebs_multiple(
     for i in range(len(volume_ids)):
         # test different volume types
         volume_id = volume_ids[i]
-        ebs_settings = _get_ebs_settings_by_name(cluster.config, f"ebs{i+1}")
+        ebs_settings = _get_ebs_settings_by_name(cluster.config, f"ebs{i + 1}")
         volume_type = ebs_settings["VolumeType"]
         volume = describe_volume(volume_id, region)
         assert_that(volume[0]).is_equal_to(volume_type)