feat(compliance): add ENSRD2022 for Azure and GCP (prowler-cloud#5746)

README.md

@@ -64,8 +64,8 @@ It contains hundreds of controls covering CIS, NIST 800, NIST CSF, CISA, RBI, Fe
 | Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) |
 |---|---|---|---|---|
 | AWS | 553 | 77 -> `prowler aws --list-services` | 30 -> `prowler aws --list-compliance` | 9 -> `prowler aws --list-categories` |
-| GCP | 77 | 13 -> `prowler gcp --list-services` | 2 -> `prowler gcp --list-compliance` | 2 -> `prowler gcp --list-categories`|
-| Azure | 138 | 17 -> `prowler azure --list-services` | 3 -> `prowler azure --list-compliance` | 2 -> `prowler azure --list-categories` |
+| GCP | 77 | 13 -> `prowler gcp --list-services` | 3 -> `prowler gcp --list-compliance` | 2 -> `prowler gcp --list-categories`|
+| Azure | 138 | 17 -> `prowler azure --list-services` | 4 -> `prowler azure --list-compliance` | 2 -> `prowler azure --list-categories` |
 | Kubernetes | 83 | 7 -> `prowler kubernetes --list-services` | 1 -> `prowler kubernetes --list-compliance` | 7 -> `prowler kubernetes --list-categories` |
 
 # 💻 Installation

dashboard/compliance/ens_rd2022_azure.py

@@ -0,0 +1,36 @@
+import warnings
+
+from dashboard.common_methods import get_section_containers_ens
+
+warnings.filterwarnings("ignore")
+
+
+def get_table(data):
+    # append the requirements_description to idgrupocontrol
+    data["REQUIREMENTS_ATTRIBUTES_IDGRUPOCONTROL"] = (
+        data["REQUIREMENTS_ATTRIBUTES_IDGRUPOCONTROL"]
+        + " - "
+        + data["REQUIREMENTS_DESCRIPTION"]
+    )
+
+    aux = data[
+        [
+            "REQUIREMENTS_ATTRIBUTES_MARCO",
+            "REQUIREMENTS_ATTRIBUTES_CATEGORIA",
+            "REQUIREMENTS_ATTRIBUTES_IDGRUPOCONTROL",
+            "REQUIREMENTS_ATTRIBUTES_TIPO",
+            "CHECKID",
+            "STATUS",
+            "REGION",
+            "ACCOUNTID",
+            "RESOURCEID",
+        ]
+    ]
+
+    return get_section_containers_ens(
+        aux,
+        "REQUIREMENTS_ATTRIBUTES_MARCO",
+        "REQUIREMENTS_ATTRIBUTES_CATEGORIA",
+        "REQUIREMENTS_ATTRIBUTES_IDGRUPOCONTROL",
+        "REQUIREMENTS_ATTRIBUTES_TIPO",
+    )

dashboard/compliance/ens_rd2022_gcp.py

@@ -0,0 +1,36 @@
+import warnings
+
+from dashboard.common_methods import get_section_containers_ens
+
+warnings.filterwarnings("ignore")
+
+
+def get_table(data):
+    # append the requirements_description to idgrupocontrol
+    data["REQUIREMENTS_ATTRIBUTES_IDGRUPOCONTROL"] = (
+        data["REQUIREMENTS_ATTRIBUTES_IDGRUPOCONTROL"]
+        + " - "
+        + data["REQUIREMENTS_DESCRIPTION"]
+    )
+
+    aux = data[
+        [
+            "REQUIREMENTS_ATTRIBUTES_MARCO",
+            "REQUIREMENTS_ATTRIBUTES_CATEGORIA",
+            "REQUIREMENTS_ATTRIBUTES_IDGRUPOCONTROL",
+            "REQUIREMENTS_ATTRIBUTES_TIPO",
+            "CHECKID",
+            "STATUS",
+            "REGION",
+            "ACCOUNTID",
+            "RESOURCEID",
+        ]
+    ]
+
+    return get_section_containers_ens(
+        aux,
+        "REQUIREMENTS_ATTRIBUTES_MARCO",
+        "REQUIREMENTS_ATTRIBUTES_CATEGORIA",
+        "REQUIREMENTS_ATTRIBUTES_IDGRUPOCONTROL",
+        "REQUIREMENTS_ATTRIBUTES_TIPO",
+    )

dashboard/pages/compliance.py

@@ -148,6 +148,7 @@ def load_csv_files(csv_files):
     select_account_dropdown_list = ["All"]
     # Append to the list the unique values of the columns ACCOUNTID, PROJECTID and SUBSCRIPTIONID if they exist
     if "ACCOUNTID" in data.columns:
+        data["ACCOUNTID"] = data["ACCOUNTID"].astype(str)
         select_account_dropdown_list = select_account_dropdown_list + list(
             data["ACCOUNTID"].unique()
         )
@@ -246,9 +247,11 @@ def load_csv_files(files):
         dfs = []
         for file in files:
             df = pd.read_csv(
-                file, sep=";", on_bad_lines="skip", encoding=encoding_format
+                file, sep=";", on_bad_lines="skip", encoding=encoding_format, dtype=str
             )
-            dfs.append(df.astype(str))
+            df = df.astype(str).fillna("nan")
+            df.columns = df.columns.astype(str)
+            dfs.append(df)
         return pd.concat(dfs, ignore_index=True)
 
     data = load_csv_files(files)
@@ -274,17 +277,24 @@ def load_csv_files(files):
         data.rename(columns={"PROJECTID": "ACCOUNTID"}, inplace=True)
         data["REGION"] = "-"
     # Rename the column SUBSCRIPTIONID to ACCOUNTID for Azure
-    if data.columns.str.contains("SUBSCRIPTIONID").any():
+    if (
+        data.columns.str.contains("SUBSCRIPTIONID").any()
+        and not data.columns.str.contains("ACCOUNTID").any()
+    ):
         data.rename(columns={"SUBSCRIPTIONID": "ACCOUNTID"}, inplace=True)
         data["REGION"] = "-"
     # Handle v3 azure cis compliance
-    if data.columns.str.contains("SUBSCRIPTION").any():
+    if (
+        data.columns.str.contains("SUBSCRIPTION").any()
+        and not data.columns.str.contains("ACCOUNTID").any()
+    ):
         data.rename(columns={"SUBSCRIPTION": "ACCOUNTID"}, inplace=True)
         data["REGION"] = "-"
 
     # Filter ACCOUNT
     if account_filter == ["All"]:
         updated_cloud_account_values = data["ACCOUNTID"].unique()
+
     elif "All" in account_filter and len(account_filter) > 1:
         # Remove 'All' from the list
         account_filter.remove("All")
@@ -299,9 +309,11 @@ def load_csv_files(files):
 
     account_filter_options = list(data["ACCOUNTID"].unique())
     account_filter_options = account_filter_options + ["All"]
-    for item in account_filter_options:
-        if "nan" in item or item.__class__.__name__ != "str" or item is None:
-            account_filter_options.remove(item)
+    account_filter_options = [
+        item
+        for item in account_filter_options
+        if isinstance(item, str) and item.lower() != "nan"
+    ]
 
     # Filter REGION
     if region_filter_analytics == ["All"]:

prowler/__main__.py

@@ -53,6 +53,8 @@
 from prowler.lib.outputs.compliance.cis.cis_kubernetes import KubernetesCIS
 from prowler.lib.outputs.compliance.compliance import display_compliance_table
 from prowler.lib.outputs.compliance.ens.ens_aws import AWSENS
+from prowler.lib.outputs.compliance.ens.ens_azure import AzureENS
+from prowler.lib.outputs.compliance.ens.ens_gcp import GCPENS
 from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
 from prowler.lib.outputs.compliance.iso27001.iso27001_aws import AWSISO27001
 from prowler.lib.outputs.compliance.kisa_ismsp.kisa_ismsp_aws import AWSKISAISMSP
@@ -511,6 +513,20 @@ def prowler():
                 )
                 generated_outputs["compliance"].append(mitre_attack)
                 mitre_attack.batch_write_data_to_file()
+            elif compliance_name.startswith("ens_"):
+                # Generate ENS Finding Object
+                filename = (
+                    f"{output_options.output_directory}/compliance/"
+                    f"{output_options.output_filename}_{compliance_name}.csv"
+                )
+                ens = AzureENS(
+                    findings=finding_outputs,
+                    compliance=bulk_compliance_frameworks[compliance_name],
+                    create_file_descriptor=True,
+                    file_path=filename,
+                )
+                generated_outputs["compliance"].append(ens)
+                ens.batch_write_data_to_file()
             else:
                 filename = (
                     f"{output_options.output_directory}/compliance/"
@@ -555,6 +571,20 @@ def prowler():
                 )
                 generated_outputs["compliance"].append(mitre_attack)
                 mitre_attack.batch_write_data_to_file()
+            elif compliance_name.startswith("ens_"):
+                # Generate ENS Finding Object
+                filename = (
+                    f"{output_options.output_directory}/compliance/"
+                    f"{output_options.output_filename}_{compliance_name}.csv"
+                )
+                ens = GCPENS(
+                    findings=finding_outputs,
+                    compliance=bulk_compliance_frameworks[compliance_name],
+                    create_file_descriptor=True,
+                    file_path=filename,
+                )
+                generated_outputs["compliance"].append(ens)
+                ens.batch_write_data_to_file()
             else:
                 filename = (
                     f"{output_options.output_directory}/compliance/"