small change for schema compare method

api/test/e2e/journeys/base_journey.py

@@ -10,7 +10,6 @@
 import pandas as pd
 from io import StringIO
 from uuid import uuid4
-from enum import Enum
 from jinja2 import Template
 from strenum import StrEnum
 from api.common.config.constants import CONTENT_ENCODING
@@ -210,3 +209,27 @@ def make_file_invalid(self, buffer_obj: StringIO) -> StringIO:
         df.to_csv(output_buffer)
         output_buffer.seek(0)
         return output_buffer
+
+    def compare_schema(self, local_schema, queried_schema, override_keys=None):
+        """
+        We're explicitly checking for local_schema values in queried_schema.
+        There will be dome queried_schema values that are not in local_schema,
+        """
+        if override_keys is None:
+            override_keys = []
+
+        for key, value in local_schema.items():
+            if key in override_keys:
+                continue
+            if isinstance(value, list):
+                for i in range(len(value)):
+                    self.compare_schema(value[i], queried_schema[key][i], override_keys)
+            elif isinstance(value, dict):
+                self.compare_schema(value, queried_schema[key], override_keys)
+            else:
+                assert (
+                    key in queried_schema
+                ), f"Key '{key}' not found in second dictionary."
+                assert (
+                    value == queried_schema[key]
+                ), f"Value for key '{key}' does not match. Expected: {value}, Actual: {queried_schema[key]}"

api/test/e2e/journeys/test_protected_domains.py

@@ -126,7 +126,7 @@ def test_delete_protected_domain(self):
         delete_url = self.protected_domain_url(self.test_domain_name)
         response = requests.delete(delete_url, headers=self.generate_auth_headers())
         assert response.status_code == HTTPStatus.ACCEPTED
-        
+
         list_url = self.list_protected_domain_url()
         response = requests.get(list_url, headers=self.generate_auth_headers())
         assert self.test_domain_name not in response.json()

api/test/e2e/journeys/test_schemas.py

@@ -33,19 +33,6 @@ def update_schema(self, schema: dict) -> Response:
             json=schema,
         )
 
-    def clean_schema(self, schema):
-        schema["metadata"].pop("version", None)
-        schema["metadata"].pop("is_latest_version", None)
-        schema["metadata"].pop("number_of_rows", None)
-        schema["metadata"].pop("number_of_columns", None)
-        schema["metadata"].pop("last_updated", None)
-
-        for column in schema["columns"]:
-            column.pop("format", None)
-            column.pop("statistics", None)
-
-        return schema
-
     def get_schema_from_info_endpoint(
         self, layer: str, domain: str, dataset: str
     ) -> dict:
@@ -56,8 +43,9 @@ def get_schema_from_info_endpoint(
                 client_name="E2E_TEST_CLIENT_READ_ALL_WRITE_ALL"
             ),
         )
-        return self.clean_schema(response.json())
+        return response.json()
 
+    @pytest.mark.focus
     def test_uploads_new_schema_version(self):
         schema_v1 = self.read_schema_version(SchemaVersion.V1)
         layer = schema_v1["metadata"]["layer"]
@@ -67,19 +55,16 @@ def test_uploads_new_schema_version(self):
         response1 = self.upload_schema(schema_v1)
         assert response1.status_code == HTTPStatus.CREATED
         uploaded_schema = self.get_schema_from_info_endpoint(layer, domain, dataset)
-        assert uploaded_schema == schema_v1
+        self.compare_schema(schema_v1, uploaded_schema, override_keys=["version"])
 
         schema_v2 = self.read_schema_version(SchemaVersion.V2)
         response2 = self.update_schema(schema_v2)
         assert response2.status_code == HTTPStatus.OK
 
         updated_schema = self.get_schema_from_info_endpoint(layer, domain, dataset)
-        # For test simplicity, assert the newcolumn column is present:
-
-        check_for_new_column = (
-            list(filter(lambda x: x["name"] == "newcolumn", updated_schema["columns"])),
+        self.compare_schema(
+            schema_v2, updated_schema, override_keys=["version", "tags"]
         )
-        assert len(check_for_new_column) == 1
 
     @pytest.mark.parametrize(
         "metadata_key, metadata_value",

api/test/e2e/journeys/test_subjects.py

@@ -9,7 +9,6 @@
 import pytest
 
 
-@pytest.mark.focus
 class TestSubjectJourneys(BaseAuthenticatedJourneyTest):
     subject_id = None