diff --git a/tests/data/mock_manifests/table_manifest.csv b/tests/data/mock_manifests/table_manifest.csv index c64988577..f0d07db9b 100644 --- a/tests/data/mock_manifests/table_manifest.csv +++ b/tests/data/mock_manifests/table_manifest.csv @@ -1,10 +1,3 @@ Component,Days to Follow Up,Adverse Event,Progression or Recurrence,Barretts Esophagus Goblet Cells Present,BMI,Cause of Response,Comorbidity,Comorbidity Method of Diagnosis,Days to Adverse Event,Days to Comorbidity,Diabetes Treatment Type,Disease Response,DLCO Ref Predictive Percent,ECOG Performance Status,FEV1 FVC Post Bronch Percent,FEV 1 FVC Pre Bronch Percent,FEV1 Ref Post Bronch Percent,FEV1 Ref Pre Bronch Percent,Height,Hepatitis Sustained Virological Response,HPV Positive Type,Karnofsky Performance Status,Menopause Status,Pancreatitis Onset Year,Reflux Treatment Type,Risk Factor,Risk Factor Treatment,Viral Hepatitis Serologies,Weight,Days to Progression,Days to Progression Free,Days to Recurrence,Progression or Recurrence Anatomic Site,Progression or Recurrence Type,entityId,HTAN Participant ID FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22249015,455.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22249915,456.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22687161,457.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22688035,458.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978930,459.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978946,460.0 FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978975,461.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22979177,462.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn21989551,454.0 diff --git a/tests/data/mock_manifests/table_manifest_replacement.csv b/tests/data/mock_manifests/table_manifest_replacement.csv index 71fd13e30..fea37d859 100644 --- a/tests/data/mock_manifests/table_manifest_replacement.csv +++ b/tests/data/mock_manifests/table_manifest_replacement.csv @@ -1,10 +1,3 @@ Component,Days to Follow Up,Adverse Event,Progression or Recurrence,Barretts Esophagus Goblet Cells Present,BMI,Cause of Response,Comorbidity,Comorbidity Method of Diagnosis,Days to Adverse Event,Days to Comorbidity,Diabetes Treatment Type,Disease Response,DLCO Ref Predictive Percent,ECOG Performance Status,FEV1 FVC Post Bronch Percent,FEV 1 FVC Pre Bronch Percent,FEV1 Ref Post Bronch Percent,FEV1 Ref Pre Bronch Percent,Height,Hepatitis Sustained Virological Response,HPV Positive Type,Karnofsky Performance Status,Menopause Status,Pancreatitis Onset Year,Reflux Treatment Type,Risk Factor,Risk Factor Treatment,Viral Hepatitis Serologies,Weight,Days to Progression,Days to Progression Free,Days to Recurrence,Progression or Recurrence Anatomic Site,Progression or Recurrence Type,entityId,HTAN Participant ID,Uuid FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22249015,455,318e2d8c-83be-4b19-9a9f-5e97ce9c9cf7 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22249915,456,600666ee-1f96-49f8-b848-2be751cb59f9 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22687161,457,1fb32068-7b6c-4d12-8031-96d91229dda0 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22688035,458,964949f3-efe1-47cd-8113-dedb909fd312 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978930,459,c562590f-359a-4ff5-bbd7-47e7ad840a32 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978946,460,a77e0882-7b6f-42a9-9e60-c4d2bfd72bb7 FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978975,461,e7705ed5-ade4-4451-ae26-9c790e35e878 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22979177,462,2897b73c-e225-4dc3-a093-96f645332ea3 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn21989551,454,26a1cdba-dd37-483a-be6e-d1a4f5a9521b diff --git a/tests/test_store.py b/tests/test_store.py index 49b79a3c0..e73c64072 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -1139,99 +1139,6 @@ def test_tidy_table_no_manifest_uploaded(self, synapse_store): @pytest.mark.table_operations class TestTableOperations: - @pytest.mark.slow_test - @pytest.mark.parametrize( - "table_column_names", - ["display_name", "display_label", "class_label"], - ids=["tcn_display_name", "tcn_display_label", "tcn_class_label"], - ) - @pytest.mark.parametrize( - "annotation_keys", - ["display_label", "class_label"], - ids=["aks_display_label", "aks_class_label"], - ) - async def test_create_table( - self, - helpers: Helpers, - synapse_store: SynapseStorage, - projectId: str, - datasetId: str, - table_column_names: str, - annotation_keys: str, - dmge: DataModelGraphExplorer, - schedule_for_cleanup: Callable[[CleanupItem], None], - ) -> None: - # GIVEN a table to create - table_manipulation = None - table_name = f"followup_synapse_storage_manifest_table_integration_test_{str(uuid.uuid4()).replace('-', '_')}" - schedule_for_cleanup(CleanupItem(name=table_name, parent_id=projectId)) - - # AND a manifest to associate metadata with files - manifest_path = "mock_manifests/table_manifest.csv" - - # AND a copy of all the folders in the manifest. Added to the dataset directory for easy cleanup - manifest = helpers.get_data_frame(manifest_path) - - async def copy_folder_and_update_manifest( - row: pd.Series, - index: int, - datasetId: str, - synapse_store: SynapseStorage, - manifest: pd.DataFrame, - schedule_for_cleanup: Callable[[CleanupItem], None], - ) -> None: - """Internal function to copy a folder and update the manifest.""" - folder_id = row["entityId"] - folder_copy = await FolderModel(id=folder_id).copy_async( - parent_id=datasetId, synapse_client=synapse_store.syn - ) - schedule_for_cleanup(CleanupItem(synapse_id=folder_copy.id)) - manifest.at[index, "entityId"] = folder_copy.id - - tasks = [] - - for index, row in manifest.iterrows(): - tasks.append( - copy_folder_and_update_manifest( - row, index, datasetId, synapse_store, manifest, schedule_for_cleanup - ) - ) - await asyncio.gather(*tasks) - - with patch.object( - synapse_store, "_generate_table_name", return_value=(table_name, "followup") - ), patch.object( - synapse_store, "getDatasetProject", return_value=projectId - ), tempfile.NamedTemporaryFile( - delete=True, - suffix=".csv", - dir=create_temp_folder(path=tempfile.gettempdir()), - ) as tmp_file: - # Write the DF to a temporary file to prevent modifying the original - manifest.to_csv(tmp_file.name, index=False) - - # WHEN I associate metadata with files - manifest_id = synapse_store.associateMetadataWithFiles( - dmge=dmge, - metadataManifestPath=tmp_file.name, - datasetId=datasetId, - manifest_record_type="table_and_file", - hideBlanks=True, - restrict_manifest=False, - table_manipulation=table_manipulation, - table_column_names=table_column_names, - annotation_keys=annotation_keys, - ) - schedule_for_cleanup(CleanupItem(synapse_id=manifest_id)) - - # THEN the table should exist - existing_table_id = synapse_store.syn.findEntityId( - name=table_name, parent=projectId - ) - - # assert table exists - assert existing_table_id is not None - @pytest.mark.slow_test @pytest.mark.parametrize( "table_column_names",