mskcc · sivkovic · Dec 19, 2024 · Dec 19, 2024 · Dec 20, 2024 · Dec 20, 2024
diff --git a/beagle/settings.py b/beagle/settings.py
@@ -445,6 +445,7 @@
 PRESERVATION_METADATA_KEY = "preservation"
 SEX_METADATA_KEY = "sex"
 RUN_DATE_METADATA_KEY = "runDate"
+DELIVERY_DATE = "deliveryDate"
 
 VALIDATE_PRIMARY_ID_FOR_GENE_PANELS = os.environ.get("BEAGLE_VALIDATE_PRIMARY_ID_FOR_GENE_PANELS", "").split(",")
 PRIMARY_ID_REGEX = os.environ.get("BEAGLE_PRIMARY_ID_REGEX", "[a-z]*_[0-9]*")

diff --git a/beagle_etl/jobs/metadb_jobs.py b/beagle_etl/jobs/metadb_jobs.py
@@ -911,9 +911,7 @@ def create_pooled_normal(filepath, file_group_id):
         f = File.objects.create(
             file_name=os.path.basename(filepath), path=filepath, file_group=file_group_obj, file_type=file_type_obj
         )
-        f.save()
-        fm = FileMetadata(file=f, metadata=metadata)
-        fm.save()
+        FileMetadata.objects.create_or_update(file=f, metadata=metadata)
     except Exception as e:
         logger.info("File already exist %s." % filepath)
 

diff --git a/beagle_etl/tests/jobs/test_lims_etl_jobs.py b/beagle_etl/tests/jobs/test_lims_etl_jobs.py
@@ -603,7 +603,7 @@ def test_request_callback(
             file_group=self.file_group,
         )
         date_now = date.today().strftime("%Y-%m-%d")
-        file_metadata = FileMetadata.objects.create(
+        file_metadata = FileMetadata.objects.create_or_update(
             file=file_conflict,
             version=1,
             metadata={
@@ -641,9 +641,8 @@ def test_request_callback_two_operators(
             file_group=self.file_group,
         )
         date_now = date.today().strftime("%Y-%m-%d")
-        file_metadata = FileMetadata.objects.create(
+        FileMetadata.objects.create_or_update(
             file=file_conflict,
-            version=1,
             metadata={
                 settings.REQUEST_ID_METADATA_KEY: "test1",
                 "recipe": "TestAssay",
@@ -680,9 +679,8 @@ def test_request_callback_unknown_assay(
             file_type=self.fastq,
             file_group=self.file_group,
         )
-        file_metadata = FileMetadata.objects.create(
+        FileMetadata.objects.create_or_update(
             file=file_conflict,
-            version=1,
             metadata={
                 settings.REQUEST_ID_METADATA_KEY: "test1",
                 "recipe": "UnknownAssay",
@@ -715,9 +713,8 @@ def test_request_callback_disabled_assay(
             file_type=self.fastq,
             file_group=self.file_group,
         )
-        file_metadata = FileMetadata.objects.create(
+        FileMetadata.objects.create_or_update(
             file=file_conflict,
-            version=1,
             metadata={
                 settings.REQUEST_ID_METADATA_KEY: "test1",
                 "recipe": "DisabledAssay1",

diff --git a/file_system/fixtures/13893_B.file.json b/file_system/fixtures/13893_B.file.json
@@ -12,7 +12,6 @@
             "file_group": "f6a547a3-cefd-4040-a16f-87289f7afdc6",
             "checksum": "sha1$fd2278acbe70b94e93553b81a2fb4baad6998a28",
             "request_id": null,
-            "sample": null,
             "samples": "[]",
             "patient_id": null
         }
@@ -30,7 +29,6 @@
             "file_group": "f6a547a3-cefd-4040-a16f-87289f7afdc6",
             "checksum": "sha1$d235041c1a415bd9bb42025ba1607828fcbbca0a",
             "request_id": null,
-            "sample": null,
             "samples": "[]",
             "patient_id": null
         }
@@ -48,7 +46,6 @@
             "file_group": "f6a547a3-cefd-4040-a16f-87289f7afdc6",
             "checksum": "sha1$fd2278acbe70b94e93553b81a2fb4baad6998a28",
             "request_id": null,
-            "sample": null,
             "samples": "[]",
             "patient_id": null
         }
@@ -66,7 +63,6 @@
             "file_group": "f6a547a3-cefd-4040-a16f-87289f7afdc6",
             "checksum": "sha1$d235041c1a415bd9bb42025ba1607828fcbbca0a",
             "request_id": null,
-            "sample": null,
             "samples": "[]",
             "patient_id": null
         }
@@ -84,7 +80,6 @@
             "file_group": "f6a547a3-cefd-4040-a16f-87289f7afdc6",
             "checksum": "sha1$f997af3fe3c580ef0700cabc8d9bef4368fc24e3",
             "request_id": null,
-            "sample": null,
             "samples": "[]",
             "patient_id": null
         }
@@ -102,7 +97,6 @@
             "file_group": "f6a547a3-cefd-4040-a16f-87289f7afdc6",
             "checksum": "sha1$f997af3fe3c580ef0700cabc8d9bef4368fc24e3",
             "request_id": null,
-            "sample": null,
             "samples": "[]",
             "patient_id": null
         }
@@ -120,7 +114,6 @@
             "file_group": "f6a547a3-cefd-4040-a16f-87289f7afdc6",
             "checksum": "sha1$1e8e606318c3c6769bfd2da7663c3d145935475d",
             "request_id": null,
-            "sample": null,
             "samples": "[]",
             "patient_id": null
         }
@@ -138,7 +131,6 @@
             "file_group": "f6a547a3-cefd-4040-a16f-87289f7afdc6",
             "checksum": "sha1$1e8e606318c3c6769bfd2da7663c3d145935475d",
             "request_id": null,
-            "sample": null,
             "samples": "[]",
             "patient_id": null
         }
@@ -156,7 +148,6 @@
             "file_group": "f6a547a3-cefd-4040-a16f-87289f7afdc6",
             "checksum": "sha1$9fa6aa23857c705c0f2c3f01fe785ab950066039",
             "request_id": null,
-            "sample": null,
             "samples": "[]",
             "patient_id": null
         }
@@ -174,7 +165,6 @@
             "file_group": "f6a547a3-cefd-4040-a16f-87289f7afdc6",
             "checksum": "sha1$619cf02b52fbcd4bbff6f9ba9f0f217b98f03491",
             "request_id": null,
-            "sample": null,
             "samples": "[]",
             "patient_id": null
         }
@@ -192,7 +182,6 @@
             "file_group": "f6a547a3-cefd-4040-a16f-87289f7afdc6",
             "checksum": "sha1$619cf02b52fbcd4bbff6f9ba9f0f217b98f03491",
             "request_id": null,
-            "sample": null,
             "samples": "[]",
             "patient_id": null
         }
@@ -210,7 +199,6 @@
             "file_group": "f6a547a3-cefd-4040-a16f-87289f7afdc6",
             "checksum": "sha1$9fa6aa23857c705c0f2c3f01fe785ab950066039",
             "request_id": null,
-            "sample": null,
             "samples": "[]",
             "patient_id": null
         }
@@ -228,7 +216,6 @@
             "file_group": "1a1b29cf-3bc2-4f6c-b376-d4c5d701166a",
             "checksum": null,
             "request_id": "13893_B",
-            "sample": null,
             "samples": "[\"13893_B_2\"]",
             "patient_id": "ALLANT3"
         }
@@ -246,7 +233,6 @@
             "file_group": "1a1b29cf-3bc2-4f6c-b376-d4c5d701166a",
             "checksum": null,
             "request_id": "13893_B",
-            "sample": null,
             "samples": "[\"13893_B_2\"]",
             "patient_id": "ALLANT3"
         }
@@ -264,7 +250,6 @@
             "file_group": "1a1b29cf-3bc2-4f6c-b376-d4c5d701166a",
             "checksum": null,
             "request_id": "13893_B",
-            "sample": null,
             "samples": "[\"13893_B_2\"]",
             "patient_id": "ALLANT3"
         }
@@ -282,7 +267,6 @@
             "file_group": "1a1b29cf-3bc2-4f6c-b376-d4c5d701166a",
             "checksum": null,
             "request_id": "13893_B",
-            "sample": null,
             "samples": "[\"13893_B_2\"]",
             "patient_id": "ALLANT3"
         }
@@ -300,7 +284,6 @@
             "file_group": "1a1b29cf-3bc2-4f6c-b376-d4c5d701166a",
             "checksum": null,
             "request_id": "13893_B",
-            "sample": null,
             "samples": "[\"13893_B_3\"]",
             "patient_id": "ALLANT"
         }
@@ -318,7 +301,6 @@
             "file_group": "1a1b29cf-3bc2-4f6c-b376-d4c5d701166a",
             "checksum": null,
             "request_id": "13893_B",
-            "sample": null,
             "samples": "[\"13893_B_3\"]",
             "patient_id": "ALLANT"
         }
@@ -336,7 +318,6 @@
             "file_group": "1a1b29cf-3bc2-4f6c-b376-d4c5d701166a",
             "checksum": null,
             "request_id": "13893_B",
-            "sample": null,
             "samples": "[\"13893_B_3\"]",
             "patient_id": "ALLANT"
         }
@@ -354,7 +335,6 @@
             "file_group": "1a1b29cf-3bc2-4f6c-b376-d4c5d701166a",
             "checksum": null,
             "request_id": "13893_B",
-            "sample": null,
             "samples": "[\"13893_B_3\"]",
             "patient_id": "ALLANT"
         }
@@ -372,7 +352,6 @@
             "file_group": "1a1b29cf-3bc2-4f6c-b376-d4c5d701166a",
             "checksum": null,
             "request_id": "13893_B",
-            "sample": null,
             "samples": "[\"13893_B_1\"]",
             "patient_id": "ALLANT2"
         }
@@ -390,7 +369,6 @@
             "file_group": "1a1b29cf-3bc2-4f6c-b376-d4c5d701166a",
             "checksum": null,
             "request_id": "13893_B",
-            "sample": null,
             "samples": "[\"13893_B_1\"]",
             "patient_id": "ALLANT2"
         }
@@ -408,7 +386,6 @@
             "file_group": "1a1b29cf-3bc2-4f6c-b376-d4c5d701166a",
             "checksum": null,
             "request_id": "13893_B",
-            "sample": null,
             "samples": "[\"13893_B_1\"]",
             "patient_id": "ALLANT2"
         }
@@ -426,7 +403,6 @@
             "file_group": "1a1b29cf-3bc2-4f6c-b376-d4c5d701166a",
             "checksum": null,
             "request_id": "13893_B",
-            "sample": null,
             "samples": "[\"13893_B_1\"]",
             "patient_id": "ALLANT2"
         }

diff --git a/file_system/migrations/0017_add_IMPACT505_b37_files.py b/file_system/migrations/0017_add_IMPACT505_b37_files.py
@@ -29,8 +29,8 @@ def create_IMPACT505_b37_reference_files(apps, schema_editor):
             file_type=txt,
             size=0,
         )
-        file_metadata_1 = FileMetadata.objects.create(
-            file=file1, version=0, metadata={"assay": "IMPACT505_b37", "data_type": "FP_genotypes"}
+        FileMetadata.objects.create_or_update(
+            file=file1, metadata={"assay": "IMPACT505_b37", "data_type": "FP_genotypes"}
         )
         print("File created")
     except Exception as e:
@@ -59,8 +59,8 @@ def create_IMPACT505_b37_reference_files(apps, schema_editor):
             file_type=ilist,
             size=0,
         )
-        file_metadata_3 = FileMetadata.objects.create(
-            file=file3, version=0, metadata={"assay": "IMPACT505_b37", "data_type": "baits_list"}
+        FileMetadata.objects.create_or_update(
+            file=file3, metadata={"assay": "IMPACT505_b37", "data_type": "baits_list"}
         )
         print("File created")
     except Exception as e:
@@ -74,8 +74,8 @@ def create_IMPACT505_b37_reference_files(apps, schema_editor):
             file_type=interval_list,
             size=0,
         )
-        file_metadata_4 = FileMetadata.objects.create(
-            file=file4, version=0, metadata={"assay": "IMPACT505_b37", "data_type": "FP_intervals"}
+        FileMetadata.objects.create_or_update(
+            file=file4, metadata={"assay": "IMPACT505_b37", "data_type": "FP_intervals"}
         )
         print("File created")
     except Exception as e:

diff --git a/file_system/migrations/0042_auto_20241220_0906.py b/file_system/migrations/0042_auto_20241220_0906.py
@@ -0,0 +1,17 @@
+# Generated by Django 2.2.28 on 2024-12-20 14:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("file_system", "0041_file_available"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="file",
+            name="sample",
+        )
+    ]
diff --git a/file_system/migrations/0043_auto_20241220_0908.py b/file_system/migrations/0043_auto_20241220_0908.py
@@ -0,0 +1,75 @@
+# Generated by Django 2.2.28 on 2024-12-20 14:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("file_system", "0042_auto_20241220_0906"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="filemetadata",
+            name="latest",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AlterField(
+            model_name="filemetadata",
+            name="version",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AlterField(
+            model_name="patient",
+            name="latest",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AlterField(
+            model_name="patient",
+            name="version",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AlterField(
+            model_name="request",
+            name="latest",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AlterField(
+            model_name="request",
+            name="request_id",
+            field=models.CharField(default="REQUEST_ID", max_length=100),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name="request",
+            name="version",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AlterField(
+            model_name="sample",
+            name="latest",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AlterField(
+            model_name="sample",
+            name="version",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AlterUniqueTogether(
+            name="filemetadata",
+            unique_together={("file", "version")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="patient",
+            unique_together={("patient_id", "version")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="request",
+            unique_together={("request_id", "version")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="sample",
+            unique_together={("sample_id", "version")},
+        ),
+    ]
diff --git a/file_system/migrations/0044_auto_20241224_1254.py b/file_system/migrations/0044_auto_20241224_1254.py
@@ -0,0 +1,17 @@
+# Generated by Django 2.2.28 on 2024-12-24 17:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("file_system", "0043_auto_20241220_0908"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="filemetadata",
+            index=models.Index(fields=["latest"], name="latest_idx"),
+        ),
+    ]