Tarfiles

.github/workflows/tests.yml

@@ -44,5 +44,9 @@ jobs:
         python -m pip install pytest
         python -m pip install -e .
     - name: Run tests
-      run: pytest
+      run: |
+        cd tests
+        python bootstrap.py
+        cd ..
+        pytest
 

tests/bootstrap.py

@@ -0,0 +1,154 @@
+#bash generate_all.sh
+from create_standard_test_output import create_standard_test_output
+from gen_mimic import generate_mimic_test_data
+from gen_chexpert import gen_chexpert
+import torchxrayvision as xrv
+from generate_test_data import generate_test_data
+import pandas as pd
+
+n = 10
+
+def create_csv_files(n):
+    print("nih")
+    create_standard_test_output(
+        xrv.datasets.NIH_Dataset(imgpath="."),
+        "nih.csv",
+        n=n
+    )
+
+    print("pc")
+    create_standard_test_output(
+        xrv.datasets.PC_Dataset(imgpath="."),
+        "pc.csv",
+        n=n
+    )
+
+    print("openi")
+    openi = xrv.datasets.Openi_Dataset(imgpath=".")
+    create_standard_test_output(
+        openi,
+        "openi.csv",
+        columns=list(openi.dicom_metadata.columns) + ["imageid"],
+        n=n
+    )
+
+    #RSNA train
+    print("rsna (just train data)")
+    rsna = xrv.datasets.RSNA_Pneumonia_Dataset(imgpath=".")
+    create_standard_test_output(
+        xrv.datasets.RSNA_Pneumonia_Dataset(imgpath="."),
+        "rsna_train.csv",
+        columns=list(rsna.raw_csv.columns) + ["patientId"],
+        n=n
+    )
+
+    #Chexpert
+    gen_chexpert(n, "test_chexpert_data.csv")
+
+    #No data is generated for the COVID19, NLM_TB (Shenzen) or
+    #NLM_TB (Montgomery) datasets.
+
+def create_images():
+
+    #python3 generate_test_data.py pc.csv ImageID 2 2 PC_test_data
+    generate_test_data(
+        pd.read_csv("pc.csv"),
+        "ImageID",
+        (2, 2),
+        "PC_test_data",
+        "",
+        "."
+    )
+
+    #python3 generate_test_data.py nih.csv "Image Index" 2 2 NIH_test_data
+    generate_test_data(
+         pd.read_csv("nih.csv"),
+        "Image Index",
+        (2, 2),
+        "NIH_test_data",
+        "",
+        "."
+    )
+
+    #python3 generate_test_data.py openi.csv imageid 2 2 Openi_test_data .png
+    generate_test_data(
+        pd.read_csv("openi.csv"),
+        "imageid",
+        (2, 2),
+        "Openi_test_data",
+        ".png",
+        "."
+    )
+
+    #python3 generate_test_data.py shenzen.csv fname 2 2 Shenzen_test_data --subfolder CXR_png
+    generate_test_data(
+        pd.read_csv("shenzen.csv"),
+        "fname",
+        (2, 2),
+        "Shenzen_test_data",
+        "",
+        "CXR_png"
+    )
+
+    #python3 generate_test_data.py montgomery.csv fname 2 2 Montgomery_test_data --subfolder CXR_png
+    generate_test_data(
+        pd.read_csv("montgomery.csv"),
+        "fname",
+        (2, 2),
+        "Montgomery_test_data",
+        "",
+        "CXR_png"
+    )
+
+    #python3 generate_test_data.py rsna_train.csv patientId 2 2 RSNA_test_data_jpg .jpg --subfolder stage_2_train_images
+    generate_test_data(
+        pd.read_csv("rsna_train.csv"),
+        "patientId",
+        (2, 2),
+        "RSNA_test_data_jpg",
+        ".jpg",
+        "stage_2_train_images"
+    )
+
+    #python3 generate_test_data.py rsna_train.csv patientId 2 2 RSNA_test_data_dcm .dcm --subfolder stage_2_train_images
+    generate_test_data(
+        pd.read_csv("rsna_train.csv"),
+        "patientId",
+        (2, 2),
+        "RSNA_test_data_dcm",
+        ".dcm",
+        "stage_2_train_images"
+    )
+
+    #python3 generate_test_data.py test_chexpert_data.csv Path 2 2 CheXpert_test_data
+    generate_test_data(
+        pd.read_csv("test_chexpert_data.csv"),
+        "Path",
+        (2, 2),
+        "CheXpert_test_data",
+        "",
+        "."
+    )
+
+    #python3 generate_test_data.py test_covid_data.csv filename 2 2 COVID_test_data --subfolder images
+    generate_test_data(
+        pd.read_csv("test_covid_data.csv"),
+        "filename",
+        (2, 2),
+        "COVID_test_data",
+        "",
+        "."
+    )
+
+def bootstrap_test_cases(n):
+    create_csv_files(n)
+    create_images()
+    #MIMIC is handled differently due to its unique structure
+    generate_mimic_test_data(
+        n,
+        directory = "gen_mimic",
+        dimensions = (2, 2)
+    )
+
+if __name__ == "__main__":
+    bootstrap_test_cases(10)

tests/create_standard_test_output.py

@@ -0,0 +1,8 @@
+import torchxrayvision as xrv
+import os
+
+def create_standard_test_output(dataset, name, columns = None, n=10):
+    minimal_dataset = dataset.csv[:n]
+    if columns is not None:
+        minimal_dataset = minimal_dataset[columns]
+    minimal_dataset.to_csv(name, index=False)

tests/gen_chexpert.py

@@ -0,0 +1,68 @@
+from random_data import random_preds
+import numpy as np
+import pandas as pd
+import random
+
+n = 10
+
+def probability(fraction):
+    gran = 100
+    return np.random.randint(0, gran) < gran * fraction
+
+def random_age():
+    return random.randint(18, 100) #Assume adult
+
+def gen_random_study():
+    views = []
+    if probability(.7):
+        views.append({
+            "Path":"view1_frontal.jpg",
+            "AP/PA":np.random.choice(["AP","PA"]),
+            "Frontal/Lateral":"Frontal"
+        })
+    if probability(.3):
+        views.append({
+            "Path":"view1_lateral.jpg",
+            "AP/PA":"",
+            "Frontal/Lateral":"Lateral"
+        })
+    return views
+
+def gen_random_data():
+    views = []
+    patient_idx = 0
+    while True:
+        n_studies = random.randint(1, 4)
+        sex = random.choice(["Female","Male"])
+        age = random_age()
+        for study_idx in range(n_studies):
+            for radiograph in gen_random_study():
+                path_format = (
+                    "CheXpert-v1.0-small/train/patient{}/study{}/{}"
+                )
+                path = path_format.format(
+                    str(patient_idx).zfill(5),
+                    str(study_idx),
+                    radiograph["Path"]
+                )
+                radiograph.update({
+                    "Path":path,
+                    "Sex":sex,
+                    "Age":age,
+                })
+                radiograph.update(random_preds())
+                yield radiograph
+        patient_idx += 1
+
+#Path,Sex,Age,Frontal/Lateral,AP/PA,No Finding,Enlarged Cardiomediastinum,Cardiomegaly,Lung Opacity,Lung Lesion,Edema,Consolidation,Pneumonia,Atelectasis,Pneumothorax,Pleural Effusion,Pleural Other,Fracture,Support Devices
+#CheXpert-v1.0-small/train/patient00001/study1/view1_frontal.jpg,Female,68,Frontal,AP,1.0,,,,,,,,,0.0,,,,1.0
+
+def gen_random_rows(nrows):
+    rows = []
+    random_data_source = gen_random_data()
+    for i in range(nrows):
+         rows.append(next(random_data_source))
+    return pd.DataFrame(rows)
+
+def gen_chexpert(n, filename):
+    gen_random_rows(n).to_csv(filename)

tests/gen_mimic.py

@@ -0,0 +1,139 @@
+import numpy as np
+import pdb
+import tarfile
+import pandas as pd
+from PIL import Image
+import random
+import argparse
+from pathlib import Path
+import os
+
+from random_data import write_random_images, gen_int, gen_hex, random_pred, random_preds
+
+def show(x):
+    print(x)
+    return x
+
+mimic_metadata_filename = "mimic-cxr-2.0.0-metadata.csv"
+mimic_csvdata_filename = "mimic-cxr-2.0.0-negbio.csv"
+
+def generate_random_metadata(n, dimensions):
+    columns = "dicom_id,subject_id,study_id,PerformedProcedureStepDescription,ViewPosition,Rows,Columns,StudyDate,StudyTime,ProcedureCodeSequence_CodeMeaning,ViewCodeSequence_CodeMeaning,PatientOrientationCodeSequence_CodeMeaning".split(",")
+    performed_procedure_step_descriptions = {
+        "CHEST (PA AND LAT)":{
+             "n_views":2,
+             "view_position":["LATERAL", "PA"],
+             "procedure_code_meaning":"CHEST (PA AND LAT)",
+             "view_code_meaning":["lateral", "postero-anterior"],
+             "orientation_code_meaning":["Erect","Recumbent"]
+        }
+    }
+
+    def generate_random_row(dimensions):
+        performed_procedure_step_description = random.choice(
+            list(performed_procedure_step_descriptions)
+        )
+        procedure = performed_procedure_step_descriptions[performed_procedure_step_description]
+        n_views = procedure["n_views"]
+        view_index = random.randint(0, n_views - 1)
+        view_position = procedure["view_position"][view_index]
+        procedure_code_meaning = procedure["procedure_code_meaning"]
+        view_code_meaning = procedure["view_code_meaning"][view_index]
+        #Currently unsure how/if view codes are mapped to orientations
+        orientation_code_meaning = random.choice(procedure["orientation_code_meaning"])
+        subject_id = gen_int(8)
+        study_id = gen_int(8)
+        meta_row = {
+            "dicom_id":"-".join([gen_hex(8) for i in range(4)]),
+            "subject_id":subject_id,
+            "study_id":study_id,
+            "PerformedProcedureStepDescription":performed_procedure_step_description,
+            "ViewPosition":view_position,
+            "Rows":dimensions[0],
+            "Columns":dimensions[1],
+            "StudyDate":0,
+            "StudyTime":0,
+            "ProcedureCodeSequence_CodeMeaning":procedure_code_meaning,
+            "ViewCodeSequence_CodeMeaning":view_code_meaning,
+            "PatientOrientationCodeSequence_CodeMeaning":orientation_code_meaning
+        }
+
+        csv_row = {
+            "subject_id":subject_id,
+            "study_id":study_id,
+        }
+        csv_row.update(random_preds())
+        return meta_row, csv_row
+
+    meta_rows, csv_rows = show(list(zip(*show([generate_random_row(dimensions) for i in range(n)]))))
+
+    return pd.DataFrame(meta_rows), pd.DataFrame(csv_rows)
+
+
+
+def generate_test_images(random_metadata, extracted, tarname, zipname, folder_of_zip_name, folder_of_tar_gz_name, dimensions):
+    paths = []
+    for _, row in random_metadata.iterrows():
+        subjectid = row["subject_id"]
+        studyid = row["study_id"]
+        dicom_id = row["dicom_id"]
+        img_fname = os.path.join("p" + subjectid[:2], "p" + subjectid, "s" + studyid, dicom_id + ".jpg")
+        paths.append(Path("files")/img_fname)
+    write_random_images(paths, extracted, tarname, zipname, folder_of_zip_name, folder_of_tar_gz_name, dimensions)
+
+def generate_mimic_test_data(n, directory, dimensions=(224, 224), tarname=None, zipname=None, folder_of_zip_name=None, folder_of_tar_gz_name = None, extracted=None):
+    directory = Path(directory)
+    if tarname is None:
+        tarname = directory/"images-224.tar"
+    if zipname is None:
+        zipname = directory/"images-224.zip"
+    if extracted is None:
+        extracted = directory/"images-224"
+    if folder_of_zip_name is None:
+        folder_of_zip_name = directory/"images-224-zips"
+    if folder_of_tar_gz_name is None:
+        folder_of_tar_gz_name = directory/"images-224-tgzs"
+    random_metadata, random_csvdata = generate_random_metadata(
+        n,
+        dimensions
+    )
+    generate_test_images(random_metadata, extracted, tarname, zipname, folder_of_zip_name, folder_of_tar_gz_name, dimensions)
+    random_metadata.to_csv(
+        directory/mimic_metadata_filename,
+        index=False
+    )
+    random_metadata.to_csv(
+        directory/(mimic_metadata_filename+".gz"),
+        compression="gzip",
+        index=False
+    )
+    random_csvdata.to_csv(
+        directory/mimic_csvdata_filename,
+        index=False
+    )
+    random_csvdata.to_csv(
+        directory/(mimic_csvdata_filename+".gz"),
+        compression="gzip",
+        index=False
+    )
+
+#./images-224/files/p17/p17387118/s56770356/b983f94c-b77ad35d-8a4aa372-2faf6503-5ec94835.jpg
+
+#if __name__ == "__main__":
+#    parser = argparse.ArgumentParser()
+#    parser.add_argument("n")
+#    parser.add_argument("directory")
+#    parser.add_argument("x")
+#    parser.add_argument("y")
+#    parser.add_argument("tarfile", default=None, nargs="?")
+#    parser.add_argument("extracted", default=None, nargs="?")
+#    args = parser.parse_args()
+#    generate_test_data(
+#        n=int(args.n),
+#        directory = args.directory,
+#        dimensions = (int(args.x), int(args.y)),
+#        tarname = args.tarfile,
+#        extracted = args.extracted
+#    )
+
+#python3 gen_mimic.py 10 gen_mimic 224 224

tests/gen_mimic/mimic-cxr-2.0.0-metadata.csv

@@ -0,0 +1,11 @@
+dicom_id,subject_id,study_id,PerformedProcedureStepDescription,ViewPosition,Rows,Columns,StudyDate,StudyTime,ProcedureCodeSequence_CodeMeaning,ViewCodeSequence_CodeMeaning,PatientOrientationCodeSequence_CodeMeaning
+dc7a8b5e-4ad1afd8-8f994433-643c39f8,346b3f01,14b48a44,CHEST (PA AND LAT),LATERAL,224,224,0,0,CHEST (PA AND LAT),lateral,Recumbent
+9622b5f2-3a3e307e-f5ca9164-375a6f70,11a8f57f,bf2ecf26,CHEST (PA AND LAT),PA,224,224,0,0,CHEST (PA AND LAT),postero-anterior,Erect
+8cf7e7b2-46bdc39f-31564b20-71ee6b6d,4eedf9fe,f8521632,CHEST (PA AND LAT),PA,224,224,0,0,CHEST (PA AND LAT),postero-anterior,Erect
+6a301286-456d8528-c59bd118-195ce98c,bc0ee611,89cfed09,CHEST (PA AND LAT),LATERAL,224,224,0,0,CHEST (PA AND LAT),lateral,Erect
+67dc7d13-32da76c2-ff22a175-5d4c3ced,54e08d2a,1c41417d,CHEST (PA AND LAT),PA,224,224,0,0,CHEST (PA AND LAT),postero-anterior,Erect
+b69fd4aa-55745241-f15af2bb-b979004a,3b3b7d36,f076c36f,CHEST (PA AND LAT),PA,224,224,0,0,CHEST (PA AND LAT),postero-anterior,Erect
+7e393cbe-6eac27c9-469708b4-dc5f22ef,310345ea,274bcf57,CHEST (PA AND LAT),LATERAL,224,224,0,0,CHEST (PA AND LAT),lateral,Erect
+a0082cb3-93689ac6-4fbbad4e-ddd68866,bfb2d1b6,2c5d98ee,CHEST (PA AND LAT),LATERAL,224,224,0,0,CHEST (PA AND LAT),lateral,Recumbent
+4197fbbc-1ed2f09e-8a82308b-971009a3,3d404d22,1e752e16,CHEST (PA AND LAT),LATERAL,224,224,0,0,CHEST (PA AND LAT),lateral,Erect
+baf27490-fe6678a0-fe0b7379-665d78ba,6145fd64,7968e508,CHEST (PA AND LAT),PA,224,224,0,0,CHEST (PA AND LAT),postero-anterior,Recumbent