Activity index report

experiments/activity_index/bucket_path.txt

@@ -0,0 +1,15 @@
+#s3://lab-nematode/raw/220503_TPS_ACR085/crawling_zoom113_10ms_100img/Default/
+#s3://lab-nematode/raw/220503_TPS_ACR085/trashing_zoom113_10ms_100img/Default/
+#s3://lab-nematode/raw/220503_TPS_ACR125/crawling_zoom113_10ms_100img/Default/
+#s3://lab-nematode/raw/220503_TPS_ACR125/thrashing_zoom113_10ms_100img/Default/
+#s3://lab-nematode/raw/220503_TPS_N2/zoom113_20ms_2/Default/
+#s3://lab-nematode/raw/220503_TPS_N2/zoom197_20ms_1/Default/
+#s3://lab-nematode/raw/220503_TPS_N2/zoom197_20ms_2/Default/
+s3://lab-nematode/220427_BF_RC7_30ms_11.3x/crawl/ACR125_50ms_1/Default/
+s3://lab-nematode/220427_BF_RC7_30ms_11.3x/crawl/ACR125_50ms_2_1/Default/
+s3://lab-nematode/220427_BF_RC7_30ms_11.3x/crawl/ACR125_burst_1/Default/
+s3://lab-nematode/220427_BF_RC7_30ms_11.3x/crawl/ACR125_burst_3_1/Default/
+s3://lab-nematode/220427_BF_RC7_30ms_11.3x/swim/acr125_burst_1/Default/
+s3://lab-nematode/220427_BF_RC7_30ms_11.3x/swim/acr125_burst_2/Default/
+s3://lab-nematode/220427_BF_RC7_30ms_11.3x/swim/acr125_burst_3/Default/
+s3://lab-nematode/220427_BF_RC7_30ms_11.3x/swim/acr125_burst_4_2/Default/

experiments/activity_index/make_videos.py

@@ -0,0 +1,43 @@
+import enum
+import os
+
+
+def extract_bucket_names(abs_file_path: str) -> list:
+ """Extract aws bucket filepath from a txt file into a list.
+
+ Args:
+ abs_file_path (str): text file with the list of bucket path
+
+ Returns:
+ list: list of bucket paths
+ """
+ bucket_list = []
+ with open(abs_file_path, "r") as f:
+ while True:
+ bucket_name = f.readline()
+ if not bucket_name:
+ break
+ else:
+ if bucket_name.find("#") == -1:
+ bucket_list.append(bucket_name.replace("\n", " "))
+ return bucket_list
+
+
+if __name__ == "__main__":
+ working_dir = os.path.dirname(__file__)
+ bucket_names_file = os.path.join(working_dir, "bucket_path.txt")
+ bucket_list = extract_bucket_names(bucket_names_file)
+ for i, bucket_path in enumerate(bucket_list, start=1):
+ # Set up new dir and cd to it
+ dir_name = f"video_{i}"
+ new_dir = os.path.join(working_dir, dir_name)
+ os.mkdir(new_dir)
+ os.chdir(new_dir)
+ # Download all tiff images from an aws bucket
+ aws_cmd = f"aws s3 cp {bucket_path} . --recursive"
+ os.system(aws_cmd)
+ # Create a video from previously downloaded images
+ ffmpeg_cmd = "ffmpeg -loglevel 0 -framerate 33 -i "
+ ffmpeg_cmd += f"img_channel000_position000_time%09d_z000.tif {dir_name}.mp4"
+ os.system(ffmpeg_cmd)
+ os.chdir("..")

experiments/activity_index/plot_activity_index.py

@@ -0,0 +1,67 @@
+import os
+import re
+import sys
+
+import matplotlib.pyplot as plt
+import pandas as pd
+import seaborn as sns
+
+
+def extract_data_from_directory(dir: str) -> pd.DataFrame:
+ """Extract data from each particles.csv file found in 'results' dir
+ and stack it in a single dataframe
+ The data is linked to each video results with the column 'dataset_name'
+ Args:
+ dir (str): dir containing the data for each video analyzed
+
+ Returns:
+ pd.DataFrame: all the data extracted and stacked from particles.csv files
+ """
+ pwd = os.path.dirname(__file__)
+ result_dir = os.path.join(pwd, dir)
+ list_subdir = os.listdir(result_dir)
+ results_subdir = [d for d in list_subdir if os.path.isdir(os.path.join(result_dir, d))]
+ files = [
+ f
+ for d in results_subdir
+ for f in os.listdir(os.path.join(pwd, dir, d))
+ if f.endswith(".csv")
+ ]
+ data = pd.DataFrame()
+ for subdir, file in zip(results_subdir, files):
+ file_path = os.path.join(pwd, dir, subdir, file)
+ try:
+ id_num = re.findall("[0-9]+", subdir)[0]
+ except Exception:
+ print("ERROR: a sub-directory in 'results' has no index")
+ sys.exit(-1)
+ name = f"results_{id_num}"
+ if os.path.isfile(file_path):
+ data_subset = pd.read_csv(file_path, index_col=0)
+ data_subset = data_subset.assign(dataset_name=name)
+ if data.empty:
+ data = data_subset
+ else:
+ data = pd.concat([data, data_subset], ignore_index=True)
+ return data
+
+
+def cleaning_data(data: pd.DataFrame) -> pd.DataFrame:
+ data = data.dropna(axis=1, how="all")
+ return data
+
+
+if __name__ == "__main__":
+ raw_data = extract_data_from_directory("results")
+ cleaned_data = cleaning_data(raw_data)
+ data = cleaned_data[["Activity Index", "BPM", "Area", "Eccentricity", "dataset_name"]].copy()
+ # Remove all rows with an activity index that is null
+ data = data[data["Activity Index"] != 0]
+ try:
+ sns.displot(
+ data=data, x=data["Activity Index"], hue="dataset_name", multiple="stack", bins=20
+ )
+ plt.show()
+ except Exception:
+ print("ERROR: cannot plot your data")
+ sys.exit(-1)

experiments/activity_index/report_overview/report_overview.md

@@ -0,0 +1,27 @@
+# Description
+The goal is to plot the distribution of the activity index yielded by wfntp from several video samples.\
+
+The images analyzed are pulled from aws buckets listed in `bucket_path.txt`.\
+`make_videos.py` is a script used to pull images from these buckets and convert them in a mp4 video.\
+This script uses the uncommented path in `bucket_path.txt`\
+The settings used for each video inputed in wfntp and the outputs at worm level are available in `results/` for each video.\
+`plot_activity_index.py` is a script used to plot the distribution of the results of the activity index from the `particles.csv` files available in `results/`
+
+# Methodology
+
+## Formula and implementation in wfntp
+The formula of the activity index used is as follow:
+$$
+activity\_index = \frac{area}{\frac{2 \times 60}{BPM}} = \frac{area \times BPM}{120}
+$$
+In WF NTP, we use 2 variables to calculate the activity index:
+* `area` corresponding to the total area covered by the worm (particle) in mm.
+* `BPM` corresponding to the number of bends per minutes.
+## Data and plotting
+The data from each `particles.csv`in each subdirectory of `results/` are stacked to obtain mutliple measure of the activity index from several video samples.
+Row with null value in the activity index field are removed from the dataset to better vizualize the distribution.
+
+# Results
+
+Plot of the distribution of the activity indeces extracted from each video (dataset_name)\
+![image](distribution_activity_index.jpeg)

experiments/activity_index/results/results_video_1.Directory/particles.csv

@@ -0,0 +1,31 @@
+,BPM,Activity Index,Bends per 15.00 s,Speed,Max speed,Dist per bend,Area,Appears in frames,Moving (non-paralyzed),Region,Round ratio,Eccentricity
+1,0.0,0.0,0.0,0.044415204809294846,0.055064567564075476,,0.0848,76.0,False,['all'],0.9125,0.8341828999152534
+3,0.0,0.0,0.0,0.019776098816120396,0.025718317713733632,,0.06720000000000001,53.0,False,['all'],0.0,0.9501249843425181
+4,0.0,0.0,0.0,0.012963557004910317,0.032733263288501034,,0.1744,58.0,False,['all'],0.3620689655172413,0.9319898872335945
+6,35.64356435643564,0.022336633663366336,8.91089108910891,0.09564937516978797,0.11165780981190442,0.10664745308856793,0.0752,102.0,True,['all'],0.6764705882352942,0.9085250781753056
+7,0.0,0.0,0.0,0.020775189275124423,0.02647626520473094,,0.052000000000000005,74.0,False,['all'],1.0,0.8494708890412466
+10,12.244897959183671,0.006775510204081631,3.0612244897959178,0.045014094115580605,0.19251276451170923,0.049450672934574634,0.0664,88.0,True,['all'],0.2525252525252525,0.9219295252397287
+11,0.0,0.0,0.0,0.004319109472409712,0.03208341493345129,,0.09920000000000001,75.0,False,['all'],0.8,0.9241313790504472
+20,63.63636363636364,0.06193939393939395,15.90909090909091,0.018617511435408424,0.03699986944098085,0.05635832667248688,0.1168,102.0,True,['all'],0.13533834586466165,0.9578986752850126
+27,0.0,0.0,0.0,0.0,0.007942705274545407,,0.0656,71.0,False,['all'],0.0,0.9687308989755009
+28,0.0,0.0,0.0,0.0,0.0008870564871774968,,0.0432,66.0,False,['all'],1.0,0.8734444257596304
+30,0.0,0.0,0.0,0.015142524122182331,0.026009433625358583,,0.0784,57.0,False,['all'],0.0,0.9725254746972853
+31,0.0,0.0,0.0,0.0019919302317849687,0.01251975493729961,,0.0512,69.0,False,['all'],0.01449275362318836,0.9471295062598326
+39,0.0,0.0,0.0,0.0027194281753848432,0.012526677764387385,,0.0944,61.0,False,['all'],0.0,0.9867183146351306
+40,0.0,0.0,0.0,0.0,0.006322023040113795,,0.048,58.0,False,['all'],0.0,0.9850162385108103
+42,0.0,0.0,0.0,0.0,0.0,,0.08,70.0,False,['all'],0.0,0.9685755558090517
+48,0.0,0.0,0.0,0.0,0.022630859066074704,,0.12000000000000001,70.0,False,['all'],0.0,0.9671527695272293
+57,0.0,0.0,0.0,0.0023589706258262265,0.010862804295598756,,0.1264,70.0,False,['all'],0.0,0.9756534614613125
+66,0.0,0.0,0.0,0.008910331222741317,0.020690393035708105,,0.0512,60.0,False,['all'],0.0,0.969888335133293
+75,0.0,0.0,0.0,0.0,0.0012714398460388565,,0.057600000000000005,75.0,False,['all'],0.013333333333333308,0.9472825148626098
+78,0.0,0.0,0.0,0.002344146486608877,0.002344146486608877,,0.046400000000000004,68.0,False,['all'],1.0,0.8277885659492056
+81,0.0,0.0,0.0,0.006806240163897196,0.008647327554910714,,0.049600000000000005,63.0,False,['all'],0.25,0.9289756596621586
+120,0.0,0.0,0.0,0.004369021721839705,0.021870769885256484,,0.0816,68.0,False,['all'],0.0,0.9855623249197767
+132,19.047619047619047,0.025142857142857144,4.761904761904762,0.0664334115845835,0.07582051524796299,0.0,0.1584,62.0,True,['all'],0.0,0.9497444195570522
+133,0.0,0.0,0.0,0.0011071255188511706,0.019346581322281516,,0.1328,64.0,False,['all'],0.0,0.9539213111419036
+147,0.0,0.0,0.0,0.0,0.009895039917240526,,0.054400000000000004,62.0,False,['all'],1.0,0.8519064269544613
+151,0.0,0.0,0.0,0.005772300254582748,0.013505166035610323,,0.064,60.0,False,['all'],1.0,0.7620919979423352
+152,0.0,0.0,0.0,0.004041911719537441,0.010540528866856464,,0.1456,60.0,False,['all'],0.0,0.9594023564427004
+159,0.0,0.0,0.0,0.025738571719786663,0.037793750705519547,,0.1056,59.0,False,['all'],0.10169491525423724,0.938150172913144
+160,0.0,0.0,0.0,0.0023609931454336586,0.008384168593091483,,0.0448,59.0,False,['all'],0.0,0.9469250422611907
+161,0.0,0.0,0.0,0.005525028200227908,0.00853818972271706,,0.0432,54.0,False,['all'],1.0,0.4119377035025761

experiments/activity_index/results/results_video_1.Directory/settings.json

@@ -0,0 +1,46 @@
+{
+ "video_filename": "/Users/constantalle/Documents/42project/pythonProjects/elegans/Elegant-Elegans/experiments/activity_index/video_1/video_1.mp4",
+ "start_frame": 0,
+ "limit_images_to": 300,
+ "fps": 20.0,
+ "px_to_mm": 0.04,
+ "darkfield": false,
+ "keep_paralyzed_method": false,
+ "std_px": 64,
+ "threshold": 9,
+ "opening": 1,
+ "closing": 3,
+ "prune_size": 0,
+ "skeletonize": false,
+ "do_full_prune": false,
+ "min_size": 25,
+ "max_size": 120,
+ "minimum_ecc": 0.93,
+ "use_average": true,
+ "lower": 0,
+ "upper": 100,
+ "Bends_max": 20.0,
+ "Speed_max": 0.035,
+ "extra_filter": false,
+ "cutoff_filter": true,
+ "max_dist_move": 10,
+ "min_track_length": 50,
+ "memory": 5,
+ "bend_threshold": 2.1,
+ "minimum_bends": 0.0,
+ "frames_to_estimate_velocity": 49,
+ "maximum_bpm": 0.5,
+ "maximum_velocity": 0.1,
+ "regions": {},
+ "save_as": "/Users/constantalle/Documents/42project/pythonProjects/elegans/Elegant-Elegans/experiments/activity_index/results/results_video_1.Directory",
+ "output_overlayed_images": 10,
+ "font_size": 8,
+ "scale_bar_size": 1.0,
+ "scale_bar_thickness": 7,
+ "max_plot_pixels": 2250000,
+ "Z_skip_images": 1,
+ "use_images": 100,
+ "use_around": 5,
+ "stop_after_example_output": false,
+ "stdout prefix": "[8]"
+}

experiments/activity_index/results/results_video_2.Directory/particles.csv

@@ -0,0 +1,9 @@
+,BPM,Activity Index,Bends per 15.00 s,Speed,Max speed,Dist per bend,Area,Appears in frames,Moving (non-paralyzed),Region,Round ratio,Eccentricity
+0,0.0,0.0,0.0,0.0,0.0,,0.0608,75.0,False,['all'],0.0,0.9305023059480323
+3,0.0,0.0,0.0,0.018060729932317266,0.04909631576693778,,0.10880000000000001,69.0,False,['all'],0.9333333333333333,0.9110423398875791
+4,0.0,0.0,0.0,0.002481024653519466,0.005076606832340354,,0.054400000000000004,75.0,False,['all'],1.0,0.762381429790187
+5,0.0,0.0,0.0,0.011735336914827654,0.04376602317413075,,0.0944,75.0,False,['all'],0.0,0.9696817030238467
+26,0.0,0.0,0.0,0.019597128953417406,0.024083251066859022,,0.18960000000000002,70.0,False,['all'],0.09999999999999998,0.934414587408587
+27,17.391304347826086,0.008579710144927536,4.3478260869565215,0.008219133589455259,0.0704802992622866,0.0,0.0592,67.0,True,['all'],0.0,0.9783770419321811
+28,48.0,0.06656000000000001,12.0,0.07694274124265732,0.08198108794938103,0.053910918462867434,0.16640000000000002,51.0,True,['all'],0.17647058823529416,0.9249621300268956
+34,0.0,0.0,0.0,0.005009727009098464,0.007998602699368048,,0.1728,66.0,False,['all'],0.0,0.9733459992362739

experiments/activity_index/results/results_video_2.Directory/settings.json

@@ -0,0 +1,46 @@
+{
+ "video_filename": "/Users/constantalle/Documents/42project/pythonProjects/elegans/Elegant-Elegans/experiments/activity_index/video_2/video_2.mp4",
+ "start_frame": 0,
+ "limit_images_to": 300,
+ "fps": 20.0,
+ "px_to_mm": 0.04,
+ "darkfield": false,
+ "keep_paralyzed_method": false,
+ "std_px": 64,
+ "threshold": 9,
+ "opening": 1,
+ "closing": 3,
+ "prune_size": 0,
+ "skeletonize": false,
+ "do_full_prune": false,
+ "min_size": 25,
+ "max_size": 120,
+ "minimum_ecc": 0.93,
+ "use_average": true,
+ "lower": 0,
+ "upper": 100,
+ "Bends_max": 20.0,
+ "Speed_max": 0.035,
+ "extra_filter": false,
+ "cutoff_filter": true,
+ "max_dist_move": 10,
+ "min_track_length": 50,
+ "memory": 5,
+ "bend_threshold": 2.1,
+ "minimum_bends": 0.0,
+ "frames_to_estimate_velocity": 49,
+ "maximum_bpm": 0.5,
+ "maximum_velocity": 0.1,
+ "regions": {},
+ "save_as": "/Users/constantalle/Documents/42project/pythonProjects/elegans/Elegant-Elegans/experiments/activity_index/results/results_video_2.Directory",
+ "output_overlayed_images": 10,
+ "font_size": 8,
+ "scale_bar_size": 1.0,
+ "scale_bar_thickness": 7,
+ "max_plot_pixels": 2250000,
+ "Z_skip_images": 1,
+ "use_images": 100,
+ "use_around": 5,
+ "stop_after_example_output": false,
+ "stdout prefix": "[6]"
+}

experiments/activity_index/results/results_video_3.Directory/particles.csv

@@ -0,0 +1,4 @@
+,BPM,Activity Index,Bends per 3.03 s,Speed,Max speed,Dist per bend,Area,Appears in frames,Moving (non-paralyzed),Region,Round ratio,Eccentricity
+1,0.0,0.0,0.0,0.035551983136684145,0.035551983136684145,,0.12480000000000001,50.0,False,['all'],0.6799999999999999,0.9017556006399596
+7,242.44897959183675,0.1616326530612245,12.244897959183673,0.005807510782206227,0.005807510782206227,0.08789839089725465,0.08,50.0,True,['all'],0.42000000000000004,0.9332199581661395
+37,32.459016393442624,0.025967213114754098,1.639344262295082,0.6150490303597141,0.6756649796485247,0.0,0.096,53.0,True,['all'],0.0,0.9754275318974297

experiments/activity_index/results/results_video_3.Directory/settings.json

@@ -0,0 +1,46 @@
+{
+ "video_filename": "/Users/constantalle/Documents/42project/pythonProjects/elegans/Elegant-Elegans/experiments/activity_index/video_3/video_3.mp4",
+ "start_frame": 0,
+ "limit_images_to": 100,
+ "fps": 33.0,
+ "px_to_mm": 0.04,
+ "darkfield": false,
+ "keep_paralyzed_method": false,
+ "std_px": 64,
+ "threshold": 9,
+ "opening": 1,
+ "closing": 3,
+ "prune_size": 0,
+ "skeletonize": false,
+ "do_full_prune": false,
+ "min_size": 25,
+ "max_size": 120,
+ "minimum_ecc": 0.93,
+ "use_average": true,
+ "lower": 0,
+ "upper": 100,
+ "Bends_max": 20.0,
+ "Speed_max": 0.035,
+ "extra_filter": false,
+ "cutoff_filter": true,
+ "max_dist_move": 10,
+ "min_track_length": 50,
+ "memory": 5,
+ "bend_threshold": 2.1,
+ "minimum_bends": 0.0,
+ "frames_to_estimate_velocity": 49,
+ "maximum_bpm": 0.5,
+ "maximum_velocity": 0.1,
+ "regions": {},
+ "save_as": "/Users/constantalle/Documents/42project/pythonProjects/elegans/Elegant-Elegans/experiments/activity_index/results/results_video_3.Directory",
+ "output_overlayed_images": 10,
+ "font_size": 8,
+ "scale_bar_size": 1.0,
+ "scale_bar_thickness": 7,
+ "max_plot_pixels": 2250000,
+ "Z_skip_images": 1,
+ "use_images": 100,
+ "use_around": 5,
+ "stop_after_example_output": false,
+ "stdout prefix": "[4]"
+}

experiments/activity_index/results/results_video_4.Directory/particles.csv

@@ -0,0 +1,5 @@
+,BPM,Activity Index,Bends per 3.03 s,Speed,Max speed,Dist per bend,Area,Appears in frames,Moving (non-paralyzed),Region,Round ratio,Eccentricity
+1,23.294117647058822,0.02376,1.1764705882352942,0.14941939747785246,0.22354889322252788,0.12573206121075722,0.12240000000000001,76.0,True,['all'],0.2441860465116279,0.9391012066883105
+9,0.0,0.0,0.0,0.04416084766721351,0.06520306136426007,,0.0688,99.0,False,['all'],1.0,0.7673145406319852
+10,20.41237113402062,0.017962886597938148,1.0309278350515463,0.02116979040198643,0.05355389770618404,0.043202600144632224,0.1056,95.0,True,['all'],0.4387755102040817,0.9282151136917394
+38,0.0,0.0,0.0,0.0667102324136727,0.0667102324136727,,0.052000000000000005,50.0,False,['all'],0.9,0.7638496942125877

experiments/activity_index/results/results_video_4.Directory/settings.json

@@ -0,0 +1,46 @@
+{
+ "video_filename": "/Users/constantalle/Documents/42project/pythonProjects/elegans/Elegant-Elegans/experiments/activity_index/video_4/video_4.mp4",
+ "start_frame": 0,
+ "limit_images_to": 100,
+ "fps": 33.0,
+ "px_to_mm": 0.04,
+ "darkfield": false,
+ "keep_paralyzed_method": false,
+ "std_px": 64,
+ "threshold": 9,
+ "opening": 1,
+ "closing": 3,
+ "prune_size": 0,
+ "skeletonize": false,
+ "do_full_prune": false,
+ "min_size": 25,
+ "max_size": 120,
+ "minimum_ecc": 0.93,
+ "use_average": true,
+ "lower": 0,
+ "upper": 100,
+ "Bends_max": 20.0,
+ "Speed_max": 0.035,
+ "extra_filter": false,
+ "cutoff_filter": false,
+ "max_dist_move": 10,
+ "min_track_length": 50,
+ "memory": 5,
+ "bend_threshold": 2.1,
+ "minimum_bends": 0.0,
+ "frames_to_estimate_velocity": 49,
+ "maximum_bpm": 0.5,
+ "maximum_velocity": 0.1,
+ "regions": {},
+ "save_as": "/Users/constantalle/Documents/42project/pythonProjects/elegans/Elegant-Elegans/experiments/activity_index/results/results_video_4.Directory",
+ "output_overlayed_images": 0,
+ "font_size": 8,
+ "scale_bar_size": 1.0,
+ "scale_bar_thickness": 7,
+ "max_plot_pixels": 2250000,
+ "Z_skip_images": 1,
+ "use_images": 100,
+ "use_around": 5,
+ "stop_after_example_output": false,
+ "stdout prefix": "[9]"
+}