Merge pull request #41 from gwaygenomics/final-pipeline

Final pipeline initialization

.gitattributes

@@ -5,3 +5,7 @@ data/pancan_scaled_zeroone_rnaseq.tsv filter=lfs diff=lfs merge=lfs -text
 models/decoder_onehidden_vae.hdf5 filter=lfs diff=lfs merge=lfs -text
 models/encoder_onehidden_vae.hdf5 filter=lfs diff=lfs merge=lfs -text
 results/parameter_sweep_full_results.tsv filter=lfs diff=lfs merge=lfs -text
+data/vae_encoded_with_clinical.tsv filter=lfs diff=lfs merge=lfs -text
+results/rnaseq_tsne_out.tsv filter=lfs diff=lfs merge=lfs -text
+results/vae_subtracted_tsne_out.tsv filter=lfs diff=lfs merge=lfs -text
+results/vae_tsne_out.tsv filter=lfs diff=lfs merge=lfs -text

get_distance.ipynb

@@ -4,9 +4,9 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Determining the distance of COAD and GBM tumors in different spaces\n",
+    "# Determining the distance of BLCA and LUAD tumors in different spaces\n",
     "\n",
-    "The script will output distance matrices for pairwise comparisons of all colon cancer (COAD) and glioblastoma (GBM) tumors. The script will output distances for raw gene expression features and gene expression features encoded by a variational autoencoder. Additionally, distance matrices for the subtraction of cancer-type specific means is also output."
+    "The script will output distance matrices for pairwise comparisons of all bladder cancer (BLCA) and lung adenocarcinoma (LUAD) tumors. The script will output distances for raw gene expression features and gene expression features encoded by a variational autoencoder. Additionally, distance matrices for the subtraction of cancer-type specific means is also output."
    ]
   },
   {

param_sweep/paramsweep_0.0005lr_100bs_100e_0.01k.tsv

@@ -0,0 +1,101 @@
+	loss	val_loss	learning_rate	batch_size	epochs	kappa
+0	3094.06242344	3003.45718296	0.0005	100	100	0.01
+1	2872.42192306	2862.50107459	0.0005	100	100	0.01
+2	2823.29870202	2817.24025211	0.0005	100	100	0.01
+3	2799.29271491	2790.84067747	0.0005	100	100	0.01
+4	2783.3950928	2779.287628	0.0005	100	100	0.01
+5	2772.98739344	2766.07224975	0.0005	100	100	0.01
+6	2763.81086981	2765.03613468	0.0005	100	100	0.01
+7	2757.17609493	2755.40132984	0.0005	100	100	0.01
+8	2752.55144214	2751.41989247	0.0005	100	100	0.01
+9	2747.96251545	2744.10740834	0.0005	100	100	0.01
+10	2743.83402548	2753.8577337	0.0005	100	100	0.01
+11	2739.79570233	2739.07239633	0.0005	100	100	0.01
+12	2736.46720237	2735.5304653	0.0005	100	100	0.01
+13	2733.90989948	2734.78306355	0.0005	100	100	0.01
+14	2730.8214979	2731.4601808	0.0005	100	100	0.01
+15	2727.97906872	2730.93978129	0.0005	100	100	0.01
+16	2725.68784558	2728.13127483	0.0005	100	100	0.01
+17	2723.54027833	2721.77257904	0.0005	100	100	0.01
+18	2721.39586854	2722.62433573	0.0005	100	100	0.01
+19	2719.063261	2717.70285659	0.0005	100	100	0.01
+20	2717.76581361	2716.40584481	0.0005	100	100	0.01
+21	2715.50071559	2717.03676347	0.0005	100	100	0.01
+22	2714.34927824	2713.26718881	0.0005	100	100	0.01
+23	2712.36097739	2716.45158827	0.0005	100	100	0.01
+24	2710.31320283	2710.02121829	0.0005	100	100	0.01
+25	2709.28993143	2708.54429589	0.0005	100	100	0.01
+26	2707.77991362	2708.59478585	0.0005	100	100	0.01
+27	2706.49259032	2706.33971258	0.0005	100	100	0.01
+28	2704.98627571	2705.68304432	0.0005	100	100	0.01
+29	2703.88576644	2704.20469534	0.0005	100	100	0.01
+30	2702.65991055	2704.05780438	0.0005	100	100	0.01
+31	2701.06392389	2701.14872132	0.0005	100	100	0.01
+32	2699.99309932	2700.07891577	0.0005	100	100	0.01
+33	2699.01525971	2700.81600293	0.0005	100	100	0.01
+34	2698.17215213	2700.3107336	0.0005	100	100	0.01
+35	2697.1144063	2697.42026825	0.0005	100	100	0.01
+36	2696.13814923	2696.4956965	0.0005	100	100	0.01
+37	2695.63311578	2697.07563598	0.0005	100	100	0.01
+38	2694.53201108	2697.12351462	0.0005	100	100	0.01
+39	2693.8017837	2694.51319993	0.0005	100	100	0.01
+40	2693.21635888	2692.78901255	0.0005	100	100	0.01
+41	2691.90991499	2692.24520495	0.0005	100	100	0.01
+42	2690.98151277	2693.17508057	0.0005	100	100	0.01
+43	2690.41313828	2691.28162811	0.0005	100	100	0.01
+44	2689.7895105	2688.69412354	0.0005	100	100	0.01
+45	2688.34813504	2688.78854668	0.0005	100	100	0.01
+46	2687.53485204	2690.00752448	0.0005	100	100	0.01
+47	2687.5519417	2687.50730648	0.0005	100	100	0.01
+48	2686.5280008	2687.24542201	0.0005	100	100	0.01
+49	2685.5005469	2686.67405686	0.0005	100	100	0.01
+50	2685.27609291	2686.10105928	0.0005	100	100	0.01
+51	2684.49155439	2684.55383277	0.0005	100	100	0.01
+52	2684.42261341	2685.67899709	0.0005	100	100	0.01
+53	2683.71455482	2693.31681471	0.0005	100	100	0.01
+54	2683.11518071	2684.14748054	0.0005	100	100	0.01
+55	2682.82254032	2683.64392533	0.0005	100	100	0.01
+56	2681.9826192	2682.56575365	0.0005	100	100	0.01
+57	2681.48145934	2682.28591995	0.0005	100	100	0.01
+58	2680.7787431	2681.7468145	0.0005	100	100	0.01
+59	2680.4867137	2681.56937188	0.0005	100	100	0.01
+60	2679.95457306	2680.82965893	0.0005	100	100	0.01
+61	2679.3111159	2681.00514329	0.0005	100	100	0.01
+62	2678.86390847	2680.34570733	0.0005	100	100	0.01
+63	2678.48110567	2680.38234709	0.0005	100	100	0.01
+64	2678.05560301	2679.50695917	0.0005	100	100	0.01
+65	2677.53936331	2678.33646919	0.0005	100	100	0.01
+66	2677.348064	2678.80605058	0.0005	100	100	0.01
+67	2676.60843522	2677.67359799	0.0005	100	100	0.01
+68	2676.58025522	2678.20310259	0.0005	100	100	0.01
+69	2675.83788806	2676.19437235	0.0005	100	100	0.01
+70	2675.33714651	2676.37586219	0.0005	100	100	0.01
+71	2675.38156399	2676.68959037	0.0005	100	100	0.01
+72	2674.51287233	2676.03645118	0.0005	100	100	0.01
+73	2675.19505524	2675.39934404	0.0005	100	100	0.01
+74	2674.32116623	2675.74001778	0.0005	100	100	0.01
+75	2673.50019504	2675.69614108	0.0005	100	100	0.01
+76	2673.56164482	2676.08423412	0.0005	100	100	0.01
+77	2672.93450117	2677.09243874	0.0005	100	100	0.01
+78	2672.69150074	2677.56753733	0.0005	100	100	0.01
+79	2672.24484864	2673.81304943	0.0005	100	100	0.01
+80	2672.02605771	2672.66292862	0.0005	100	100	0.01
+81	2671.85494678	2671.64199321	0.0005	100	100	0.01
+82	2671.1761405	2672.88837172	0.0005	100	100	0.01
+83	2671.3705303	2674.47967284	0.0005	100	100	0.01
+84	2671.03527623	2674.02044386	0.0005	100	100	0.01
+85	2670.42649497	2672.1221268	0.0005	100	100	0.01
+86	2670.62767432	2672.39696799	0.0005	100	100	0.01
+87	2670.17414141	2670.89534583	0.0005	100	100	0.01
+88	2669.89150778	2674.58749524	0.0005	100	100	0.01
+89	2669.51301187	2673.88811917	0.0005	100	100	0.01
+90	2669.56478205	2670.90477209	0.0005	100	100	0.01
+91	2669.15656124	2669.87581831	0.0005	100	100	0.01
+92	2668.8067402	2670.18330993	0.0005	100	100	0.01
+93	2668.49051643	2668.92435095	0.0005	100	100	0.01
+94	2668.42625244	2670.54645441	0.0005	100	100	0.01
+95	2668.48396562	2670.0734023	0.0005	100	100	0.01
+96	2668.14597345	2669.73092622	0.0005	100	100	0.01
+97	2667.9709324	2669.50732935	0.0005	100	100	0.01
+98	2667.43319163	2670.14731343	0.0005	100	100	0.01
+99	2667.64566903	2669.13442112	0.0005	100	100	0.01

param_sweep/paramsweep_0.0005lr_100bs_100e_0.05k.tsv

@@ -0,0 +1,101 @@
+	loss	val_loss	learning_rate	batch_size	epochs	kappa
+0	3097.11178406	3008.44479248	0.0005	100	100	0.05
+1	2872.34623591	2862.94395269	0.0005	100	100	0.05
+2	2823.33514638	2818.60281961	0.0005	100	100	0.05
+3	2799.35796857	2791.18899939	0.0005	100	100	0.05
+4	2783.4193617	2778.67019122	0.0005	100	100	0.05
+5	2772.95235909	2765.6060602	0.0005	100	100	0.05
+6	2763.61091817	2764.26616323	0.0005	100	100	0.05
+7	2757.03062049	2754.44841257	0.0005	100	100	0.05
+8	2752.41493449	2751.13730039	0.0005	100	100	0.05
+9	2747.79318822	2744.1753938	0.0005	100	100	0.05
+10	2743.78220952	2752.77514602	0.0005	100	100	0.05
+11	2739.72442855	2739.50521938	0.0005	100	100	0.05
+12	2736.36563496	2736.38136306	0.0005	100	100	0.05
+13	2734.00085194	2735.72911827	0.0005	100	100	0.05
+14	2730.78856597	2731.89314763	0.0005	100	100	0.05
+15	2727.94358679	2730.78813215	0.0005	100	100	0.05
+16	2725.64573141	2728.62590934	0.0005	100	100	0.05
+17	2723.51539383	2722.788725	0.0005	100	100	0.05
+18	2721.36835055	2723.88602134	0.0005	100	100	0.05
+19	2719.04761889	2717.13309305	0.0005	100	100	0.05
+20	2717.8332068	2716.45714188	0.0005	100	100	0.05
+21	2715.49449365	2716.50303005	0.0005	100	100	0.05
+22	2714.54627867	2713.74808002	0.0005	100	100	0.05
+23	2712.4038959	2716.50959431	0.0005	100	100	0.05
+24	2710.33932445	2709.66562817	0.0005	100	100	0.05
+25	2709.28798839	2709.21594495	0.0005	100	100	0.05
+26	2707.75152917	2708.7129149	0.0005	100	100	0.05
+27	2706.47750654	2706.40728025	0.0005	100	100	0.05
+28	2705.1382882	2706.1235599	0.0005	100	100	0.05
+29	2703.92821073	2705.22266419	0.0005	100	100	0.05
+30	2702.62851816	2704.7899583	0.0005	100	100	0.05
+31	2701.22719169	2701.1287312	0.0005	100	100	0.05
+32	2700.10840194	2700.39474645	0.0005	100	100	0.05
+33	2699.07730727	2701.76769903	0.0005	100	100	0.05
+34	2698.30281294	2700.36524931	0.0005	100	100	0.05
+35	2697.14273431	2697.18445594	0.0005	100	100	0.05
+36	2696.20263143	2695.96625398	0.0005	100	100	0.05
+37	2695.55818297	2697.3056818	0.0005	100	100	0.05
+38	2694.62745585	2697.84206809	0.0005	100	100	0.05
+39	2693.84814028	2694.60393435	0.0005	100	100	0.05
+40	2693.19422716	2693.81882618	0.0005	100	100	0.05
+41	2691.8454214	2693.13112686	0.0005	100	100	0.05
+42	2691.08539164	2693.88614178	0.0005	100	100	0.05
+43	2690.3448487	2691.50397767	0.0005	100	100	0.05
+44	2689.72822174	2688.7966472	0.0005	100	100	0.05
+45	2688.41490168	2688.8355944	0.0005	100	100	0.05
+46	2687.57919698	2689.53305888	0.0005	100	100	0.05
+47	2687.57901172	2688.39509889	0.0005	100	100	0.05
+48	2686.59371789	2686.3143789	0.0005	100	100	0.05
+49	2685.50297697	2686.58138192	0.0005	100	100	0.05
+50	2685.20364223	2686.05774416	0.0005	100	100	0.05
+51	2684.52489228	2685.093321	0.0005	100	100	0.05
+52	2684.3731861	2686.81313673	0.0005	100	100	0.05
+53	2683.79682658	2692.33870567	0.0005	100	100	0.05
+54	2683.10157806	2683.31212562	0.0005	100	100	0.05
+55	2682.78161264	2684.15543262	0.0005	100	100	0.05
+56	2681.95612521	2683.26577251	0.0005	100	100	0.05
+57	2681.4737889	2682.64809813	0.0005	100	100	0.05
+58	2680.81184375	2682.52402101	0.0005	100	100	0.05
+59	2680.54383913	2682.28181811	0.0005	100	100	0.05
+60	2680.03535383	2681.48895625	0.0005	100	100	0.05
+61	2679.33892378	2681.95383315	0.0005	100	100	0.05
+62	2678.84368928	2680.91800049	0.0005	100	100	0.05
+63	2678.42669318	2682.7256667	0.0005	100	100	0.05
+64	2678.14520124	2680.05773669	0.0005	100	100	0.05
+65	2677.437878	2678.43063559	0.0005	100	100	0.05
+66	2677.35895293	2679.88700584	0.0005	100	100	0.05
+67	2676.68769385	2678.52437718	0.0005	100	100	0.05
+68	2676.64749079	2678.97878404	0.0005	100	100	0.05
+69	2675.97869609	2676.39055451	0.0005	100	100	0.05
+70	2675.37447359	2676.36239898	0.0005	100	100	0.05
+71	2675.47513884	2676.39470163	0.0005	100	100	0.05
+72	2674.62990452	2675.68113227	0.0005	100	100	0.05
+73	2675.29443533	2677.3468174	0.0005	100	100	0.05
+74	2674.59936614	2675.40573511	0.0005	100	100	0.05
+75	2673.56490497	2675.39409992	0.0005	100	100	0.05
+76	2673.62986624	2675.29000779	0.0005	100	100	0.05
+77	2672.87737566	2676.4107701	0.0005	100	100	0.05
+78	2672.76394255	2677.77474596	0.0005	100	100	0.05
+79	2672.32034378	2673.89510729	0.0005	100	100	0.05
+80	2672.19888323	2672.55240668	0.0005	100	100	0.05
+81	2671.94630831	2671.89373674	0.0005	100	100	0.05
+82	2671.21014026	2671.82696405	0.0005	100	100	0.05
+83	2671.30000969	2671.99282749	0.0005	100	100	0.05
+84	2671.02151586	2676.78777831	0.0005	100	100	0.05
+85	2670.42983158	2672.22510092	0.0005	100	100	0.05
+86	2670.66184114	2673.78968242	0.0005	100	100	0.05
+87	2670.16148731	2670.52586163	0.0005	100	100	0.05
+88	2669.94087593	2673.34015138	0.0005	100	100	0.05
+89	2669.56341766	2674.59914583	0.0005	100	100	0.05
+90	2669.67481924	2670.45271748	0.0005	100	100	0.05
+91	2669.26982962	2670.5991253	0.0005	100	100	0.05
+92	2668.79803597	2669.7479143	0.0005	100	100	0.05
+93	2668.62076949	2669.10376817	0.0005	100	100	0.05
+94	2668.56069334	2671.19700888	0.0005	100	100	0.05
+95	2668.54228779	2669.73327147	0.0005	100	100	0.05
+96	2668.25179118	2669.76856496	0.0005	100	100	0.05
+97	2668.12308116	2669.43906614	0.0005	100	100	0.05
+98	2667.52439588	2669.09757596	0.0005	100	100	0.05
+99	2667.80568422	2668.97299422	0.0005	100	100	0.05

param_sweep/paramsweep_0.0005lr_100bs_100e_0.1k.tsv

@@ -0,0 +1,101 @@
+	loss	val_loss	learning_rate	batch_size	epochs	kappa
+0	3094.65494508	3007.30595348	0.0005	100	100	0.1
+1	2872.32298451	2860.72647241	0.0005	100	100	0.1
+2	2823.30266538	2817.4887882	0.0005	100	100	0.1
+3	2799.2929624	2790.36420413	0.0005	100	100	0.1
+4	2783.30361373	2779.63704832	0.0005	100	100	0.1
+5	2772.84144235	2766.88591071	0.0005	100	100	0.1
+6	2763.71724412	2765.74823313	0.0005	100	100	0.1
+7	2756.99544881	2754.96362912	0.0005	100	100	0.1
+8	2752.30251345	2750.50284193	0.0005	100	100	0.1
+9	2747.83618794	2744.14408218	0.0005	100	100	0.1
+10	2743.7226058	2752.51400518	0.0005	100	100	0.1
+11	2739.68706069	2739.06694075	0.0005	100	100	0.1
+12	2736.33871274	2735.75624169	0.0005	100	100	0.1
+13	2733.79012836	2735.06507165	0.0005	100	100	0.1
+14	2730.73241469	2730.73334942	0.0005	100	100	0.1
+15	2727.89812375	2730.91006242	0.0005	100	100	0.1
+16	2725.59226856	2727.55287582	0.0005	100	100	0.1
+17	2723.38806213	2722.08294199	0.0005	100	100	0.1
+18	2721.27899881	2723.30311996	0.0005	100	100	0.1
+19	2718.95251326	2717.21718806	0.0005	100	100	0.1
+20	2717.70179342	2716.29750286	0.0005	100	100	0.1
+21	2715.4816306	2716.80331462	0.0005	100	100	0.1
+22	2714.33292342	2713.0274049	0.0005	100	100	0.1
+23	2712.29023685	2715.9092343	0.0005	100	100	0.1
+24	2710.30484696	2709.58312451	0.0005	100	100	0.1
+25	2709.1767437	2708.53956759	0.0005	100	100	0.1
+26	2707.68645216	2709.10622078	0.0005	100	100	0.1
+27	2706.40755414	2706.49326489	0.0005	100	100	0.1
+28	2704.93508349	2706.43864555	0.0005	100	100	0.1
+29	2703.80211825	2704.283662	0.0005	100	100	0.1
+30	2702.58957759	2704.2481239	0.0005	100	100	0.1
+31	2701.03375482	2700.98670204	0.0005	100	100	0.1
+32	2699.963126	2699.8472604	0.0005	100	100	0.1
+33	2699.00217008	2699.81954367	0.0005	100	100	0.1
+34	2698.06908345	2700.61521524	0.0005	100	100	0.1
+35	2697.03072069	2698.24909019	0.0005	100	100	0.1
+36	2696.10028328	2696.99505697	0.0005	100	100	0.1
+37	2695.58996109	2695.87622677	0.0005	100	100	0.1
+38	2694.47824832	2696.88078562	0.0005	100	100	0.1
+39	2693.7243366	2693.96732577	0.0005	100	100	0.1
+40	2693.2004588	2692.67858677	0.0005	100	100	0.1
+41	2691.79353256	2692.93819788	0.0005	100	100	0.1
+42	2690.99649226	2694.56324503	0.0005	100	100	0.1
+43	2690.28621752	2691.1659456	0.0005	100	100	0.1
+44	2689.69866791	2688.28625978	0.0005	100	100	0.1
+45	2688.32316726	2689.00856733	0.0005	100	100	0.1
+46	2687.42305579	2688.9708273	0.0005	100	100	0.1
+47	2687.46425471	2688.0580966	0.0005	100	100	0.1
+48	2686.46937494	2686.89301412	0.0005	100	100	0.1
+49	2685.46612219	2686.75172252	0.0005	100	100	0.1
+50	2685.16750262	2685.60336671	0.0005	100	100	0.1
+51	2684.44820401	2684.40275501	0.0005	100	100	0.1
+52	2684.35826437	2685.79684372	0.0005	100	100	0.1
+53	2683.62164778	2693.54492608	0.0005	100	100	0.1
+54	2683.02005779	2684.02515956	0.0005	100	100	0.1
+55	2682.78461986	2683.71100285	0.0005	100	100	0.1
+56	2681.93599501	2683.19109302	0.0005	100	100	0.1
+57	2681.45177731	2682.01859157	0.0005	100	100	0.1
+58	2680.65357293	2681.32472477	0.0005	100	100	0.1
+59	2680.47890636	2681.32662935	0.0005	100	100	0.1
+60	2679.86630291	2682.00285827	0.0005	100	100	0.1
+61	2679.24258508	2681.27758369	0.0005	100	100	0.1
+62	2678.75727497	2680.81020704	0.0005	100	100	0.1
+63	2678.4132033	2681.48845257	0.0005	100	100	0.1
+64	2678.02115244	2679.47737521	0.0005	100	100	0.1
+65	2677.4383482	2678.13777467	0.0005	100	100	0.1
+66	2677.30040084	2678.72573718	0.0005	100	100	0.1
+67	2676.59795535	2679.48431291	0.0005	100	100	0.1
+68	2676.53825987	2678.54768164	0.0005	100	100	0.1
+69	2675.87911628	2675.74561947	0.0005	100	100	0.1
+70	2675.29074625	2676.27544291	0.0005	100	100	0.1
+71	2675.37568867	2676.64002609	0.0005	100	100	0.1
+72	2674.55780453	2676.77623041	0.0005	100	100	0.1
+73	2675.20706881	2675.87153582	0.0005	100	100	0.1
+74	2674.25056425	2676.19541613	0.0005	100	100	0.1
+75	2673.42116213	2675.21677634	0.0005	100	100	0.1
+76	2673.57935123	2675.64221495	0.0005	100	100	0.1
+77	2672.79818526	2676.50704787	0.0005	100	100	0.1
+78	2672.69575949	2677.26789135	0.0005	100	100	0.1
+79	2672.28531524	2674.6467794	0.0005	100	100	0.1
+80	2671.98634685	2673.30995916	0.0005	100	100	0.1
+81	2671.84033691	2671.87530156	0.0005	100	100	0.1
+82	2671.14821917	2672.96224083	0.0005	100	100	0.1
+83	2671.33461668	2674.39299545	0.0005	100	100	0.1
+84	2670.98238473	2675.01505503	0.0005	100	100	0.1
+85	2670.40466523	2672.24081228	0.0005	100	100	0.1
+86	2670.65405665	2673.24740688	0.0005	100	100	0.1
+87	2670.16545519	2671.48225896	0.0005	100	100	0.1
+88	2669.85530784	2674.72507572	0.0005	100	100	0.1
+89	2669.48667508	2674.07565839	0.0005	100	100	0.1
+90	2669.58889593	2670.44968276	0.0005	100	100	0.1
+91	2669.14983781	2670.05397889	0.0005	100	100	0.1
+92	2668.78202112	2669.40588822	0.0005	100	100	0.1
+93	2668.48614063	2669.24111151	0.0005	100	100	0.1
+94	2668.43295615	2669.76806641	0.0005	100	100	0.1
+95	2668.52228701	2670.28766814	0.0005	100	100	0.1
+96	2668.29258243	2669.30058752	0.0005	100	100	0.1
+97	2667.97017182	2669.52495649	0.0005	100	100	0.1
+98	2667.48843443	2669.9926165	0.0005	100	100	0.1
+99	2667.66731284	2668.74265944	0.0005	100	100	0.1