thin_output.py

#!/usr/bin/env python

import pandas as pd
import numpy as np
import sys, os

yyyymmdd = sys.argv[1]

Step_ID,Track_ID,Run_Date,Valid_Date,Forecast_Hour,Valid_Hour_UTC,Duration,Centroid_Lon,Centroid_Lat,Storm_Motion_U,Storm_Motion_V,UP_HELI_MAX_mean,UP_HELI_MAX_max,UP_HELI_MAX_min,GRPL_MAX_mean,GRPL_MAX_max,GRPL_MAX_min,WSPD10MAX_mean,WSPD10MAX_max,WSPD10MAX_min,W_UP_MAX_mean,W_UP_MAX_max,W_UP_MAX_min,W_DN_MAX_mean,W_DN_MAX_max,W_DN_MAX_min,RVORT1_MAX_mean,RVORT1_MAX_max,RVORT1_MAX_min,RVORT5_MAX_mean,RVORT5_MAX_max,RVORT5_MAX_min,UP_HELI_MAX03_mean,UP_HELI_MAX03_max,UP_HELI_MAX03_min,UP_HELI_MAX01_mean,UP_HELI_MAX01_max,UP_HELI_MAX01_min,UP_HELI_MIN_mean,UP_HELI_MIN_max,UP_HELI_MIN_min,REFL_COM_mean,REFL_COM_max,REFL_COM_min,REFL_1KM_AGL_mean,REFL_1KM_AGL_max,REFL_1KM_AGL_min,REFD_MAX_mean,REFD_MAX_max,REFD_MAX_min,PSFC_mean,PSFC_max,PSFC_min,T2_mean,T2_max,T2_min,T2_std,T2_percentile_10,T2_percentile_25,T2_percentile_50,T2_percentile_75,T2_percentile_90,Q2_mean,Q2_max,Q2_min,Q2_std,Q2_percentile_10,Q2_percentile_25,Q2_percentile_50,Q2_percentile_75,Q2_percentile_90,TD2_mean,TD2_max,TD2_min,TD2_std,TD2_percentile_10,TD2_percentile_25,TD2_percentile_50,TD2_percentile_75,TD2_percentile_90,U10_mean,U10_max,U10_min,U10_std,U10_percentile_10,U10_percentile_25,U10_percentile_50,U10_percentile_75,U10_percentile_90,V10_mean,V10_max,V10_min,V10_std,V10_percentile_10,V10_percentile_25,V10_percentile_50,V10_percentile_75,V10_percentile_90,SBLCL-potential_mean,SBLCL-potential_max,SBLCL-potential_min,SBLCL-potential_std,SBLCL-potential_percentile_10,SBLCL-potential_percentile_25,SBLCL-potential_percentile_50,SBLCL-potential_percentile_75,SBLCL-potential_percentile_90,MLLCL-potential_mean,MLLCL-potential_max,MLLCL-potential_min,MLLCL-potential_std,MLLCL-potential_percentile_10,MLLCL-potential_percentile_25,MLLCL-potential_percentile_50,MLLCL-potential_percentile_75,MLLCL-potential_percentile_90,SBCAPE-potential_mean,SBCAPE-potential_max,SBCAPE-potential_min,SBCAPE-potential_std,SBCAPE-potential_percentile_10,SBCAPE-potential_percentile_25,SBCAPE-potential_percentile_50,SBCAPE-potential_percentile_75,SBCAPE-potential_percentile_90,MLCAPE-potential_mean,MLCAPE-potential_max,MLCAPE-potential_min,MLCAPE-potential_std,MLCAPE-potential_percentile_10,MLCAPE-potential_percentile_25,MLCAPE-potential_percentile_50,MLCAPE-potential_percentile_75,MLCAPE-potential_percentile_90,MUCAPE-potential_mean,MUCAPE-potential_max,MUCAPE-potential_min,MUCAPE-potential_std,MUCAPE-potential_percentile_10,MUCAPE-potential_percentile_25,MUCAPE-potential_percentile_50,MUCAPE-potential_percentile_75,MUCAPE-potential_percentile_90,SBCINH-potential_mean,SBCINH-potential_max,SBCINH-potential_min,SBCINH-potential_std,SBCINH-potential_percentile_10,SBCINH-potential_percentile_25,SBCINH-potential_percentile_50,SBCINH-potential_percentile_75,SBCINH-potential_percentile_90,MLCINH-potential_mean,MLCINH-potential_max,MLCINH-potential_min,MLCINH-potential_std,MLCINH-potential_percentile_10,MLCINH-potential_percentile_25,MLCINH-potential_percentile_50,MLCINH-potential_percentile_75,MLCINH-potential_percentile_90,USHR1-potential_mean,USHR1-potential_max,USHR1-potential_min,USHR1-potential_std,USHR1-potential_percentile_10,USHR1-potential_percentile_25,USHR1-potential_percentile_50,USHR1-potential_percentile_75,USHR1-potential_percentile_90,VSHR1-potential_mean,VSHR1-potential_max,VSHR1-potential_min,VSHR1-potential_std,VSHR1-potential_percentile_10,VSHR1-potential_percentile_25,VSHR1-potential_percentile_50,VSHR1-potential_percentile_75,VSHR1-potential_percentile_90,USHR6-potential_mean,USHR6-potential_max,USHR6-potential_min,USHR6-potential_std,USHR6-potential_percentile_10,USHR6-potential_percentile_25,USHR6-potential_percentile_50,USHR6-potential_percentile_75,USHR6-potential_percentile_90,VSHR6-potential_mean,VSHR6-potential_max,VSHR6-potential_min,VSHR6-potential_std,VSHR6-potential_percentile_10,VSHR6-potential_percentile_25,VSHR6-potential_percentile_50,VSHR6-potential_percentile_75,VSHR6-potential_percentile_90,U_BUNK-potential_mean,U_BUNK-potential_max,U_BUNK-potential_min,U_BUNK-potential_std,U_BUNK-potential_percentile_10,U_BUNK-potential_percentile_25,U_BUNK-potential_percentile_50,U_BUNK-potential_percentile_75,U_BUNK-potential_percentile_90,V_BUNK-potential_mean,V_BUNK-potential_max,V_BUNK-potential_min,V_BUNK-potential_std,V_BUNK-potential_percentile_10,V_BUNK-potential_percentile_25,V_BUNK-potential_percentile_50,V_BUNK-potential_percentile_75,V_BUNK-potential_percentile_90,SRH03-potential_mean,SRH03-potential_max,SRH03-potential_min,SRH03-potential_std,SRH03-potential_percentile_10,SRH03-potential_percentile_25,SRH03-potential_percentile_50,SRH03-potential_percentile_75,SRH03-potential_percentile_90,SRH01-potential_mean,SRH01-potential_max,SRH01-potential_min,SRH01-potential_std,SRH01-potential_percentile_10,SRH01-potential_percentile_25,SRH01-potential_percentile_50,SRH01-potential_percentile_75,SRH01-potential_percentile_90,PSFC-potential_mean,PSFC-potential_max,PSFC-potential_min,PSFC-potential_std,PSFC-potential_percentile_10,PSFC-potential_percentile_25,PSFC-potential_percentile_50,PSFC-potential_percentile_75,PSFC-potential_percentile_90,T2-potential_mean,T2-potential_max,T2-potential_min,T2-potential_std,T2-potential_percentile_10,T2-potential_percentile_25,T2-potential_percentile_50,T2-potential_percentile_75,T2-potential_percentile_90,Q2-potential_mean,Q2-potential_max,Q2-potential_min,Q2-potential_std,Q2-potential_percentile_10,Q2-potential_percentile_25,Q2-potential_percentile_50,Q2-potential_percentile_75,Q2-potential_percentile_90,TD2-potential_mean,TD2-potential_max,TD2-potential_min,TD2-potential_std,TD2-potential_percentile_10,TD2-potential_percentile_25,TD2-potential_percentile_50,TD2-potential_percentile_75,TD2-potential_percentile_90,U10-potential_mean,U10-potential_max,U10-potential_min,U10-potential_std,U10-potential_percentile_10,U10-potential_percentile_25,U10-potential_percentile_50,U10-potential_percentile_75,U10-potential_percentile_90,V10-potential_mean,V10-potential_max,V10-potential_min,V10-potential_std,V10-potential_percentile_10,V10-potential_percentile_25,V10-potential_percentile_50,V10-potential_percentile_75,V10-potential_percentile_90,area,eccentricity,major_axis_length,minor_axis_length,orientation,Matched,Max_Hail_Size,Num_Matches,Shape,Location,Scale

# retain mean/max/min for each field
usecols = 'Step_ID,Track_ID,Run_Date,Valid_DForecast_Hour,Valid_Hour_UTC,Duration,Centroid_Lon,Centroid_Lat,Storm_Motion_U,Storm_Motion_V,UP_HELI_MAX_mean,UP_HELI_MAX_max,UP_HELI_MAX_min,GRPL_MAX_mean,GRPL_MAX_max,GRPL_MAX_min,W_UP_MAX_mean,W_UP_MAX_max,W_UP_MAX_min,W_DN_MAX_mean,W_DN_MAX_max,W_DN_MAX_min,HAIL_MAX2D_mean,HAIL_MAX2D_max,HAIL_MAX2D_min,HAIL_MAXK1_mean,HAIL_MAXK1_max,HAIL_MAXK1_min,LTG3_MAX_mean,LTG3_MAX_max,LTG3_MAX_min,RVORT1_MAX_mean,RVORT1_MAX_max,RVORT1_MAX_min,UP_HELI_MAX03_mean,UP_HELI_MAX03_max,UP_HELI_MAX03_min,UP_HELI_MIN_mean,UP_HELI_MIN_max,UP_HELI_MIN_min,WSPD10MAX_mean,WSPD10MAX_max,WSPD10MAX_min,REFD_MAX_mean,REFD_MAX_max,REFD_MAX_min,UBSHR1-potential_mean,UBSHR1-potential_max,UBSHR1-potential_min,VBSHR1-potential_mean,VBSHR1-potential_max,VBSHR1-potential_min,UBSHR6-potential_mean,UBSHR6-potential_max,UBSHR6-potential_min,VBSHR6-potential_mean,VBSHR6-potential_max,VBSHR6-potential_min,PWAT-potential_mean,PWAT-potential_max,PWAT-potential_min,SRH3-potential_mean,SRH3-potential_max,SRH3-potential_min,LCL_HEIGHT-potential_mean,LCL_HEIGHT-potential_max,LCL_HEIGHT-potential_min,CAPE_SFC-potential_mean,CAPE_SFC-potential_max,CAPE_SFC-potential_min,CIN_SFC-potential_mean,CIN_SFC-potential_max,CIN_SFC-potential_min,MUCAPE-potential_mean,MUCAPE-potential_max,MUCAPE-potential_min,area,eccentricity,major_axis_length,minor_axis_length,orientation,extent'

for mem in range(1,11):
    fname = './track_data_ncar_2016_csv/track_step_NCAR_mem%d_%s.csv'%(mem,yyyymmdd)
    if os.path.exists(fname):
        print 'thinning', fname
        df = pd.read_csv(fname, usecols=usecols.split(','))

        # multiply by 1000 so precision not lost
        for f in ['RVORT1_MAX_min', 'RVORT1_MAX_max', 'RVORT1_MAX_mean', 'HAIL_MAX2D_mean',\
                  'HAIL_MAX2D_max', 'HAIL_MAX2D_min', 'HAIL_MAXK1_mean', 'HAIL_MAXK1_max', 'HAIL_MAXK1_min']:
            df[f]  = df[f]*1000

        df.to_csv('./track_data_ncar_2016_csv_thin/track_step_NCAR_mem%d_%s.csv'%(mem,yyyymmdd), float_format='%.2f', index=False)