Add a pre and post parser hook to the interactive parser (#46)

* add an optional pre and post parser hook and and add_parser_arg hook

* update arrivial plotting

* fix font size and add pre/post/add_parser_args tests

* fix type conversion error

* cleanup testing error

* more scale factor cleanup

* fix more numpy min max type errors

* fix min max type casting

* revert changes

* alternative min max calls for dumps

* prevent agressive numpy data type casting

* more numpy1-numpy2 min/max fixes

* add a diagnostic print

* fix ambiguous 1d/2d/3d dump parsing

---------

Co-authored-by: Cleveland <[email protected]>

Author: clevelam

opppy/dump_utils.py

@@ -515,7 +515,7 @@ def thread_all(file_name, key_words, result_d):
             result_d[file_name.split('/')[-1]] = opppy_parser.build_data_dictionary(file_name,key_words)
         print("Number of threads used for processing: ",nthreads)
         for stride in range(math.ceil(float(total)/float(nthreads))):
-            files = dump_files[nthreads*stride:min(nthreads*(stride+1),len(dump_files))]
+            files = dump_files[nthreads*stride:array([nthreads*(stride+1),len(dump_files)]).min()]
             with Manager() as manager:
                 result_d = manager.dict()
                 threads = []

opppy/interactive_utils.py

@@ -146,7 +146,7 @@ def plot_dict(self, args, dictionaries, data_names):
 
                     print('# ', header_xlabel, header_ylabel, file=outputfile)
                     for x_value, y_value in zip(x, y):
-                        outstring = "%.9e"%(x_value*scale_x)+" %.9e"%(y_value*scale_y)+"\n"
+                        outstring = "%.9e"%(x_value)+" %.9e"%(y_value)+"\n"
                         outputfile.write(outstring)
                 data_name = ''                    
                 if(args.data_file_name is not None):
@@ -199,12 +199,13 @@ def plot_dict(self, args, dictionaries, data_names):
                     if(len(x)>0):
                         last_x.append(x[-1])
                         last_y.append(y[-1])
+
+                if(args.font_size is not None):
+                    PyPloter.rcParams.update({'font_size':args.font_size})
 
                 if(args.plot_max):
                     last_y.append(sum(sorted(y,reverse=True)[0:2])/3.0)
                     last_x.append(x[-1])
-                elif(args.plot_arrival):
-                    continue
                 elif(data_line_color != '' and data_line_type != ''):
                     PyPloter.plot(x,y,label = data_name, linestyle = data_line_type, color = data_line_color)
                 elif(data_line_color != '' ):
@@ -214,17 +215,22 @@ def plot_dict(self, args, dictionaries, data_names):
                 else:
                     PyPloter.plot(x,y,label = data_name)
 
+                interp_x = x[0]
                 if(args.plot_arrival):
+                    args.plot_arrival = False
                     last_x = x[0]
                     last_y = y[0]
                     for x_value, y_value in zip(x,y):
                         if(y_value>args.y_exceeds_value):
+                            args.plot_arrival = True
                             dy = (args.y_exceeds_value - last_y)/(y_value - last_y)
                             interp_x = last_x + dy*(x_value-last_x)
-                            print(data_name, "first exceeds ", y_exceeds_value, " at ", interp_x)
+                            print(data_name, "first exceeds ", args.y_exceeds_value, " at ", interp_x)
                             break
                         last_x = x_value
                         last_y = y_value
+                    if(not args.plot_arrival):
+                        print("WARNING: y never exceeds y_exceeds_value="+str(args.y_exceeds_value));
 
                 if(args.find_max_y):
                     print(data_name, "max y value ", x[y.index(max(y))], max(y))
@@ -233,9 +239,13 @@ def plot_dict(self, args, dictionaries, data_names):
 
             if(args.last_point_only):
                 PyPloter.plot(last_x, last_y, label = data_name)
-            elif(args.plot_arrival or args.plot_max):
-                print(last_x, last_y)
-                PyPloter.plot(last_x, last_y, label = data_name)
+            elif(args.plot_arrival):
+                color=PyPloter.gca().lines[-1].get_color()
+                PyPloter.plot(interp_x, args.y_exceeds_value, linestyle=None, color=color, marker='o', label = data_name + " exceeds y="+str(args.y_exceeds_value)+" @ x="+str(interp_x))
+            elif(args.plot_max):
+                print("max y= "+str(last_y)+" @ x="+str(last_x))
+                color=PyPloter.gca().lines[-1].get_color()
+                PyPloter.plot(last_x, last_y, linestyle=None, color=color, marker='x', label = data_name + " max y="+str(last_y)+" @ x="+str(last_x))
 
         if(args.x_label is not None):
             PyPloter.xlabel(args.x_label)

opppy/plot_dictionary.py

@@ -773,17 +773,23 @@ def init_lines():
                 if(args.data_file_name is not None):
                     outputfile = open(args.data_file_name+'_'+filename.split('/')[-1]+'.'+re.sub(r'[^\w]','',yname)+'.dat', 'w')
 
-                xmin=(np.array(series_data[0][xname])*scale_x).min()
-                xmax=(np.array(series_data[0][xname])*scale_x).max()
-                ymin=(np.array(series_data[0][yname])*scale_y).min()
-                ymax=(np.array(series_data[0][yname])*scale_y).max()
+                xmin=(np.array(series_data[0][xname])).min()
+                xmax=(np.array(series_data[0][xname])).max()
+                ymin=(np.array(series_data[0][yname])).min()
+                ymax=(np.array(series_data[0][yname])).max()
                 for data, index_value in zip(series_data, series_pair.index[index_key]):
-                    x = np.array(data[xname])*scale_x
-                    y = np.array(data[yname])*scale_y
-                    xmin = min(x.min(),xmin)
-                    xmax = max(x.max(),xmax)
-                    ymin = min(y.min(),ymin)
-                    ymax = max(y.max(),ymax)
+                    x = np.array(data[xname])
+                    y = np.array(data[yname])
+                    xmin = np.array([x.min(),xmin]).min()
+                    xmax = np.array([x.max(),xmax]).max()
+                    ymin = np.array([y.min(),ymin]).min()
+                    ymax = np.array([y.max(),ymax]).max()
+                    x = x*scale_x
+                    y = y*scale_y
+                    xmin = xmin*scale_x
+                    xmax = xmax*scale_x
+                    ymin = ymin*scale_y
+                    ymax = ymax*scale_y
                     if(args.data_file_name is not None):
                         if(args.x_label is not None):
                             header_xlabel = args.x_label
@@ -797,7 +803,7 @@ def init_lines():
 
                         print('# ', index_key, header_xlabel, header_ylabel, file=outputfile)
                         for x_value, y_value in zip(x, y):
-                            outstring = "%.9e"%(index_value)+" %.9e"%(x_value*scale_x)+" %.9e"%(y_value*scale_y)+"\n"
+                            outstring = "%.9e"%(index_value)+" %.9e"%(x_value)+" %.9e"%(y_value)+"\n"
                             outputfile.write(outstring)
                 if(args.data_file_name is not None):
                     print("data saved as -- "+args.data_file_name+'_'+filename.split('/')[-1]+'.'+re.sub(r'[^\w]','',yname)+'.dat')
@@ -1003,28 +1009,38 @@ def init_contour():
             if(args.data_file_name is not None):
                 outputfile = open(args.data_file_name+'_'+filename.split('/')[-1]+'.'+re.sub(r'[^\w]','',yname)+'.dat', 'w')
 
-            vmin=(np.array(series_data[0][dname])*args.scale_value).min()
-            vmax=(np.array(series_data[0][dname])*args.scale_value).max()
-            xmin=(np.array(series_data[0][xname])*args.scale_x).min()
-            xmax=(np.array(series_data[0][xname])*args.scale_x).max()
-            ymin=(np.array(series_data[0][yname])*args.scale_y).min()
-            ymax=(np.array(series_data[0][yname])*args.scale_y).max()
+            vmin=(np.array(series_data[0][dname])).min()
+            vmax=(np.array(series_data[0][dname])).max()
+            xmin=(np.array(series_data[0][xname])).min()
+            xmax=(np.array(series_data[0][xname])).max()
+            ymin=(np.array(series_data[0][yname])).min()
+            ymax=(np.array(series_data[0][yname])).max()
             bias = 0.0
             for data, index_value in zip(series_data, series_pair.index[index_key]):
-                v = np.array(data[dname])*args.scale_value
+                v = np.array(data[dname])
                 if(args.log_scale):
                     bias = v.min()
                     bias = 0.0 if bias>0.0 else abs(bias)
                     v = np.array([ [log10(val+bias) if (val+bias)>0.0 else 0.0 
                         for val in vals] for vals in v])
-                x = np.array(data[xname])*args.scale_x
-                y = np.array(data[yname])*args.scale_y
-                vmin = min(v.min(),vmin)
-                vmax = max(v.max(),vmax)
-                xmin = min(x.min(),xmin)
-                xmax = max(x.max(),xmax)
-                ymin = min(y.min(),ymin)
-                ymax = max(y.max(),ymax)
+                x = np.array(data[xname])
+                y = np.array(data[yname])
+                vmin = np.array([v.min(),vmin]).min()
+                vmax = np.array([v.max(),vmax]).max()
+                xmin = np.array([x.min(),xmin]).min()
+                xmax = np.array([x.max(),xmax]).max()
+                ymin = np.array([y.min(),ymin]).min()
+                ymax = np.array([y.max(),ymax]).max()
+                # apply scaling
+                v = v*args.scale_value
+                x = x*args.scale_x
+                y = y*args.scale_y
+                vmin = vmin*args.scale_value
+                vmax = vmax*args.scale_value
+                xmin = xmin*args.scale_x
+                xmax = xmax*args.scale_x
+                ymin = ymin*args.scale_y
+                ymax = ymax*args.scale_y
                 if(args.data_file_name is not None):
                     if(args.x_label is not None):
                         header_xlabel = args.x_label
@@ -1038,7 +1054,7 @@ def init_contour():
 
                     print('# ', index_key, header_xlabel, header_ylabel, dname, file=outputfile)
                     for x_value, y_value, v_value in zip(x, y, v):
-                        outstring = "%.9e"%(index_value)+" %.9e"%(x_value*args.scale_x)+" %.9e"%(y_value*args.scale_y)+" %.9e"%(v_value*args.scale_value)+"\n"
+                        outstring = "%.9e"%(index_value)+" %.9e"%(x_value)+" %.9e"%(y_value)+" %.9e"%(v_value)+"\n"
                         outputfile.write(outstring)
             if(args.data_file_name is not None):
                 print("data saved as -- "+args.data_file_name+'_'+filename.split('/')[-1]+'.'+re.sub(r'[^\w]','',dname)+'.dat')

opppy/plot_dump_dictionary.py

@@ -276,6 +276,7 @@ def add_plot_options(parser):
     parser.add_argument('-lpo','--last_point_only', dest='last_point_only', help='only plot the last data point', nargs='?', type=bool, default=False, const=True )
     parser.add_argument('-ltv','--last_time_value', dest='last_time_value', help='only plot the last time value', nargs=1, type=float)
     parser.add_argument('-fr','--figure_resolution', dest='figure_resolution', help='figure resolution in  dpi', nargs='?', type=float, default=300.0)
+    parser.add_argument('-fs','--font_size', dest='font_size', help='set the plot font size', type=float, default=None, nargs="?")
     parser.add_argument('-yev','--y_exceeds_value', dest='y_exceeds_value', help='y arrival value', nargs=1, type=float, default=0.0)
     parser.add_argument('-sy','--scale_y', dest='scale_y', help='scale y values', type=float, default=[], action="append")
     parser.add_argument('-sx','--scale_x', dest='scale_x', help='scale x values', type=float, default=[], action="append")
@@ -301,6 +302,7 @@ def add_2d_plot_options(parser):
     parser.add_argument('-xlab','--xlab', dest='x_label', help='x axis label')
     parser.add_argument('-ylab','--ylabl', dest='y_label', help='y axis label')
     parser.add_argument('-fr','--figure_resolution', dest='figure_resolution', help='figure resolution in  dpi', nargs='?', type=float, default=300.0)
+    parser.add_argument('-fs','--font_size', dest='font_size', help='set the plot font size', type=float, default=None, nargs="?")
     parser.add_argument('-sv','--scale_value', dest='scale_value', help='scale values', type=float, default=1.0)
     parser.add_argument('-sx','--scale_x', dest='scale_x', help='scale x values', type=float, default=1.0)
     parser.add_argument('-sy','--scale_y', dest='scale_y', help='scale y values', type=float, default=1.0)

opppy/plotting_help.py

@@ -0,0 +1,12 @@
+keys: time cell_id x y z density temperature pressure
+
+time: 1.0
+
+cell_id: 1 2 3 4
+x: 1 2 3 4 5
+y: 1 1 1 1 1
+z: 1 1 1 1 1
+density: 1.1 1.1 1.1 1.1 1.1
+temperature: 3.1 3.1 3.1 3.1 3.1
+pressure: 5.1 5.1 5.1 5.1 5.1
+

tests/example_1d_dump.txt

@@ -0,0 +1,10 @@
+keys: time cell_id x y z density temperature pressure
+
+time: 2.0
+
+cell_id: 1 2 3 4 5
+x: 1 2 3 4 5
+density: 1.2 1.2 1.2 1.2 1.2
+temperature: 3.2 3.2 3.2 3.2 3.2
+pressure: 5.2 5.2 5.2 5.2 5.2
+

tests/example_1d_dump_2.txt

@@ -0,0 +1,10 @@
+keys: time cell_id x y z density temperature pressure
+
+time: 3.0
+
+cell_id: 1 2 3 4 5
+x: 1 2 3 4 5
+density: 1.3 1.3 1.3 1.3 1.3
+temperature: 3.3 3.3 3.3 3.3 3.3
+pressure: 5.3 5.3 5.3 5.3 5.3
+

tests/example_1d_dump_3.txt

@@ -0,0 +1,12 @@
+keys: time cell_id x y z density temperature pressure
+
+time: 1.0
+
+cell_id: 1 2 3 4 5 6 7 8 9 10
+x: 1 2 3 4 5 1 2 3 4 5
+y: 1 1 1 1 1 2 2 2 2 2
+z: 1 1 1 1 1 1 1 1 1 1
+density: 1.1 1.1 1.1 1.1 1.1 2.1 2.1 2.1 2.1 2.1
+temperature: 3.1 3.1 3.1 3.1 3.1 4.1 4.1 4.1 4.1 4.1
+pressure: 5.1 5.1 5.1 5.1 5.1 6.1 6.1 6.1 6.1 6.1
+

tests/example_2d_dump.txt

@@ -0,0 +1,11 @@
+keys: time cell_id x y z density temperature pressure
+
+time: 2.0
+
+cell_id: 1 2 3 4 5 6 7 8 9 10
+x: 1 2 3 4 5 1 2 3 4 5
+y: 1 1 1 1 1 2 2 2 2 2
+density: 1.2 1.2 1.2 1.2 1.2 2.2 2.2 2.2 2.2 2.2
+temperature: 3.2 3.2 3.2 3.2 3.2 4.2 4.2 4.2 4.2 4.2
+pressure: 5.2 5.2 5.2 5.2 5.2 6.2 6.2 6.2 6.2 6.2
+

tests/example_2d_dump_2.txt

@@ -0,0 +1,12 @@
+keys: time cell_id x y z density temperature pressure
+
+time: 3.0
+
+cell_id: 1 2 3 4 5 6 7 8 9 10
+x: 1 2 3 4 5 1 2 3 4 5
+y: 1 1 1 1 1 2 2 2 2 2
+z: 1 1 1 1 1 1 1 1 1 1
+density: 1.3 1.3 1.3 1.3 1.3 2.3 2.3 2.3 2.3 2.3
+temperature: 3.3 3.3 3.3 3.3 3.3 4.3 4.3 4.3 4.3 4.3
+pressure: 5.3 5.3 5.3 5.3 5.3 6.3 6.3 6.3 6.3 6.3
+

tests/example_2d_dump_3.txt

@@ -32,7 +32,7 @@ def build_data_dictionary(self, filename, dump_keys=None):
         for line in lines:
             for key in keys:
                 if key in line and len(key) is len(line.split(':')[0]):
-                    data[key] = array(str_vector_to_float_vector(line.strip('\n').split(' ')[1:]))
+                    data[key] = array(str_vector_to_float_vector(line.strip('\n').split(' ')[1:])).astype(float)
 
 
         # build xy_verts for 2d mesh plotting example
@@ -45,3 +45,4 @@ def build_data_dictionary(self, filename, dump_keys=None):
         return data
 
 
+

tests/gold_dumps.p

@@ -14,6 +14,15 @@ def __init__(self):
       self.file_end_string = None
       print("Initializing my_test_opppy_parser")
 
+    def add_parser_args(self, parser):
+        parser.add_argument('-ppt', '--pre_parser_test', dest="pre_parser_test", nargs="?", default=None)
+
+    def pre_parse(self, args):
+        if(args.pre_parser_test is not None):
+            print("pre_parse hook works: args.pre_parser_test")
+        else:
+            print("pre_parse hook works: None")
+
     def parse_cycle_string(self,cycle_string):
       # return dictionary of dictionaries 
       data_dict = {}
@@ -53,8 +62,13 @@ def parse_cycle_string(self,cycle_string):
       # append dictionary with multiple entries
       data_dict['density'] = density_data
 
+      return data_dict
 
+    def post_parse(self, args, data):
+       if(args.pre_parser_test is not None):
+           data["post_parse_test_"+args.pre_parser_test]=data['test_data1']
+       else:
+           data['post_parser_test']=data['test_data1']
 
 
-      return data_dict
-        
+       

tests/gold_line_data.p

@@ -20,6 +20,16 @@ def __init__(self):
       self.file_end_string = None
       print("Initializing my_test_opppy_tally_parser")
 
+    def add_parser_args(self, parser):
+        parser.add_argument('-ppt', '--pre_parser_test', dest="pre_parser_test", nargs="?", default=None)
+
+    def pre_parse(self, args):
+        if(args.pre_parser_test is not None):
+            print("pre_parse hook works: args.pre_parser_test")
+        else:
+            print("pre_parse hook works: None")
+
+
     def parse_cycle_string(self,cycle_string):
       cycle_data_keys = ['bins','odd_counts','even_counts']
       cycle_info_keys = ['time', 'cycle']
@@ -58,4 +68,8 @@ def parse_cycle_string(self,cycle_string):
       data_dict['cool_counts'] = counts
 
       return data_dict
-        
+
+    def post_parse(self, args, data):
+       print("Post Parse Test Data keys: ",data.keys())
+
+