update testing to reflect thread variable passing to dump, output, an…

…d tally parsers

.github/workflows/testing.yml

@@ -26,11 +26,6 @@ jobs:
 ##         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
 ##         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
 ##         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Threaded Test
+    - name: Testing
       run: |
         python -m pytest --cov-report term-missing --cov=opppy  tests/
-    - name: Serial Test
-      run: |
-        python -m pytest --cov-report term-missing --cov=opppy  tests/
-      env: 
-          OPPPY_USE_THREADS: false

opppy/interactive_utils.py

@@ -611,7 +611,7 @@ def pickle_dumps_parser(self, subparser):
         input_type_parser.add_argument('-cf','--case_file', dest='case_file', help='Case file to be pickled', nargs='?')
         pickle_parser.add_argument('-pf','--pickle_file', dest='pickle_name', help='Pickle file name to be created or appended to', required=True )
         pickle_parser.add_argument('-kw','--key_words', dest='key_words', help='Only extract the specified key_words', nargs='+', default=None )
-        pickle_parser.add_argument('-nt','--nthreads', dest='nthreads', help='Specify number of threads for dump parsing', nargs='?', default=0 )
+        pickle_parser.add_argument('-nt','--nthreads', dest='nthreads', help='Specify number of threads for dump parsing', nargs='?', type=int, default=0 )
         pickle_parser.set_defaults(func=self.pickle_dumps)
 
     def pickle_dumps(self, args):
@@ -665,6 +665,7 @@ def plot_series_point_parser(self, subparser):
         input_type_parser.add_argument('-pf','--pickle_files', dest='pickle_files', help='pickle files to be plotted (run1.p run2.p etc...)', nargs='+')
         input_type_parser.add_argument('-df','--dump_files', dest='dump_files', help='dump files to be parsed and plotted (output_file1.txt output_file2.txt etc...)', nargs='+', action='append')
         input_type_parser.add_argument('-cf','--case_files', dest='case_files', help='Case file to be ploted', nargs='+')
+        plot_parser.add_argument('-nt','--nthreads', dest='nthreads', help='Specify number of threads for dump parsing', nargs='?', type=int, default=0 )
         plot_parser.add_argument('-kw','--key_words', dest='key_words', help='Only extract the specified key_words', nargs='+', default=None )
         plot_parser.add_argument('-dk','--dimension_keys', dest='dimension_keys', help='keys used to extract the points (e.g. [x], [x,y], or [x,y,x]', nargs='+', required=True )
         plot_parser.add_argument('-p','--point', dest='point', help='point location to extract data (e.g. [1], [1,2], or [1,2,3]', nargs='+', required=True, type=float )
@@ -730,6 +731,7 @@ def plot_series_line_parser(self, subparser):
         input_type_parser.add_argument('-pf','--pickle_files', dest='pickle_files', help='pickle files to be plotted (run1.p run2.p etc...)', nargs='+')
         input_type_parser.add_argument('-cf','--case_files', dest='case_files', help='Case file to be ploted', nargs='+')
         input_type_parser.add_argument('-df','--dump_files', dest='dump_files', help='dump files to be parsed and plotted (output_file1.txt output_file2.txt etc...)', nargs='+', action='append')
+        plot_parser.add_argument('-nt','--nthreads', dest='nthreads', help='Specify number of threads for dump parsing', nargs='?', type=int, default=0 )
         plot_parser.add_argument('-kw','--key_words', dest='key_words', help='Only extract the specified key_words', nargs='+', default=None )
         plot_parser.add_argument('-dk','--dimension_keys', dest='dimension_keys', help='keys used to extract the points (e.g. [x], [x,y], or [x,y,x]', nargs='+', required=True )
         plot_parser.add_argument('-p0','--point0', dest='point0', help='beginning point location to extract data (e.g. [1], [1,2], or [1,2,3]', nargs='+', required=True, type=float )
@@ -788,6 +790,7 @@ def plot_series_contour_parser(self, subparser):
         input_type_parser.add_argument('-pf','--pickle_file', dest='pickle_file', help='pickle file to be plotted', nargs='?')
         input_type_parser.add_argument('-cf','--case_file', dest='case_file', help='Case file to be ploted', nargs='?')
         input_type_parser.add_argument('-df','--dump_files', dest='dump_files', help='dump files to be parsed and plotted (output_file1.txt output_file2.txt etc...)', nargs='+')
+        plot_parser.add_argument('-nt','--nthreads', dest='nthreads', help='Specify number of threads for dump parsing', nargs='?', type=int, default=0 )
         plot_parser.add_argument('-kw','--key_words', dest='key_words', help='Only extract the specified key_words', nargs='+', default=None )
         plot_parser.add_argument('-dk','--dimension_keys', dest='dimension_keys', help='keys used to extract the points dimensions in the dictionary (e.g. [x,y], or [my_x,my_y,my_z]', nargs='+', required=True )
         plot_parser.add_argument('-zs','--z_slice_location', dest='z_slice_location', help='location along the z plane to slice the data', nargs='?', type=float, default=None)

tests/test_dump_utils.py

@@ -155,6 +155,30 @@ def test_append_dump_dictionary(self):
                 except:
                     assert(data2[dump_name][k]==v)
 
+        # TEST WITH THREADS
+        # initialize a new data dictionary
+        data3 = {}
+        data3['version'] = __version__
+
+
+        # Append the thrid output file
+        filenames = [dir_path+"example_dump.txt",dir_path+"example_dump2.txt",dir_path+"example_dump3.txt"]
+        # Build initial pickle
+        append_dumps(data3, filenames, dump_parser, nthreads=-1)
+
+        print(data3)
+        # Don't check the version for a match
+        data3.pop('version')
+        assert(data3.keys()==gold_data.keys())
+        for dump_name, dump_dic in gold_data.items():
+            assert(data3[dump_name].keys()==dump_dic.keys())
+            for k, v in dump_dic.items():
+                try:
+                    np.testing.assert_allclose(data3[dump_name][k],v)
+                except:
+                    assert(data3[dump_name][k]==v)
+
+
         # extract subset of dump data
         sub_data = {}
         sub_data['version'] = __version__
@@ -182,6 +206,25 @@ def test_append_dump_dictionary(self):
                 except:
                     assert(sub_data[dump_name][k]==v)
 
+        # TEST WITH THREADS
+        # extract subset of dump data
+        sub_data2 = {}
+        sub_data2['version'] = __version__
+        filenames = [dir_path+"example_dump.txt",dir_path+"example_dump2.txt",dir_path+"example_dump3.txt"]
+        # Build initial pickle
+        append_dumps(sub_data2, filenames, dump_parser, ['time','density'], nthreads=-1)
+
+        sub_data2.pop('version')
+        assert(sub_data2.keys()==gold_data.keys())
+        for dump_name, dump_dic in gold_data.items():
+            assert(sub_data2[dump_name].keys()==dump_dic.keys())
+            for k, v in dump_dic.items():
+                try:
+                    np.testing.assert_allclose(sub_data2[dump_name][k],v)
+                except:
+                    assert(sub_data2[dump_name][k]==v)
+
+
 
     def test_point_value(self):
         '''

tests/test_interactive.py

@@ -44,10 +44,12 @@ def test_help(self):
         assert(os.system("python my_interactive_parser.py dump line -h")==0)
 
     def test_pickle_output(self):
-        # This uses glob to pickle all the output data
-        assert(os.system("python my_interactive_parser.py output pickle -pf interactive.p -of "+dir_path+"output_example*.txt")==0)
+        # This uses glob to pickle all the output data using all threads
+        assert(os.system("python my_interactive_parser.py output pickle -nt -1 -pf interactive.p -of "+dir_path+"output_example*.txt")==0)
+        # This uses glob to pickle all the output data using 2 threads
+        assert(os.system("python my_interactive_parser.py output pickle -nt 2 -pf interactive.p -of "+dir_path+"output_example*.txt")==0)
         # Test no threads
-        assert(os.system("OPPPY_USE_THREADS=False python my_interactive_parser.py output pickle -pf interactive.p -of "+dir_path+"output_example*.txt")==0)
+        assert(os.system("python my_interactive_parser.py output pickle -pf interactive.p -of "+dir_path+"output_example*.txt")==0)
 
     def test_plot_pickle(self):
         # This uses glob to pickle all the output data
@@ -56,6 +58,8 @@ def test_plot_pickle(self):
         assert(os.system("python my_interactive_parser.py output iplot -pf interactive.p < "+dir_path+"interactive_input.txt")==0)
         # parse and plot the output data files
         assert(os.system("python my_interactive_parser.py output iplot -of "+dir_path+"output_example*.txt < "+dir_path+"interactive_input.txt")==0)
+        # parse and plot the output data files with threads
+        assert(os.system("python my_interactive_parser.py output iplot -nt -1 -of "+dir_path+"output_example*.txt < "+dir_path+"interactive_input.txt")==0)
 
     def test_plot_dictionary(self):
         # This uses glob to pickle all the output data
@@ -64,12 +68,16 @@ def test_plot_dictionary(self):
         assert(os.system("python my_interactive_parser.py output plot -pf interactive.p -dn density -x time -y mat1 -sa density_mat1.png -hp")==0)
         # parse and plot the output data files
         assert(os.system("python my_interactive_parser.py output plot -of "+dir_path+"output_example*.txt -dn density -x time -y mat1 -sa density_mat1_pp.png -hp")==0)
+        # parse and plot the output data files with threads 
+        assert(os.system("python my_interactive_parser.py output plot -nt -1 -of "+dir_path+"output_example*.txt -dn density -x time -y mat1 -sa density_mat1_pp.png -hp")==0)
 
     def test_pickle_dumps(self):
-        # This uses glob to pickle all the dump data
-        assert(os.system("python my_interactive_parser.py dump pickle -pf interactive_dump.p -df "+dir_path+"example_dump*.txt")==0)
+        # This uses glob to pickle all the dump data with all threads
+        assert(os.system("python my_interactive_parser.py dump pickle -nt -1 -pf interactive_dump.p -df "+dir_path+"example_dump*.txt")==0)
+        # This uses glob to pickle all the dump data with 2 threads
+        assert(os.system("python my_interactive_parser.py dump pickle -nt 2 -pf interactive_dump.p -df "+dir_path+"example_dump*.txt")==0)
         # Test serial parsing
-        assert(os.system("OPPPY_USE_THREADS=False python my_interactive_parser.py dump pickle -pf interactive_dump.p -df "+dir_path+"example_dump*.txt")==0)
+        assert(os.system("python my_interactive_parser.py dump pickle -pf interactive_dump.p -df "+dir_path+"example_dump*.txt")==0)
 
     def test_plot_dump(self):
         # This uses glob to pickle all the dump data
@@ -106,6 +114,9 @@ def test_dump_series(self):
         # Parse and plot a point series
         assert(os.system("python my_interactive_parser.py dump point -df "+dir_path+"example_dump*.txt -df "+dir_path+"example_dump*.txt -dk x -p 5 -s time -d temperature -sy 10.0 -sy 1.0")==0)
 
+        # Parse and plot a point series with threads
+        assert(os.system("python my_interactive_parser.py dump point -nt -1 -df "+dir_path+"example_dump*.txt -df "+dir_path+"example_dump*.txt -dk x -p 5 -s time -d temperature -sy 10.0 -sy 1.0")==0)
+
         # test individual scale x
         assert(os.system("python my_interactive_parser.py dump point -df "+dir_path+"example_dump*.txt -df "+dir_path+"example_dump*.txt -dk x -p 5 -s time -d temperature -sx 10.0 -sx 1.0")==0)
 
@@ -128,10 +139,12 @@ def test_dump_series(self):
         assert(os.system("python my_interactive_parser.py dump contour -pf interactive_dump.p -dk z y x -zs 5 -s time -d temperature -ls")==0)
 
     def test_pickle_tally(self):
-        # This uses glob to pickle all the output data
-        assert(os.system("python my_interactive_parser.py tally pickle -pf interactive_tally.p -tf "+dir_path+"example_tally*.txt")==0)
+        # This uses glob to pickle all the output data with threads
+        assert(os.system("python my_interactive_parser.py tally pickle -nt -1 -pf interactive_tally.p -tf "+dir_path+"example_tally*.txt")==0)
+        # This uses glob to pickle all the output data with 2 threads
+        assert(os.system("python my_interactive_parser.py tally pickle -nt 2 -pf interactive_tally.p -tf "+dir_path+"example_tally*.txt")==0)
         # Test Serial parsing
-        assert(os.system("OPPPY_USE_THREADS=False python my_interactive_parser.py tally pickle -pf interactive_tally.p -tf "+dir_path+"example_tally*.txt")==0)
+        assert(os.system("python my_interactive_parser.py tally pickle -pf interactive_tally.p -tf "+dir_path+"example_tally*.txt")==0)
 
     def test_plot_tally(self):
         # This uses glob to pickle all the output data
@@ -140,8 +153,12 @@ def test_plot_tally(self):
         assert(os.system("python my_interactive_parser.py tally iplot -pf interactive_tally.p < "+dir_path+"interactive_tally_input.txt")==0)
         # parse and plot the output data files
         assert(os.system("python my_interactive_parser.py tally iplot -tf "+dir_path+"example_tally*.txt < "+dir_path+"interactive_tally_input.txt")==0)
+        # parse and plot the output data files with threads
+        assert(os.system("python my_interactive_parser.py tally iplot -nt -1 -tf "+dir_path+"example_tally*.txt < "+dir_path+"interactive_tally_input.txt")==0)
         # parse and plot the output data files
         assert(os.system("python my_interactive_parser.py tally plot -tf "+dir_path+"example_tally*.txt -sk cycle -dn cool_counts -x bins -xlab 'bin [#]'  -y odd_counts  -ylab 'Counts [#]'")==0)
         assert(os.system("python my_interactive_parser.py tally plot -pf interactive_tally.p -sk time -sv 5.0 -dn cool_counts -x bins -xlab 'bin [#]'  -y even_counts  -ylab 'Counts [#]'")==0)
+        # parse and plot the output data files with threads
+        assert(os.system("python my_interactive_parser.py tally plot -nt -1 -tf "+dir_path+"example_tally*.txt -sk cycle -dn cool_counts -x bins -xlab 'bin [#]'  -y odd_counts  -ylab 'Counts [#]'")==0)
         # test scaling and log axis
         assert(os.system("python my_interactive_parser.py tally plot -tf "+dir_path+"example_tally*.txt -tf "+dir_path+"example_tally*.txt -sk cycle -dn cool_counts -x bins -xlab 'bin [#]'  -y odd_counts  -ylab 'Counts  [#]' -lx -ly -sx 10 -sx 1e-3 -sy 10 -sy 1e-3 -xl 0.00001 1000 -yl 0.00001 10000")==0)

tests/test_output.py

@@ -142,3 +142,18 @@ def test_append_output_dictionary(self):
     data2.pop('version')
     assert(data2==gold_data)
 
+    # initialize a new data dictionary
+    data3 = {}
+    data3['version'] = __version__
+
+
+    # Append the thrid output file
+    output_files = [dir_path+"output_example1.txt",dir_path+"output_example2.txt",dir_path+"output_example3.txt"]
+    # Build initial pickle
+    append_output_dictionary(data3, output_files, opppy_parser, nthreads=-1)
+
+    print(data3)
+    # Don't check the version for a match
+    data3.pop('version')
+    assert(data3==gold_data)
+

tests/test_tally.py

@@ -115,3 +115,20 @@ def test_append_tally_dictionary(self):
     data2.pop('version')
     for dic, gold_dic in zip(data2,gold_data):
         assert(dic==gold_dic)
+
+    # initialize a new data dictionary
+    data3 = {}
+    data3['version'] = __version__
+
+
+    # WITH THREADS
+    # Append the thrid tally file
+    tally_files = [dir_path+"example_tally1.txt",dir_path+"example_tally2.txt",dir_path+"example_tally3.txt"]
+    # Build initial pickle
+    append_tally_dictionary(data3, tally_files, opppy_parser, nthreads=-1)
+
+    print(data3)
+    # Don't check the version for a match
+    data3.pop('version')
+    for dic, gold_dic in zip(data3,gold_data):
+        assert(dic==gold_dic)