Fix filter_tabular tool if skip_lines is left unset (galaxyprojec…

…t#6108) * Fix filter_tabular tool if ``skip_lines`` is left unset Fixes https://sentry.galaxyproject.org/share/issue/2fdb36e276fd4e21889ced01da9683da/: ``` NotFound: cannot find 'tbl' File "galaxy/jobs/runners/__init__.py", line 297, in prepare_job job_wrapper.prepare() File "galaxy/jobs/__init__.py", line 1260, in prepare ) = tool_evaluator.build() File "galaxy/tools/evaluation.py", line 586, in build global_tool_logs(self._build_config_files, config_file, "Building Config Files") File "galaxy/tools/evaluation.py", line 98, in global_tool_logs raise e File "galaxy/tools/evaluation.py", line 94, in global_tool_logs return func() File "galaxy/tools/evaluation.py", line 658, in _build_config_files self.__write_workdir_file(config_filename, config_text, param_dict, is_template=is_template) File "galaxy/tools/evaluation.py", line 791, in __write_workdir_file value = fill_template(content, context=context, python_template_version=self.tool.python_template_version) File "galaxy/util/template.py", line 115, in fill_template return fill_template( File "galaxy/util/template.py", line 115, in fill_template return fill_template( File "galaxy/util/template.py", line 115, in fill_template return fill_template( File "galaxy/util/template.py", line 115, in fill_template return fill_template( File "galaxy/util/template.py", line 115, in fill_template return fill_template( File "galaxy/util/template.py", line 115, in fill_template return fill_template( File "galaxy/util/template.py", line 115, in fill_template return fill_template( File "galaxy/util/template.py", line 115, in fill_template return fill_template( File "galaxy/util/template.py", line 115, in fill_template return fill_template( File "galaxy/util/template.py", line 115, in fill_template return fill_template( File "galaxy/util/template.py", line 123, in fill_template raise first_exception or e File "galaxy/util/template.py", line 87, in fill_template return unicodify(t, log_exception=False) File "galaxy/util/__init__.py", line 1183, in unicodify value = str(value) File "Cheetah/Template.py", line 1053, in __unicode__ return getattr(self, mainMethName)() File "cheetah_DynamicallyCompiledCheetahTemplate_1719172856_827265_49661.py", line 100, in respond ``` * Add homepage_url and expect_num_outputs * Fix flake8 * fix regular expressions * improve regexes the colnames enclosed in `"` should not contain `"` * Bump other tools --------- Co-authored-by: Matthias Bernt <[email protected]>
pavanvidem · Jun 27, 2024 · dd35055 · dd35055
1 parent 934f4b6
commit dd35055
Show file tree

Hide file tree

Showing 7 changed files with 51 additions and 39 deletions.
diff --git a/tools/query_tabular/.shed.yml b/tools/query_tabular/.shed.yml
@@ -24,6 +24,7 @@ long_description: |
 
 owner: iuc
 remote_repository_url: https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular
+homepage_url: https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular
 type: unrestricted
 auto_tool_repositories:
   name_template: "{{ tool_id }}"

diff --git a/tools/query_tabular/filter_tabular.xml b/tools/query_tabular/filter_tabular.xml
@@ -1,4 +1,4 @@
-<tool id="filter_tabular" name="Filter Tabular" version="3.3.0">
+<tool id="filter_tabular" name="Filter Tabular" version="3.3.1">
     <description></description>
 
     <macros>
@@ -16,10 +16,9 @@
         <configfile name="filter_json">
 #import json
 #set $dataset_name = $input.element_identifier
+#set $table=$input
 @LINEFILTERS@
-#if $input_filters:
 #echo $json.dumps($input_filters)
-#end if
         </configfile>
     </configfiles>
     <inputs>
@@ -166,6 +165,17 @@
             </repeat>
             <output name="output" file="filtered_IEDB.tsv"/>
         </test>
+        <test>
+            <param name="input" ftype="tabular" value="math_input.tsv"/>
+            <param name="comment_char" value="False"/>
+            <repeat name="linefilters">
+                <conditional name="filter">
+                    <param name="filter_type" value="skip"/>
+                    <param name="skiplines" value=""/>
+                </conditional>
+            </repeat>
+            <output name="output" file="math_input.tsv"/>
+        </test>
     </tests>
     <help><![CDATA[
 ==============

diff --git a/tools/query_tabular/filters.py b/tools/query_tabular/filters.py
@@ -11,7 +11,7 @@ class LineFilter(object):
     def __init__(self, source, filter_dict):
         self.source = source
         self.filter_dict = filter_dict
-        self.func = lambda i, l: l.rstrip('\r\n') if l else None
+        self.func = lambda i, line: line.rstrip('\r\n') if line else None
         self.src_lines = []
         self.src_line_cnt = 0
 
@@ -28,23 +28,23 @@ def xint(x):
         if filter_dict['filter'] == 'regex':
             rgx = re.compile(filter_dict['pattern'])
             if filter_dict['action'] == 'exclude_match':
-                self.func = lambda i, l: l if not rgx.match(l) else None
+                self.func = lambda i, line: line if not rgx.match(line) else None
             elif filter_dict['action'] == 'include_match':
-                self.func = lambda i, l: l if rgx.match(l) else None
+                self.func = lambda i, line: line if rgx.match(line) else None
             elif filter_dict['action'] == 'exclude_find':
-                self.func = lambda i, l: l if not rgx.search(l) else None
+                self.func = lambda i, line: line if not rgx.search(line) else None
             elif filter_dict['action'] == 'include_find':
-                self.func = lambda i, l: l if rgx.search(l) else None
+                self.func = lambda i, line: line if rgx.search(line) else None
         elif filter_dict['filter'] == 'select_columns':
             cols = [int(c) - 1 for c in filter_dict['columns']]
-            self.func = lambda i, l: self.select_columns(l, cols)
+            self.func = lambda i, line: self.select_columns(line, cols)
         elif filter_dict['filter'] == 'select_column_slices':
             cols = [x if isinstance(x, int) else [y if y is not None else None for y in [xint(k) for k in x.split(':')]] for x in [xint(c) for c in filter_dict['columns']]]
             if all([isinstance(x, int) for x in cols]):
-                self.func = lambda i, l: self.select_columns(l, cols)
+                self.func = lambda i, line: self.select_columns(line, cols)
             else:
                 cols = [slice(x[0], x[1], x[2] if len(x) > 2 else None) if isinstance(x, list) else x for x in cols]
-                self.func = lambda i, l: self.select_slices(l, cols)
+                self.func = lambda i, line: self.select_slices(line, cols)
         elif filter_dict['filter'] == 'replace':
             p = filter_dict['pattern']
             r = filter_dict['replace']
@@ -54,32 +54,32 @@ def xint(x):
                                               'append',
                                               'before',
                                               'after']:
-                self.func = lambda i, l: '\t'.join(
+                self.func = lambda i, line: '\t'.join(
                     [x if j != c else re.sub(p, r, x)
-                     for j, x in enumerate(l.split('\t'))])
+                     for j, x in enumerate(line.split('\t'))])
             else:
                 a = 0 if filter_dict['add'] == 'prepend'\
                     else min(0, c - 1) if filter_dict['add'] == 'before'\
                     else c + 1 if filter_dict['add'] == 'after'\
                     else None
-                self.func = lambda i, l: self.replace_add(l, p, r, c, a)
+                self.func = lambda i, line: self.replace_add(line, p, r, c, a)
         elif filter_dict['filter'] == 'prepend_line_num':
-            self.func = lambda i, l: '%d\t%s' % (i, l)
+            self.func = lambda i, line: '%d\t%s' % (i, line)
         elif filter_dict['filter'] == 'append_line_num':
-            self.func = lambda i, l: '%s\t%d' % (l.rstrip('\r\n'), i)
+            self.func = lambda i, line: '%s\t%d' % (line.rstrip('\r\n'), i)
         elif filter_dict['filter'] == 'prepend_text':
             s = filter_dict['column_text']
-            self.func = lambda i, l: '%s\t%s' % (s, l)
+            self.func = lambda i, line: '%s\t%s' % (s, line)
         elif filter_dict['filter'] == 'append_text':
             s = filter_dict['column_text']
-            self.func = lambda i, l: '%s\t%s' % (l.rstrip('\r\n'), s)
+            self.func = lambda i, line: '%s\t%s' % (line.rstrip('\r\n'), s)
         elif filter_dict['filter'] == 'skip':
             cnt = filter_dict['count']
-            self.func = lambda i, l: l if i > cnt else None
+            self.func = lambda i, line: line if i > cnt else None
         elif filter_dict['filter'] == 'normalize':
             cols = [int(c) - 1 for c in filter_dict['columns']]
             sep = filter_dict['separator']
-            self.func = lambda i, l: self.normalize(l, cols, sep)
+            self.func = lambda i, line: self.normalize(line, cols, sep)
 
     def __iter__(self):
         return self

diff --git a/tools/query_tabular/load_db.py b/tools/query_tabular/load_db.py
@@ -328,4 +328,4 @@ def create_index(conn, table_name, index_name, index_columns, unique=False):
         c.close()
     except Exception as e:
         print('Failed: %s err: %s' % (index_def, e), file=sys.stderr)
-        raise(e)
+        raise e
diff --git a/tools/query_tabular/macros.xml b/tools/query_tabular/macros.xml
@@ -8,8 +8,8 @@
       #set $skip_lines = None
       #if str($fi.filter.skip_lines) != '':
         #set $skip_lines = int($fi.filter.skip_lines)
-      #elif $tbl.table.metadata.comment_lines and $tbl.table.metadata.comment_lines > 0:
-        #set $skip_lines = int($tbl.table.metadata.comment_lines)
+      #elif $table.metadata.comment_lines and int($table.metadata.comment_lines) > 0:
+        #set $skip_lines = int($table.metadata.comment_lines)
       #end if
       #if $skip_lines is not None:
         #set $filter_dict = dict()

diff --git a/tools/query_tabular/query_tabular.xml b/tools/query_tabular/query_tabular.xml
@@ -1,4 +1,4 @@
-<tool id="query_tabular" name="Query Tabular" version="3.3.1">
+<tool id="query_tabular" name="Query Tabular" version="3.3.2">
     <description>using sqlite sql</description>
 
     <macros>
@@ -37,6 +37,7 @@ $sqlquery
 #set $jtbls = []
 #set $jtbldef['tables'] = $jtbls
 #for $i,$tbl in enumerate($tables):
+  #set $table = $tbl.table
   #set $jtbl = dict()
   #set $jtbl['file_path'] = str($tbl.table)
   #if $tbl.tbl_opts.table_name:
@@ -137,7 +138,7 @@ $sqlquery
                           You can override the default names by entering a comma -separated list of names, e.g. ',name1,,,name2' would rename the second and fifth columns.
                     </help>
                     <sanitizer sanitize="False"/>
-                    <validator type="regex" message="A List of names separated by commas: Column names should start with a letter and may contain additional letters, digits, and underscores. Otherwise, the name must be eclosed in: double quotes, back quotes, or square brackets.">^([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|[[]\S+[^,"]*[]])?(,([A-Za-z]\w*|"\S+.*"|`\S+[^,`]*`|[[]\S+[^,"]*[]])?)*$</validator>
+                    <validator type="regex" message="A List of names separated by commas: Column names should start with a letter and may contain additional letters, digits, and underscores. Otherwise, the name must be eclosed in: double quotes, back quotes, or square brackets.">^([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|\[\S+[^,"]*\])?(,([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|\[\S+[^,"]*\])?)*$</validator>
                 </param>
                 <param name="load_named_columns" type="boolean" truevalue="load_named_columns" falsevalue="" checked="false" label="Only load the columns you have named into database"/>
                 <param name="pkey_autoincr" type="text" value="" optional="true" label="Add an auto increment primary key column with this name"
@@ -148,7 +149,7 @@ $sqlquery
                     <param name="unique" type="boolean" truevalue="yes" falsevalue="no" checked="False" label="This is a unique index"/>
                     <param name="index_columns" type="text" value="" label="Index on Columns">
                         <help>Create an index on the column names: e.g. for default column names:  c1  or c2,c4 ( use the names you gave for columns)</help>
-                        <validator type="regex" message="Column name, separated by commes if more than one">^([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|[[]\S+[^,"]*[]])(,([A-Za-z]\w*|"\S+.*"|`\S+[^,`]*`|[[]\S+[^,"]*[]])?)*$</validator>
+                        <validator type="regex" message="Column name, separated by commes if more than one">^([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|\[\S+[^,"]*\])(,([A-Za-z]\w*|"\S+[^,"]"|`\S+[^,`]*`|\[\S+[^,"]*\])?)*$</validator>
                         <sanitizer sanitize="False"/>
                     </param>
                 </repeat>
@@ -192,7 +193,7 @@ $sqlquery
     </outputs>
     <tests>
         <!-- Test 1 -->
-        <test>
+        <test expect_num_outputs="1">
             <repeat name="tables">
                 <param name="table" ftype="tabular" value="customers.tsv"/>
                 <section name="input_opts">
@@ -228,7 +229,7 @@ $sqlquery
         </test>
 
         <!-- Test 2 -->
-        <test>
+        <test expect_num_outputs="1">
             <repeat name="tables">
                 <param name="table" ftype="tabular" value="customers.tsv"/>
                 <section name="input_opts">
@@ -259,7 +260,7 @@ $sqlquery
         </test>
 
         <!-- Test 3 -->
-        <test>
+        <test expect_num_outputs="1">
             <repeat name="tables">
                 <param name="table" ftype="tabular" value="customers.tsv"/>
                 <section name="input_opts">
@@ -279,7 +280,7 @@ $sqlquery
         </test>
 
         <!-- Test 4 -->
-        <test>
+        <test expect_num_outputs="1">
             <repeat name="tables">
                 <param name="table" ftype="tabular" value="IEDB.tsv"/>
                 <section name="input_opts">
@@ -315,7 +316,7 @@ $sqlquery
         </test>
 
         <!-- Test 5 -->
-        <test>
+        <test expect_num_outputs="1">
             <section name="add_to_database">
                 <param name="withdb" ftype="sqlite" value="testdb.sqlite"/>
             </section>
@@ -365,7 +366,7 @@ $sqlquery
         </test>
 
         <!-- Test 6 -->
-        <test>
+        <test expect_num_outputs="1">
             <repeat name="tables">
                 <param name="table" ftype="tabular" value="pets.tsv"/>
                 <section name="input_opts">
@@ -446,7 +447,7 @@ $sqlquery
         </test>
 
         <!-- Test 7 -->
-        <test>
+        <test expect_num_outputs="1">
             <repeat name="tables">
                 <param name="table" ftype="tabular" value="psm_report.tsv"/>
                 <section name="input_opts">
@@ -472,7 +473,7 @@ $sqlquery
         </test>
 
         <!-- Test 8 column_names_from_first_line -->
-        <test>
+        <test expect_num_outputs="1">
             <repeat name="tables">
                 <param name="table" ftype="tabular" value="psm_report.tsv"/>
                 <section name="input_opts">
@@ -498,7 +499,7 @@ $sqlquery
         </test>
 
         <!-- Test 9  modify database -->
-        <test>
+        <test expect_num_outputs="2">
             <repeat name="tables">
                 <param name="table" ftype="tabular" value="psm_report.tsv"/>
                 <section name="input_opts">
@@ -539,7 +540,7 @@ $sqlquery
         </test>
 
         <!-- Test 10 column_names_from_first_line -->
-        <test>
+        <test expect_num_outputs="1">
             <repeat name="tables">
                 <param name="table" ftype="tabular" value="netMHC_summary.tsv"/>
                 <section name="input_opts">
@@ -565,7 +566,7 @@ $sqlquery
         </test>
 
         <!-- Test 11 -->
-        <test>
+        <test expect_num_outputs="1">
             <repeat name="tables">
                 <param name="table" ftype="tabular" value="netMHC_summary.tsv"/>
                 <section name="input_opts">
@@ -592,7 +593,7 @@ $sqlquery
         </test>
 
         <!-- Test 12 math functions -->
-        <test>
+        <test expect_num_outputs="1">
             <repeat name="tables">
                 <param name="table" ftype="tabular" value="math_input.tsv"/>
             </repeat>

diff --git a/tools/query_tabular/sqlite_to_tabular.xml b/tools/query_tabular/sqlite_to_tabular.xml
@@ -1,5 +1,5 @@
 <?xml version="1.0"?>
-<tool id="sqlite_to_tabular" name="SQLite to tabular" version="3.2.0">
+<tool id="sqlite_to_tabular" name="SQLite to tabular" version="3.2.1">
     <description>for SQL query</description>
 
     <macros>