Cleanup hidden button handling; enable name sorting in import parties; check for empty method step description; move Missing Value Codes sections up on pages; reduce out-of-memory failures in Check Data Tables (fixes #150, fixes #157, closes #154, closes #153, closes #152)

Author: jon-ide

webapp/config.py.template

@@ -98,4 +98,7 @@ class Config(object):
     COLLABORATION_BETA_TESTERS_ONLY = False
     COLLABORATION_BETA_TESTERS = []
 
-    LOG_FILE_HANDLING_DETAILS = False
+    LOG_FILE_HANDLING_DETAILS = False
+    LOG_MEMORY_USAGE = False
+    LOG_REQUESTS = False
+    LOG_RESPONSES = False

webapp/home/forms.py

@@ -161,6 +161,12 @@ class ImportEMLItemsForm(FlaskForm):
     target = RadioField('Target', choices=[], validators=[])
 
 
+class ImportPartiesFromTemplateForm(FlaskForm):
+    to_import = MultiCheckboxField('Import', choices=[], validators=[])
+    to_import_sorted = MultiCheckboxField('Import', choices=[], validators=[])
+    target = RadioField('Target', choices=[], validators=[])
+
+
 class SelectUserForm(FlaskForm):
     user = RadioField('User', choices=[], validators=[])
 

webapp/home/home_utils.py

@@ -1,13 +1,19 @@
 """ Basic utility functions. """
 
-import sys, daiquiri
+import sys
+import daiquiri
 
 from flask import session
 from flask_login import current_user
 
+import psutil
+from pympler import muppy, summary
+
+from webapp.home.utils.hidden_buttons import is_hidden_button, handle_hidden_buttons
+
 from metapype.model.node import Node
 
-RELEASE_NUMBER = '2023.11.29'
+RELEASE_NUMBER = '2024.01.03'
 
 
 def extract_caller_module_name():
@@ -68,8 +74,106 @@ def log_available_memory():
     """
     Log the available system memory.
     """
-    import psutil
     available_memory = psutil.virtual_memory().available / 1024 / 1024
     process_usage = psutil.Process().memory_info().rss / 1024 / 1024
     log_info(f"Memory usage:   available system memory:{available_memory:.1f} MB   process usage:{process_usage:.1f} MB")
 
+
+# def profile_and_save(func, *args, **kwargs):
+#     # Profile the function and get memory usage
+#     mem_usage = memory_usage((func, args, kwargs))
+#
+#     # Save the results to a file
+#     with open('memory_profile.txt', 'a') as file:
+#         file.write("Memory usage (in MB):\n")
+#         for point in mem_usage:
+#             file.write(f"{point}\n")
+#
+#
+# def profile_and_save_with_return(func, *args, **kwargs):
+#     # Function to wrap the original function and capture its return value
+#     def wrapper():
+#         return func(*args, **kwargs)
+#
+#     # Profile the memory usage of the wrapper function
+#     mem_usage, retval = memory_usage((wrapper,), retval=True, max_usage=True)
+#
+#     # Save the memory usage data to a file
+#     with open('memory_profile.txt', 'w') as file:
+#         file.write("Memory usage (in MB):\n")
+#         file.write(f"{mem_usage}\n")
+#
+#     # Return the original function's return value
+#     return retval
+
+
+def log_profile_details(all_objects_before, all_objects_after):
+    from pympler import asizeof
+
+    ids_before = {id(obj) for obj in all_objects_before}
+    ids_after = {id(obj) for obj in all_objects_after}
+
+    # Find new object IDs
+    new_ids = ids_after - ids_before
+
+    # Retrieve new objects
+    new_objects = [obj for obj in all_objects_after if id(obj) in new_ids]
+
+    # Optionally, filter by type (e.g., list)
+    new_lists = [obj for obj in new_objects if isinstance(obj, list)]
+
+    # Sort by size and save the results to a file
+    # original_stdout = sys.stdout
+    with open('memory_profile.txt', 'a') as file:
+        sys.stdout = file
+        file.write("*********** Details ***********\n")
+        obj = new_lists[0]
+        print(f"List size: {asizeof.asizeof(obj)} bytes, Length: {len(obj)}, Example content: {str(obj[:10])}...")
+
+        for obj in sorted(new_lists, key=lambda x: asizeof.asizeof(x), reverse=True)[:5]:
+            print(f"List size: {asizeof.asizeof(obj)} bytes, Length: {len(obj)}, Example content: {str(obj[:10])}...")
+        file.write("*********** End of details ***********\n")
+        # sys.stdout = original_stdout
+
+
+def profile_and_save(func, *args, **kwargs):
+    # Profile the function and get memory usage
+
+    # Start tracking memory
+    all_objects_before = muppy.get_objects()
+    summary_1 = summary.summarize(all_objects_before)
+
+    # Execute the function
+    retval = func(*args, **kwargs)
+
+    # Check memory after the function execution
+    all_objects_after = muppy.get_objects()
+    summary_2 = summary.summarize(all_objects_after)
+
+    # Compare before and after snapshots
+    diff = summary.get_diff(summary_1, summary_2)
+
+    original_stdout = sys.stdout
+
+    try:
+        # Save the results to a file
+        with open('memory_profile.txt', 'a') as file:
+            sys.stdout = file
+            file.write(f"*********** Summary of memory usage: {func_name} ***********\n")
+            summary.print_(diff)
+            file.write("*********** End of summary ***********\n")
+
+        # Log details about the new objects
+        # log_profile_details(all_objects_before, all_objects_after)
+    except Exception as e:
+        log_error(f"Error writing memory profile: {e}")
+        raise
+    finally:
+        sys.stdout = original_stdout
+
+    return retval
+
+
+
+
+

webapp/home/templates/base.html

@@ -11,7 +11,7 @@
 {% import '_macros.html' as macros %}
 
 {# The following must agree with RELEASE_NUMBER in home_utils.py #}
-{% set release_number = '2023.11.29' %}
+{% set release_number = '2024.01.03' %}
 {% set optional = 'Black' %}
 
 {% block head %}

webapp/home/templates/import_parties_2.html

@@ -6,7 +6,7 @@
     {% set help_import_responsible_parties_2_btn = help_import_responsible_parties_2_id ~ '_btn' %}
     {% set help_import_responsible_parties_2_dialog = help_import_responsible_parties_2_id ~ '_dialog' %}
 
-    <body onload="setTarget('{{ target }}');">
+    <body onload="setTarget('{{ target }};');hideSorted(true);">
     <script>
         function setTarget(target) {
             let target_list = document.getElementsByName('target');
@@ -16,6 +16,19 @@
                 }
             }
         }
+        function hideSorted(val) {
+            let unsorted = document.getElementById('to_import');
+            let sorted = document.getElementById('to_import_sorted');
+            if (sorted) {
+                if (val) {
+                    unsorted.style.display = 'block';
+                    sorted.style.display = 'none';
+                } else {
+                    unsorted.style.display = 'none';
+                    sorted.style.display = 'block';
+                }
+            }
+        }
     </script>
 
     <table>
@@ -27,7 +40,78 @@
             <form method="POST" action="" class="form" role="form">
                 {{ form.csrf_token }}
                 <table>
-                    {{ macros.import_selection(form, source_filename, "responsible parties") }}
+
+    <script>
+        function choose_options(all) {
+            $('input:checkbox').not(this).prop('checked', all);
+        }
+        function reconcile_checkbox_statuses(sorted) {
+            // sorted is a boolean indicating whether the sorted list is the one displayed, i.e., the one whose
+            // checkbox statuses should be copied to the other list.
+            let from_items = null;
+            let to_items = null;
+            if (sorted) {
+                from_items = Array.from(document.getElementById('to_import_sorted').getElementsByTagName('li'));
+                to_items = Array.from(document.getElementById('to_import').getElementsByTagName('li'));
+            } else {
+                from_items = Array.from(document.getElementById('to_import').getElementsByTagName('li'));
+                to_items = Array.from(document.getElementById('to_import_sorted').getElementsByTagName('li'));
+            }
+            from_items.forEach(function(item, index) {
+                let checkbox = item.querySelector('input[type="checkbox"]');
+                checked = from_items[index].querySelector('input[type="checkbox"]').checked;
+                default_value = checkbox.defaultValue;
+                // Find the item in the other list
+                let other_item = to_items.find(function(element) {
+                    return element.querySelector('input[type="checkbox"]').defaultValue == default_value;
+                });
+                if (other_item) {
+                    other_item.querySelector('input[type="checkbox"]').checked = checked;
+                }
+            });
+        }
+        function toggle_order() {
+            let sorted = document.getElementById('sorted');
+
+            let to_import = document.getElementById('to_import');
+            let to_import_sorted = document.getElementById('to_import_sorted');
+            let items = null;
+
+            reconcile_checkbox_statuses(sorted.value == 'true');
+
+            if (sorted.value == 'false') {
+                items = Array.from(to_import.getElementsByTagName('li'));
+                to_import.style.display = 'none';
+                to_import_sorted.style.display = 'block';
+                sorted.value = 'true';
+            } else {
+                items = Array.from(to_import_sorted.getElementsByTagName('li'));
+                to_import.style.display = 'block';
+                to_import_sorted.style.display = 'none';
+                sorted.value = 'false';
+            }
+        }
+        function sort_state() {
+            let sorted = document.getElementById('sorted');
+            return sorted.value;
+        }
+    </script>
+                    <tr><h5>Select responsible parties to import from "{{ source_filename }}":
+                &nbsp;&nbsp;&nbsp;&nbsp;[&nbsp;<button class="link" type="button" onclick="choose_options(true); return false;">Select All</button>&nbsp;]
+                &nbsp;&nbsp;&nbsp;&nbsp;[&nbsp;<button class="link" type="button" onclick="choose_options(false); return false;">Clear All</button>&nbsp;]
+                &nbsp;&nbsp;&nbsp;&nbsp;[&nbsp;<button class="link" type="button" onclick="toggle_order(); return false;">Toggle Sort</button>&nbsp;]
+                        <input type="hidden" id="sorted" value="false" >
+                    </h5>
+                    </tr>
+                    {% if form.to_import.choices %}
+                        <tr>{{ form.to_import(style="list-style:none;") }}</tr>
+                        {% if form.to_import_sorted.choices %}
+                            <tr>{{ form.to_import_sorted(style="list-style:none;") }}</tr>
+                        {% endif %}
+                    {% else %}
+                    <tr><span style="font-style: italic;">&nbsp;&nbsp;"{{ target_filename }}" contains no {{ item_name }}.<p></p></span><br></tr>
+                    {% endif %}
+
                     {% if form.to_import.choices %}
                     <tr><h5>Import as:</h5></tr>
                     <tr>{{ form.target(style="list-style:none;") }}</tr>

webapp/home/utils/hidden_buttons.py

@@ -13,7 +13,10 @@
 so the Title method can save the title data and then go to the Open page.
 """
 
-from flask import request
+from flask import request, redirect, url_for
+from flask_login import current_user
+
+from functools import wraps
 
 from webapp.buttons import *
 from webapp.pages import *
@@ -106,4 +109,20 @@ def check_val_for_hidden_buttons(val, target_page):
     for button in HIDDEN_TARGETS:
         if val == button:
             return HIDDEN_TARGETS[button]
-    return target_page
+    return target_page
+
+
+def non_saving_hidden_buttons_decorator(func):
+    """
+    Decorator to check for hidden buttons and handle them for routes that do not save changes to the document,
+    i.e., routes that don't involve the user filling out a form whose contents need to be saved.
+    For routes that do save changes, we need to go to the route and save the changes before handling the hidden
+    buttons.
+    """
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        if is_hidden_button():
+            current_document = current_user.get_filename()
+            return redirect(url_for(handle_hidden_buttons(), filename=current_document))
+        return func(*args, **kwargs)
+    return wrapper

webapp/home/utils/import_nodes.py

@@ -300,15 +300,15 @@ def import_related_project_nodes(target_package, node_ids_to_import):
     load_and_save.save_both_formats(target_package, target_eml_node)
 
 
-def compose_rp_label(rp_node:Node=None):
+def compose_rp_label(rp_node:Node=None, last_name_first:bool=False):
     """
     Compose a label for a responsible party node. The label is a string that can be displayed to the user.
 
     What we display for a responsible party depends on the type of responsible party, i.e. whether it is an individual,
     organization, or position. We also display the role, if any.
     """
 
-    def compose_individual_name_label(rp_node: Node = None):
+    def compose_individual_name_label(rp_node: Node = None, last_name_first: bool = False):
         label = ''
         if rp_node:
             salutation_nodes = rp_node.find_all_children(names.SALUTATION)
@@ -317,17 +317,22 @@ def compose_individual_name_label(rp_node: Node = None):
                     if salutation_node and salutation_node.content:
                         label = label + " " + salutation_node.content
 
+            given_name = ''
             given_name_nodes = rp_node.find_all_children(names.GIVENNAME)
             if given_name_nodes:
                 for given_name_node in given_name_nodes:
                     if given_name_node and given_name_node.content:
-                        label = label + " " + given_name_node.content
+                        given_name = given_name + " " + given_name_node.content
 
+            surname = ''
             surname_node = rp_node.find_child(names.SURNAME)
             if surname_node and surname_node.content:
-                label = label + " " + surname_node.content
+                surname = surname_node.content
 
-        return label
+            if last_name_first:
+                return surname + "," + given_name
+            else:
+                return given_name + " " + surname
 
     def compose_simple_label(rp_node: Node = None, child_node_name: str = ''):
         label = ''
@@ -341,7 +346,7 @@ def compose_simple_label(rp_node: Node = None, child_node_name: str = ''):
     if rp_node:
         individual_name_node = rp_node.find_child(names.INDIVIDUALNAME)
         individual_name_label = (
-            compose_individual_name_label(individual_name_node))
+            compose_individual_name_label(individual_name_node, last_name_first))
         role_node = rp_node.find_child(names.ROLE)
         if role_node:
             role_label = role_node.content