Merge pull request #1601 from DDMAL/staging

Merge staging into production, 14 Aug 2024

.github/workflows/broken-link-checker.yaml

@@ -31,7 +31,7 @@ jobs:
         id: lychee
         uses: lycheeverse/[email protected]
         with:
-          args: --exclude http:\/\/cantus\.sk.* ${{ matrix.links }}
+          args: --exclude http:\/\/cantus\.sk.* --exclude https:\/\/us06web\.zoom\.us* ${{ matrix.links }}
           format: json
           output: /tmp/link-checker-output.txt
       - name: Curating Link Checker Output

.github/workflows/django_tests.yml

@@ -24,6 +24,6 @@ jobs:
                 envkey_AWS_EMAIL_HOST_PASSWORD: test_password
                 directory: config/envs
                 file_name: dev_env
-            - run: docker compose -f docker-compose-development.yml build
-            - run: docker compose -f docker-compose-development.yml up -d
-            - run: docker compose -f docker-compose-development.yml exec -T django python manage.py test main_app.tests
+            - run: docker compose -f docker-compose-test-runner.yml build
+            - run: docker compose -f docker-compose-test-runner.yml up -d
+            - run: docker compose -f docker-compose-test-runner.yml exec -T django python manage.py test main_app.tests

config/nginx/conf.d/cantusdb.conf.development

@@ -10,7 +10,7 @@ server {
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header Host $host;
         proxy_redirect off;
-        proxy_intercept_errors on;
+        proxy_intercept_errors off;
     }
 
     location /static {

cron/postgres/db_backup.sh

@@ -19,7 +19,7 @@ BACKUP_FILENAME=$(date "+%Y-%m-%dT%H:%M:%S").sql.gz                 # This is th
 
 
 # Create the backup and copy it to the daily backup directory
-mkdir -p $BACKUP_DIR/daily $BACKUP_DIR/weekly $BACKUP_DIR/monthly $BACKUP_DIR/yearly
+mkdir -p $BACKUP_DIR/daily $BACKUP_DIR/weekly $BACKUP_DIR/monthly $BACKUP_DIR/yearly $BACKUP_DIR/rism
 /usr/bin/docker exec cantusdb-postgres-1 /usr/local/bin/postgres_backup.sh $BACKUP_FILENAME
 /usr/bin/docker cp cantusdb-postgres-1:/var/lib/postgresql/backups/$BACKUP_FILENAME $BACKUP_DIR/daily
 /usr/bin/docker exec cantusdb-postgres-1 rm /var/lib/postgresql/backups/$BACKUP_FILENAME
@@ -34,10 +34,14 @@ MONTH_OF_YEAR=$(date "+%m")
 
 # Retain weekly backups on Mondays
 # Manage retention of weekly backups
+# Copy the partial export for RISM purposes (created weekly on Mondays)
 if [ $DAY_OF_WEEK -eq 1 ]; then
     cp $BACKUP_DIR/daily/$BACKUP_FILENAME $BACKUP_DIR/weekly
     FILES_TO_REMOVE=$(ls -td $BACKUP_DIR/weekly/* | tail -n +9)
     [[ ! -z "$FILES_TO_REMOVE" ]] && rm $FILES_TO_REMOVE
+    # Copy the partial DB backup for RISM purposes to $BACKUP_DIR/rism
+    /usr/bin/docker cp cantusdb-postgres-1:/var/lib/postgresql/backups/cantusdb_rism_export.sql.gz $BACKUP_DIR/rism
+    /usr/bin/docker exec cantusdb-postgres-1 rm /var/lib/postgresql/backups/cantusdb_rism_export.sql.gz
 fi
 
 # Retain a monthly backup on the first day of the month

django/cantusdb_project/cantusdb/settings.py

@@ -208,5 +208,6 @@
 
 if DEBUG:
     INSTALLED_APPS.append("debug_toolbar")
+    INSTALLED_APPS.append("django_extensions")
     # debug toolbar must be inserted as early in the middleware as possible
     MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware")

django/cantusdb_project/cantusindex.py

@@ -3,131 +3,117 @@
 Cantus Index's (CI's) various APIs.
 """
 
-import requests
-from typing import Optional, Union, Callable
-from main_app.models import Genre
 import json
+from typing import Optional, Union, Callable, TypedDict, Any
+
+import requests
 from requests.exceptions import SSLError, Timeout, HTTPError
 
+from main_app.models import Genre
+
 CANTUS_INDEX_DOMAIN: str = "https://cantusindex.uwaterloo.ca"
 OLD_CANTUS_INDEX_DOMAIN: str = "https://cantusindex.org"
 DEFAULT_TIMEOUT: float = 2  # seconds
-NUMBER_OF_SUGGESTED_CHANTS: int = 3  # this number can't be too large,
-# since for each suggested chant, we make a request to Cantus Index.
-# We haven't yet parallelized this process, so setting this number
-# too high will cause the Chant Create page to take a very long time
-# to load. If/when we parallelize this process, we want to limit
-# the size of the burst of requests sent to CantusIndex.
+NUMBER_OF_SUGGESTED_CHANTS: int = 5  # default number of suggested chants to return
+# with the get_suggested_chants function
+
+
+class SuggestedChant(TypedDict):
+    """
+    Dictionary containing information required for
+    the suggested chants feature on the Chant Create form.
+    """
+
+    cantus_id: str
+    occurrences: int
+    fulltext: Optional[str]
+    genre_name: Optional[str]
+    genre_id: Optional[int]
 
 
 def get_suggested_chants(
     cantus_id: str, number_of_suggestions: int = NUMBER_OF_SUGGESTED_CHANTS
-) -> Optional[list[dict]]:
+) -> Optional[list[SuggestedChant]]:
+    """
+    Given a Cantus ID, query Cantus Index's /nextchants API for a list of
+    Cantus IDs that follow the given Cantus ID in existing manuscripts.
+    Sort the list by the number of occurrences of each Cantus ID, and return
+    a list of dictionaries containing information about the suggested Cantus IDs
+    with the highest number of occurrences.
+
+    Args:
+        cantus_id (str): a Cantus ID
+        number_of_suggestions (int): the number of suggested Cantus IDs to return
+
+    Returns:
+        Optional[list[dict]]: A list of dictionaries, each containing information
+        about a suggested Cantus ID:
+            - "cantus_id": the suggested Cantus ID
+            - "occurrences": the number of times the suggested Cantus ID follows
+                the given Cantus ID in existing manuscripts
+            - "fulltext": the full text of the suggested Cantus ID
+            - "genre_name": the genre of the suggested Cantus ID
+            - "genre_id": the ID of the genre of the suggested Cantus ID
+            If no suggestions are available, returns None.
+    """
     endpoint_path: str = f"/json-nextchants/{cantus_id}"
-    all_suggestions: Union[list, dict, None] = get_json_from_ci_api(endpoint_path)
+    all_suggestions = get_json_from_ci_api(endpoint_path)
 
-    if not isinstance(all_suggestions, list):
-        # get_json_from_ci_api timed out
-        # or CI returned a response with no suggestions.
+    if all_suggestions is None:
         return None
 
-    # when Cantus ID doesn't exist within CI, CI's api returns a 200 response with `['Cantus ID is not valid']`
+    # when Cantus ID doesn't exist within CI, CI's api returns a
+    # 200 response with `['Cantus ID is not valid']`
     first_suggestion = all_suggestions[0]
     if not isinstance(first_suggestion, dict):
         return None
 
-    sort_by_occurrences: Callable[[dict], int] = lambda suggestion: int(
+    sort_by_occurrences: Callable[[dict[Any, Any]], int] = lambda suggestion: int(
         suggestion["count"]
     )
-    sorted_suggestions: list = sorted(
+    sorted_suggestions: list[dict[Any, Any]] = sorted(
         all_suggestions, key=sort_by_occurrences, reverse=True
     )
-    trimmed_suggestions: list = sorted_suggestions[:number_of_suggestions]
+    trimmed_suggestions = sorted_suggestions[:number_of_suggestions]
 
-    suggested_chants: list[Optional[dict]] = []
+    suggested_chants: list[SuggestedChant] = []
     for suggestion in trimmed_suggestions:
-        cantus_id: str = suggestion["cid"]
-        occurrences: int = int(suggestion["count"])
-        suggested_chants.append(get_suggested_chant(cantus_id, occurrences))
-
-    # filter out Cantus IDs where get_suggested_chant timed out
-    filtered_suggestions: list[dict] = [
-        sugg for sugg in suggested_chants if sugg is not None
-    ]
-
-    return filtered_suggestions
-
-
-def get_suggested_chant(
-    cantus_id: str, occurrences: int, timeout: float = DEFAULT_TIMEOUT
-) -> Optional[dict]:
-    """Given a Cantus ID and a number of occurrences, query one of Cantus Index's
-    APIs for information on that Cantus ID and return a dictionary
-    containing a full text, an incipit, the ID of that Cantus ID's genre, and
-    the number of occurrences for that Cantus ID
-
-    (Number of occurrences: this function is used on the Chant Create page,
-    to suggest Cantus IDs of chants that might follow a chant with the Cantus ID
-    of the most recently created chant within the current source. Number of occurrences
-    is provided by Cantus Index's /nextchants API, based on which chants follow which
-    other chants in existing manuscripts)
-
-    Args:
-        cantus_id (str): a Cantus ID
-        occurrences (int): the number of times chants with this Cantus ID follow chants
-            with the Cantus ID of the most recently created chant.
-
-    Returns:
-        Optional[dict]: A dictionary with the following keys:
-            - "cantus_id"
-            - "occurrences"
-            - "fulltext"
-            - "incipit"
-            - "genre_id"
-            ...but if get_json_from_ci_api timed out, returns None instead
-    """
-    endpoint_path: str = f"/json-cid/{cantus_id}"
-    json: Union[dict, list, None] = get_json_from_ci_api(endpoint_path, timeout=timeout)
-
-    if not isinstance(json, dict):
-        # mostly, in case of a timeout within get_json_from_ci_api
-        return None
+        sugg_cantus_id = suggestion["cid"]
+        occurences = int(suggestion["count"])
+        suggestion_info = suggestion.get("info")
+        if suggestion_info:
+            fulltext = suggestion_info.get("field_full_text")
+            genre_name = suggestion_info.get("field_genre")
+        else:
+            fulltext = None
+            genre_name = None
+        try:
+            genre_id = Genre.objects.get(name=genre_name).id
+        except Genre.DoesNotExist:
+            genre_id = None
+        suggested_chants.append(
+            {
+                "cantus_id": sugg_cantus_id,
+                "occurrences": occurences,
+                "fulltext": fulltext,
+                "genre_name": genre_name,
+                "genre_id": genre_id,
+            }
+        )
 
-    try:
-        fulltext: str = json["info"]["field_full_text"]
-        incipit: str = " ".join(fulltext.split(" ")[:5])
-        genre_name: str = json["info"]["field_genre"]
-    except TypeError:
-        return None
-    genre_id: Optional[int] = None
-    try:
-        genre_id = Genre.objects.get(name=genre_name).id
-    except Genre.DoesNotExist:
-        pass
-
-    clean_cantus_id = cantus_id.replace(".", "d").replace(":", "c")
-    #                                        "d"ot             "c"olon
-    return {
-        "cantus_id": cantus_id,
-        "occurrences": occurrences,
-        "fulltext": fulltext,
-        "incipit": incipit,
-        "genre_name": genre_name,
-        "genre_id": genre_id,
-        "clean_cantus_id": clean_cantus_id,
-    }
+    return suggested_chants
 
 
 def get_suggested_fulltext(cantus_id: str) -> Optional[str]:
     endpoint_path: str = f"/json-cid/{cantus_id}"
-    json: Union[dict, list, None] = get_json_from_ci_api(endpoint_path)
+    json_response: Union[dict, list, None] = get_json_from_ci_api(endpoint_path)
 
-    if not isinstance(json, dict):
+    if not isinstance(json_response, dict):
         # mostly, in case of a timeout within get_json_from_ci_api
         return None
 
     try:
-        suggested_fulltext = json["info"]["field_full_text"]
+        suggested_fulltext = json_response["info"]["field_full_text"]
     except KeyError:
         return None
 
@@ -205,7 +191,7 @@ def get_ci_text_search(search_term: str) -> Optional[list[Optional[dict]]]:
 
 def get_json_from_ci_api(
     path: str, timeout: float = DEFAULT_TIMEOUT
-) -> Union[dict, list, None]:
+) -> Union[dict[Any, Any], list[Any], None]:
     """Given a path, send a request to Cantus Index at that path,
     decode the response to remove its Byte Order Marker, parse it,
     and return it as a dictionary or list.
@@ -219,7 +205,7 @@ def get_json_from_ci_api(
         Union[dict, list, None]:
             If the JSON returned from Cantus Index is a JSON object, returns a dict.
             If the JSON returned is a JSON array, returns a list.
-            In case the request times out, returns None.
+            If the request times out, or other types are returned, returns None.
     """
 
     if not path.startswith("/"):
@@ -241,4 +227,9 @@ def get_json_from_ci_api(
         # there are no suggested chants
         return None
 
-    return response.json()
+    parsed_response = response.json()
+
+    if not isinstance(parsed_response, (dict, list)):
+        return None
+
+    return parsed_response

django/cantusdb_project/main_app/admin/__init__.py

@@ -4,7 +4,7 @@
 from main_app.admin.feast import FeastAdmin
 from main_app.admin.genre import GenreAdmin
 from main_app.admin.notation import NotationAdmin
-from main_app.admin.office import OfficeAdmin
+from main_app.admin.service import ServiceAdmin
 from main_app.admin.provenance import ProvenanceAdmin
 from main_app.admin.segment import SegmentAdmin
 from main_app.admin.sequence import SequenceAdmin

django/cantusdb_project/main_app/admin/chant.py

@@ -1,17 +1,34 @@
 from django.contrib import admin
 
 from main_app.admin.base_admin import EXCLUDE, READ_ONLY, BaseModelAdmin
+from main_app.admin.filters import InputFilter
 from main_app.forms import AdminChantForm
 from main_app.models import Chant
 
 
+class SourceKeyFilter(InputFilter):
+    parameter_name = "source_id"
+    title = "Source ID"
+
+    def queryset(self, request, queryset):
+        if self.value():
+            return queryset.filter(source_id=self.value())
+
+
 @admin.register(Chant)
 class ChantAdmin(BaseModelAdmin):
 
+    def get_queryset(self, request):
+        return (
+            super()
+            .get_queryset(request)
+            .select_related("source__holding_institution", "genre", "service")
+        )
+
     @admin.display(description="Source Siglum")
     def get_source_siglum(self, obj):
         if obj.source:
-            return obj.source.siglum
+            return obj.source.short_heading
 
     list_display = (
         "incipit",
@@ -23,13 +40,15 @@ def get_source_siglum(self, obj):
         "incipit",
         "cantus_id",
         "id",
+        "source__holding_institution__siglum",
     )
 
     readonly_fields = READ_ONLY + ("incipit",)
 
     list_filter = (
+        SourceKeyFilter,
         "genre",
-        "office",
+        "service",
     )
     exclude = EXCLUDE + (
         "col1",
@@ -50,4 +69,4 @@ def get_source_siglum(self, obj):
         "source",
         "feast",
     )
-    ordering = ("source__siglum",)
+    ordering = ("source__holding_institution__siglum", "source__shelfmark")

django/cantusdb_project/main_app/admin/filters.py

@@ -0,0 +1,17 @@
+from django.contrib.admin import SimpleListFilter
+
+
+class InputFilter(SimpleListFilter):
+    template = "admin/input_filter.html"
+
+    def lookups(self, request, model_admin):
+        return ((),)
+
+    def choices(self, changelist):
+        all_choice = next(super().choices(changelist))
+        all_choice["query_parts"] = (
+            (key, value)
+            for key, value in changelist.get_filters_params().items()
+            if key != self.parameter_name
+        )
+        yield all_choice