Core 1190 ephemeral books (#651)

* Add support for ephemeral books

* Add ancillaries consumer

* Small fix for not showing corrupted build artifacts

This shouldn't happen for real, but it does happen when running integration tests, so let's fix it!

* Filter ABL on REX consumer by default

* Remove extra print statement

Oops

* Use more specific prefix for identifying super documents

* Ensure build artifacts frame does not get too large

Author: TylerZeroMaster

backend/app/app/api/endpoints/abl.py

@@ -26,6 +26,10 @@ async def get_abl_info(
     version: Optional[str] = None,
     code_version: Optional[str] = None,
 ):
+    if consumer is None:
+        consumer = "REX"
+    elif consumer == "ANY":
+        consumer = None
     return abl_service.get_abl_info_database(
         db, consumer, repo_name, version, code_version
     )

backend/app/app/service/abl.py

@@ -159,8 +159,8 @@ def update_versions_by_consumer(
 
 
 def guess_consumer(book_slug: str) -> str:
-    if book_slug.endswith("-ancillary") or book_slug.endswith("-ancillaries"):
-        return "ancillary"
+    if book_slug.startswith("super--"):
+        return "ancillaries"
     return "REX"
 
 

backend/app/app/service/jobs.py

@@ -2,7 +2,7 @@
 import logging
 from datetime import datetime
 from typing import Dict, Generator, List, Optional, Union, cast
-from uuid import UUID
+from uuid import NAMESPACE_OID, UUID, uuid5
 
 from lxml import etree
 from sqlalchemy.exc import IntegrityError
@@ -75,6 +75,20 @@ def add_books_to_commit(
         db.add(db_book)
 
 
+def get_or_add_book(db: Session, book: Book):
+    existing_book = cast(
+        Optional[Book],
+        db.query(Book)
+        .filter(Book.uuid == book.uuid, Book.commit_id == book.commit_id)
+        .first(),
+    )
+    if existing_book is not None:
+        return existing_book
+    db.add(book)
+    db.flush()
+    return book
+
+
 def add_books_to_job(
     db: Session,
     job: JobSchema,
@@ -190,11 +204,38 @@ async def insert_job():
 
     def update(self, db_session: Session, job: JobSchema, job_in: JobUpdate):
         if isinstance(job_in.artifact_urls, list):
-            book_job_by_book_slug = {b.book.slug: b for b in job.books}
-            for artifact_url in job_in.artifact_urls:
-                book_job_by_book_slug[
-                    artifact_url.slug
-                ].artifact_url = artifact_url.url
+            artifact_url_by_book_slug = {
+                art.slug: art.url for art in job_in.artifact_urls
+            }
+            job_books = {b.book.slug for b in job.books}
+            # books that are created during the build
+            ephemeral_book_slugs = {
+                artifact_url.slug
+                for artifact_url in job_in.artifact_urls
+                if artifact_url.slug not in job_books
+            }
+            if ephemeral_book_slugs:
+                for book_slug in ephemeral_book_slugs:
+                    book_uuid = str(uuid5(NAMESPACE_OID, book_slug))
+                    style = (
+                        "super"
+                        if book_slug.startswith("super--")
+                        else "unknown"
+                    )
+                    book = Book(
+                        uuid=book_uuid,
+                        commit_id=job.books[0].book.commit_id,
+                        edition=0,
+                        slug=book_slug,
+                        style=style,
+                    )
+                    book = get_or_add_book(db_session, book)
+                    add_books_to_job(db_session, job, [book])
+                db_session.flush()
+            for book_job in job.books:
+                artifact_url = artifact_url_by_book_slug.get(book_job.book.slug)
+                if artifact_url:
+                    book_job.artifact_url = artifact_url
         elif job_in.artifact_urls is not None:
             job.books[0].artifact_url = job_in.artifact_urls
         return super().update(db_session, job, job_in, JobUpdate)

backend/app/migrations/versions/a853887ac719_add_ancillaries_consumer.py

@@ -0,0 +1,60 @@
+"""add ancillaries consumer
+
+Revision ID: a853887ac719
+Revises: aa75305665c6
+Create Date: 2025-09-03 19:22:17.308206
+
+"""
+
+from datetime import datetime, timezone
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a853887ac719"
+down_revision = "aa75305665c6"
+branch_labels = None
+depends_on = None
+
+
+utcnow = datetime.now(timezone.utc)
+consumer_table = sa.table(
+    "consumer",
+    sa.column("id", sa.Integer),
+    sa.column("name", sa.String),
+    sa.column("created_at", sa.DateTime),
+    sa.column("updated_at", sa.DateTime),
+)
+approved_book_table = sa.table(
+    "approved_book",
+    sa.column("book_id", sa.Integer()),
+    sa.column("consumer_id", sa.Integer()),
+    sa.column("code_version_id", sa.Integer()),
+    sa.column("created_at", sa.DateTime()),
+    sa.column("updated_at", sa.DateTime()),
+)
+
+
+new_consumers = [
+    {"id": 2, "name": "ancillaries", "created_at": utcnow, "updated_at": utcnow}
+]
+
+
+def upgrade():
+    bind = op.get_bind()
+    insert = consumer_table.insert().values(new_consumers)
+    bind.execute(insert)
+
+
+def downgrade():
+    bind = op.get_bind()
+    new_consumer_ids = {c["id"] for c in new_consumers}
+    delete_abl_entries = approved_book_table.delete().where(
+        approved_book_table.c.consumer_id.in_(new_consumer_ids)
+    )
+    bind.execute(delete_abl_entries)
+    delete = consumer_table.delete().where(
+        consumer_table.c.id.in_(new_consumer_ids)
+    )
+    bind.execute(delete)

backend/app/tests/integration/api/endpoints/test_jobs.py

@@ -95,3 +95,55 @@ def test_jobs_cru(
 
     # AND: We get the error as a string
     assert error_text == error_text_from_server
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize(
+    "status_id, job_type_id, version, repo_name, repo_owner, book_name",
+    [
+        ("1", "4", None, "tiny-book", "openstax", "book-slug1"),
+        ("1", "4", None, "tiny-book", "openstax", "book-slug1"),
+    ],
+)
+def test_ephemeral_book(
+    testclient,
+    api_url,
+    status_id,
+    job_type_id,
+    version,
+    repo_owner,
+    repo_name,
+    book_name,
+):
+    # GIVEN: An api url to the jobs endpoint
+    # AND: Data for job is ready to be submitted.
+    data = {
+        "status_id": status_id,
+        "job_type_id": job_type_id,
+        "version": version,
+        "repository": {"name": repo_name, "owner": repo_owner},
+        "book": book_name,
+    }
+
+    # WHEN: A POST request is made to the url with data
+    response = testclient.post(f"{api_url}/{ENDPOINT}/", json=data)
+
+    # THEN: A 200 code is returned
+    assert response.status_code == 200
+    job = response.json()
+    job_id = job["id"]
+
+    # WHEN: A PUT request is made to the url with data
+    # In the first case the book is newly added, it is reused in the second
+    data = {
+        "status_id": "5",
+        "artifact_urls": [
+            {"slug": "super-cool-book", "url": "http://example.com"}
+        ],
+        "worker_version": "dev",
+    }
+    job_update_response = testclient.put(
+        f"{api_url}/{ENDPOINT}/{job_id}", json=data
+    )
+    # THEN: A 200 code is returned
+    assert job_update_response.status_code == 200

frontend/specs/abl.spec.ts

@@ -194,7 +194,7 @@ describe("fetchABL", () => {
     const url = fetchSpy.mock.lastCall?.[0];
     expect(fetchSpy).toHaveBeenCalledTimes(1);
     expect(errors[0]).toBeUndefined();
-    expect(url).toBe("/api/abl/");
+    expect(url).toBe("/api/abl/?consumer=ANY");
     expect(value).toStrictEqual([]);
   });
   it("sends errors to error store", async () => {

frontend/src/components/BuildArtifacts.svelte

@@ -7,7 +7,7 @@
 <div id="build-artifacts-frame">
   <h3>Build Artifacts</h3>
   <ul>
-    {#each selectedJob.artifact_urls as artifact}
+    {#each selectedJob.artifact_urls.filter(art => art.slug && art.url) as artifact}
       <li>
         <a href={artifact.url} target="_blank" rel="noreferrer"
           >{artifact.slug}</a
@@ -22,6 +22,8 @@
     padding: 10px;
     margin: 10px;
     background-color: rgba(255, 128, 0, 0.25);
+    max-height: 15rem;
+    overflow: scroll;
   }
 
   #build-artifacts-frame h3:nth-of-type(1) {

frontend/src/ts/abl.ts

@@ -68,7 +68,7 @@ export function getLatestCodeVersionForJob(
 export async function fetchABL(): Promise<ApprovedBookWithDate[]> {
   let abl: ApprovedBookWithDate[];
   try {
-    abl = await RequireAuth.fetchJson("/api/abl/");
+    abl = await RequireAuth.fetchJson("/api/abl/?consumer=ANY");
   } catch (error) {
     handleError(error);
     abl = [];