LinkedDataHub v5 WIP

.github/workflows/http-tests.yml

@@ -7,37 +7,45 @@ jobs:
     name: Build Docker image and run HTTP test suite against it
     runs-on: ubuntu-latest
     env:
-      ASF_ARCHIVE: http://archive.apache.org/dist/
+      ASF_ARCHIVE: https://archive.apache.org/dist/
       JENA_VERSION: 4.7.0
       BASE_URI: https://localhost:4443/
-      OWNER_CERT_PWD: changeit
-      SECRETARY_CERT_PWD: LinkedDataHub
     steps:
       - name: Install Linux packages
         run: sudo apt-get update && sudo apt-get install -qq raptor2-utils && sudo apt-get install curl
       - name: Download Jena
-        run: curl -sS --fail "${{env.ASF_ARCHIVE}}jena/binaries/apache-jena-${{env.JENA_VERSION}}.tar.gz" -o "${{runner.temp}}/jena.tar.gz"
+        run: curl -sS --fail "${{ env.ASF_ARCHIVE }}jena/binaries/apache-jena-${{ env.JENA_VERSION }}.tar.gz" -o "${{ runner.temp }}/jena.tar.gz"
       - name: Unpack Jena
         run: tar -zxf jena.tar.gz
-        working-directory: ${{runner.temp}}
-      - run: echo "$JENA_HOME/bin" >> $GITHUB_PATH
-        env:
-            JENA_HOME: "${{runner.temp}}/apache-jena-${{env.JENA_VERSION}}"
+        working-directory: ${{ runner.temp }}
+      - name: Set JENA_HOME and Update PATH
+        run: |
+          echo "${{ runner.temp }}/apache-jena-${{ env.JENA_VERSION }}/bin" >> $GITHUB_PATH
       - name: Checkout code
         uses: actions/checkout@v3
-      - name: Generate owner's and secretary's certificates/public keys
-        run: ../scripts/setup.sh .env ssl "${{env.OWNER_CERT_PWD}}" "${{env.SECRETARY_CERT_PWD}}" 3650
-        shell: bash
+      - name: Generating server certificate
+        run: ../scripts/server-cert-gen.sh .env nginx ssl
         working-directory: http-tests
+      - name: Writing secrets to files
+        run: |
+          mkdir -p ./secrets
+          printf "%s" "${{ secrets.HTTP_TEST_OWNER_CERT_PASSWORD }}" > ./secrets/owner_cert_password.txt
+          printf "%s" "${{ secrets.HTTP_TEST_SECRETARY_CERT_PASSWORD }}" > ./secrets/secretary_cert_password.txt
+          printf "%s" "${{ secrets.HTTP_TEST_SECRETARY_CERT_PASSWORD }}" > ./secrets/client_truststore_password.txt
+        shell: bash
       - name: Build Docker image & Run Docker containers
-        run: docker-compose -f docker-compose.yml -f ./http-tests/docker-compose.http-tests.yml --env-file ./http-tests/.env up --build -d
+        run: docker compose -f docker-compose.yml -f ./http-tests/docker-compose.http-tests.yml --env-file ./http-tests/.env up --build -d
       - name: Wait for the server to start...
         run: while ! (status=$(curl -k -s -w "%{http_code}\n" https://localhost:4443 -o /dev/null) && echo "$status" && echo "$status" | grep "403") ; do sleep 1 ; done # wait for the webapp to start (returns 403 by default)
+      - name: Fix certificate permissions on the host
+        run: |
+          sudo chmod 644 ./ssl/owner/cert.pem ./ssl/secretary/cert.pem
+        working-directory: http-tests
       - name: Run HTTP test scripts
-        run: ./run.sh "$PWD/ssl/owner/cert.pem" "${{env.OWNER_CERT_PWD}}" "$PWD/ssl/secretary/cert.pem" "${{env.SECRETARY_CERT_PWD}}"
+        run: ./run.sh "$PWD/ssl/owner/cert.pem" "${{ secrets.HTTP_TEST_OWNER_CERT_PASSWORD }}" "$PWD/ssl/secretary/cert.pem" "${{ secrets.HTTP_TEST_SECRETARY_CERT_PASSWORD }}"
         shell: bash
         working-directory: http-tests
       - name: Stop Docker containers
-        run: docker-compose --env-file ./http-tests/.env down
+        run: docker compose --env-file ./http-tests/.env down
       - name: Remove Docker containers
-        run: docker-compose --env-file ./http-tests/.env rm -f
+        run: docker compose --env-file ./http-tests/.env rm -f

.github/workflows/image.yml

@@ -0,0 +1,44 @@
+name: ci
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract version parts
+        id: version
+        run: |
+          VERSION="${{ github.ref_name }}"
+          MAJOR="${VERSION%%.*}"
+          MINOR="${VERSION%.*}"
+          MINOR="${MINOR#*.}"
+          echo "MAJOR=$MAJOR" >> $GITHUB_ENV
+          echo "MINOR=$MAJOR.$MINOR" >> $GITHUB_ENV
+          echo "FULL_VERSION=$VERSION" >> $GITHUB_ENV
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            atomgraph/linkeddatahub:latest
+            atomgraph/linkeddatahub:${{ env.FULL_VERSION }}
+            atomgraph/linkeddatahub:${{ env.MINOR }}
+            atomgraph/linkeddatahub:${{ env.MAJOR }}

.gitignore

@@ -1,10 +1,15 @@
+.DS_Store
 /target
-/certs
 /.env
 /data
 /uploads
 /nb-configuration.xml
 /node
 /node_modules
 /docker-compose.override.yml
+/secrets
+/datasets
 /ssl
+/http-tests/ssl
+/http-tests/datasets
+/http-tests/uploads

Dockerfile

@@ -1,4 +1,4 @@
-FROM maven:3.8.4-openjdk-17 as maven
+FROM maven:3.8.4-openjdk-17 AS maven
 
 # download and extract Jena
 
@@ -22,7 +22,7 @@ RUN mvn -Pstandalone clean install
 
 # ==============================
 
-FROM atomgraph/letsencrypt-tomcat:10.1.4
+FROM atomgraph/letsencrypt-tomcat:10.1.34
 
 LABEL maintainer="[email protected]"
 
@@ -38,10 +38,6 @@ ENV SOURCE_COMMIT=$SOURCE_COMMIT
 
 WORKDIR $CATALINA_HOME
 
-# add XSLT stylesheet that makes changes to ROOT.xml
-
-COPY platform/context.xsl conf/context.xsl
-
 ENV CACHE_MODEL_LOADS=true
 
 ENV STYLESHEET=static/com/atomgraph/linkeddatahub/xsl/bootstrap/2.3.2/layout.xsl
@@ -72,17 +68,26 @@ ENV HTTPS=false
 
 ENV SERVER_CERT=/var/linkeddatahub/ssl/server/server.crt
 
-ENV SECRETARY_CERT=/var/linkeddatahub/ssl/secretary/cert.pem
+ENV OWNER_CERT_ALIAS=root-owner
+ENV OWNER_KEYSTORE=/var/linkeddatahub/ssl/owner/keystore.p12
+ENV OWNER_CERT=/var/linkeddatahub/ssl/owner/cert.pem
+ENV OWNER_PUBLIC_KEY=/var/linkeddatahub/ssl/owner/public.pem
+ENV OWNER_PRIVATE_KEY=/var/linkeddatahub/ssl/owner/private.key
 
-ENV SECRETARY_CERT_ALIAS=secretary
+ENV SECRETARY_COMMON_NAME=LinkedDataHub
+ENV SECRETARY_CERT_ALIAS=root-secretary
+ENV SECRETARY_KEYSTORE=/var/linkeddatahub/ssl/secretary/keystore.p12
+ENV SECRETARY_CERT=/var/linkeddatahub/ssl/secretary/cert.pem
+ENV SECRETARY_PUBLIC_KEY=/var/linkeddatahub/ssl/secretary/public.pem
+ENV SECRETARY_PRIVATE_KEY=/var/linkeddatahub/ssl/secretary/private.key
 
 ENV CLIENT_KEYSTORE_MOUNT=/var/linkeddatahub/ssl/secretary/keystore.p12
-
 ENV CLIENT_KEYSTORE="$CATALINA_HOME/webapps/ROOT/WEB-INF/keystore.p12"
-
 ENV CLIENT_TRUSTSTORE="$CATALINA_HOME/webapps/ROOT/WEB-INF/client.truststore"
 
-ENV OWNER_PUBLIC_KEY=/var/linkeddatahub/ssl/owner/public.pem
+ENV CERT_VALIDITY=3650
+
+ENV SIGN_UP_CERT_VALIDITY=
 
 ENV LOAD_DATASETS=
 
@@ -102,12 +107,18 @@ ENV MAX_CONN_PER_ROUTE=20
 
 ENV MAX_TOTAL_CONN=40
 
+ENV MAX_REQUEST_RETRIES=3
+
 ENV IMPORT_KEEPALIVE=
 
+ENV MAX_IMPORT_THREADS=10
+
 ENV GOOGLE_CLIENT_ID=
 
 ENV GOOGLE_CLIENT_SECRET=
 
+ENV SERVLET_NAME=
+
 ENV GENERATE_SITEMAP=true
 
 # remove default Tomcat webapps and install xmlstarlet (used for XPath queries) and envsubst (for variable substitution)
@@ -120,6 +131,14 @@ RUN apt-get update --allow-releaseinfo-change && \
     rm -rf webapps/* && \
     rm -rf /var/lib/apt/lists/*
 
+# add XSLT stylesheet that makes changes to ROOT.xml
+
+COPY platform/context.xsl /var/linkeddatahub/xsl/context.xsl
+
+# add XSLT stylesheet that makes changes to web.xml
+
+COPY platform/web.xsl /var/linkeddatahub/xsl/web.xsl
+
 # copy entrypoint
 
 COPY platform/entrypoint.sh entrypoint.sh
@@ -172,6 +191,7 @@ ENV PATH="${PATH}:${JENA_HOME}/bin"
 
 RUN useradd --no-log-init -U ldh && \
     chown -R ldh:ldh . && \
+    mkdir -p "$(dirname "$OIDC_REFRESH_TOKENS")" && \
     chown -R ldh:ldh /var/linkeddatahub && \
     mkdir -p "${UPLOAD_ROOT}/${UPLOAD_CONTAINER_PATH}" && \
     chown -R ldh:ldh "$UPLOAD_ROOT" && \
@@ -180,7 +200,7 @@ RUN useradd --no-log-init -U ldh && \
 
 RUN ./import-letsencrypt-stg-roots.sh
 
-HEALTHCHECK --start-period=80s --interval=20s --timeout=10s \
+HEALTHCHECK --start-period=80s --retries=5 \
     CMD curl -f -I "http://localhost:${HTTP_PORT}/ns" -H "Accept: application/n-triples" || exit 1 # relies on public access to the namespace document
 
 USER ldh

README.md

@@ -57,19 +57,23 @@ It takes a few clicks and filling out a form to install the product into your ow
      OWNER_STATE_OR_PROVINCE=Denmark
      OWNER_COUNTRY_NAME=DK
      ```
-  3. Setup SSL certificates/keys by running this from command line (replace `$owner_cert_pwd` and `$secretary_cert_pwd` with your own passwords):
-     ```
-     ./scripts/setup.sh .env ssl $owner_cert_pwd $secretary_cert_pwd 3650
+  3. Setup server's SSL certificates by running this from command line:
+     ```shell
+     ./scripts/server-cert-gen.sh .env nginx ssl
      ```
      The script will create an `ssl` sub-folder where the SSL certificates and/or public keys will be placed.
   4. Launch the application services by running this from command line:
-     ```
+     ```shell
      docker-compose up --build
      ```
      It will build LinkedDataHub's Docker image, start its container and mount the following sub-folders:
+     - `ssl`
+       * `owner` stores root owner's WebID certificate, keystore, and public key
+       * `secretary` stores root application's WebID certificate, keystore, and public key
+       * `server` stores the server's certificate (also used by nginx)
      - `data` where the triplestore(s) will persist RDF data
      - `uploads` where LDH stores content-hashed file uploads
-     The first should take around half a minute as datasets are being loaded into triplestores. After a successful startup, the last line of the Docker log should read something like:
+     It should take up to half a minute as datasets are being loaded into triplestores. After a successful startup, the last line of the Docker log should read something like:
      ```
      linkeddatahub_1     | 09-Feb-2021 14:18:10.536 INFO [main] org.apache.catalina.startup.Catalina.start Server startup in [32609] milliseconds
      ```
@@ -78,7 +82,15 @@ It takes a few clicks and filling out a form to install the product into your ow
      - Mozilla Firefox: `Options > Privacy > Security > View Certificates... > Import...`
      - Apple Safari: The file is installed directly into the operating system. Open the file and import it using the [Keychain Access](https://support.apple.com/guide/keychain-access/what-is-keychain-access-kyca1083/mac) tool (drag it to the `local` section).
      - Microsoft Edge: Does not support certificate management, you need to install the file into Windows. [Read more here](https://social.technet.microsoft.com/Forums/en-US/18301fff-0467-4e41-8dee-4e44823ed5bf/microsoft-edge-browser-and-ssl-certificates?forum=win10itprogeneral).
-  6. Open **https://localhost:4443/** in that web browser
+  6. For authenticated API access use the `ssl/owner/cert.pem` HTTPS client certificate.
+     If you are running Linux with user other than `root`, you might need to fix the certificate permissions because Docker bind mounts are owned by `root` by default. For example:
+     ```shell
+     sudo setfacl -m u:$(whoami):r ./ssl/owner/*
+     ```
+  7. Open **https://localhost:4443/** in the web browser or use `curl` for API access, for example:
+     ```shell
+     curl -k -E ./ssl/owner/cert.pem:<your cert password> -H "Accept: text/turtle" 'https://localhost:4443/'
+     ```
 
   ### Notes
 

docker-compose.debug.yml

@@ -1,4 +1,3 @@
-version: "2.3"
 services:
   linkeddatahub:
     ports: