diff --git a/.github/workflows/azure_e2e_run_workflow.yml b/.github/workflows/azure_e2e_run_workflow.yml new file mode 100644 index 00000000000..5071fc37e8b --- /dev/null +++ b/.github/workflows/azure_e2e_run_workflow.yml @@ -0,0 +1,117 @@ +name: 'Azure e2e - Run Workflow' +on: + schedule: + - cron: '0 16 * * *' # UTC 4pm, EST 11am, EDT 12pm + workflow_dispatch: + +env: + BROADBOT_TOKEN: '${{ secrets.BROADBOT_GITHUB_TOKEN }}' # github token for access to kick off a job in the private repo + RUN_NAME_SUFFIX: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}' + +jobs: + + # This job provisions useful parameters for e2e tests + params-gen: + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + outputs: + project-name: ${{ steps.gen.outputs.project_name }} + bee-name: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt}}-dev' + steps: + - name: Generate a random billing project name + id: 'gen' + run: | + project_name=$(echo "tmp-billing-project-$(uuidgen)" | cut -c -30) + echo "project_name=${project_name}" >> $GITHUB_OUTPUT + + create-bee-workflow: + runs-on: ubuntu-latest + needs: [params-gen] + permissions: + contents: 'read' + id-token: 'write' + steps: + - name: Dispatch to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v3 + with: + workflow: bee-create + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ env.BROADBOT_TOKEN }} + # NOTE: Opting to use "prod" instead of custom tag since I specifically want to test against the current prod state + # NOTE: For testing/development purposes I'm using dev + inputs: '{ "bee-name": "${{ needs.params-gen.outputs.bee-name }}", "version-template": "dev", "bee-template-name": "rawls-e2e-azure-tests"}' + + create-and-attach-billing-project-to-landing-zone-workflow: + runs-on: ubuntu-latest + needs: [create-bee-workflow, params-gen] + steps: + - name: dispatch to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v3 + with: + workflow: attach-billing-project-to-landing-zone.yaml + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ env.BROADBOT_TOKEN }} + inputs: '{ + "run-name": "attach-billing-project-to-landing-zone-${{ env.RUN_NAME_SUFFIX }}", + "bee-name": "${{ needs.params-gen.outputs.bee-name }}", + "billing-project": "${{ needs.params-gen.outputs.project-name }}", + "service-account": "firecloud-qa@broad-dsde-qa.iam.gserviceaccount.com" }' + + run-cromwell-az-e2e: + needs: [params-gen, create-and-attach-billing-project-to-landing-zone-workflow] + permissions: + contents: read + id-token: write + uses: "broadinstitute/dsp-reusable-workflows/.github/workflows/cromwell-az-e2e-test.yaml@main" + with: + bee-name: "${{ needs.params-gen.outputs.bee-name }}" + billing-project-name: "${{ needs.params-gen.outputs.project-name }}" + + delete-billing-project-v2-from-bee-workflow: + continue-on-error: true + runs-on: ubuntu-latest + needs: [run-cromwell-az-e2e, create-and-attach-billing-project-to-landing-zone-workflow, params-gen] + if: always() + steps: + - name: dispatch to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v3 + with: + workflow: .github/workflows/delete-billing-project-v2-from-bee.yaml + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ env.BROADBOT_TOKEN }} + inputs: '{ + "run-name": "delete-billing-project-v2-from-bee-${{ env.RUN_NAME_SUFFIX }}", + "bee-name": "${{ needs.params-gen.outputs.bee-name }}", + "billing-project": "${{ needs.params-gen.outputs.project-name }}", + "service-account": "firecloud-qa@broad-dsde-qa.iam.gserviceaccount.com", + "silent-on-failure": "false" }' + + destroy-bee-workflow: + runs-on: ubuntu-latest + needs: [params-gen, create-bee-workflow, delete-billing-project-v2-from-bee-workflow] + if: always() + permissions: + contents: 'read' + id-token: 'write' + steps: + - name: dispatch to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v3 + with: + workflow: bee-destroy.yaml + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ env.BROADBOT_TOKEN }} + inputs: '{ "bee-name": "${{ needs.params-gen.outputs.bee-name }}" }' + wait-for-completion: true + + report-workflow: + uses: broadinstitute/sherlock/.github/workflows/client-report-workflow.yaml@main + with: + notify-slack-channels-upon-workflow-failure: "#cromwell_jenkins_ci_errors" + permissions: + id-token: write diff --git a/.github/workflows/chart_update_on_merge.yml b/.github/workflows/chart_update_on_merge.yml index a2b14f2ec65..a17dd971436 100644 --- a/.github/workflows/chart_update_on_merge.yml +++ b/.github/workflows/chart_update_on_merge.yml @@ -60,7 +60,7 @@ jobs: run: | set -e cd cromwell - sbt -Dproject.isSnapshot=false -Dproject.isRelease=false dockerBuildAndPush + sbt -Dproject.isSnapshot=false dockerBuildAndPush - name: Deploy to dev and board release train (Cromwell) uses: broadinstitute/repository-dispatch@master with: diff --git a/.github/workflows/consumer_contract_tests.yml b/.github/workflows/consumer_contract_tests.yml index 0970e45e863..4635abaa765 100644 --- a/.github/workflows/consumer_contract_tests.yml +++ b/.github/workflows/consumer_contract_tests.yml @@ -29,18 +29,25 @@ name: Consumer contract tests # Consumer kicks off can-i-deploy on process to determine if changes can be promoted and used for deployment. # # NOTE: The publish-contracts workflow will use the latest commit of the branch that triggers this workflow to publish the unique consumer contract version to Pact Broker. - on: pull_request: + branches: + - develop paths-ignore: - 'README.md' push: + branches: + - develop paths-ignore: - 'README.md' merge_group: branches: - develop +env: + PUBLISH_CONTRACTS_RUN_NAME: 'publish-contracts-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}' + CAN_I_DEPLOY_RUN_NAME: 'can-i-deploy-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}' + jobs: init-github-context: runs-on: ubuntu-latest @@ -52,6 +59,23 @@ jobs: steps: - uses: actions/checkout@v3 + # Construct a version string like `87-9c6c439`. Adapted from `chart_update_on_merge.yml`. + - name: Find Cromwell short SHA + run: | + set -e + echo "CROMWELL_SHORT_SHA=`git rev-parse --short $GITHUB_SHA`" >> $GITHUB_ENV + - name: Find Cromwell release number + run: | + set -e + previous_version=$(curl -X GET https://api.github.com/repos/broadinstitute/cromwell/releases/latest | jq .tag_name | xargs) + if ! [[ "${previous_version}" =~ ^[0-9][0-9]+$ ]]; then + exit 1 + fi + echo "CROMWELL_NUMBER=$((previous_version + 1))" >> $GITHUB_ENV + - name: Save complete image ID + run: | + echo "CROMWELL_VERSION=`echo "$CROMWELL_NUMBER-$CROMWELL_SHORT_SHA"`" >> $GITHUB_ENV + - name: Extract branch id: extract-branch run: | @@ -70,9 +94,9 @@ jobs: fi echo "CURRENT_BRANCH=${GITHUB_REF/refs\/heads\//""}" >> $GITHUB_ENV echo "CURRENT_SHA=$GITHUB_SHA" >> $GITHUB_ENV - + echo "repo-branch=${GITHUB_REF/refs\/heads\//""}" >> $GITHUB_OUTPUT - echo "repo-version=${GITHUB_SHA}" >> $GITHUB_OUTPUT + echo "repo-version=${CROMWELL_VERSION}" >> $GITHUB_OUTPUT echo "fork=${FORK}" >> $GITHUB_OUTPUT - name: Is PR triggered by forked repo? @@ -88,11 +112,12 @@ jobs: echo "repo-version=${{ steps.extract-branch.outputs.repo-version }}" echo "fork=${{ steps.extract-branch.outputs.fork }}" - cromwell-consumer-contract-tests: + cromwell-contract-tests: runs-on: ubuntu-latest needs: [init-github-context] outputs: - pact-b64: ${{ steps.encode-pact.outputs.pact-b64 }} + pact-b64-drshub: ${{ steps.encode-pact.outputs.pact-b64-drshub }} + pact-b64-cbas: ${{ steps.encode-pact.outputs.pact-b64-cbas }} steps: - uses: actions/checkout@v3 @@ -108,25 +133,52 @@ jobs: - name: Output consumer contract as non-breaking base64 string id: encode-pact run: | + set -e cd pact4s - NON_BREAKING_B64=$(cat target/pacts/cromwell-consumer-drshub-provider.json | base64 -w 0) - echo "pact-b64=${NON_BREAKING_B64}" >> $GITHUB_OUTPUT + NON_BREAKING_B64_DRSHUB=$(cat target/pacts/cromwell-drshub.json | base64 -w 0) + NON_BREAKING_B64_CBAS=$(cat target/pacts/cromwell-cbas.json | base64 -w 0) + echo "pact-b64-drshub=${NON_BREAKING_B64_DRSHUB}" >> $GITHUB_OUTPUT + echo "pact-b64-cbas=${NON_BREAKING_B64_CBAS}" >> $GITHUB_OUTPUT # Prevent untrusted sources from using PRs to publish contracts # since access to secrets is not allowed. publish-contracts: runs-on: ubuntu-latest if: ${{ needs.init-github-context.outputs.fork == 'false' || needs.init-github-context.outputs.fork == ''}} - needs: [init-github-context, cromwell-consumer-contract-tests] + needs: [init-github-context, cromwell-contract-tests] steps: - - name: Dispatch to terra-github-workflows - uses: broadinstitute/workflow-dispatch@v3 + - name: Dispatch drshub to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v4.0.0 + with: + run-name: "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}" + workflow: .github/workflows/publish-contracts.yaml + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # github token for access to kick off a job in the private repo + inputs: '{ + "run-name": "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}", + "pact-b64": "${{ needs.cromwell-contract-tests.outputs.pact-b64-drshub }}", + "repo-owner": "${{ github.repository_owner }}", + "repo-name": "${{ github.event.repository.name }}", + "repo-branch": "${{ needs.init-github-context.outputs.repo-branch }}", + "release-tag": "${{ needs.init-github-context.outputs.repo-version }}" + }' + - name: Dispatch cbas to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v4.0.0 with: + run-name: "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}" workflow: .github/workflows/publish-contracts.yaml repo: broadinstitute/terra-github-workflows ref: refs/heads/main token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # github token for access to kick off a job in the private repo - inputs: '{ "pact-b64": "${{ needs.cromwell-consumer-contract-tests.outputs.pact-b64 }}", "repo-owner": "${{ github.repository_owner }}", "repo-name": "${{ github.event.repository.name }}", "repo-branch": "${{ needs.init-github-context.outputs.repo-branch }}" }' + inputs: '{ + "run-name": "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}", + "pact-b64": "${{ needs.cromwell-contract-tests.outputs.pact-b64-cbas }}", + "repo-owner": "${{ github.repository_owner }}", + "repo-name": "${{ github.event.repository.name }}", + "repo-branch": "${{ needs.init-github-context.outputs.repo-branch }}", + "release-tag": "${{ needs.init-github-context.outputs.repo-version }}" + }' can-i-deploy: runs-on: ubuntu-latest @@ -134,10 +186,15 @@ jobs: needs: [ init-github-context, publish-contracts ] steps: - name: Dispatch to terra-github-workflows - uses: broadinstitute/workflow-dispatch@v3 + uses: broadinstitute/workflow-dispatch@v4.0.0 with: + run-name: "${{ env.CAN_I_DEPLOY_RUN_NAME }}" workflow: .github/workflows/can-i-deploy.yaml repo: broadinstitute/terra-github-workflows ref: refs/heads/main token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # github token for access to kick off a job in the private repo - inputs: '{ "pacticipant": "cromwell-consumer", "version": "${{ needs.init-github-context.outputs.repo-version }}" }' + inputs: '{ + "run-name": "${{ env.CAN_I_DEPLOY_RUN_NAME }}", + "pacticipant": "cromwell", + "version": "${{ needs.init-github-context.outputs.repo-version }}" + }' diff --git a/.github/workflows/docker_build_test.yml b/.github/workflows/docker_build_test.yml index 01c2ea502c9..b4d373f330e 100644 --- a/.github/workflows/docker_build_test.yml +++ b/.github/workflows/docker_build_test.yml @@ -34,4 +34,10 @@ jobs: run: | set -e cd cromwell - sbt -Dproject.isSnapshot=false -Dproject.isRelease=false docker + sbt -Dproject.isSnapshot=false docker + # Rarely used but we really want it always working for emergencies + - name: Build Cromwell Debug Docker + run: | + set -e + cd cromwell + sbt -Dproject.isDebug=true docker diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index ebafe51064c..5ff831a4b23 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -16,6 +16,12 @@ on: permissions: contents: read +concurrency: + # Don't run this workflow concurrently on the same branch + group: ${{ github.workflow }}-${{ github.ref }} + # For PRs, don't wait for completion of existing runs, cancel them instead + cancel-in-progress: ${{ github.ref != 'develop' }} + jobs: integration-tests: strategy: @@ -32,6 +38,9 @@ jobs: - build_type: centaurPapiV2beta build_mysql: 5.7 friendly_name: Centaur Papi V2 Beta with MySQL 5.7 + - build_type: centaurPapiV2betaRestart + build_mysql: 5.7 + friendly_name: Centaur Papi V2 Beta (restart) - build_type: dbms friendly_name: DBMS - build_type: centaurTes @@ -101,6 +110,10 @@ jobs: set -e echo Running test.sh ./src/ci/bin/test.sh + - uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false # Tolerate missing codecov reports, since not all suites generate them. # always() is some github magic that forces the following step to run, even when the previous fails. # Without it, the if statement won't be evaluated on a test failure. - uses: ravsamhq/notify-slack-action@v2 diff --git a/.github/workflows/scalafmt-check.yml b/.github/workflows/scalafmt-check.yml new file mode 100644 index 00000000000..3730d2ffc8f --- /dev/null +++ b/.github/workflows/scalafmt-check.yml @@ -0,0 +1,31 @@ +name: 'ScalaFmt Check' + +# This GitHub Action runs the ScalaFmt linting tool on the entire codebase. +# It fails if any files are not formatted properly. +# If it is triggered by someone commenting 'scalafmt' on a PR, it will first format, commit, and push formatted code +# to the branch. + +run-name: ${{ format('ScalaFmt Check on {0}', github.ref_name) }} + +on: + workflow_dispatch: + push: + +permissions: + contents: read + +jobs: + run-scalafmt-check: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.target-branch }} + - uses: ./.github/set_up_cromwell_action + with: + cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + - name: Run ScalaFmt + run: | + sbt scalafmtCheckAll + working-directory: ${{ github.workspace }} diff --git a/.github/workflows/scalafmt-fix.yml b/.github/workflows/scalafmt-fix.yml new file mode 100644 index 00000000000..a20eab6dbee --- /dev/null +++ b/.github/workflows/scalafmt-fix.yml @@ -0,0 +1,64 @@ +name: 'ScalaFmt Fix' + +# This GitHub Action runs the ScalaFmt linting tool on the entire codebase. +# It will fix, commit, and push linted code. +# It will only run when someone comments "scalafmt" on a PR. + +run-name: ScalaFmt Fix + +on: + issue_comment: + types: + - created + workflow_dispatch: + branch_name: + description: 'Branch to run ScalaFmt against' + required: true + pull_request_target: + types: + - opened + - synchronize + +jobs: + run-scalafmt-fix: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Determine Target Branch + id: determine-branch + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "::set-output name=target_branch::${{ inputs.branch_name }}" + else + echo "::set-output name=target_branch::${{ github.event.pull_request.head.ref }}" + fi + shell: bash + env: + inputs.branch_name: ${{ inputs.branch_name }} + - name: Check for ScalaFmt Comment + id: check-comment + run: | + if [[ "${{ github.event_name }}" == "issue_comment" && "${{ github.event.comment.body }}" == *"scalafmt"* ]]; then + echo "::set-output name=comment-triggered::true" + else + echo "::set-output name=comment-triggered::false" + fi + shell: bash + - uses: actions/checkout@v3 # checkout the cromwell repo + with: + ref: ${{ inputs.target-branch }} + - uses: ./.github/set_up_cromwell_action + with: + cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + - name: Run ScalaFmt Fixup + if: steps.check-comment.outputs.comment-triggered == 'true' || github.event_name == 'workflow_dispatch' + env: + BROADBOT_GITHUB_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + run: | + sbt scalafmtAll + git config --global user.email "broadbot@broadinstitute.org" + git config --global user.name "Broad Bot" + git add . + git commit -m "ScalaFmt fixup via Broad Bot" + git push origin ${{ steps.determine-branch.outputs.target_branch }} + working-directory: ${{ github.workspace }} diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index b005da65041..d353eae4e58 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -12,7 +12,6 @@ jobs: project: - cromiam - cromwell-drs-localizer - - perf - server - womtool diff --git a/.gitignore b/.gitignore index a5b72f6b263..94accae9038 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,9 @@ console_output.txt expected.json run_mode_metadata.json +#bloop files +/.bloop + # custom config cromwell-executions cromwell-test-executions @@ -55,3 +58,10 @@ tesk_application.conf **/venv/ exome_germline_single_sample_v1.3/ **/*.pyc +src/ci/resources/*.temp + +# GHA credentials +gha-creds-*.json + +# jenv +.java-version diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000000..376dcc3a901 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,14 @@ +# Read the Docs configuration file for MkDocs projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + +mkdocs: + configuration: mkdocs.yml diff --git a/.scalafmt.conf b/.scalafmt.conf new file mode 100644 index 00000000000..336b0fd7145 --- /dev/null +++ b/.scalafmt.conf @@ -0,0 +1,18 @@ +version = 3.7.17 +align.preset = none +align.openParenCallSite = true +align.openParenDefnSite = true +maxColumn = 120 +continuationIndent.defnSite = 2 +assumeStandardLibraryStripMargin = true +align.stripMargin = true +danglingParentheses.preset = true +rewrite.rules = [Imports, RedundantBraces, RedundantParens, SortModifiers] +rewrite.imports.sort = scalastyle +docstrings.style = keep +project.excludeFilters = [ + Dependencies.scala, + Settings.scala, + build.sbt +] +runner.dialect = scala213 diff --git a/CHANGELOG.md b/CHANGELOG.md index a581852c02e..aecc6361de4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Cromwell Change Log +## 87 Release Notes + +### `upgrade` command removed from Womtool + +Womtool previously supported a `womtool upgrade` command for upgrading draft-2 WDLs to 1.0. With WDL 1.1 soon to become the latest supported version, this functionality is retiring. + +### Replacement of `gsutil` with `gcloud storage` + +In this release, all **localization** functionality on the GCP backend migrates to use the more modern and performant `gcloud storage`. With sufficiently powerful worker VMs, Cromwell can now localize at up to 1200 MB/s [0][1][2]. + +In a future release, **delocalization** will also migrate to `gcloud storage`. As part of that upcoming change, we are considering turning on [parallel composite uploads](https://cromwell.readthedocs.io/en/stable/backends/Google/#parallel-composite-uploads) by default to maximize performance. Delocalized composite objects will no longer have an md5 checksum in their metadata; refer to the matrix below [3]. If you have compatibility concerns for your workflow, please [submit an issue](https://github.com/broadinstitute/cromwell/issues). + +| Delocalization Strategy | Performance | crc32c | md5 | +|-------------------------|---------------|--------|-----| +| Classic | Baseline/slow | ✅ | ✅ | +| Parallel Composite | Fast | ✅ | ❌ | + +[0] Tested with Intel Ice Lake CPU platform, 16 vCPU, 32 GB RAM, 2500 GB SSD + +[1] [Throughput scales with vCPU count](https://cloud.google.com/compute/docs/disks/performance#n2_vms) with a plateau at 16 vCPUs. + +[2] [Throughput scales with disk size and type](https://cloud.google.com/compute/docs/disks/performance#throughput_limits_for_zonal) with at a plateau at 2.5 TB SSD. Worked example: 1200 MB/s ÷ 0.48 MB/s per GB = 2500 GB. + +[3] Cromwell itself uses crc32c hashes for call caching and is not affected + ## 86 Release Notes ### GCP Batch diff --git a/CromIAM/src/main/scala/cromiam/auth/Collection.scala b/CromIAM/src/main/scala/cromiam/auth/Collection.scala index 84e72194ab2..47785ae2005 100644 --- a/CromIAM/src/main/scala/cromiam/auth/Collection.scala +++ b/CromIAM/src/main/scala/cromiam/auth/Collection.scala @@ -9,6 +9,7 @@ import scala.util.{Success, Try} final case class Collection(name: String) extends AnyVal object Collection { + /** * Parses a raw JSON string to make sure it fits the standard pattern (see below) for labels, * performs some CromIAM-specific checking to ensure the user isn't attempting to manipulate the @@ -19,13 +20,14 @@ object Collection { */ def validateLabels(labelsJson: Option[String]): Directive1[Option[Map[String, JsValue]]] = { - val labels = labelsJson map { l => - Try(l.parseJson) match { - case Success(JsObject(json)) if json.keySet.contains(CollectionLabelName) => throw new LabelContainsCollectionException - case Success(JsObject(json)) => json - case _ => throw InvalidLabelsException(l) - } + val labels = labelsJson map { l => + Try(l.parseJson) match { + case Success(JsObject(json)) if json.keySet.contains(CollectionLabelName) => + throw new LabelContainsCollectionException + case Success(JsObject(json)) => json + case _ => throw InvalidLabelsException(l) } + } provide(labels) } @@ -34,15 +36,16 @@ object Collection { val LabelsKey = "labels" // LabelContainsCollectionException is a class because of ScalaTest, some of the constructs don't play well w/ case objects - final class LabelContainsCollectionException extends Exception(s"Submitted labels contain the key $CollectionLabelName, which is not allowed\n") - final case class InvalidLabelsException(labels: String) extends Exception(s"Labels must be a valid JSON object, received: $labels\n") + final class LabelContainsCollectionException + extends Exception(s"Submitted labels contain the key $CollectionLabelName, which is not allowed\n") + final case class InvalidLabelsException(labels: String) + extends Exception(s"Labels must be a valid JSON object, received: $labels\n") /** * Returns the default collection for a user. */ - def forUser(user: User): Collection = { + def forUser(user: User): Collection = Collection(user.userId.value) - } implicit val collectionJsonReader = new JsonReader[Collection] { import spray.json.DefaultJsonProtocol._ diff --git a/CromIAM/src/main/scala/cromiam/auth/User.scala b/CromIAM/src/main/scala/cromiam/auth/User.scala index d123f8fa2f7..fec64ebc5f9 100644 --- a/CromIAM/src/main/scala/cromiam/auth/User.scala +++ b/CromIAM/src/main/scala/cromiam/auth/User.scala @@ -7,4 +7,3 @@ import org.broadinstitute.dsde.workbench.model.WorkbenchUserId * Wraps the concept of an authenticated workbench user including their numeric ID as well as their bearer token */ final case class User(userId: WorkbenchUserId, authorization: Authorization) - diff --git a/CromIAM/src/main/scala/cromiam/cromwell/CromwellClient.scala b/CromIAM/src/main/scala/cromiam/cromwell/CromwellClient.scala index a95b8df89b5..4a419b62255 100644 --- a/CromIAM/src/main/scala/cromiam/cromwell/CromwellClient.scala +++ b/CromIAM/src/main/scala/cromiam/cromwell/CromwellClient.scala @@ -25,10 +25,16 @@ import scala.concurrent.{ExecutionContextExecutor, Future} * * FIXME: Look for ways to synch this up with the mothership */ -class CromwellClient(scheme: String, interface: String, port: Int, log: LoggingAdapter, serviceRegistryActorRef: ActorRef)(implicit system: ActorSystem, - ece: ExecutionContextExecutor, - materializer: ActorMaterializer) - extends SprayJsonSupport with DefaultJsonProtocol with StatusCheckedSubsystem with CromIamInstrumentation{ +class CromwellClient(scheme: String, + interface: String, + port: Int, + log: LoggingAdapter, + serviceRegistryActorRef: ActorRef +)(implicit system: ActorSystem, ece: ExecutionContextExecutor, materializer: ActorMaterializer) + extends SprayJsonSupport + with DefaultJsonProtocol + with StatusCheckedSubsystem + with CromIamInstrumentation { val cromwellUrl = new URL(s"$scheme://$interface:$port") val cromwellApiVersion = "v1" @@ -41,21 +47,23 @@ class CromwellClient(scheme: String, interface: String, port: Int, log: LoggingA def collectionForWorkflow(workflowId: String, user: User, - cromIamRequest: HttpRequest): FailureResponseOrT[Collection] = { + cromIamRequest: HttpRequest + ): FailureResponseOrT[Collection] = { import CromwellClient.EnhancedWorkflowLabels log.info("Requesting collection for " + workflowId + " for user " + user.userId + " from metadata") // Look up in Cromwell what the collection is for this workflow. If it doesn't exist, fail the Future - val cromwellApiLabelFunc = () => cromwellApiClient.labels(WorkflowId.fromString(workflowId), headers = List(user.authorization)) flatMap { - _.caasCollection match { - case Some(c) => FailureResponseOrT.pure[IO, HttpResponse](c) - case None => - val exception = new IllegalArgumentException(s"Workflow $workflowId has no associated collection") - val failure = IO.raiseError[Collection](exception) - FailureResponseOrT.right[HttpResponse](failure) + val cromwellApiLabelFunc = () => + cromwellApiClient.labels(WorkflowId.fromString(workflowId), headers = List(user.authorization)) flatMap { + _.caasCollection match { + case Some(c) => FailureResponseOrT.pure[IO, HttpResponse](c) + case None => + val exception = new IllegalArgumentException(s"Workflow $workflowId has no associated collection") + val failure = IO.raiseError[Collection](exception) + FailureResponseOrT.right[HttpResponse](failure) + } } - } instrumentRequest(cromwellApiLabelFunc, cromIamRequest, wfCollectionPrefix) } @@ -63,13 +71,14 @@ class CromwellClient(scheme: String, interface: String, port: Int, log: LoggingA def forwardToCromwell(httpRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { val future = { // See CromwellClient's companion object for info on these header modifications - val headers = httpRequest.headers.filterNot(header => header.name == TimeoutAccessHeader || header.name == HostHeader) + val headers = + httpRequest.headers.filterNot(header => header.name == TimeoutAccessHeader || header.name == HostHeader) val cromwellRequest = httpRequest .copy(uri = httpRequest.uri.withAuthority(interface, port).withScheme(scheme)) .withHeaders(headers) Http().singleRequest(cromwellRequest) - } recoverWith { - case e => Future.failed(CromwellConnectionFailure(e)) + } recoverWith { case e => + Future.failed(CromwellConnectionFailure(e)) } future.asFailureResponseOrT } @@ -86,7 +95,7 @@ class CromwellClient(scheme: String, interface: String, port: Int, log: LoggingA use the current workflow id. This is all called from inside the context of a Future, so exceptions will be properly caught. - */ + */ metadata.value.parseJson.asJsObject.fields.get("rootWorkflowId").map(_.convertTo[String]).getOrElse(workflowId) } @@ -96,11 +105,13 @@ class CromwellClient(scheme: String, interface: String, port: Int, log: LoggingA Grab the metadata from Cromwell filtered down to the rootWorkflowId. Then transform the response to get just the root workflow ID itself */ - val cromwellApiMetadataFunc = () => cromwellApiClient.metadata( - WorkflowId.fromString(workflowId), - args = Option(Map("includeKey" -> List("rootWorkflowId"))), - headers = List(user.authorization)).map(metadataToRootWorkflowId - ) + val cromwellApiMetadataFunc = () => + cromwellApiClient + .metadata(WorkflowId.fromString(workflowId), + args = Option(Map("includeKey" -> List("rootWorkflowId"))), + headers = List(user.authorization) + ) + .map(metadataToRootWorkflowId) instrumentRequest(cromwellApiMetadataFunc, cromIamRequest, rootWfIdPrefix) } @@ -120,14 +131,14 @@ object CromwellClient { // See: https://broadworkbench.atlassian.net/browse/DDO-2190 val HostHeader = "Host" - final case class CromwellConnectionFailure(f: Throwable) extends Exception(s"Unable to connect to Cromwell (${f.getMessage})", f) + final case class CromwellConnectionFailure(f: Throwable) + extends Exception(s"Unable to connect to Cromwell (${f.getMessage})", f) implicit class EnhancedWorkflowLabels(val wl: WorkflowLabels) extends AnyVal { - import Collection.{CollectionLabelName, collectionJsonReader} + import Collection.{collectionJsonReader, CollectionLabelName} - def caasCollection: Option[Collection] = { + def caasCollection: Option[Collection] = wl.labels.fields.get(CollectionLabelName).map(_.convertTo[Collection]) - } } } diff --git a/CromIAM/src/main/scala/cromiam/instrumentation/CromIamInstrumentation.scala b/CromIAM/src/main/scala/cromiam/instrumentation/CromIamInstrumentation.scala index 65b164f00f6..c79e0b6e14c 100644 --- a/CromIAM/src/main/scala/cromiam/instrumentation/CromIamInstrumentation.scala +++ b/CromIAM/src/main/scala/cromiam/instrumentation/CromIamInstrumentation.scala @@ -27,10 +27,11 @@ trait CromIamInstrumentation extends CromwellInstrumentation { val rootWfIdPrefix = NonEmptyList.one("root-workflow-id") val wfCollectionPrefix = NonEmptyList.one("workflow-collection") - def convertRequestToPath(httpRequest: HttpRequest): NonEmptyList[String] = NonEmptyList.of( // Returns the path of the URI only, without query parameters (e.g: api/engine/workflows/metadata) - httpRequest.uri.path.toString().stripPrefix("/") + httpRequest.uri.path + .toString() + .stripPrefix("/") // Replace UUIDs with [id] to keep paths same regardless of the workflow .replaceAll(CromIamInstrumentation.UUIDRegex, "[id]"), // Name of the method (e.g: GET) @@ -43,15 +44,19 @@ trait CromIamInstrumentation extends CromwellInstrumentation { def makePathFromRequestAndResponse(httpRequest: HttpRequest, httpResponse: HttpResponse): InstrumentationPath = convertRequestToPath(httpRequest).concatNel(NonEmptyList.of(httpResponse.status.intValue.toString)) - def sendTimingApi(statsDPath: InstrumentationPath, timing: FiniteDuration, prefixToStatsd: NonEmptyList[String]): Unit = { + def sendTimingApi(statsDPath: InstrumentationPath, + timing: FiniteDuration, + prefixToStatsd: NonEmptyList[String] + ): Unit = sendTiming(prefixToStatsd.concatNel(statsDPath), timing, CromIamPrefix) - } - def instrumentationPrefixForSam(methodPrefix: NonEmptyList[String]): NonEmptyList[String] = samPrefix.concatNel(methodPrefix) + def instrumentationPrefixForSam(methodPrefix: NonEmptyList[String]): NonEmptyList[String] = + samPrefix.concatNel(methodPrefix) def instrumentRequest[A](func: () => FailureResponseOrT[A], httpRequest: HttpRequest, - prefix: NonEmptyList[String]): FailureResponseOrT[A] = { + prefix: NonEmptyList[String] + ): FailureResponseOrT[A] = { def now(): Deadline = Deadline.now val startTimestamp = now() diff --git a/CromIAM/src/main/scala/cromiam/sam/SamClient.scala b/CromIAM/src/main/scala/cromiam/sam/SamClient.scala index d6a315f8241..8fa0cc8fd87 100644 --- a/CromIAM/src/main/scala/cromiam/sam/SamClient.scala +++ b/CromIAM/src/main/scala/cromiam/sam/SamClient.scala @@ -33,20 +33,21 @@ class SamClient(scheme: String, port: Int, checkSubmitWhitelist: Boolean, log: LoggingAdapter, - serviceRegistryActorRef: ActorRef) - (implicit system: ActorSystem, ece: ExecutionContextExecutor, materializer: ActorMaterializer) extends StatusCheckedSubsystem with CromIamInstrumentation { + serviceRegistryActorRef: ActorRef +)(implicit system: ActorSystem, ece: ExecutionContextExecutor, materializer: ActorMaterializer) + extends StatusCheckedSubsystem + with CromIamInstrumentation { - private implicit val cs = IO.contextShift(ece) + implicit private val cs = IO.contextShift(ece) override val statusUri = uri"$samBaseUri/status" override val serviceRegistryActor: ActorRef = serviceRegistryActorRef - def isSubmitWhitelisted(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = { + def isSubmitWhitelisted(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = checkSubmitWhitelist.fold( isSubmitWhitelistedSam(user, cromIamRequest), FailureResponseOrT.pure(true) ) - } def isSubmitWhitelistedSam(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = { val request = HttpRequest( @@ -64,7 +65,7 @@ class SamClient(scheme: String, whitelisted <- response.status match { case StatusCodes.OK => // Does not seem to be already provided? - implicit val entityToBooleanUnmarshaller : Unmarshaller[HttpEntity, Boolean] = + implicit val entityToBooleanUnmarshaller: Unmarshaller[HttpEntity, Boolean] = (Unmarshaller.stringUnmarshaller flatMap Unmarshaller.booleanFromStringUnmarshaller).asScala val unmarshal = IO.fromFuture(IO(Unmarshal(response.entity).to[Boolean])) FailureResponseOrT.right[HttpResponse](unmarshal) @@ -95,14 +96,19 @@ class SamClient(scheme: String, userInfo.enabled } case _ => - log.error("Could not verify access with Sam for user {}, error was {} {}", user.userId, response.status, response.toString().take(100)) + log.error("Could not verify access with Sam for user {}, error was {} {}", + user.userId, + response.status, + response.toString().take(100) + ) FailureResponseOrT.pure[IO, HttpResponse](false) } } yield userEnabled } def collectionsForUser(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[List[Collection]] = { - val request = HttpRequest(method = HttpMethods.GET, uri = samBaseCollectionUri, headers = List[HttpHeader](user.authorization)) + val request = + HttpRequest(method = HttpMethods.GET, uri = samBaseCollectionUri, headers = List[HttpHeader](user.authorization)) for { response <- instrumentRequest( @@ -120,24 +126,25 @@ class SamClient(scheme: String, * @return Successful future if the auth is accepted, a Failure otherwise. */ def requestAuth(authorizationRequest: CollectionAuthorizationRequest, - cromIamRequest: HttpRequest): FailureResponseOrT[Unit] = { + cromIamRequest: HttpRequest + ): FailureResponseOrT[Unit] = { val logString = authorizationRequest.action + " access for user " + authorizationRequest.user.userId + - " on a request to " + authorizationRequest.action + " for collection " + authorizationRequest.collection.name + " on a request to " + authorizationRequest.action + " for collection " + authorizationRequest.collection.name - def validateEntityBytes(byteString: ByteString): FailureResponseOrT[Unit] = { + def validateEntityBytes(byteString: ByteString): FailureResponseOrT[Unit] = if (byteString.utf8String == "true") { Monad[FailureResponseOrT].unit } else { log.warning("Sam denied " + logString) FailureResponseOrT[IO, HttpResponse, Unit](IO.raiseError(new SamDenialException)) } - } log.info("Requesting authorization for " + logString) val request = HttpRequest(method = HttpMethods.GET, - uri = samAuthorizeActionUri(authorizationRequest), - headers = List[HttpHeader](authorizationRequest.user.authorization)) + uri = samAuthorizeActionUri(authorizationRequest), + headers = List[HttpHeader](authorizationRequest.user.authorization) + ) for { response <- instrumentRequest( @@ -158,10 +165,7 @@ class SamClient(scheme: String, - If user has the 'add' permission we're ok - else fail the future */ - def requestSubmission(user: User, - collection: Collection, - cromIamRequest: HttpRequest - ): FailureResponseOrT[Unit] = { + def requestSubmission(user: User, collection: Collection, cromIamRequest: HttpRequest): FailureResponseOrT[Unit] = { log.info("Verifying user " + user.userId + " can submit a workflow to collection " + collection.name) val createCollection = registerCreation(user, collection, cromIamRequest) @@ -169,15 +173,20 @@ class SamClient(scheme: String, case r if r.status == StatusCodes.NoContent => Monad[FailureResponseOrT].unit case r => FailureResponseOrT[IO, HttpResponse, Unit](IO.raiseError(SamRegisterCollectionException(r.status))) } recoverWith { - case r if r.status == StatusCodes.Conflict => requestAuth(CollectionAuthorizationRequest(user, collection, "add"), cromIamRequest) + case r if r.status == StatusCodes.Conflict => + requestAuth(CollectionAuthorizationRequest(user, collection, "add"), cromIamRequest) case r => FailureResponseOrT[IO, HttpResponse, Unit](IO.raiseError(SamRegisterCollectionException(r.status))) } } protected def registerCreation(user: User, collection: Collection, - cromIamRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { - val request = HttpRequest(method = HttpMethods.POST, uri = samRegisterUri(collection), headers = List[HttpHeader](user.authorization)) + cromIamRequest: HttpRequest + ): FailureResponseOrT[HttpResponse] = { + val request = HttpRequest(method = HttpMethods.POST, + uri = samRegisterUri(collection), + headers = List[HttpHeader](user.authorization) + ) instrumentRequest( () => Http().singleRequest(request).asFailureResponseOrT, @@ -186,9 +195,9 @@ class SamClient(scheme: String, ) } - private def samAuthorizeActionUri(authorizationRequest: CollectionAuthorizationRequest) = { - akka.http.scaladsl.model.Uri(s"${samBaseUriForWorkflow(authorizationRequest.collection)}/action/${authorizationRequest.action}") - } + private def samAuthorizeActionUri(authorizationRequest: CollectionAuthorizationRequest) = + akka.http.scaladsl.model + .Uri(s"${samBaseUriForWorkflow(authorizationRequest.collection)}/action/${authorizationRequest.action}") private def samRegisterUri(collection: Collection) = akka.http.scaladsl.model.Uri(samBaseUriForWorkflow(collection)) @@ -207,15 +216,18 @@ object SamClient { class SamDenialException extends Exception("Access Denied") - final case class SamConnectionFailure(phase: String, f: Throwable) extends Exception(s"Unable to connect to Sam during $phase (${f.getMessage})", f) + final case class SamConnectionFailure(phase: String, f: Throwable) + extends Exception(s"Unable to connect to Sam during $phase (${f.getMessage})", f) - final case class SamRegisterCollectionException(errorCode: StatusCode) extends Exception(s"Can't register collection with Sam. Status code: ${errorCode.value}") + final case class SamRegisterCollectionException(errorCode: StatusCode) + extends Exception(s"Can't register collection with Sam. Status code: ${errorCode.value}") final case class CollectionAuthorizationRequest(user: User, collection: Collection, action: String) val SamDenialResponse = HttpResponse(status = StatusCodes.Forbidden, entity = new SamDenialException().getMessage) - def SamRegisterCollectionExceptionResp(statusCode: StatusCode) = HttpResponse(status = statusCode, entity = SamRegisterCollectionException(statusCode).getMessage) + def SamRegisterCollectionExceptionResp(statusCode: StatusCode) = + HttpResponse(status = statusCode, entity = SamRegisterCollectionException(statusCode).getMessage) case class UserStatusInfo(adminEnabled: Boolean, enabled: Boolean, userEmail: String, userSubjectId: String) diff --git a/CromIAM/src/main/scala/cromiam/server/CromIamServer.scala b/CromIAM/src/main/scala/cromiam/server/CromIamServer.scala index 9f5af038b12..b18366490c5 100644 --- a/CromIAM/src/main/scala/cromiam/server/CromIamServer.scala +++ b/CromIAM/src/main/scala/cromiam/server/CromIamServer.scala @@ -15,7 +15,6 @@ import org.broadinstitute.dsde.workbench.util.health.Subsystems.{Cromwell, Sam} import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future, Promise} - object CromIamServer extends HttpApp with CromIamApiService with SwaggerService { final val rootConfig: Config = ConfigFactory.load() @@ -35,21 +34,28 @@ object CromIamServer extends HttpApp with CromIamApiService with SwaggerService If there is a reason then leave a comment why there should be two actor systems. https://github.com/broadinstitute/cromwell/issues/3851 */ - CromIamServer.startServer(configuration.cromIamConfig.http.interface, configuration.cromIamConfig.http.port, configuration.cromIamConfig.serverSettings) + CromIamServer.startServer(configuration.cromIamConfig.http.interface, + configuration.cromIamConfig.http.port, + configuration.cromIamConfig.serverSettings + ) } - override implicit val system: ActorSystem = ActorSystem() - override implicit lazy val executor: ExecutionContextExecutor = system.dispatcher - override implicit val materializer: ActorMaterializer = ActorMaterializer() + implicit override val system: ActorSystem = ActorSystem() + implicit override lazy val executor: ExecutionContextExecutor = system.dispatcher + implicit override val materializer: ActorMaterializer = ActorMaterializer() override val log = Logging(system, getClass) override val routes: Route = allRoutes ~ swaggerUiResourceRoute - override val statusService: StatusService = new StatusService(() => Map(Cromwell -> cromwellClient.subsystemStatus(), Sam -> samClient.subsystemStatus())) + override val statusService: StatusService = new StatusService(() => + Map(Cromwell -> cromwellClient.subsystemStatus(), Sam -> samClient.subsystemStatus()) + ) // Override default shutdownsignal which was just "hit return/enter" - override def waitForShutdownSignal(actorSystem: ActorSystem)(implicit executionContext: ExecutionContext): Future[Done] = { + override def waitForShutdownSignal( + actorSystem: ActorSystem + )(implicit executionContext: ExecutionContext): Future[Done] = { val promise = Promise[Done]() sys.addShutdownHook { // we can add anything we want the server to do when someone shutdowns the server (Ctrl-c) diff --git a/CromIAM/src/main/scala/cromiam/server/config/CromIamServerConfig.scala b/CromIAM/src/main/scala/cromiam/server/config/CromIamServerConfig.scala index 8d27c7b980d..ce8c5934acd 100644 --- a/CromIAM/src/main/scala/cromiam/server/config/CromIamServerConfig.scala +++ b/CromIAM/src/main/scala/cromiam/server/config/CromIamServerConfig.scala @@ -15,7 +15,8 @@ import scala.util.{Failure, Success, Try} final case class CromIamServerConfig(cromIamConfig: CromIamConfig, cromwellConfig: ServiceConfig, samConfig: SamClientConfig, - swaggerOauthConfig: SwaggerOauthConfig) + swaggerOauthConfig: SwaggerOauthConfig +) object CromIamServerConfig { def getFromConfig(conf: Config): ErrorOr[CromIamServerConfig] = { @@ -27,22 +28,28 @@ object CromIamServerConfig { (cromIamConfig, cromwellConfig, samConfig, googleConfig) mapN CromIamServerConfig.apply } - private[config] def getValidatedConfigPath[A](conf: Config, path: String, getter: (Config, String) => A, default: Option[A] = None): ErrorOr[A] = { + private[config] def getValidatedConfigPath[A](conf: Config, + path: String, + getter: (Config, String) => A, + default: Option[A] = None + ): ErrorOr[A] = if (conf.hasPath(path)) { Try(getter.apply(conf, path)) match { case Success(s) => s.validNel case Failure(e) => s"Unable to read valid value at '$path': ${e.getMessage}".invalidNel } - } else default match { - case Some(d) => d.validNel - case None => s"Configuration does not have path $path".invalidNel - } - } + } else + default match { + case Some(d) => d.validNel + case None => s"Configuration does not have path $path".invalidNel + } - private[config] implicit final class ValidatingConfig(val conf: Config) extends AnyVal { - def getValidatedString(path: String, default: Option[String] = None): ErrorOr[String] = getValidatedConfigPath(conf, path, (c, p) => c.getString(p), default) + implicit final private[config] class ValidatingConfig(val conf: Config) extends AnyVal { + def getValidatedString(path: String, default: Option[String] = None): ErrorOr[String] = + getValidatedConfigPath(conf, path, (c, p) => c.getString(p), default) def getValidatedInt(path: String): ErrorOr[Int] = getValidatedConfigPath(conf, path, (c, p) => c.getInt(p)) - def getValidatedStringList(path: String): ErrorOr[List[String]] = getValidatedConfigPath[List[String]](conf, path, (c, p) => c.getStringList(p).asScala.toList) + def getValidatedStringList(path: String): ErrorOr[List[String]] = + getValidatedConfigPath[List[String]](conf, path, (c, p) => c.getStringList(p).asScala.toList) } } @@ -50,13 +57,12 @@ final case class CromIamConfig(http: ServiceConfig, serverSettings: ServerSettin object CromIamConfig { - private def getValidatedServerSettings(conf: Config): ErrorOr[ServerSettings] = { + private def getValidatedServerSettings(conf: Config): ErrorOr[ServerSettings] = Try(ServerSettings(conf)) match { case Success(serverSettings) => serverSettings.validNel case Failure(e) => s"Unable to generate server settings from configuration file: ${e.getMessage}".invalidNel } - } private[config] def getFromConfig(conf: Config, basePath: String): ErrorOr[CromIamConfig] = { val serviceConfig = ServiceConfig.getFromConfig(conf, basePath) @@ -94,6 +100,9 @@ object SwaggerOauthConfig { private[config] def getFromConfig(conf: Config, basePath: String): ErrorOr[SwaggerOauthConfig] = { def getValidatedOption(option: String) = conf.getValidatedString(s"$basePath.$option") - (getValidatedOption("client_id"), getValidatedOption("realm"), getValidatedOption("app_name")) mapN SwaggerOauthConfig.apply - } + (getValidatedOption("client_id"), + getValidatedOption("realm"), + getValidatedOption("app_name") + ) mapN SwaggerOauthConfig.apply + } } diff --git a/CromIAM/src/main/scala/cromiam/server/status/StatusCheckedSubsystem.scala b/CromIAM/src/main/scala/cromiam/server/status/StatusCheckedSubsystem.scala index 7aa6c0af752..b267b201bb0 100644 --- a/CromIAM/src/main/scala/cromiam/server/status/StatusCheckedSubsystem.scala +++ b/CromIAM/src/main/scala/cromiam/server/status/StatusCheckedSubsystem.scala @@ -1,7 +1,7 @@ package cromiam.server.status import com.softwaremill.sttp.asynchttpclient.future.AsyncHttpClientFutureBackend -import com.softwaremill.sttp.{Uri, sttp} +import com.softwaremill.sttp.{sttp, Uri} import org.broadinstitute.dsde.workbench.util.health.SubsystemStatus import scala.concurrent.{ExecutionContext, Future} @@ -18,12 +18,11 @@ trait StatusCheckedSubsystem { * Make a call to the status endpoint. If we receive a 200 OK fill in the SubsystemStatus w/ OK = true and no * error messages, otherwise OK = false and include the response body */ - def subsystemStatus()(implicit ec: ExecutionContext): Future[SubsystemStatus] = { + def subsystemStatus()(implicit ec: ExecutionContext): Future[SubsystemStatus] = sttp.get(statusUri).send() map { x => x.body match { case Right(_) => SubsystemStatus(true, None) case Left(errors) => SubsystemStatus(false, Option(List(errors))) } } - } } diff --git a/CromIAM/src/main/scala/cromiam/server/status/StatusService.scala b/CromIAM/src/main/scala/cromiam/server/status/StatusService.scala index 97b16e98a5e..c3f38865224 100644 --- a/CromIAM/src/main/scala/cromiam/server/status/StatusService.scala +++ b/CromIAM/src/main/scala/cromiam/server/status/StatusService.scala @@ -15,13 +15,14 @@ import scala.concurrent.duration._ */ class StatusService(checkStatus: () => Map[Subsystem, Future[SubsystemStatus]], initialDelay: FiniteDuration = Duration.Zero, - pollInterval: FiniteDuration = 1.minute)(implicit system: ActorSystem, executionContext: ExecutionContext) { + pollInterval: FiniteDuration = 1.minute +)(implicit system: ActorSystem, executionContext: ExecutionContext) { implicit val askTimeout = Timeout(5.seconds) private val healthMonitor = system.actorOf(HealthMonitor.props(Set(Cromwell, Sam))(checkStatus), "HealthMonitorActor") system.scheduler.schedule(initialDelay, pollInterval, healthMonitor, HealthMonitor.CheckAll) - def status(): Future[StatusCheckResponse] = healthMonitor.ask(GetCurrentStatus).asInstanceOf[Future[StatusCheckResponse]] + def status(): Future[StatusCheckResponse] = + healthMonitor.ask(GetCurrentStatus).asInstanceOf[Future[StatusCheckResponse]] } - diff --git a/CromIAM/src/main/scala/cromiam/webservice/CromIamApiService.scala b/CromIAM/src/main/scala/cromiam/webservice/CromIamApiService.scala index 7a16e5ea797..d046694b0f9 100644 --- a/CromIAM/src/main/scala/cromiam/webservice/CromIamApiService.scala +++ b/CromIAM/src/main/scala/cromiam/webservice/CromIamApiService.scala @@ -16,7 +16,12 @@ import cromiam.auth.{Collection, User} import cromiam.cromwell.CromwellClient import cromiam.instrumentation.CromIamInstrumentation import cromiam.sam.SamClient -import cromiam.sam.SamClient.{CollectionAuthorizationRequest, SamConnectionFailure, SamDenialException, SamDenialResponse} +import cromiam.sam.SamClient.{ + CollectionAuthorizationRequest, + SamConnectionFailure, + SamDenialException, + SamDenialResponse +} import cromiam.server.config.CromIamServerConfig import cromiam.server.status.StatusService import cromiam.webservice.CromIamApiService._ @@ -30,12 +35,13 @@ trait SwaggerService extends SwaggerUiResourceHttpService { } // NB: collection name *must* follow label value rules in cromwell. This needs to be documented somewhere. (although those restrictions are soon to die) -trait CromIamApiService extends RequestSupport - with EngineRouteSupport - with SubmissionSupport - with QuerySupport - with WomtoolRouteSupport - with CromIamInstrumentation { +trait CromIamApiService + extends RequestSupport + with EngineRouteSupport + with SubmissionSupport + with QuerySupport + with WomtoolRouteSupport + with CromIamInstrumentation { implicit val system: ActorSystem implicit def executor: ExecutionContextExecutor @@ -44,23 +50,23 @@ trait CromIamApiService extends RequestSupport protected def rootConfig: Config protected def configuration: CromIamServerConfig - override lazy val serviceRegistryActor: ActorRef = system.actorOf(ServiceRegistryActor.props(rootConfig), "ServiceRegistryActor") + override lazy val serviceRegistryActor: ActorRef = + system.actorOf(ServiceRegistryActor.props(rootConfig), "ServiceRegistryActor") val log: LoggingAdapter - val CromIamExceptionHandler: ExceptionHandler = { - ExceptionHandler { - case e: Exception => - log.error(e, "Request failed {}", e) - complete(HttpResponse(InternalServerError, entity = e.getMessage)) // FIXME: use workbench-model ErrorReport + val CromIamExceptionHandler: ExceptionHandler = + ExceptionHandler { case e: Exception => + log.error(e, "Request failed {}", e) + complete(HttpResponse(InternalServerError, entity = e.getMessage)) // FIXME: use workbench-model ErrorReport } - } lazy val cromwellClient = new CromwellClient(configuration.cromwellConfig.scheme, - configuration.cromwellConfig.interface, - configuration.cromwellConfig.port, - log, - serviceRegistryActor) + configuration.cromwellConfig.interface, + configuration.cromwellConfig.port, + log, + serviceRegistryActor + ) lazy val samClient = new SamClient( configuration.samConfig.http.scheme, @@ -68,7 +74,8 @@ trait CromIamApiService extends RequestSupport configuration.samConfig.http.port, configuration.samConfig.checkSubmitWhitelist, log, - serviceRegistryActor) + serviceRegistryActor + ) val statusService: StatusService @@ -76,8 +83,7 @@ trait CromIamApiService extends RequestSupport workflowLogsRoute ~ abortRoute ~ metadataRoute ~ timingRoute ~ statusRoute ~ backendRoute ~ labelPatchRoute ~ callCacheDiffRoute ~ labelGetRoute ~ releaseHoldRoute - - val allRoutes: Route = handleExceptions(CromIamExceptionHandler) { workflowRoutes ~ engineRoutes ~ womtoolRoutes } + val allRoutes: Route = handleExceptions(CromIamExceptionHandler)(workflowRoutes ~ engineRoutes ~ womtoolRoutes) def abortRoute: Route = path("api" / "workflows" / Segment / Segment / Abort) { (_, workflowId) => post { @@ -90,8 +96,8 @@ trait CromIamApiService extends RequestSupport } } - //noinspection MutatorLikeMethodIsParameterless - def releaseHoldRoute: Route = path("api" / "workflows" / Segment / Segment / ReleaseHold) { (_, workflowId) => + // noinspection MutatorLikeMethodIsParameterless + def releaseHoldRoute: Route = path("api" / "workflows" / Segment / Segment / ReleaseHold) { (_, workflowId) => post { extractUserAndStrictRequest { (user, req) => logUserWorkflowAction(user, workflowId, ReleaseHold) @@ -109,22 +115,22 @@ trait CromIamApiService extends RequestSupport def statusRoute: Route = workflowGetRouteWithId("status") def labelGetRoute: Route = workflowGetRouteWithId(Labels) - def labelPatchRoute: Route = { + def labelPatchRoute: Route = path("api" / "workflows" / Segment / Segment / Labels) { (_, workflowId) => patch { extractUserAndStrictRequest { (user, req) => entity(as[String]) { labels => logUserWorkflowAction(user, workflowId, Labels) - validateLabels(Option(labels)) { _ => // Not using the labels, just using this to verify they didn't specify labels we don't want them to - complete { - authorizeUpdateThenForwardToCromwell(user, workflowId, req).asHttpResponse - } + validateLabels(Option(labels)) { + _ => // Not using the labels, just using this to verify they didn't specify labels we don't want them to + complete { + authorizeUpdateThenForwardToCromwell(user, workflowId, req).asHttpResponse + } } } } } } - } def backendRoute: Route = workflowGetRoute("backends") @@ -137,7 +143,10 @@ trait CromIamApiService extends RequestSupport complete { (paramMap.get("workflowA"), paramMap.get("workflowB")) match { case (Some(a), Some(b)) => authorizeReadThenForwardToCromwell(user, List(a, b), req).asHttpResponse - case _ => HttpResponse(status = BadRequest, entity = "Must supply both workflowA and workflowB to the /callcaching/diff endpoint") + case _ => + HttpResponse(status = BadRequest, + entity = "Must supply both workflowA and workflowB to the /callcaching/diff endpoint" + ) } } } @@ -162,31 +171,31 @@ trait CromIamApiService extends RequestSupport */ private def workflowGetRouteWithId(urlSuffix: String): Route = workflowRoute(urlSuffix, get) - private def workflowRoute(urlSuffix: String, method: Directive0): Route = path("api" / "workflows" / Segment / Segment / urlSuffix) { (_, workflowId) => - method { - extractUserAndStrictRequest { (user, req) => - logUserWorkflowAction(user, workflowId, urlSuffix) - complete { - authorizeReadThenForwardToCromwell(user, List(workflowId), req).asHttpResponse + private def workflowRoute(urlSuffix: String, method: Directive0): Route = + path("api" / "workflows" / Segment / Segment / urlSuffix) { (_, workflowId) => + method { + extractUserAndStrictRequest { (user, req) => + logUserWorkflowAction(user, workflowId, urlSuffix) + complete { + authorizeReadThenForwardToCromwell(user, List(workflowId), req).asHttpResponse + } } } } - } private def authorizeThenForwardToCromwell(user: User, workflowIds: List[String], action: String, request: HttpRequest, - cromwellClient: CromwellClient): - FailureResponseOrT[HttpResponse] = { - def authForCollection(collection: Collection): FailureResponseOrT[Unit] = { + cromwellClient: CromwellClient + ): FailureResponseOrT[HttpResponse] = { + def authForCollection(collection: Collection): FailureResponseOrT[Unit] = samClient.requestAuth(CollectionAuthorizationRequest(user, collection, action), request) mapErrorWith { case e: SamDenialException => IO.raiseError(e) case e => log.error(e, "Unable to connect to Sam {}", e) IO.raiseError(SamConnectionFailure("authorization", e)) } - } val cromwellResponseT = for { rootWorkflowIds <- workflowIds.traverse(cromwellClient.getRootWorkflow(_, user, request)) @@ -209,31 +218,29 @@ trait CromIamApiService extends RequestSupport private def authorizeReadThenForwardToCromwell(user: User, workflowIds: List[String], request: HttpRequest - ): FailureResponseOrT[HttpResponse] = { - authorizeThenForwardToCromwell( - user = user, - workflowIds = workflowIds, - action = "view", - request = request, - cromwellClient = cromwellClient) - } + ): FailureResponseOrT[HttpResponse] = + authorizeThenForwardToCromwell(user = user, + workflowIds = workflowIds, + action = "view", + request = request, + cromwellClient = cromwellClient + ) private def authorizeUpdateThenForwardToCromwell(user: User, workflowId: String, request: HttpRequest - ): FailureResponseOrT[HttpResponse] = { - authorizeThenForwardToCromwell( - user = user, - workflowIds = List(workflowId), - action = "update", - request = request, - cromwellClient = cromwellClient) - } + ): FailureResponseOrT[HttpResponse] = + authorizeThenForwardToCromwell(user = user, + workflowIds = List(workflowId), + action = "update", + request = request, + cromwellClient = cromwellClient + ) private def authorizeAbortThenForwardToCromwell(user: User, workflowId: String, request: HttpRequest - ): FailureResponseOrT[HttpResponse] = { + ): FailureResponseOrT[HttpResponse] = // Do all the authing for the abort with "this" cromwell instance (cromwellClient), but the actual abort command // must go to the dedicated abort server (cromwellAbortClient). authorizeThenForwardToCromwell( @@ -243,13 +250,11 @@ trait CromIamApiService extends RequestSupport request = request, cromwellClient = cromwellClient ) - } private def logUserAction(user: User, action: String) = log.info("User " + user.userId + " requesting " + action) - private def logUserWorkflowAction(user: User, wfId: String, action: String) = { + private def logUserWorkflowAction(user: User, wfId: String, action: String) = log.info("User " + user.userId + " requesting " + action + " with " + wfId) - } } object CromIamApiService { diff --git a/CromIAM/src/main/scala/cromiam/webservice/EngineRouteSupport.scala b/CromIAM/src/main/scala/cromiam/webservice/EngineRouteSupport.scala index 5719bda3479..42f69ee6449 100644 --- a/CromIAM/src/main/scala/cromiam/webservice/EngineRouteSupport.scala +++ b/CromIAM/src/main/scala/cromiam/webservice/EngineRouteSupport.scala @@ -13,7 +13,6 @@ import org.broadinstitute.dsde.workbench.util.health.StatusJsonSupport._ import scala.concurrent.ExecutionContextExecutor - trait EngineRouteSupport extends RequestSupport with SprayJsonSupport { val statusService: StatusService val cromwellClient: CromwellClient @@ -25,7 +24,7 @@ trait EngineRouteSupport extends RequestSupport with SprayJsonSupport { def versionRoute: Route = path("engine" / Segment / "version") { _ => get { extractStrictRequest { req => - complete { cromwellClient.forwardToCromwell(req).asHttpResponse } + complete(cromwellClient.forwardToCromwell(req).asHttpResponse) } } } @@ -39,10 +38,11 @@ trait EngineRouteSupport extends RequestSupport with SprayJsonSupport { } } - def statsRoute: Route = path("engine" / Segment / "stats") { _ => complete(CromIamStatsForbidden) } + def statsRoute: Route = path("engine" / Segment / "stats")(_ => complete(CromIamStatsForbidden)) } object EngineRouteSupport { - private[webservice] val CromIamStatsForbidden = HttpResponse(status = Forbidden, entity = "CromIAM does not allow access to the /stats endpoint") + private[webservice] val CromIamStatsForbidden = + HttpResponse(status = Forbidden, entity = "CromIAM does not allow access to the /stats endpoint") } diff --git a/CromIAM/src/main/scala/cromiam/webservice/QuerySupport.scala b/CromIAM/src/main/scala/cromiam/webservice/QuerySupport.scala index e9397605c6a..68ad1f21230 100644 --- a/CromIAM/src/main/scala/cromiam/webservice/QuerySupport.scala +++ b/CromIAM/src/main/scala/cromiam/webservice/QuerySupport.scala @@ -43,7 +43,7 @@ trait QuerySupport extends RequestSupport { post { preprocessQuery { (user, collections, request) => processLabelsForPostQuery(user, collections) { entity => - complete { cromwellClient.forwardToCromwell(request.withEntity(entity)).asHttpResponse } + complete(cromwellClient.forwardToCromwell(request.withEntity(entity)).asHttpResponse) } } } @@ -54,7 +54,7 @@ trait QuerySupport extends RequestSupport { * retrieves the collections for the user, grabs the underlying HttpRequest and forwards it on to the specific * directive */ - private def preprocessQuery: Directive[(User, List[Collection], HttpRequest)] = { + private def preprocessQuery: Directive[(User, List[Collection], HttpRequest)] = extractUserAndStrictRequest tflatMap { case (user, cromIamRequest) => log.info("Received query " + cromIamRequest.method.value + " request for user " + user.userId) @@ -71,13 +71,12 @@ trait QuerySupport extends RequestSupport { throw new RuntimeException(s"Unable to look up collections for user ${user.userId}: ${e.getMessage}", e) } } - } /** * Will verify that none of the GET query parameters are specifying the collection label, and then tack * on query parameters for the user's collections on to the query URI */ - private def processLabelsForGetQuery(user: User, collections: List[Collection]): Directive1[Uri] = { + private def processLabelsForGetQuery(user: User, collections: List[Collection]): Directive1[Uri] = extractUri flatMap { uri => val query = uri.query() @@ -95,7 +94,6 @@ trait QuerySupport extends RequestSupport { provide(uri.withQuery(newQueryBuilder.result())) } - } /** * Will verify that none of the POSTed query parameters are specifying the collection label, and then tack @@ -115,7 +113,7 @@ trait QuerySupport extends RequestSupport { case jsObject if jsObject.fields.keySet.exists(key => key.equalsIgnoreCase(LabelOrKey)) => jsObject.fields.values.map(_.convertTo[String]) } - ).flatten + ).flatten // DO NOT REMOVE THE NEXT LINE WITHOUT READING THE SCALADOC ON ensureNoLabelOrs ensureNoLabelOrs(user, labelOrs) @@ -152,12 +150,11 @@ trait QuerySupport extends RequestSupport { * - https://github.com/persvr/rql#rql-rules * - https://github.com/jirutka/rsql-parser#grammar-and-semantic */ - protected[this] def ensureNoLabelOrs(user: User, labelOrs: Iterable[String]): Unit = { + protected[this] def ensureNoLabelOrs(user: User, labelOrs: Iterable[String]): Unit = labelOrs.toList match { case Nil => () case head :: tail => throw new LabelContainsOrException(user, NonEmptyList(head, tail)) } - } /** * Returns the user's collections as a set of labels @@ -169,12 +166,14 @@ trait QuerySupport extends RequestSupport { } object QuerySupport { - final case class InvalidQueryException(e: Throwable) extends - Exception(s"Invalid JSON in query POST body: ${e.getMessage}", e) - - final class LabelContainsOrException(val user: User, val labelOrs: NonEmptyList[String]) extends - Exception(s"User ${user.userId} submitted a labels query containing an OR which CromIAM is blocking: " + - labelOrs.toList.mkString("LABELS CONTAIN '", "' OR LABELS CONTAIN '", "'")) + final case class InvalidQueryException(e: Throwable) + extends Exception(s"Invalid JSON in query POST body: ${e.getMessage}", e) + + final class LabelContainsOrException(val user: User, val labelOrs: NonEmptyList[String]) + extends Exception( + s"User ${user.userId} submitted a labels query containing an OR which CromIAM is blocking: " + + labelOrs.toList.mkString("LABELS CONTAIN '", "' OR LABELS CONTAIN '", "'") + ) val LabelAndKey = "label" val LabelOrKey = "labelor" diff --git a/CromIAM/src/main/scala/cromiam/webservice/RequestSupport.scala b/CromIAM/src/main/scala/cromiam/webservice/RequestSupport.scala index c9b6a196368..22575738ae7 100644 --- a/CromIAM/src/main/scala/cromiam/webservice/RequestSupport.scala +++ b/CromIAM/src/main/scala/cromiam/webservice/RequestSupport.scala @@ -15,32 +15,34 @@ import cromiam.sam.SamClient import scala.util.{Failure, Success} trait RequestSupport { - def extractStrictRequest: Directive1[HttpRequest] = { + def extractStrictRequest: Directive1[HttpRequest] = toStrictEntity(Timeout) tflatMap { _ => extractRequest flatMap { request => provide(request) } } - } /** * Obtain both the user id header from the proxy as well as the bearer token and pass that back * into the route logic as a User object */ - def extractUser: Directive1[User] = { - (headerValueByName("OIDC_CLAIM_user_id") & headerValuePF { case a: Authorization => a }) tmap { case (userId, auth) => - User(WorkbenchUserId(userId), auth) + def extractUser: Directive1[User] = + (headerValueByName("OIDC_CLAIM_user_id") & headerValuePF { case a: Authorization => a }) tmap { + case (userId, auth) => + User(WorkbenchUserId(userId), auth) } - } - def extractUserAndStrictRequest: Directive[(User, HttpRequest)] = { + def extractUserAndStrictRequest: Directive[(User, HttpRequest)] = for { user <- extractUser request <- extractStrictRequest } yield (user, request) - } - def forwardIfUserEnabled(user: User, req: HttpRequest, cromwellClient: CromwellClient, samClient: SamClient): Route = { + def forwardIfUserEnabled(user: User, + req: HttpRequest, + cromwellClient: CromwellClient, + samClient: SamClient + ): Route = { import cromwell.api.model.EnhancedFailureResponseOrHttpResponseT onComplete(samClient.isUserEnabledSam(user, req).value.unsafeToFuture()) { @@ -52,7 +54,8 @@ trait RequestSupport { } } case Failure(e) => - val message = s"Unable to look up enablement status for user ${user.userId}: ${e.getMessage}. Please try again later." + val message = + s"Unable to look up enablement status for user ${user.userId}: ${e.getMessage}. Please try again later." throw new RuntimeException(message, e) } } diff --git a/CromIAM/src/main/scala/cromiam/webservice/SubmissionSupport.scala b/CromIAM/src/main/scala/cromiam/webservice/SubmissionSupport.scala index 52a05d1cdc7..79c66a77313 100644 --- a/CromIAM/src/main/scala/cromiam/webservice/SubmissionSupport.scala +++ b/CromIAM/src/main/scala/cromiam/webservice/SubmissionSupport.scala @@ -7,7 +7,7 @@ import akka.http.scaladsl.server._ import akka.stream.ActorMaterializer import akka.util.ByteString import cats.effect.IO -import cromiam.auth.Collection.{CollectionLabelName, LabelsKey, validateLabels} +import cromiam.auth.Collection.{validateLabels, CollectionLabelName, LabelsKey} import cromiam.auth.{Collection, User} import cromiam.cromwell.CromwellClient import cromiam.sam.SamClient @@ -57,16 +57,18 @@ trait SubmissionSupport extends RequestSupport { private def forwardSubmissionToCromwell(user: User, collection: Collection, - submissionRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { - log.info("Forwarding submission request for " + user.userId + " with collection " + collection.name + " to Cromwell") + submissionRequest: HttpRequest + ): FailureResponseOrT[HttpResponse] = { + log.info( + "Forwarding submission request for " + user.userId + " with collection " + collection.name + " to Cromwell" + ) - def registerWithSam(collection: Collection, httpRequest: HttpRequest): FailureResponseOrT[Unit] = { + def registerWithSam(collection: Collection, httpRequest: HttpRequest): FailureResponseOrT[Unit] = samClient.requestSubmission(user, collection, httpRequest) mapErrorWith { case e: SamDenialException => IO.raiseError(e) case SamRegisterCollectionException(statusCode) => IO.raiseError(SamRegisterCollectionException(statusCode)) case e => IO.raiseError(SamConnectionFailure("new workflow registration", e)) } - } FailureResponseOrT( (for { @@ -81,36 +83,34 @@ trait SubmissionSupport extends RequestSupport { } object SubmissionSupport { - def extractCollection(user: User): Directive1[Collection] = { + def extractCollection(user: User): Directive1[Collection] = formField(CollectionNameKey.?) map { maybeCollectionName => maybeCollectionName.map(Collection(_)).getOrElse(Collection.forUser(user)) } - } - def extractSubmission(user: User): Directive1[WorkflowSubmission] = { + def extractSubmission(user: User): Directive1[WorkflowSubmission] = ( extractCollection(user) & - formFields(( - WorkflowSourceKey.?, - WorkflowUrlKey.?, - WorkflowTypeKey.?, - WorkflowTypeVersionKey.?, - WorkflowInputsKey.?, - WorkflowOptionsKey.?, - WorkflowOnHoldKey.as[Boolean].?, - WorkflowDependenciesKey.as[ByteString].?)) & - extractLabels & - extractInputAux + formFields( + (WorkflowSourceKey.?, + WorkflowUrlKey.?, + WorkflowTypeKey.?, + WorkflowTypeVersionKey.?, + WorkflowInputsKey.?, + WorkflowOptionsKey.?, + WorkflowOnHoldKey.as[Boolean].?, + WorkflowDependenciesKey.as[ByteString].? + ) + ) & + extractLabels & + extractInputAux ).as(WorkflowSubmission) - } - def extractLabels: Directive1[Option[Map[String, JsValue]]] = { + def extractLabels: Directive1[Option[Map[String, JsValue]]] = formField(LabelsKey.?) flatMap validateLabels - } - def extractInputAux: Directive1[Map[String, String]] = { + def extractInputAux: Directive1[Map[String, String]] = formFieldMap.map(_.view.filterKeys(_.startsWith(WorkflowInputsAuxPrefix)).toMap) - } // FIXME: Much like CromwellClient see if there are ways of unifying this a bit w/ the mothership final case class WorkflowSubmission(collection: Collection, @@ -123,32 +123,61 @@ object SubmissionSupport { workflowOnHold: Option[Boolean], workflowDependencies: Option[ByteString], origLabels: Option[Map[String, JsValue]], - workflowInputsAux: Map[String, String]) { + workflowInputsAux: Map[String, String] + ) { // For auto-validation, if origLabels defined, can't have CaaS collection label set. Was checked previously, but ... require(origLabels.forall(!_.keySet.contains(CollectionLabelName))) // Inject the collection name into the labels and convert to a String private val collectionLabels = Map(CollectionLabelName -> JsString(collection.name)) - private val labels: String = JsObject(origLabels.map(o => o ++ collectionLabels).getOrElse(collectionLabels)).toString + private val labels: String = JsObject( + origLabels.map(o => o ++ collectionLabels).getOrElse(collectionLabels) + ).toString val entity: MessageEntity = { - val sourcePart = workflowSource map { s => Multipart.FormData.BodyPart(WorkflowSourceKey, HttpEntity(MediaTypes.`application/json`, s)) } - val urlPart = workflowUrl map { u => Multipart.FormData.BodyPart(WorkflowUrlKey, HttpEntity(MediaTypes.`application/json`, u))} - val typePart = workflowType map { t => Multipart.FormData.BodyPart(WorkflowTypeKey, HttpEntity(MediaTypes.`application/json`, t)) } - val typeVersionPart = workflowTypeVersion map { v => Multipart.FormData.BodyPart(WorkflowTypeVersionKey, HttpEntity(MediaTypes.`application/json`, v)) } - val inputsPart = workflowInputs map { i => Multipart.FormData.BodyPart(WorkflowInputsKey, HttpEntity(MediaTypes.`application/json`, i)) } - val optionsPart = workflowOptions map { o => Multipart.FormData.BodyPart(WorkflowOptionsKey, HttpEntity(MediaTypes.`application/json`, o)) } - val importsPart = workflowDependencies map { d => Multipart.FormData.BodyPart(WorkflowDependenciesKey, HttpEntity(MediaTypes.`application/octet-stream`, d)) } - val onHoldPart = workflowOnHold map { h => Multipart.FormData.BodyPart(WorkflowOnHoldKey, HttpEntity(h.toString)) } + val sourcePart = workflowSource map { s => + Multipart.FormData.BodyPart(WorkflowSourceKey, HttpEntity(MediaTypes.`application/json`, s)) + } + val urlPart = workflowUrl map { u => + Multipart.FormData.BodyPart(WorkflowUrlKey, HttpEntity(MediaTypes.`application/json`, u)) + } + val typePart = workflowType map { t => + Multipart.FormData.BodyPart(WorkflowTypeKey, HttpEntity(MediaTypes.`application/json`, t)) + } + val typeVersionPart = workflowTypeVersion map { v => + Multipart.FormData.BodyPart(WorkflowTypeVersionKey, HttpEntity(MediaTypes.`application/json`, v)) + } + val inputsPart = workflowInputs map { i => + Multipart.FormData.BodyPart(WorkflowInputsKey, HttpEntity(MediaTypes.`application/json`, i)) + } + val optionsPart = workflowOptions map { o => + Multipart.FormData.BodyPart(WorkflowOptionsKey, HttpEntity(MediaTypes.`application/json`, o)) + } + val importsPart = workflowDependencies map { d => + Multipart.FormData.BodyPart(WorkflowDependenciesKey, HttpEntity(MediaTypes.`application/octet-stream`, d)) + } + val onHoldPart = workflowOnHold map { h => + Multipart.FormData.BodyPart(WorkflowOnHoldKey, HttpEntity(h.toString)) + } val labelsPart = Multipart.FormData.BodyPart(LabelsKey, HttpEntity(MediaTypes.`application/json`, labels)) - val parts = List(sourcePart, urlPart, typePart, typeVersionPart, inputsPart, optionsPart, importsPart, onHoldPart, Option(labelsPart)).flatten ++ auxParts + val parts = List(sourcePart, + urlPart, + typePart, + typeVersionPart, + inputsPart, + optionsPart, + importsPart, + onHoldPart, + Option(labelsPart) + ).flatten ++ auxParts Multipart.FormData(parts: _*).toEntity() } - private def auxParts = { - workflowInputsAux map { case (k, v) => Multipart.FormData.BodyPart(k, HttpEntity(MediaTypes.`application/json`, v)) } - } + private def auxParts = + workflowInputsAux map { case (k, v) => + Multipart.FormData.BodyPart(k, HttpEntity(MediaTypes.`application/json`, v)) + } } // FIXME: Unify these w/ Cromwell.PartialWorkflowSources (via common?) diff --git a/CromIAM/src/main/scala/cromiam/webservice/SwaggerUiHttpService.scala b/CromIAM/src/main/scala/cromiam/webservice/SwaggerUiHttpService.scala index 9fed12ca163..874949a5244 100644 --- a/CromIAM/src/main/scala/cromiam/webservice/SwaggerUiHttpService.scala +++ b/CromIAM/src/main/scala/cromiam/webservice/SwaggerUiHttpService.scala @@ -27,7 +27,7 @@ trait SwaggerUiHttpService extends Directives { s"META-INF/resources/webjars/swagger-ui/$swaggerUiVersion" } - private val serveIndex: server.Route = { + private val serveIndex: server.Route = mapResponseEntity { entityFromJar => entityFromJar.transformDataBytes(Flow.fromFunction[ByteString, ByteString] { original: ByteString => ByteString(rewriteSwaggerIndex(original.utf8String)) @@ -35,14 +35,13 @@ trait SwaggerUiHttpService extends Directives { } { getFromResource(s"$resourceDirectory/index.html") } - } /** * Serves up the swagger UI only. Redirects requests to the root of the UI path to the index.html. * * @return Route serving the swagger UI. */ - final def swaggerUiRoute: Route = { + final def swaggerUiRoute: Route = pathEndOrSingleSlash { get { serveIndex @@ -68,16 +67,15 @@ trait SwaggerUiHttpService extends Directives { } } - } /** Rewrite the swagger index.html. Default passes through the origin data. */ protected def rewriteSwaggerIndex(original: String): String = { val swaggerOptions = s""" - | validatorUrl: null, - | apisSorter: "alpha", - | oauth2RedirectUrl: window.location.origin + "/swagger/oauth2-redirect.html", - | operationsSorter: "alpha" + | validatorUrl: null, + | apisSorter: "alpha", + | oauth2RedirectUrl: window.location.origin + "/swagger/oauth2-redirect.html", + | operationsSorter: "alpha" """.stripMargin val initOAuthOriginal = "window.ui = ui" @@ -94,7 +92,6 @@ trait SwaggerUiHttpService extends Directives { |$initOAuthOriginal |""".stripMargin - original .replace(initOAuthOriginal, initOAuthReplacement) .replace("""url: "https://petstore.swagger.io/v2/swagger.json"""", "url: 'cromiam.yaml'") @@ -109,6 +106,7 @@ trait SwaggerUiHttpService extends Directives { * swagger UI, but defaults to "yaml". This is an alternative to spray-swagger's SwaggerHttpService. */ trait SwaggerResourceHttpService { + /** * @return The directory for the resource under the classpath, and in the url */ @@ -134,7 +132,8 @@ trait SwaggerResourceHttpService { */ final def swaggerResourceRoute: Route = { // Serve CromIAM API docs from either `/swagger/cromiam.yaml` or just `cromiam.yaml`. - val swaggerDocsDirective = path(separateOnSlashes(swaggerDocsPath)) | path(s"$swaggerServiceName.$swaggerResourceType") + val swaggerDocsDirective = + path(separateOnSlashes(swaggerDocsPath)) | path(s"$swaggerServiceName.$swaggerResourceType") val route = get { swaggerDocsDirective { // Return /uiPath/serviceName.resourceType from the classpath resources. diff --git a/CromIAM/src/test/scala/cromiam/auth/CollectionSpec.scala b/CromIAM/src/test/scala/cromiam/auth/CollectionSpec.scala index 5f3f4fd3791..0ee89b9bbb6 100644 --- a/CromIAM/src/test/scala/cromiam/auth/CollectionSpec.scala +++ b/CromIAM/src/test/scala/cromiam/auth/CollectionSpec.scala @@ -42,7 +42,6 @@ class CollectionSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers } } - behavior of "validateLabels" it should "not throw exception when labels are valid" in { val labels = """{"key-1":"foo","key-2":"bar"}""" diff --git a/CromIAM/src/test/scala/cromiam/cromwell/CromwellClientSpec.scala b/CromIAM/src/test/scala/cromiam/cromwell/CromwellClientSpec.scala index ce6f16935d6..303aff01dc4 100644 --- a/CromIAM/src/test/scala/cromiam/cromwell/CromwellClientSpec.scala +++ b/CromIAM/src/test/scala/cromiam/cromwell/CromwellClientSpec.scala @@ -39,25 +39,35 @@ class CromwellClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfter } "CromwellClient" should "eventually return a subworkflow's root workflow id" in { - cromwellClient.getRootWorkflow(SubworkflowId.id.toString, FictitiousUser, fakeHttpRequest).map(w => assert(w == RootWorkflowId.id.toString)) - .asIo.unsafeToFuture() + cromwellClient + .getRootWorkflow(SubworkflowId.id.toString, FictitiousUser, fakeHttpRequest) + .map(w => assert(w == RootWorkflowId.id.toString)) + .asIo + .unsafeToFuture() } it should "eventually return a top level workflow's ID when requesting root workflow id" in { - cromwellClient.getRootWorkflow(RootWorkflowId.id.toString, FictitiousUser, fakeHttpRequest).map(w => assert(w == RootWorkflowId.id.toString)) - .asIo.unsafeToFuture() + cromwellClient + .getRootWorkflow(RootWorkflowId.id.toString, FictitiousUser, fakeHttpRequest) + .map(w => assert(w == RootWorkflowId.id.toString)) + .asIo + .unsafeToFuture() } it should "properly fetch the collection for a workflow with a collection name" in { - cromwellClient.collectionForWorkflow(RootWorkflowId.id.toString, FictitiousUser, fakeHttpRequest).map(c => - assert(c.name == CollectionName) - ).asIo.unsafeToFuture() + cromwellClient + .collectionForWorkflow(RootWorkflowId.id.toString, FictitiousUser, fakeHttpRequest) + .map(c => assert(c.name == CollectionName)) + .asIo + .unsafeToFuture() } it should "throw an exception if the workflow doesn't have a collection" in { recoverToExceptionIf[IllegalArgumentException] { - cromwellClient.collectionForWorkflow(WorkflowIdWithoutCollection.id.toString, FictitiousUser, fakeHttpRequest) - .asIo.unsafeToFuture() + cromwellClient + .collectionForWorkflow(WorkflowIdWithoutCollection.id.toString, FictitiousUser, fakeHttpRequest) + .asIo + .unsafeToFuture() } map { exception => assert(exception.getMessage == s"Workflow $WorkflowIdWithoutCollection has no associated collection") } @@ -65,24 +75,25 @@ class CromwellClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfter } object CromwellClientSpec { - final class MockCromwellClient()(implicit system: ActorSystem, - ece: ExecutionContextExecutor, - materializer: ActorMaterializer) - extends CromwellClient("http", "bar", 1, NoLogging, ActorRef.noSender) { + final class MockCromwellClient()(implicit + system: ActorSystem, + ece: ExecutionContextExecutor, + materializer: ActorMaterializer + ) extends CromwellClient("http", "bar", 1, NoLogging, ActorRef.noSender) { override val cromwellApiClient: CromwellApiClient = new MockCromwellApiClient() override def sendTimingApi(statsDPath: InstrumentationPath, timing: FiniteDuration, prefixToStatsd: NonEmptyList[String] - ): Unit = () + ): Unit = () } final class MockCromwellApiClient()(implicit actorSystem: ActorSystem, materializer: ActorMaterializer) - extends CromwellApiClient(new URL("http://foo.com"), "bar") { + extends CromwellApiClient(new URL("http://foo.com"), "bar") { - - override def labels(workflowId: WorkflowId, headers: List[HttpHeader] = defaultHeaders) - (implicit ec: ExecutionContext): FailureResponseOrT[WorkflowLabels] = { + override def labels(workflowId: WorkflowId, headers: List[HttpHeader] = defaultHeaders)(implicit + ec: ExecutionContext + ): FailureResponseOrT[WorkflowLabels] = if (workflowId == RootWorkflowId) { FailureResponseOrT.pure(FictitiousWorkflowLabelsWithCollection) } else if (workflowId == WorkflowIdWithoutCollection) { @@ -92,18 +103,17 @@ object CromwellClientSpec { IO.raiseError(new RuntimeException("Unexpected workflow ID sent to MockCromwellApiClient")) } } - } override def metadata(workflowId: WorkflowId, - args: Option[Map[String, List[String]]] = None, - headers: List[HttpHeader] = defaultHeaders - )(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowMetadata] = { + args: Option[Map[String, List[String]]] = None, + headers: List[HttpHeader] = defaultHeaders + )(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowMetadata] = if (workflowId == RootWorkflowId) FailureResponseOrT.pure(RootWorkflowMetadata) else if (workflowId == SubworkflowId) FailureResponseOrT.pure(SubWorkflowMetadata) - else FailureResponseOrT[IO, HttpResponse, WorkflowMetadata] { - IO.raiseError(new RuntimeException("Unexpected workflow ID sent to MockCromwellApiClient")) - } - } + else + FailureResponseOrT[IO, HttpResponse, WorkflowMetadata] { + IO.raiseError(new RuntimeException("Unexpected workflow ID sent to MockCromwellApiClient")) + } } private val SubworkflowId = WorkflowId.fromString("58114f5c-f439-4488-8d73-092273cf92d9") @@ -126,7 +136,8 @@ object CromwellClientSpec { }""") val CollectionName = "foo" - val FictitiousWorkflowLabelsWithCollection = WorkflowLabels(RootWorkflowId.id.toString, JsObject(Map("caas-collection-name" -> JsString(CollectionName)))) - val FictitiousWorkflowLabelsWithoutCollection = WorkflowLabels(RootWorkflowId.id.toString, JsObject(Map("something" -> JsString("foo")))) + val FictitiousWorkflowLabelsWithCollection = + WorkflowLabels(RootWorkflowId.id.toString, JsObject(Map("caas-collection-name" -> JsString(CollectionName)))) + val FictitiousWorkflowLabelsWithoutCollection = + WorkflowLabels(RootWorkflowId.id.toString, JsObject(Map("something" -> JsString("foo")))) } - diff --git a/CromIAM/src/test/scala/cromiam/sam/SamClientSpec.scala b/CromIAM/src/test/scala/cromiam/sam/SamClientSpec.scala index 40f0cda2e86..95869718fb5 100644 --- a/CromIAM/src/test/scala/cromiam/sam/SamClientSpec.scala +++ b/CromIAM/src/test/scala/cromiam/sam/SamClientSpec.scala @@ -28,7 +28,8 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { HttpResponse(StatusCodes.InternalServerError, entity = HttpEntity("expected error")) private val authorization = Authorization(OAuth2BearerToken("my-token")) - private val authorizedUserWithCollection = User(WorkbenchUserId(MockSamClient.AuthorizedUserCollectionStr), authorization) + private val authorizedUserWithCollection = + User(WorkbenchUserId(MockSamClient.AuthorizedUserCollectionStr), authorization) private val unauthorizedUserWithNoCollection = User(WorkbenchUserId(MockSamClient.UnauthorizedUserCollectionStr), authorization) private val notWhitelistedUser = User(WorkbenchUserId(MockSamClient.NotWhitelistedUser), authorization) @@ -47,25 +48,25 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { super.afterAll() } - behavior of "SamClient" it should "return true if user is whitelisted" in { val samClient = new MockSamClient() - samClient.isSubmitWhitelisted(authorizedUserWithCollection, emptyHttpRequest).map(v => assert(v)) - .asIo.unsafeToFuture() + samClient + .isSubmitWhitelisted(authorizedUserWithCollection, emptyHttpRequest) + .map(v => assert(v)) + .asIo + .unsafeToFuture() } it should "return false if user is not whitelisted" in { val samClient = new MockSamClient() - samClient.isSubmitWhitelisted(notWhitelistedUser, emptyHttpRequest).map(v => assert(!v)) - .asIo.unsafeToFuture() + samClient.isSubmitWhitelisted(notWhitelistedUser, emptyHttpRequest).map(v => assert(!v)).asIo.unsafeToFuture() } it should "return sam errors while checking is whitelisted" in { val samClient = new MockSamClient() { - override def isSubmitWhitelistedSam(user: User, cromiamRequest: HttpRequest): FailureResponseOrT[Boolean] = { + override def isSubmitWhitelistedSam(user: User, cromiamRequest: HttpRequest): FailureResponseOrT[Boolean] = MockSamClient.returnResponse(expectedErrorResponse) - } } samClient.isSubmitWhitelisted(notWhitelistedUser, emptyHttpRequest).value.unsafeToFuture() map { _ should be(Left(expectedErrorResponse)) @@ -74,32 +75,33 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { it should "eventually return the collection(s) of user" in { val samClient = new MockSamClient() - samClient.collectionsForUser(authorizedUserWithCollection, emptyHttpRequest).map(collectionList => - assert(collectionList == MockSamClient.UserCollectionList) - ).asIo.unsafeToFuture() + samClient + .collectionsForUser(authorizedUserWithCollection, emptyHttpRequest) + .map(collectionList => assert(collectionList == MockSamClient.UserCollectionList)) + .asIo + .unsafeToFuture() } it should "fail if user doesn't have any collections" in { val samClient = new MockSamClient() recoverToExceptionIf[Exception] { - samClient.collectionsForUser(unauthorizedUserWithNoCollection, emptyHttpRequest) - .asIo.unsafeToFuture() - } map(exception => - assert(exception.getMessage == s"Unable to look up collections for user ${unauthorizedUserWithNoCollection.userId.value}!") + samClient.collectionsForUser(unauthorizedUserWithNoCollection, emptyHttpRequest).asIo.unsafeToFuture() + } map (exception => + assert( + exception.getMessage == s"Unable to look up collections for user ${unauthorizedUserWithNoCollection.userId.value}!" + ) ) } it should "return true if user is authorized to perform action on collection" in { val samClient = new MockSamClient() - samClient.requestAuth(authorizedCollectionRequest, emptyHttpRequest).map(_ => succeed) - .asIo.unsafeToFuture() + samClient.requestAuth(authorizedCollectionRequest, emptyHttpRequest).map(_ => succeed).asIo.unsafeToFuture() } it should "throw SamDenialException if user is not authorized to perform action on collection" in { val samClient = new MockSamClient() recoverToExceptionIf[SamDenialException] { - samClient.requestAuth(unauthorizedCollectionRequest, emptyHttpRequest) - .asIo.unsafeToFuture() + samClient.requestAuth(unauthorizedCollectionRequest, emptyHttpRequest).asIo.unsafeToFuture() } map { exception => assert(exception.getMessage == "Access Denied") } @@ -107,15 +109,21 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { it should "register collection to Sam if user has authorization to create/add to collection" in { val samClient = new MockSamClient() - samClient.requestSubmission(authorizedUserWithCollection, authorizedCollection, emptyHttpRequest).map(_ => succeed) - .asIo.unsafeToFuture() + samClient + .requestSubmission(authorizedUserWithCollection, authorizedCollection, emptyHttpRequest) + .map(_ => succeed) + .asIo + .unsafeToFuture() } it should "throw SamRegisterCollectionException if user doesn't have authorization to create/add to collection" in { val samClient = new MockSamClient() recoverToExceptionIf[SamRegisterCollectionException] { - samClient.requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest).map(_ => succeed) - .asIo.unsafeToFuture() + samClient + .requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) + .map(_ => succeed) + .asIo + .unsafeToFuture() } map { exception => assert(exception.getMessage == "Can't register collection with Sam. Status code: 400 Bad Request") } @@ -125,15 +133,16 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { val samClient = new BaseMockSamClient() { override protected def registerCreation(user: User, collection: Collection, - cromiamRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { + cromiamRequest: HttpRequest + ): FailureResponseOrT[HttpResponse] = { val conflictResponse = HttpResponse(StatusCodes.Conflict, entity = HttpEntity("expected conflict")) returnResponse(conflictResponse) } override def requestAuth(authorizationRequest: CollectionAuthorizationRequest, - cromiamRequest: HttpRequest): FailureResponseOrT[Unit] = { + cromiamRequest: HttpRequest + ): FailureResponseOrT[Unit] = Monad[FailureResponseOrT].unit - } } samClient .requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) @@ -146,19 +155,22 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { val samClient = new BaseMockSamClient() { override protected def registerCreation(user: User, collection: Collection, - cromiamRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { + cromiamRequest: HttpRequest + ): FailureResponseOrT[HttpResponse] = { val conflictResponse = HttpResponse(StatusCodes.Conflict, entity = HttpEntity("expected conflict")) returnResponse(conflictResponse) } override def requestAuth(authorizationRequest: CollectionAuthorizationRequest, - cromiamRequest: HttpRequest): FailureResponseOrT[Unit] = { + cromiamRequest: HttpRequest + ): FailureResponseOrT[Unit] = returnResponse(expectedErrorResponse) - } } recoverToExceptionIf[UnsuccessfulRequestException] { - samClient.requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) - .asIo.unsafeToFuture() + samClient + .requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) + .asIo + .unsafeToFuture() } map { exception => assert(exception.getMessage == "expected error") } @@ -168,14 +180,17 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { val samClient = new BaseMockSamClient() { override protected def registerCreation(user: User, collection: Collection, - cromiamRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { + cromiamRequest: HttpRequest + ): FailureResponseOrT[HttpResponse] = { val unexpectedOkResponse = HttpResponse(StatusCodes.OK, entity = HttpEntity("elided ok message")) returnResponse(unexpectedOkResponse) } } recoverToExceptionIf[SamRegisterCollectionException] { - samClient.requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) - .asIo.unsafeToFuture() + samClient + .requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) + .asIo + .unsafeToFuture() } map { exception => exception.getMessage should be("Can't register collection with Sam. Status code: 200 OK") } @@ -185,14 +200,17 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { val samClient = new BaseMockSamClient() { override protected def registerCreation(user: User, collection: Collection, - cromiamRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { + cromiamRequest: HttpRequest + ): FailureResponseOrT[HttpResponse] = { val unexpectedFailureResponse = HttpResponse(StatusCodes.ImATeapot, entity = HttpEntity("elided error message")) returnResponse(unexpectedFailureResponse) } } recoverToExceptionIf[SamRegisterCollectionException] { - samClient.requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) - .asIo.unsafeToFuture() + samClient + .requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) + .asIo + .unsafeToFuture() } map { exception => exception.getMessage should be("Can't register collection with Sam. Status code: 418 I'm a teapot") } diff --git a/CromIAM/src/test/scala/cromiam/server/status/MockStatusService.scala b/CromIAM/src/test/scala/cromiam/server/status/MockStatusService.scala index 9010da8bda0..c7866c68089 100644 --- a/CromIAM/src/test/scala/cromiam/server/status/MockStatusService.scala +++ b/CromIAM/src/test/scala/cromiam/server/status/MockStatusService.scala @@ -6,8 +6,10 @@ import org.broadinstitute.dsde.workbench.util.health.Subsystems.{Cromwell, Sam, import scala.concurrent.{ExecutionContext, Future} -class MockStatusService(checkStatus: () => Map[Subsystem, Future[SubsystemStatus]])(implicit system: ActorSystem, executionContext: ExecutionContext) extends - StatusService(checkStatus)(system, executionContext) { +class MockStatusService(checkStatus: () => Map[Subsystem, Future[SubsystemStatus]])(implicit + system: ActorSystem, + executionContext: ExecutionContext +) extends StatusService(checkStatus)(system, executionContext) { override def status(): Future[StatusCheckResponse] = { val subsystemStatus: SubsystemStatus = SubsystemStatus(ok = true, None) @@ -16,4 +18,3 @@ class MockStatusService(checkStatus: () => Map[Subsystem, Future[SubsystemStatus Future.successful(StatusCheckResponse(ok = true, subsystems)) } } - diff --git a/CromIAM/src/test/scala/cromiam/webservice/CromIamApiServiceSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/CromIamApiServiceSpec.scala index 89945c5bfcb..e93acd51a2a 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/CromIamApiServiceSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/CromIamApiServiceSpec.scala @@ -13,14 +13,23 @@ import cromiam.server.status.StatusService import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with CromIamApiService with ScalatestRouteTest { +class CromIamApiServiceSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with CromIamApiService + with ScalatestRouteTest { override def testConfigSource = "akka.loglevel = DEBUG" val log = NoLogging - override def rootConfig: Config = throw new UnsupportedOperationException("This spec shouldn't need to access the real config") + override def rootConfig: Config = throw new UnsupportedOperationException( + "This spec shouldn't need to access the real config" + ) - override def configuration = throw new UnsupportedOperationException("This spec shouldn't need to access the real interface/port") + override def configuration = throw new UnsupportedOperationException( + "This spec shouldn't need to access the real interface/port" + ) override lazy val cromwellClient = new MockCromwellClient() override lazy val samClient = new MockSamClient() @@ -29,13 +38,15 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma val version = "v1" val authorization = Authorization(OAuth2BearerToken("my-token")) - val badAuthHeaders: List[HttpHeader] = List(authorization, RawHeader("OIDC_CLAIM_user_id", cromwellClient.unauthorizedUserCollectionStr)) - val goodAuthHeaders: List[HttpHeader] = List(authorization, RawHeader("OIDC_CLAIM_user_id", cromwellClient.authorizedUserCollectionStr)) - + val badAuthHeaders: List[HttpHeader] = + List(authorization, RawHeader("OIDC_CLAIM_user_id", cromwellClient.unauthorizedUserCollectionStr)) + val goodAuthHeaders: List[HttpHeader] = + List(authorization, RawHeader("OIDC_CLAIM_user_id", cromwellClient.authorizedUserCollectionStr)) behavior of "Status endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/status").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/status") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -43,7 +54,8 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 200 for authorized user who has collection associated with subworkflow" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/status").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/status") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -51,15 +63,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/status").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/status") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have view permissions" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/status").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/status") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -72,10 +88,10 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "Outputs endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/outputs").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/outputs") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -83,7 +99,8 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 200 for authorized user who has collection associated with subworkflow" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/outputs").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/outputs") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -91,15 +108,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/outputs").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/outputs") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have view permissions" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/outputs").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/outputs") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -112,10 +133,10 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "Metadata endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/metadata").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/metadata") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -123,7 +144,8 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 200 for authorized user who has collection associated with subworkflow" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/metadata").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/metadata") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -131,15 +153,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/metadata").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/metadata") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have view permissions" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/metadata").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/metadata") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -152,10 +178,10 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "Logs endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/logs").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/logs") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -163,7 +189,8 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 200 for authorized user who has collection associated with subworkflow" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/logs").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/logs") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -171,15 +198,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/logs").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/logs") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have view permissions" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/logs").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/logs") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -192,10 +223,10 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "GET Labels endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -203,7 +234,8 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 200 for authorized user who has collection associated with subworkflow" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/labels").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/labels") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -211,15 +243,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/labels").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/labels") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have view permissions" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/labels").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/labels") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -232,12 +268,13 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "PATCH Labels endpoint" it should "successfully forward request to Cromwell if nothing is untoward" in { val labels = """{"key-1":"foo","key-2":"bar"}""" val labelEntity = HttpEntity(ContentTypes.`application/json`, labels) - Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels").withHeaders(goodAuthHeaders).withEntity(labelEntity) ~> allRoutes ~> check { + Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels") + .withHeaders(goodAuthHeaders) + .withEntity(labelEntity) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -248,7 +285,9 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma val labels = """{"key-1":"foo","caas-collection-name":"bar"}""" val labelEntity = HttpEntity(ContentTypes.`application/json`, labels) - Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels").withHeaders(goodAuthHeaders).withEntity(labelEntity) ~> allRoutes ~> check { + Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels") + .withHeaders(goodAuthHeaders) + .withEntity(labelEntity) ~> allRoutes ~> check { status shouldBe InternalServerError responseAs[String] shouldBe "Submitted labels contain the key caas-collection-name, which is not allowed\n" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -259,7 +298,9 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma val labels = """"key-1":"foo"""" val labelEntity = HttpEntity(ContentTypes.`application/json`, labels) - Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels").withHeaders(goodAuthHeaders).withEntity(labelEntity) ~> allRoutes ~> check { + Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels") + .withHeaders(goodAuthHeaders) + .withEntity(labelEntity) ~> allRoutes ~> check { status shouldBe InternalServerError responseAs[String] shouldBe "Labels must be a valid JSON object, received: \"key-1\":\"foo\"\n" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -269,9 +310,13 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma it should "return 500 for authorized user who doesn't have collection associated with workflow" in { val labels = """{"key-1":"foo","key-2":"bar"}""" val labelEntity = HttpEntity(ContentTypes.`application/json`, labels) - Patch(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/labels").withHeaders(goodAuthHeaders).withEntity(labelEntity) ~> allRoutes ~> check { + Patch(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/labels") + .withHeaders(goodAuthHeaders) + .withEntity(labelEntity) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } @@ -279,7 +324,9 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma it should "return SamDenialException for user who doesn't have update permissions" in { val labels = """{"key-1":"foo","key-2":"bar"}""" val labelEntity = HttpEntity(ContentTypes.`application/json`, labels) - Patch(s"/api/workflows/$version/${cromwellClient.subworkflowId}/labels").withHeaders(badAuthHeaders).withEntity(labelEntity) ~> allRoutes ~> check { + Patch(s"/api/workflows/$version/${cromwellClient.subworkflowId}/labels") + .withHeaders(badAuthHeaders) + .withEntity(labelEntity) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -289,12 +336,12 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma it should "reject request if it doesn't contain OIDC_CLAIM_user_id in header" in { val labels = """{"key-1":"foo","key-2":"bar"}""" val labelEntity = HttpEntity(ContentTypes.`application/json`, labels) - Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels").withEntity(labelEntity) ~> allRoutes ~> check { + Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels") + .withEntity(labelEntity) ~> allRoutes ~> check { rejection shouldEqual MissingHeaderRejection("OIDC_CLAIM_user_id") } } - behavior of "Backends endpoint" it should "successfully forward request to Cromwell if auth header is provided" in { Get(s"/api/workflows/$version/backends").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { @@ -346,7 +393,8 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma behavior of "ReleaseHold endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Post(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/releaseHold").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Post(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/releaseHold") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -354,15 +402,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Post(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/releaseHold").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Post(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/releaseHold") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have update permissions" in { - Post(s"/api/workflows/$version/${cromwellClient.subworkflowId}/releaseHold").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Post(s"/api/workflows/$version/${cromwellClient.subworkflowId}/releaseHold") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -375,10 +427,10 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "Abort endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Post(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/abort").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Post(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/abort") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -386,15 +438,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Post(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/abort").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Post(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/abort") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have abort permissions" in { - Post(s"/api/workflows/$version/${cromwellClient.subworkflowId}/abort").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Post(s"/api/workflows/$version/${cromwellClient.subworkflowId}/abort") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -407,11 +463,13 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "CallCacheDiff endpoint" it should "return 200 for authorized user who has collection associated with both workflows" in { - val callCacheDiffParams = s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.anotherRootWorkflowIdWithCollection}&callB=helloCall" - Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + val callCacheDiffParams = + s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.anotherRootWorkflowIdWithCollection}&callB=helloCall" + Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders( + goodAuthHeaders + ) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -420,7 +478,9 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma it should "return BadRequest if request is malformed" in { val callCacheDiffParams = s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall" - Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders( + goodAuthHeaders + ) ~> allRoutes ~> check { status shouldBe BadRequest responseAs[String] shouldBe "Must supply both workflowA and workflowB to the /callcaching/diff endpoint" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -428,17 +488,25 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who has doesn't have collection associated with any one workflow" in { - val callCacheDiffParams = s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.workflowIdWithoutCollection}&callB=helloCall" - Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + val callCacheDiffParams = + s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.workflowIdWithoutCollection}&callB=helloCall" + Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders( + goodAuthHeaders + ) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have read permissions" in { - val callCacheDiffParams = s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.anotherRootWorkflowIdWithCollection}&callB=helloCall" - Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + val callCacheDiffParams = + s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.anotherRootWorkflowIdWithCollection}&callB=helloCall" + Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders( + badAuthHeaders + ) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -446,7 +514,8 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "reject request if it doesn't contain OIDC_CLAIM_user_id in header" in { - val callCacheDiffParams = s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.anotherRootWorkflowIdWithCollection}&callB=helloCall" + val callCacheDiffParams = + s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.anotherRootWorkflowIdWithCollection}&callB=helloCall" Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams") ~> allRoutes ~> check { rejection shouldEqual MissingHeaderRejection("OIDC_CLAIM_user_id") } diff --git a/CromIAM/src/test/scala/cromiam/webservice/EngineRouteSupportSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/EngineRouteSupportSpec.scala index f6f1e3b75de..9d431f6af05 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/EngineRouteSupportSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/EngineRouteSupportSpec.scala @@ -9,8 +9,12 @@ import cromiam.server.status.{MockStatusService, StatusService} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - -class EngineRouteSupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with EngineRouteSupport { +class EngineRouteSupportSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalatestRouteTest + with EngineRouteSupport { override val cromwellClient = new MockCromwellClient() val samClient = new MockSamClient() override val statusService: StatusService = new MockStatusService(() => Map.empty) diff --git a/CromIAM/src/test/scala/cromiam/webservice/MockClients.scala b/CromIAM/src/test/scala/cromiam/webservice/MockClients.scala index 59e75347159..ff976e24b4b 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/MockClients.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/MockClients.scala @@ -16,10 +16,8 @@ import cromwell.api.model._ import scala.concurrent.ExecutionContextExecutor -class MockCromwellClient()(implicit system: ActorSystem, - ece: ExecutionContextExecutor, - materializer: ActorMaterializer) - extends CromwellClient("http", "bar", 1, NoLogging, ActorRef.noSender)(system, ece, materializer) { +class MockCromwellClient()(implicit system: ActorSystem, ece: ExecutionContextExecutor, materializer: ActorMaterializer) + extends CromwellClient("http", "bar", 1, NoLogging, ActorRef.noSender)(system, ece, materializer) { val version = "v1" val unauthorizedUserCollectionStr: String = "987654321" @@ -56,7 +54,7 @@ class MockCromwellClient()(implicit system: ActorSystem, val womtoolRoutePath = s"/api/womtool/$version/describe" httpRequest.uri.path.toString match { - //version endpoint doesn't require authentication + // version endpoint doesn't require authentication case `versionRoutePath` => FailureResponseOrT.pure(HttpResponse(status = OK, entity = "Response from Cromwell")) // womtool endpoint requires authn which it gets for free from the proxy, does not care about authz @@ -68,18 +66,19 @@ class MockCromwellClient()(implicit system: ActorSystem, override def getRootWorkflow(workflowId: String, user: User, - cromIamRequest: HttpRequest): FailureResponseOrT[String] = { + cromIamRequest: HttpRequest + ): FailureResponseOrT[String] = workflowId match { case `subworkflowId` | `rootWorkflowIdWithCollection` => FailureResponseOrT.pure(rootWorkflowIdWithCollection) case `anotherRootWorkflowIdWithCollection` => FailureResponseOrT.pure(anotherRootWorkflowIdWithCollection) case _ => FailureResponseOrT.pure(workflowIdWithoutCollection) } - } override def collectionForWorkflow(workflowId: String, user: User, - cromIamRequest: HttpRequest): FailureResponseOrT[Collection] = { + cromIamRequest: HttpRequest + ): FailureResponseOrT[Collection] = workflowId match { case `rootWorkflowIdWithCollection` | `anotherRootWorkflowIdWithCollection` => FailureResponseOrT.pure(userCollection) @@ -87,33 +86,32 @@ class MockCromwellClient()(implicit system: ActorSystem, val exception = new IllegalArgumentException(s"Workflow $workflowId has no associated collection") FailureResponseOrT.left(IO.raiseError[HttpResponse](exception)) } - } } /** * Overrides some values, but doesn't override methods. */ -class BaseMockSamClient(checkSubmitWhitelist: Boolean = true) - (implicit system: ActorSystem, - ece: ExecutionContextExecutor, - materializer: ActorMaterializer) - extends SamClient( - "http", - "bar", - 1, - checkSubmitWhitelist, - NoLogging, - ActorRef.noSender - )(system, ece, materializer) +class BaseMockSamClient(checkSubmitWhitelist: Boolean = true)(implicit + system: ActorSystem, + ece: ExecutionContextExecutor, + materializer: ActorMaterializer +) extends SamClient( + "http", + "bar", + 1, + checkSubmitWhitelist, + NoLogging, + ActorRef.noSender + )(system, ece, materializer) /** * Extends the base mock client with overriden methods. */ -class MockSamClient(checkSubmitWhitelist: Boolean = true) - (implicit system: ActorSystem, - ece: ExecutionContextExecutor, - materializer: ActorMaterializer) - extends BaseMockSamClient(checkSubmitWhitelist) { +class MockSamClient(checkSubmitWhitelist: Boolean = true)(implicit + system: ActorSystem, + ece: ExecutionContextExecutor, + materializer: ActorMaterializer +) extends BaseMockSamClient(checkSubmitWhitelist) { override def collectionsForUser(user: User, httpRequest: HttpRequest): FailureResponseOrT[List[Collection]] = { val userId = user.userId.value @@ -127,7 +125,8 @@ class MockSamClient(checkSubmitWhitelist: Boolean = true) override def requestSubmission(user: User, collection: Collection, - cromIamRequest: HttpRequest): FailureResponseOrT[Unit] = { + cromIamRequest: HttpRequest + ): FailureResponseOrT[Unit] = collection match { case c if c.name.equalsIgnoreCase(UnauthorizedUserCollectionStr) => val exception = SamRegisterCollectionException(StatusCodes.BadRequest) @@ -135,28 +134,25 @@ class MockSamClient(checkSubmitWhitelist: Boolean = true) case c if c.name.equalsIgnoreCase(AuthorizedUserCollectionStr) => Monad[FailureResponseOrT].unit case _ => Monad[FailureResponseOrT].unit } - } - override def isSubmitWhitelistedSam(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = { + override def isSubmitWhitelistedSam(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = FailureResponseOrT.pure(!user.userId.value.equalsIgnoreCase(NotWhitelistedUser)) - } - override def isUserEnabledSam(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = { + override def isUserEnabledSam(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = if (user.userId.value == "enabled@example.com" || user.userId.value == MockSamClient.AuthorizedUserCollectionStr) FailureResponseOrT.pure(true) else if (user.userId.value == "disabled@example.com") FailureResponseOrT.pure(false) else throw new Exception("Misconfigured test") - } override def requestAuth(authorizationRequest: CollectionAuthorizationRequest, - cromIamRequest: HttpRequest): FailureResponseOrT[Unit] = { + cromIamRequest: HttpRequest + ): FailureResponseOrT[Unit] = authorizationRequest.user.userId.value match { case AuthorizedUserCollectionStr => Monad[FailureResponseOrT].unit case _ => FailureResponseOrT.left(IO.raiseError[HttpResponse](new SamDenialException)) } - } } object MockSamClient { @@ -165,7 +161,6 @@ object MockSamClient { val NotWhitelistedUser: String = "ABC123" val UserCollectionList: List[Collection] = List(Collection("col1"), Collection("col2")) - def returnResponse[T](response: HttpResponse): FailureResponseOrT[T] = { + def returnResponse[T](response: HttpResponse): FailureResponseOrT[T] = FailureResponseOrT.left(IO.pure(response)) - } } diff --git a/CromIAM/src/test/scala/cromiam/webservice/QuerySupportSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/QuerySupportSpec.scala index 216691c1c35..57dedbaefdf 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/QuerySupportSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/QuerySupportSpec.scala @@ -11,8 +11,12 @@ import org.broadinstitute.dsde.workbench.model.WorkbenchUserId import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - -class QuerySupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with QuerySupport { +class QuerySupportSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalatestRouteTest + with QuerySupport { override val cromwellClient = new MockCromwellClient() override val samClient = new MockSamClient() override val log: LoggingAdapter = NoLogging @@ -27,7 +31,8 @@ class QuerySupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matcher val getQuery = s"$queryPath?status=Submitted&label=foo:bar&label=foo:baz" val badGetQuery = s"$queryPath?status=Submitted&labelor=foo:bar&label=foo:baz" - val goodPostEntity = HttpEntity(ContentTypes.`application/json`, + val goodPostEntity = HttpEntity( + ContentTypes.`application/json`, """|[ | { | "start": "2015-11-01T00:00:00-04:00" @@ -50,7 +55,8 @@ class QuerySupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matcher |] |""".stripMargin ) - val badPostEntity = HttpEntity(ContentTypes.`application/json`, + val badPostEntity = HttpEntity( + ContentTypes.`application/json`, """|[ | { | "start": "2015-11-01T00:00:00-04:00" diff --git a/CromIAM/src/test/scala/cromiam/webservice/SubmissionSupportSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/SubmissionSupportSpec.scala index c4a424747e6..bfbd38075f5 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/SubmissionSupportSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/SubmissionSupportSpec.scala @@ -12,8 +12,12 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.duration._ - -class SubmissionSupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with SubmissionSupport { +class SubmissionSupportSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalatestRouteTest + with SubmissionSupport { override val cromwellClient = new MockCromwellClient() override val samClient = new MockSamClient() override val log: LoggingAdapter = NoLogging @@ -56,7 +60,6 @@ class SubmissionSupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma val workflowSource = Multipart.FormData.BodyPart("workflowSource", HttpEntity(helloWorldWdl)) val formData = Multipart.FormData(workflowSource).toEntity() - "Submit endpoint" should "forward the request to Cromwell for authorized SAM user" in { Post(submitPath).withHeaders(goodAuthHeaders).withEntity(formData) ~> submitRoute ~> check { status shouldEqual StatusCodes.OK diff --git a/CromIAM/src/test/scala/cromiam/webservice/SwaggerServiceSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/SwaggerServiceSpec.scala index 838a523f4eb..32a7511c6de 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/SwaggerServiceSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/SwaggerServiceSpec.scala @@ -16,9 +16,13 @@ import org.yaml.snakeyaml.{LoaderOptions, Yaml => SnakeYaml} import scala.jdk.CollectionConverters._ - -class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with SwaggerService with ScalatestRouteTest with Matchers - with TableDrivenPropertyChecks { +class SwaggerServiceSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with SwaggerService + with ScalatestRouteTest + with Matchers + with TableDrivenPropertyChecks { def actorRefFactory = system override def oauthConfig: SwaggerOauthConfig = SwaggerOauthConfig("clientId", "realm", "appName") val yamlLoaderOptions = new LoaderOptions @@ -33,7 +37,8 @@ class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Swagg contentType should be(ContentTypes.`application/octet-stream`) val body = responseAs[String] - val yaml = new SnakeYaml(new UniqueKeyConstructor(new LoaderOptions)).loadAs(body, classOf[java.util.Map[String, AnyRef]]) + val yaml = new SnakeYaml(new UniqueKeyConstructor(new LoaderOptions)) + .loadAs(body, classOf[java.util.Map[String, AnyRef]]) yaml.get("swagger") should be("2.0") } @@ -62,27 +67,42 @@ class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Swagg resultWithInfo.getSwagger.getDefinitions.asScala foreach { // If no properties, `getProperties` returns `null` instead of an empty map - case (defKey, defVal) => Option(defVal.getProperties).map(_.asScala).getOrElse(Map.empty) foreach { - /* + case (defKey, defVal) => + Option(defVal.getProperties).map(_.asScala).getOrElse(Map.empty) foreach { + /* Two against one. Swagger parser implementation lets a RefProperty have descriptions. http://swagger.io/specification/#referenceObject & http://editor.swagger.io both say it's ref ONLY! - */ - case (propKey, propVal: RefProperty) => - withClue(s"RefProperty $defKey.$propKey has a description: ") { - propVal.getDescription should be(null) - } - case _ => /* ignore */ - } + */ + case (propKey, propVal: RefProperty) => + withClue(s"RefProperty $defKey.$propKey has a description: ") { + propVal.getDescription should be(null) + } + case _ => /* ignore */ + } } } } it should "return status OK when getting OPTIONS on paths" in { - val pathExamples = Table("path", "/", "/swagger", "/swagger/cromwell.yaml", "/swagger/index.html", "/api", - "/api/workflows/", "/api/workflows/v1", "/workflows/v1/outputs", "/workflows/v1/status", - "/api/workflows/v1/validate", "/workflows", "/workflows/v1", "/workflows/v1/outputs", "/workflows/v1/status", - "/workflows/v1/validate") + val pathExamples = Table( + "path", + "/", + "/swagger", + "/swagger/cromwell.yaml", + "/swagger/index.html", + "/api", + "/api/workflows/", + "/api/workflows/v1", + "/workflows/v1/outputs", + "/workflows/v1/status", + "/api/workflows/v1/validate", + "/workflows", + "/workflows/v1", + "/workflows/v1/outputs", + "/workflows/v1/status", + "/workflows/v1/validate" + ) forAll(pathExamples) { path => Options(path) ~> diff --git a/CromIAM/src/test/scala/cromiam/webservice/SwaggerUiHttpServiceSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/SwaggerUiHttpServiceSpec.scala index 1d0103a09fd..b39466ec4f0 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/SwaggerUiHttpServiceSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/SwaggerUiHttpServiceSpec.scala @@ -10,19 +10,38 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks - -trait SwaggerUiHttpServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with SwaggerUiHttpService { +trait SwaggerUiHttpServiceSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalatestRouteTest + with SwaggerUiHttpService { def actorRefFactory = system } -trait SwaggerResourceHttpServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with -TableDrivenPropertyChecks with SwaggerResourceHttpService { - - val testPathsForOptions = Table("endpoint", "/", "/swagger", "/swagger/index.html", "/api", "/api/example", - "/api/example?with=param", "/api/example/path") +trait SwaggerResourceHttpServiceSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalatestRouteTest + with TableDrivenPropertyChecks + with SwaggerResourceHttpService { + + val testPathsForOptions = Table("endpoint", + "/", + "/swagger", + "/swagger/index.html", + "/api", + "/api/example", + "/api/example?with=param", + "/api/example/path" + ) } -trait SwaggerUiResourceHttpServiceSpec extends SwaggerUiHttpServiceSpec with SwaggerResourceHttpServiceSpec with SwaggerUiResourceHttpService +trait SwaggerUiResourceHttpServiceSpec + extends SwaggerUiHttpServiceSpec + with SwaggerResourceHttpServiceSpec + with SwaggerUiResourceHttpService object SwaggerUiHttpServiceSpec { // TODO: Re-common-ize swagger out of cromwell's engine and reuse. @@ -40,7 +59,7 @@ class BasicSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec { behavior of "SwaggerUiHttpService" override protected def rewriteSwaggerIndex(data: String): String = - // Replace same magic string used in SwaggerUiResourceHttpService.rewriteSwaggerIndex + // Replace same magic string used in SwaggerUiResourceHttpService.rewriteSwaggerIndex data.replace("window.ui = ui", "replaced-client-id") it should "redirect /swagger to /" in { @@ -80,7 +99,9 @@ class BasicSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec { } override def oauthConfig: SwaggerOauthConfig = SwaggerOauthConfig( - clientId = "test-client-id", realm = "test-realm", appName = "test-appname" + clientId = "test-client-id", + realm = "test-realm", + appName = "test-appname" ) } @@ -195,7 +216,6 @@ class YamlSwaggerUiResourceHttpServiceSpec extends SwaggerUiResourceHttpServiceS } } - class JsonSwaggerUiResourceHttpServiceSpec extends SwaggerUiResourceHttpServiceSpec { override def oauthConfig: SwaggerOauthConfig = SwaggerOauthConfig("clientId", "realm", "appName") diff --git a/CromIAM/src/test/scala/cromiam/webservice/WomtoolRouteSupportSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/WomtoolRouteSupportSpec.scala index 785c887c374..a229f2db255 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/WomtoolRouteSupportSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/WomtoolRouteSupportSpec.scala @@ -10,8 +10,12 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - -class WomtoolRouteSupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with WomtoolRouteSupport with ScalatestRouteTest { +class WomtoolRouteSupportSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with WomtoolRouteSupport + with ScalatestRouteTest { override lazy val cromwellClient = new MockCromwellClient() override lazy val samClient = new MockSamClient() diff --git a/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala b/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala index 54849df4250..549bf761e38 100644 --- a/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala @@ -6,13 +6,19 @@ import cromwell.core.simpleton.WomValueSimpleton import cromwell.services.CallCaching.CallCachingEntryId object BackendCacheHitCopyingActor { - final case class CopyOutputsCommand(womValueSimpletons: Seq[WomValueSimpleton], jobDetritusFiles: Map[String, String], cacheHit: CallCachingEntryId, returnCode: Option[Int]) + final case class CopyOutputsCommand(womValueSimpletons: Seq[WomValueSimpleton], + jobDetritusFiles: Map[String, String], + cacheHit: CallCachingEntryId, + returnCode: Option[Int] + ) final case class CopyingOutputsFailedResponse(jobKey: JobKey, cacheCopyAttempt: Int, failure: CacheCopyFailure) sealed trait CacheCopyFailure + /** A cache hit copy was attempted but failed. */ final case class CopyAttemptError(failure: Throwable) extends CacheCopyFailure + /** Copying was requested for a blacklisted cache hit, however the cache hit copying actor found the hit had already * been blacklisted so no novel copy attempt was made. */ final case class BlacklistSkip(failureCategory: MetricableCacheCopyErrorCategory) extends CacheCopyFailure diff --git a/backend/src/main/scala/cromwell/backend/BackendInitializationData.scala b/backend/src/main/scala/cromwell/backend/BackendInitializationData.scala index b1099b07155..8a9daa3cc39 100644 --- a/backend/src/main/scala/cromwell/backend/BackendInitializationData.scala +++ b/backend/src/main/scala/cromwell/backend/BackendInitializationData.scala @@ -15,12 +15,11 @@ object BackendInitializationData { * @tparam A The type to cast the initialization data. * @return The initialization data as the type A. */ - def as[A <: BackendInitializationData](initializationDataOption: Option[BackendInitializationData]): A = { + def as[A <: BackendInitializationData](initializationDataOption: Option[BackendInitializationData]): A = initializationDataOption match { case Some(initializationData) => initializationData.asInstanceOf[A] case None => throw new RuntimeException("Initialization data was not found.") } - } } object AllBackendInitializationData { @@ -30,5 +29,6 @@ object AllBackendInitializationData { // Holds initialization data for all backends initialized for a workflow. case class AllBackendInitializationData(data: Map[String, Option[BackendInitializationData]]) { def get(backendName: String): Option[BackendInitializationData] = data.get(backendName).flatten - def getWorkflowRoots(): Set[Path] = data.values.collect({case Some(i: StandardInitializationData) => i.workflowPaths.workflowRoot}).toSet[Path] + def getWorkflowRoots(): Set[Path] = + data.values.collect { case Some(i: StandardInitializationData) => i.workflowPaths.workflowRoot }.toSet[Path] } diff --git a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala index a58131363d3..493221ec3ce 100644 --- a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala @@ -33,7 +33,8 @@ object BackendJobExecutionActor { jobDetritusFiles: Option[Map[String, Path]], executionEvents: Seq[ExecutionEvent], dockerImageUsed: Option[String], - resultGenerationMode: ResultGenerationMode) extends BackendJobExecutionResponse + resultGenerationMode: ResultGenerationMode + ) extends BackendJobExecutionResponse sealed trait ResultGenerationMode case object RunOnBackend extends ResultGenerationMode @@ -41,25 +42,30 @@ object BackendJobExecutionActor { case object FetchedFromJobStore extends ResultGenerationMode case class JobAbortedResponse(jobKey: BackendJobDescriptorKey) extends BackendJobExecutionResponse - - sealed trait BackendJobFailedResponse extends BackendJobExecutionResponse { def throwable: Throwable; def returnCode: Option[Int] } - case class JobFailedNonRetryableResponse(jobKey: JobKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse - case class JobFailedRetryableResponse(jobKey: BackendJobDescriptorKey, - throwable: Throwable, - returnCode: Option[Int]) extends BackendJobFailedResponse - - // Reconnection Exceptions - case class JobReconnectionNotSupportedException(jobKey: BackendJobDescriptorKey) extends Exception( - s"This backend does not support job reconnection. The status of the underlying job for ${jobKey.tag} cannot be known." - ) with CromwellFatalExceptionMarker - case class JobNotFoundException(jobKey: BackendJobDescriptorKey) extends Exception ( - s"No backend job for ${jobKey.tag} could be found. The status of the underlying job cannot be known." - ) with CromwellFatalExceptionMarker + sealed trait BackendJobFailedResponse extends BackendJobExecutionResponse { + def throwable: Throwable; def returnCode: Option[Int] + } + case class JobFailedNonRetryableResponse(jobKey: JobKey, throwable: Throwable, returnCode: Option[Int]) + extends BackendJobFailedResponse + case class JobFailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) + extends BackendJobFailedResponse - def buildJobExecutionActorName(workflowId: WorkflowId, jobKey: BackendJobDescriptorKey) = { + // Reconnection Exceptions + case class JobReconnectionNotSupportedException(jobKey: BackendJobDescriptorKey) + extends Exception( + s"This backend does not support job reconnection. The status of the underlying job for ${jobKey.tag} cannot be known." + ) + with CromwellFatalExceptionMarker + + case class JobNotFoundException(jobKey: BackendJobDescriptorKey) + extends Exception( + s"No backend job for ${jobKey.tag} could be found. The status of the underlying job cannot be known." + ) + with CromwellFatalExceptionMarker + + def buildJobExecutionActorName(workflowId: WorkflowId, jobKey: BackendJobDescriptorKey) = s"$workflowId-BackendJobExecutionActor-${jobKey.tag}" - } } /** @@ -92,7 +98,9 @@ trait BackendJobExecutionActor extends BackendJobLifecycleActor with ActorLoggin */ def recover: Future[BackendJobExecutionResponse] = { log.warning("{} backend currently doesn't support recovering jobs. Starting {} again.", - jobTag, jobDescriptor.key.call.fullyQualifiedName) + jobTag, + jobDescriptor.key.call.fullyQualifiedName + ) execute } @@ -100,28 +108,24 @@ trait BackendJobExecutionActor extends BackendJobLifecycleActor with ActorLoggin * Tries to reconnect to a previously started job. This method differs from recover by sending a ReconnectionFailure * if it can't reconnect to the job for whatever reason. It should NOT execute the job if reconnection is impossible. */ - def reconnect: Future[BackendJobExecutionResponse] = { + def reconnect: Future[BackendJobExecutionResponse] = Future.failed(JobReconnectionNotSupportedException(jobDescriptor.key)) - } /** * Similar to reconnect, except that if the reconnection succeeds and the job is still running, * an abort attempt should be made. */ - def reconnectToAborting: Future[BackendJobExecutionResponse] = { + def reconnectToAborting: Future[BackendJobExecutionResponse] = Future.failed(JobReconnectionNotSupportedException(jobDescriptor.key)) - } /** * Abort a running job. */ - def abort(): Unit = { - log.warning("{} backend currently doesn't support abort for {}.", - jobTag, jobDescriptor.key.call.fullyQualifiedName) - } + def abort(): Unit = + log.warning("{} backend currently doesn't support abort for {}.", jobTag, jobDescriptor.key.call.fullyQualifiedName) - def evaluateOutputs(wdlFunctions: IoFunctionSet, - postMapper: WomValue => Try[WomValue] = v => Success(v))(implicit ec: ExecutionContext): EvaluatedJobOutputs = { + def evaluateOutputs(wdlFunctions: IoFunctionSet, postMapper: WomValue => Try[WomValue] = v => Success(v))(implicit + ec: ExecutionContext + ): EvaluatedJobOutputs = Await.result(OutputEvaluator.evaluateOutputs(jobDescriptor, wdlFunctions, postMapper), Duration.Inf) - } } diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala index 72b0c24a800..450179c43e4 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala @@ -30,7 +30,7 @@ trait BackendLifecycleActor extends Actor { /** * The execution context for the actor */ - protected implicit def ec: ExecutionContext = context.dispatcher + implicit protected def ec: ExecutionContext = context.dispatcher /** * The configuration for the backend, in the context of the entire Cromwell configuration file. @@ -39,7 +39,8 @@ trait BackendLifecycleActor extends Actor { protected def performActionThenRespond(operation: => Future[BackendWorkflowLifecycleActorResponse], onFailure: Throwable => BackendWorkflowLifecycleActorResponse, - andThen: => Unit = ()) = { + andThen: => Unit = () + ) = { val respondTo: ActorRef = sender() operation onComplete { case Success(r) => @@ -54,9 +55,9 @@ trait BackendLifecycleActor extends Actor { trait BackendWorkflowLifecycleActor extends BackendLifecycleActor with WorkflowLogging { - //For Logging and boilerplate - override lazy final val workflowIdForLogging = workflowDescriptor.possiblyNotRootWorkflowId - override lazy final val rootWorkflowIdForLogging = workflowDescriptor.rootWorkflowId + // For Logging and boilerplate + final override lazy val workflowIdForLogging = workflowDescriptor.possiblyNotRootWorkflowId + final override lazy val rootWorkflowIdForLogging = workflowDescriptor.rootWorkflowId /** * The workflow descriptor for the workflow in which this Backend is being used diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala index 0d38d87b01f..dc06ec94f09 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala @@ -25,7 +25,8 @@ trait BackendLifecycleActorFactory { */ def name: String - def nameForCallCachingPurposes: String = configurationDescriptor.backendConfig.getOrElse("name-for-call-caching-purposes", name) + def nameForCallCachingPurposes: String = + configurationDescriptor.backendConfig.getOrElse("name-for-call-caching-purposes", name) /** * Config values for the backend, and a pointer to the global config. @@ -44,7 +45,8 @@ trait BackendLifecycleActorFactory { ioActor: ActorRef, calls: Set[CommandCallNode], serviceRegistryActor: ActorRef, - restarting: Boolean): Option[Props] = None + restarting: Boolean + ): Option[Props] = None /* ****************************** */ /* Job Execution */ @@ -54,7 +56,8 @@ trait BackendLifecycleActorFactory { initializationData: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef, - backendSingletonActor: Option[ActorRef]): Props + backendSingletonActor: Option[ActorRef] + ): Props lazy val jobExecutionTokenType: JobTokenType = { val concurrentJobLimit = configurationDescriptor.backendConfig.as[Option[Int]]("concurrent-job-limit") @@ -67,7 +70,8 @@ trait BackendLifecycleActorFactory { } lazy val jobRestartCheckTokenType: JobTokenType = { - val concurrentRestartCheckLimit = configurationDescriptor.globalConfig.as[Option[Int]]("system.job-restart-check-rate-control.max-jobs") + val concurrentRestartCheckLimit = + configurationDescriptor.globalConfig.as[Option[Int]]("system.job-restart-check-rate-control.max-jobs") // if defined, use per-backend hog-factor, otherwise use system-level value val hogFactor = configurationDescriptor.backendConfig.as[Option[Int]]("hog-factor") match { case Some(backendHogFactorValue) => backendHogFactorValue @@ -85,13 +89,16 @@ trait BackendLifecycleActorFactory { calls: Set[CommandCallNode], jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, - initializationData: Option[BackendInitializationData]): Option[Props] = None + initializationData: Option[BackendInitializationData] + ): Option[Props] = None /* ****************************** */ /* Call Caching */ /* ****************************** */ - def fileHashingActorProps: Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Option[ActorRef]) => Props] = None + def fileHashingActorProps + : Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Option[ActorRef]) => Props] = + None /** * Providing this method to generate Props for a cache hit copying actor is optional. @@ -102,7 +109,9 @@ trait BackendLifecycleActorFactory { * * Simples! */ - def cacheHitCopyingActorProps: Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Int, Option[BlacklistCache]) => Props] = None + def cacheHitCopyingActorProps: Option[ + (BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Int, Option[BlacklistCache]) => Props + ] = None /* ****************************** */ /* Misc. */ @@ -114,19 +123,26 @@ trait BackendLifecycleActorFactory { jobKey: BackendJobDescriptorKey, initializationData: Option[BackendInitializationData], ioActor: ActorRef, - ec: ExecutionContext): IoFunctionSet = NoIoFunctionSet + ec: ExecutionContext + ): IoFunctionSet = NoIoFunctionSet def pathBuilders(initializationDataOption: Option[BackendInitializationData]): PathBuilders = List.empty - def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, initializationData: Option[BackendInitializationData]): Path = { + def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, + backendConfig: Config, + initializationData: Option[BackendInitializationData] + ): Path = new WorkflowPathsWithDocker(workflowDescriptor, backendConfig).executionRoot - } - def getWorkflowExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, initializationData: Option[BackendInitializationData]): Path = { + def getWorkflowExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, + backendConfig: Config, + initializationData: Option[BackendInitializationData] + ): Path = new WorkflowPathsWithDocker(workflowDescriptor, backendConfig).workflowRoot - } - def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]): Set[RuntimeAttributeDefinition] = Set.empty + def runtimeAttributeDefinitions( + initializationDataOption: Option[BackendInitializationData] + ): Set[RuntimeAttributeDefinition] = Set.empty /** * A set of KV store keys that this backend requests that the engine lookup before running each job. @@ -136,13 +152,21 @@ trait BackendLifecycleActorFactory { /** * A set of KV store keys that are requested and looked up on behalf of all backends before running each job. */ - def defaultKeyValueStoreKeys: Seq[String] = Seq(BackendLifecycleActorFactory.FailedRetryCountKey, BackendLifecycleActorFactory.MemoryMultiplierKey) + def defaultKeyValueStoreKeys: Seq[String] = + Seq(BackendLifecycleActorFactory.FailedRetryCountKey, BackendLifecycleActorFactory.MemoryMultiplierKey) /* * Returns credentials that can be used to authenticate to a docker registry server * in order to obtain a docker hash. */ - def dockerHashCredentials(workflowDescriptor: BackendWorkflowDescriptor, initializationDataOption: Option[BackendInitializationData]): List[Any] = List.empty + def dockerHashCredentials(workflowDescriptor: BackendWorkflowDescriptor, + initializationDataOption: Option[BackendInitializationData] + ): List[Any] = List.empty + + /** + * Allows Cromwell to self-identify which cloud it's running on for runtime attribute purposes + */ + def platform: Option[Platform] = None } object BackendLifecycleActorFactory { diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowFinalizationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowFinalizationActor.scala index 2f77c2ebc2a..04abe76e372 100644 --- a/backend/src/main/scala/cromwell/backend/BackendWorkflowFinalizationActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowFinalizationActor.scala @@ -27,8 +27,8 @@ object BackendWorkflowFinalizationActor { */ trait BackendWorkflowFinalizationActor extends BackendWorkflowLifecycleActor with ActorLogging { - def receive: Receive = LoggingReceive { - case Finalize => performActionThenRespond(afterAll() map { _ => FinalizationSuccess }, onFailure = FinalizationFailed) + def receive: Receive = LoggingReceive { case Finalize => + performActionThenRespond(afterAll() map { _ => FinalizationSuccess }, onFailure = FinalizationFailed) } /** diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala index bc3dd963f63..9f59a451478 100644 --- a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala @@ -34,7 +34,8 @@ object BackendWorkflowInitializationActor { // Responses sealed trait BackendWorkflowInitializationActorResponse extends BackendWorkflowLifecycleActorResponse sealed trait InitializationResponse extends BackendWorkflowInitializationActorResponse - case class InitializationSuccess(backendInitializationData: Option[BackendInitializationData]) extends InitializationResponse + case class InitializationSuccess(backendInitializationData: Option[BackendInitializationData]) + extends InitializationResponse case class InitializationFailed(reason: Throwable) extends Exception with InitializationResponse /** @@ -70,24 +71,25 @@ object BackendWorkflowInitializationActor { * - It would be nice to memoize as much of the work that gets done here as possible so it doesn't have to all be * repeated when the various `FooRuntimeAttributes` classes are created, in the spirit of #1076. */ - def validateRuntimeAttributes( - taskName: String, - defaultRuntimeAttributes: Map[String, WomValue], - runtimeAttributes: Map[String, WomExpression], - runtimeAttributeValidators: Map[String, Option[WomExpression] => Boolean] - ): ValidatedNel[RuntimeAttributeValidationFailure, Unit] = { - - //This map append will overwrite default key/values with runtime settings upon key collisions - val lookups = defaultRuntimeAttributes.safeMapValues(_.asWomExpression) ++ runtimeAttributes - - runtimeAttributeValidators.toList.traverse{ - case (attributeName, validator) => - val runtimeAttributeValue: Option[WomExpression] = lookups.get(attributeName) - validator(runtimeAttributeValue).fold( - validNel(()), - Invalid(NonEmptyList.of(RuntimeAttributeValidationFailure(taskName, attributeName, runtimeAttributeValue))) - ) - }.map(_ => ()) + def validateRuntimeAttributes( + taskName: String, + defaultRuntimeAttributes: Map[String, WomValue], + runtimeAttributes: Map[String, WomExpression], + runtimeAttributeValidators: Map[String, Option[WomExpression] => Boolean] + ): ValidatedNel[RuntimeAttributeValidationFailure, Unit] = { + + // This map append will overwrite default key/values with runtime settings upon key collisions + val lookups = defaultRuntimeAttributes.safeMapValues(_.asWomExpression) ++ runtimeAttributes + + runtimeAttributeValidators.toList + .traverse { case (attributeName, validator) => + val runtimeAttributeValue: Option[WomExpression] = lookups.get(attributeName) + validator(runtimeAttributeValue).fold( + validNel(()), + Invalid(NonEmptyList.of(RuntimeAttributeValidationFailure(taskName, attributeName, runtimeAttributeValue))) + ) + } + .map(_ => ()) } } @@ -110,7 +112,9 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w * declarations will fail evaluation and return `true` from this predicate, even if the type could be determined * to be wrong with consideration of task declarations or inputs. */ - protected def womTypePredicate(valueRequired: Boolean, predicate: WomType => Boolean)(womExpressionMaybe: Option[WomExpression]): Boolean = { + protected def womTypePredicate(valueRequired: Boolean, predicate: WomType => Boolean)( + womExpressionMaybe: Option[WomExpression] + ): Boolean = womExpressionMaybe match { case None => !valueRequired case Some(womExpression: WomExpression) => @@ -119,29 +123,31 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w case Invalid(_) => true // If we can't evaluate it, we'll let it pass for now... } } - } /** * This predicate is only appropriate for validation during workflow initialization. The logic does not differentiate * between evaluation failures due to missing call inputs or evaluation failures due to malformed expressions, and will * return `true` in both cases. */ - protected def continueOnReturnCodePredicate(valueRequired: Boolean)(womExpressionMaybe: Option[WomValue]): Boolean = { - ContinueOnReturnCodeValidation.default(configurationDescriptor.backendRuntimeAttributesConfig).validateOptionalWomValue(womExpressionMaybe) - } + protected def continueOnReturnCodePredicate(valueRequired: Boolean)(womExpressionMaybe: Option[WomValue]): Boolean = + ContinueOnReturnCodeValidation + .default(configurationDescriptor.backendRuntimeAttributesConfig) + .validateOptionalWomValue(womExpressionMaybe) protected def runtimeAttributeValidators: Map[String, Option[WomExpression] => Boolean] // FIXME: If a workflow executes jobs using multiple backends, // each backend will try to write its own workflow root and override any previous one. // They should be structured differently or at least be prefixed by the backend name - protected def publishWorkflowRoot(workflowRoot: String): Unit = { - serviceRegistryActor ! PutMetadataAction(MetadataEvent(MetadataKey(workflowDescriptor.id, None, WorkflowMetadataKeys.WorkflowRoot), MetadataValue(workflowRoot))) - } + protected def publishWorkflowRoot(workflowRoot: String): Unit = + serviceRegistryActor ! PutMetadataAction( + MetadataEvent(MetadataKey(workflowDescriptor.id, None, WorkflowMetadataKeys.WorkflowRoot), + MetadataValue(workflowRoot) + ) + ) protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WomValue]] - def receive: Receive = LoggingReceive { case Initialize => performActionThenRespond(initSequence(), onFailure = InitializationFailed) case Abort => abortInitialization() @@ -154,10 +160,11 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w for { defaultRuntimeAttributes <- coerceDefaultRuntimeAttributes(workflowDescriptor.workflowOptions) |> Future.fromTry _ taskList = calls.toList.map(_.callable).map(t => t.name -> t.runtimeAttributes.attributes) - _ <- taskList. - traverse{ - case (name, runtimeAttributes) => validateRuntimeAttributes(name, defaultRuntimeAttributes, runtimeAttributes, runtimeAttributeValidators) - }.toFuture(errors => RuntimeAttributeValidationFailures(errors.toList)) + _ <- taskList + .traverse { case (name, runtimeAttributes) => + validateRuntimeAttributes(name, defaultRuntimeAttributes, runtimeAttributes, runtimeAttributeValidators) + } + .toFuture(errors => RuntimeAttributeValidationFailures(errors.toList)) _ <- validate() data <- beforeAll() } yield InitializationSuccess(data) diff --git a/backend/src/main/scala/cromwell/backend/Command.scala b/backend/src/main/scala/cromwell/backend/Command.scala index 40310c68485..c523500b7ca 100644 --- a/backend/src/main/scala/cromwell/backend/Command.scala +++ b/backend/src/main/scala/cromwell/backend/Command.scala @@ -3,7 +3,6 @@ package cromwell.backend import common.validation.ErrorOr._ import common.validation.Validation._ import wom.InstantiatedCommand -import wom.callable.RuntimeEnvironment import wom.expression.IoFunctionSet import wom.values.{WomEvaluatedCallInputs, WomValue} @@ -24,11 +23,11 @@ object Command { */ def instantiate(jobDescriptor: BackendJobDescriptor, callEngineFunction: IoFunctionSet, - inputsPreProcessor: WomEvaluatedCallInputs => Try[WomEvaluatedCallInputs] = (i: WomEvaluatedCallInputs) => Success(i), - valueMapper: WomValue => WomValue, - runtimeEnvironment: RuntimeEnvironment): ErrorOr[InstantiatedCommand] = { + inputsPreProcessor: WomEvaluatedCallInputs => Try[WomEvaluatedCallInputs] = + (i: WomEvaluatedCallInputs) => Success(i), + valueMapper: WomValue => WomValue + ): ErrorOr[InstantiatedCommand] = inputsPreProcessor(jobDescriptor.evaluatedTaskInputs).toErrorOr flatMap { mappedInputs => - jobDescriptor.taskCall.callable.instantiateCommand(mappedInputs, callEngineFunction, valueMapper, runtimeEnvironment) + jobDescriptor.taskCall.callable.instantiateCommand(mappedInputs, callEngineFunction, valueMapper) } - } } diff --git a/backend/src/main/scala/cromwell/backend/FileSizeTooBig.scala b/backend/src/main/scala/cromwell/backend/FileSizeTooBig.scala index 2580d9218fc..79ccc698b05 100644 --- a/backend/src/main/scala/cromwell/backend/FileSizeTooBig.scala +++ b/backend/src/main/scala/cromwell/backend/FileSizeTooBig.scala @@ -1,4 +1,3 @@ package cromwell.backend case class FileSizeTooBig(override val getMessage: String) extends Exception - diff --git a/backend/src/main/scala/cromwell/backend/MinimumRuntimeSettings.scala b/backend/src/main/scala/cromwell/backend/MinimumRuntimeSettings.scala new file mode 100644 index 00000000000..67c18324ef8 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/MinimumRuntimeSettings.scala @@ -0,0 +1,13 @@ +package cromwell.backend + +import eu.timepit.refined.api.Refined +import eu.timepit.refined.numeric.Positive +import eu.timepit.refined.refineMV +import wdl4s.parser.MemoryUnit +import wom.format.MemorySize + +case class MinimumRuntimeSettings(cores: Int Refined Positive = refineMV(1), + ram: MemorySize = MemorySize(4, MemoryUnit.GB), + outputPathSize: Long = Long.MaxValue, + tempPathSize: Long = Long.MaxValue +) diff --git a/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala b/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala index 2bf215fb0fa..8c799e36ad5 100644 --- a/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala +++ b/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala @@ -26,15 +26,20 @@ object OutputEvaluator { def evaluateOutputs(jobDescriptor: BackendJobDescriptor, ioFunctions: IoFunctionSet, - postMapper: WomValue => Try[WomValue] = v => Success(v))(implicit ec: ExecutionContext): Future[EvaluatedJobOutputs] = { + postMapper: WomValue => Try[WomValue] = v => Success(v) + )(implicit ec: ExecutionContext): Future[EvaluatedJobOutputs] = { val taskInputValues: Map[String, WomValue] = jobDescriptor.localInputs - def foldFunction(accumulatedOutputs: Try[ErrorOr[List[(OutputPort, WomValue)]]], output: ExpressionBasedOutputPort) = accumulatedOutputs flatMap { accumulated => + def foldFunction(accumulatedOutputs: Try[ErrorOr[List[(OutputPort, WomValue)]]], + output: ExpressionBasedOutputPort + ) = accumulatedOutputs flatMap { accumulated => // Extract the valid pairs from the job outputs accumulated so far, and add to it the inputs (outputs can also reference inputs) val allKnownValues: Map[String, WomValue] = accumulated match { case Valid(outputs) => // The evaluateValue methods needs a Map[String, WomValue], use the output port name for already computed outputs - outputs.toMap[OutputPort, WomValue].map({ case (port, value) => port.internalName -> value }) ++ taskInputValues + outputs.toMap[OutputPort, WomValue].map { case (port, value) => + port.internalName -> value + } ++ taskInputValues case Invalid(_) => taskInputValues } @@ -45,22 +50,24 @@ object OutputEvaluator { } // Attempt to coerce the womValue to the desired output type - def coerceOutputValue(womValue: WomValue, coerceTo: WomType): OutputResult[WomValue] = { + def coerceOutputValue(womValue: WomValue, coerceTo: WomType): OutputResult[WomValue] = fromEither[Try]( // TODO WOM: coerceRawValue should return an ErrorOr - coerceTo.coerceRawValue(womValue).toEither.leftMap(t => NonEmptyList.one(t.getClass.getSimpleName + ": " + t.getMessage)) + coerceTo + .coerceRawValue(womValue) + .toEither + .leftMap(t => NonEmptyList.one(t.getClass.getSimpleName + ": " + t.getMessage)) ) - } /* - * Go through evaluation, coercion and post processing. - * Transform the result to a validated Try[ErrorOr[(String, WomValue)]] with toValidated - * If we have a valid pair, add it to the previously accumulated outputs, otherwise combine the Nels of errors + * Go through evaluation, coercion and post processing. + * Transform the result to a validated Try[ErrorOr[(String, WomValue)]] with toValidated + * If we have a valid pair, add it to the previously accumulated outputs, otherwise combine the Nels of errors */ val evaluated = for { evaluated <- evaluateOutputExpression coerced <- coerceOutputValue(evaluated, output.womType) - postProcessed <- EitherT { postMapper(coerced).map(_.validNelCheck) }: OutputResult[WomValue] + postProcessed <- EitherT(postMapper(coerced).map(_.validNelCheck)): OutputResult[WomValue] pair = output -> postProcessed } yield pair @@ -73,24 +80,27 @@ object OutputEvaluator { val emptyValue = Success(List.empty[(OutputPort, WomValue)].validNel): Try[ErrorOr[List[(OutputPort, WomValue)]]] // Fold over the outputs to evaluate them in order, map the result to an EvaluatedJobOutputs - def fromOutputPorts: EvaluatedJobOutputs = jobDescriptor.taskCall.expressionBasedOutputPorts.foldLeft(emptyValue)(foldFunction) match { - case Success(Valid(outputs)) => ValidJobOutputs(CallOutputs(outputs.toMap)) - case Success(Invalid(errors)) => InvalidJobOutputs(errors) - case Failure(exception) => JobOutputsEvaluationException(exception) - } + def fromOutputPorts: EvaluatedJobOutputs = + jobDescriptor.taskCall.expressionBasedOutputPorts.foldLeft(emptyValue)(foldFunction) match { + case Success(Valid(outputs)) => ValidJobOutputs(CallOutputs(outputs.toMap)) + case Success(Invalid(errors)) => InvalidJobOutputs(errors) + case Failure(exception) => JobOutputsEvaluationException(exception) + } /* - * Because Cromwell doesn't trust anyone, if custom evaluation is provided, - * still make sure that all the output ports have been filled with values + * Because Cromwell doesn't trust anyone, if custom evaluation is provided, + * still make sure that all the output ports have been filled with values */ def validateCustomEvaluation(outputs: Map[OutputPort, WomValue]): EvaluatedJobOutputs = { - def toError(outputPort: OutputPort) = s"Missing output value for ${outputPort.identifier.fullyQualifiedName.value}" + def toError(outputPort: OutputPort) = + s"Missing output value for ${outputPort.identifier.fullyQualifiedName.value}" jobDescriptor.taskCall.expressionBasedOutputPorts.diff(outputs.keySet.toList) match { case Nil => val errorMessagePrefix = "Error applying postMapper in short-circuit output evaluation" - TryUtil.sequenceMap(outputs map { case (k, v) => (k, postMapper(v))}, errorMessagePrefix) match { - case Failure(e) => InvalidJobOutputs(NonEmptyList.of(e.getMessage, e.getStackTrace.take(5).toIndexedSeq.map(_.toString):_*)) + TryUtil.sequenceMap(outputs map { case (k, v) => (k, postMapper(v)) }, errorMessagePrefix) match { + case Failure(e) => + InvalidJobOutputs(NonEmptyList.of(e.getMessage, e.getStackTrace.take(5).toIndexedSeq.map(_.toString): _*)) case Success(postMappedOutputs) => ValidJobOutputs(CallOutputs(postMappedOutputs)) } case head :: tail => InvalidJobOutputs(NonEmptyList.of(toError(head), tail.map(toError): _*)) @@ -98,18 +108,20 @@ object OutputEvaluator { } /* - * See if the task definition has "short-circuit" for the default output evaluation. - * In the case of CWL for example, this gives a chance to look for cwl.output.json and use it as the output of the tool, - * instead of the default behavior of going over each output port of the task and evaluates their expression. - * If the "customOutputEvaluation" returns None (which will happen if the cwl.output.json is not there, as well as for all WDL workflows), - * we fallback to the default behavior. + * See if the task definition has "short-circuit" for the default output evaluation. + * In the case of CWL for example, this gives a chance to look for cwl.output.json and use it as the output of the tool, + * instead of the default behavior of going over each output port of the task and evaluates their expression. + * If the "customOutputEvaluation" returns None (which will happen if the cwl.output.json is not there, as well as for all WDL workflows), + * we fallback to the default behavior. */ - jobDescriptor.taskCall.customOutputEvaluation(taskInputValues, ioFunctions, ec).value - .map({ + jobDescriptor.taskCall + .customOutputEvaluation(taskInputValues, ioFunctions, ec) + .value + .map { case Some(Right(outputs)) => validateCustomEvaluation(outputs) case Some(Left(errors)) => InvalidJobOutputs(errors) // If it returns an empty value, fallback to canonical output evaluation case None => fromOutputPorts - }) + } } } diff --git a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala index 33e6e2db020..37bfa3aa7b8 100644 --- a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala +++ b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala @@ -6,6 +6,7 @@ import cromwell.util.JsonFormatting.WomValueJsonFormatter import common.validation.ErrorOr.ErrorOr import wom.callable.Callable.InputDefinition import wom.expression.IoFunctionSet +import wom.values.WomObject import wom.values.WomValue import wom.{RuntimeAttributes, WomExpressionException} @@ -20,30 +21,80 @@ case class RuntimeAttributeDefinition(name: String, factoryDefault: Option[WomVa object RuntimeAttributeDefinition { + /** + * "Evaluate" means hydrating the runtime attributes with information from the inputs + * @param unevaluated WOM expressions that may or may not reference inputs + * @param wdlFunctions The set of IO for the current backend + * @param evaluatedInputs The inputs + * @param platform Optional, directs platform-based prioritization + * @return Evaluated + */ def evaluateRuntimeAttributes(unevaluated: RuntimeAttributes, wdlFunctions: IoFunctionSet, - evaluatedInputs: Map[InputDefinition, WomValue]): ErrorOr[Map[String, WomValue]] = { + evaluatedInputs: Map[InputDefinition, WomValue], + platform: Option[Platform] = None + ): ErrorOr[Map[String, WomValue]] = { import common.validation.ErrorOr._ val inputsMap = evaluatedInputs map { case (x, y) => x.name -> y } - unevaluated.attributes.traverseValues(_.evaluateValue(inputsMap, wdlFunctions)) + val evaluated = unevaluated.attributes.traverseValues(_.evaluateValue(inputsMap, wdlFunctions)) + + // Platform mapping must come after evaluation because we need to evaluate + // e.g. `gcp: userDefinedObject` to find out what its runtime value is. + // The type system informs us of this because a `WomExpression` in `unevaluated` + // cannot be safely read as a `WomObject` with a `values` map until evaluation + evaluated.map(e => applyPlatform(e, platform)) + } + + def applyPlatform(attributes: Map[String, WomValue], maybePlatform: Option[Platform]): Map[String, WomValue] = { + + def extractPlatformAttributes(platform: Platform): Map[String, WomValue] = + attributes.get(platform.runtimeKey) match { + case Some(obj: WomObject) => + // WDL spec: "Use objects to avoid collisions" + // https://github.com/openwdl/wdl/blob/wdl-1.1/SPEC.md#conventions-and-best-practices + obj.values + case _ => + // A malformed non-object override such as gcp: "banana" is ignored + Map.empty + } + + val platformAttributes = maybePlatform match { + case Some(platform) => + extractPlatformAttributes(platform) + case None => + Map.empty + } + + // We've scooped our desired platform, now delete "azure", "gcp", etc. + val originalAttributesWithoutPlatforms: Map[String, WomValue] = + attributes -- Platform.all.map(_.runtimeKey) + + // With `++` keys from the RHS overwrite duplicates in LHS, which is what we want + // RHS `Map.empty` is a no-op + originalAttributesWithoutPlatforms ++ platformAttributes } def buildMapBasedLookup(evaluatedDeclarations: Map[InputDefinition, Try[WomValue]])(identifier: String): WomValue = { val successfulEvaluations = evaluatedDeclarations collect { case (k, v) if v.isSuccess => k.name -> v.get } - successfulEvaluations.getOrElse(identifier, throw new WomExpressionException(s"Could not resolve variable $identifier as a task input")) + successfulEvaluations.getOrElse( + identifier, + throw new WomExpressionException(s"Could not resolve variable $identifier as a task input") + ) } - def addDefaultsToAttributes(runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition], workflowOptions: WorkflowOptions) - (specifiedAttributes: Map[LocallyQualifiedName, WomValue]): Map[LocallyQualifiedName, WomValue] = { + def addDefaultsToAttributes(runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition], + workflowOptions: WorkflowOptions + )(specifiedAttributes: Map[LocallyQualifiedName, WomValue]): Map[LocallyQualifiedName, WomValue] = { import WomValueJsonFormatter._ def isUnspecifiedAttribute(name: String) = !specifiedAttributes.contains(name) val missing = runtimeAttributeDefinitions filter { x => isUnspecifiedAttribute(x.name) } val defaults = missing map { x => (x, workflowOptions.getDefaultRuntimeOption(x.name)) } collect { - case (runtimeAttributeDefinition, Success(jsValue)) => runtimeAttributeDefinition.name -> jsValue.convertTo[WomValue] + case (runtimeAttributeDefinition, Success(jsValue)) => + runtimeAttributeDefinition.name -> jsValue.convertTo[WomValue] case (RuntimeAttributeDefinition(name, Some(factoryDefault), _), _) => name -> factoryDefault } specifiedAttributes ++ defaults diff --git a/backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala b/backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala deleted file mode 100644 index c6c4cd22c21..00000000000 --- a/backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala +++ /dev/null @@ -1,59 +0,0 @@ -package cromwell.backend - -import java.util.UUID - -import cromwell.backend.io.JobPaths -import cromwell.backend.validation.{CpuValidation, MemoryValidation} -import cromwell.core.path.Path -import eu.timepit.refined.api.Refined -import eu.timepit.refined.numeric.Positive -import eu.timepit.refined.refineMV -import wdl4s.parser.MemoryUnit -import wom.callable.RuntimeEnvironment -import wom.format.MemorySize -import wom.values.WomValue - -object RuntimeEnvironmentBuilder { - - def apply(runtimeAttributes: Map[String, WomValue], callRoot: Path, callExecutionRoot: Path): MinimumRuntimeSettings => RuntimeEnvironment = { - minimums => - - val outputPath: String = callExecutionRoot.pathAsString - - val tempPath: String = { - val uuid = UUID.randomUUID().toString - val hash = uuid.substring(0, uuid.indexOf('-')) - callRoot.resolve(s"tmp.$hash").pathAsString - } - - val cores: Int Refined Positive = CpuValidation.instanceMin.validate(runtimeAttributes).getOrElse(minimums.cores) - - val memoryInMB: Double = - MemoryValidation.instance(). - validate(runtimeAttributes). - map(_.to(MemoryUnit.MB).amount). - getOrElse(minimums.ram.amount) - - //TODO: Read these from somewhere else - val outputPathSize: Long = minimums.outputPathSize - - val tempPathSize: Long = minimums.outputPathSize - - RuntimeEnvironment(outputPath, tempPath, cores, memoryInMB, outputPathSize, tempPathSize) - } - - /** - * Per the spec: - * - * "For cores, ram, outdirSize and tmpdirSize, if an implementation can't provide the actual number of reserved cores - * during the expression evaluation time, it should report back the minimal requested amount." - */ - def apply(runtimeAttributes: Map[String, WomValue], jobPaths: JobPaths): MinimumRuntimeSettings => RuntimeEnvironment = { - this.apply(runtimeAttributes, jobPaths.callRoot, jobPaths.callExecutionRoot) - } -} - -case class MinimumRuntimeSettings(cores: Int Refined Positive = refineMV(1), - ram: MemorySize = MemorySize(4, MemoryUnit.GB), - outputPathSize: Long = Long.MaxValue, - tempPathSize: Long = Long.MaxValue) diff --git a/backend/src/main/scala/cromwell/backend/SlowJobWarning.scala b/backend/src/main/scala/cromwell/backend/SlowJobWarning.scala index 074895e366a..22db6d7f92a 100644 --- a/backend/src/main/scala/cromwell/backend/SlowJobWarning.scala +++ b/backend/src/main/scala/cromwell/backend/SlowJobWarning.scala @@ -12,20 +12,21 @@ trait SlowJobWarning { this: Actor with ActorLogging => def slowJobWarningReceive: Actor.Receive = { case WarnAboutSlownessAfter(jobId, duration) => alreadyWarned = false - warningDetails = Option(WarningDetails(jobId, OffsetDateTime.now(), OffsetDateTime.now().plusSeconds(duration.toSeconds))) + warningDetails = Option( + WarningDetails(jobId, OffsetDateTime.now(), OffsetDateTime.now().plusSeconds(duration.toSeconds)) + ) case WarnAboutSlownessIfNecessary => handleWarnMessage() } var warningDetails: Option[WarningDetails] = None var alreadyWarned: Boolean = false - def warnAboutSlowJobIfNecessary(jobId: String) = { + def warnAboutSlowJobIfNecessary(jobId: String) = // Don't do anything here because we might need to update state. // Instead, send a message and handle this in the receive block. self ! WarnAboutSlownessIfNecessary - } - private def handleWarnMessage(): Unit = { + private def handleWarnMessage(): Unit = if (!alreadyWarned) { warningDetails match { case Some(WarningDetails(jobId, startTime, warningTime)) if OffsetDateTime.now().isAfter(warningTime) => @@ -34,7 +35,6 @@ trait SlowJobWarning { this: Actor with ActorLogging => case _ => // Nothing to do } } - } } diff --git a/backend/src/main/scala/cromwell/backend/WriteFunctions.scala b/backend/src/main/scala/cromwell/backend/WriteFunctions.scala index 572a417c047..5a8224d37a6 100644 --- a/backend/src/main/scala/cromwell/backend/WriteFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/WriteFunctions.scala @@ -24,13 +24,14 @@ trait WriteFunctions extends PathFactory with IoFunctionSet with AsyncIoFunction */ def writeDirectory: Path - private lazy val _writeDirectory = if (isDocker) writeDirectory.createPermissionedDirectories() else writeDirectory.createDirectories() + private lazy val _writeDirectory = + if (isDocker) writeDirectory.createPermissionedDirectories() else writeDirectory.createDirectories() override def createTemporaryDirectory(name: Option[String]) = { val tempDirPath = _writeDirectory / name.getOrElse(UUID.randomUUID().toString) // This is evil, but has the added advantage to work both for cloud and local val tempDirHiddenFile = tempDirPath / ".file" - asyncIo.writeAsync(tempDirHiddenFile, "", OpenOptions.default) as { tempDirPath.pathAsString } + asyncIo.writeAsync(tempDirHiddenFile, "", OpenOptions.default) as tempDirPath.pathAsString } protected def writeAsync(file: Path, content: String) = asyncIo.writeAsync(file, content, OpenOptions.default) @@ -38,21 +39,22 @@ trait WriteFunctions extends PathFactory with IoFunctionSet with AsyncIoFunction override def writeFile(path: String, content: String): Future[WomSingleFile] = { val file = _writeDirectory / path asyncIo.existsAsync(file) flatMap { - case false => writeAsync(file, content) as { WomSingleFile(file.pathAsString) } + case false => writeAsync(file, content) as WomSingleFile(file.pathAsString) case true => Future.successful(WomSingleFile(file.pathAsString)) } } private val relativeToLocal = System.getProperty("user.dir").ensureSlashed - def relativeToAbsolutePath(pathFrom: String): String = if (buildPath(pathFrom).isAbsolute) pathFrom else relativeToLocal + pathFrom + def relativeToAbsolutePath(pathFrom: String): String = + if (buildPath(pathFrom).isAbsolute) pathFrom else relativeToLocal + pathFrom override def copyFile(pathFrom: String, targetName: String): Future[WomSingleFile] = { val source = buildPath(relativeToAbsolutePath(pathFrom)) val destination = _writeDirectory / targetName - asyncIo.copyAsync(source, destination).as(WomSingleFile(destination.pathAsString)) recoverWith { - case e => Future.failed(new Exception(s"Could not copy ${source.toAbsolutePath} to ${destination.toAbsolutePath}", e)) + asyncIo.copyAsync(source, destination).as(WomSingleFile(destination.pathAsString)) recoverWith { case e => + Future.failed(new Exception(s"Could not copy ${source.toAbsolutePath} to ${destination.toAbsolutePath}", e)) } } } diff --git a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala index 241081f5a02..2872633d9bd 100644 --- a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala @@ -1,6 +1,5 @@ package cromwell.backend.async - import java.util.concurrent.ExecutionException import akka.actor.{Actor, ActorLogging, ActorRef} @@ -18,10 +17,12 @@ import scala.util.{Failure, Success} object AsyncBackendJobExecutionActor { sealed trait AsyncBackendJobExecutionActorMessage - private final case class IssuePollRequest(executionHandle: ExecutionHandle) extends AsyncBackendJobExecutionActorMessage - private final case class PollResponseReceived(executionHandle: ExecutionHandle) extends AsyncBackendJobExecutionActorMessage - private final case class FailAndStop(reason: Throwable) extends AsyncBackendJobExecutionActorMessage - private final case class Finish(executionHandle: ExecutionHandle) extends AsyncBackendJobExecutionActorMessage + final private case class IssuePollRequest(executionHandle: ExecutionHandle) + extends AsyncBackendJobExecutionActorMessage + final private case class PollResponseReceived(executionHandle: ExecutionHandle) + extends AsyncBackendJobExecutionActorMessage + final private case class FailAndStop(reason: Throwable) extends AsyncBackendJobExecutionActorMessage + final private case class Finish(executionHandle: ExecutionHandle) extends AsyncBackendJobExecutionActorMessage trait JobId @@ -57,27 +58,24 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging with SlowJob def isTransient(throwable: Throwable): Boolean = false - private def withRetry[A](work: () => Future[A], backOff: SimpleExponentialBackoff): Future[A] = { + private def withRetry[A](work: () => Future[A], backOff: SimpleExponentialBackoff): Future[A] = Retry.withRetry(work, isTransient = isTransient, isFatal = isFatal, backoff = backOff)(context.system) - } - private def robustExecuteOrRecover(mode: ExecutionMode) = { + private def robustExecuteOrRecover(mode: ExecutionMode) = withRetry(() => executeOrRecover(mode), executeOrRecoverBackOff) onComplete { case Success(h) => self ! IssuePollRequest(h) case Failure(t) => self ! FailAndStop(t) } - } def pollBackOff: SimpleExponentialBackoff def executeOrRecoverBackOff: SimpleExponentialBackoff - private def robustPoll(handle: ExecutionHandle) = { + private def robustPoll(handle: ExecutionHandle) = withRetry(() => poll(handle), pollBackOff) onComplete { case Success(h) => self ! PollResponseReceived(h) case Failure(t) => self ! FailAndStop(t) } - } private def failAndStop(t: Throwable) = { completionPromise.success(JobFailedNonRetryableResponse(jobDescriptor.key, t, None)) @@ -94,7 +92,16 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging with SlowJob context.system.scheduler.scheduleOnce(pollBackOff.backoffMillis.millis, self, IssuePollRequest(handle)) () case Finish(SuccessfulExecutionHandle(outputs, returnCode, jobDetritusFiles, executionEvents, _)) => - completionPromise.success(JobSucceededResponse(jobDescriptor.key, Some(returnCode), outputs, Option(jobDetritusFiles), executionEvents, dockerImageUsed, resultGenerationMode = RunOnBackend)) + completionPromise.success( + JobSucceededResponse(jobDescriptor.key, + Some(returnCode), + outputs, + Option(jobDetritusFiles), + executionEvents, + dockerImageUsed, + resultGenerationMode = RunOnBackend + ) + ) context.stop(self) case Finish(FailedNonRetryableExecutionHandle(throwable, returnCode, _)) => completionPromise.success(JobFailedNonRetryableResponse(jobDescriptor.key, throwable, returnCode)) @@ -133,5 +140,5 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging with SlowJob def jobDescriptor: BackendJobDescriptor - protected implicit def ec: ExecutionContext + implicit protected def ec: ExecutionContext } diff --git a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala index ab1354e4323..ff6a088b1f9 100644 --- a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala +++ b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala @@ -15,8 +15,7 @@ sealed trait ExecutionHandle { def result: ExecutionResult } -final case class PendingExecutionHandle[BackendJobId <: JobId, BackendRunInfo, BackendRunState] -( +final case class PendingExecutionHandle[BackendJobId <: JobId, BackendRunInfo, BackendRunState]( jobDescriptor: BackendJobDescriptor, pendingJob: BackendJobId, runInfo: Option[BackendRunInfo], @@ -30,7 +29,8 @@ final case class SuccessfulExecutionHandle(outputs: CallOutputs, returnCode: Int, jobDetritusFiles: Map[String, Path], executionEvents: Seq[ExecutionEvent], - resultsClonedFrom: Option[BackendJobDescriptor] = None) extends ExecutionHandle { + resultsClonedFrom: Option[BackendJobDescriptor] = None +) extends ExecutionHandle { override val isDone = true override val result = SuccessfulExecution(outputs, returnCode, jobDetritusFiles, executionEvents, resultsClonedFrom) } @@ -41,7 +41,8 @@ sealed trait FailedExecutionHandle extends ExecutionHandle { final case class FailedNonRetryableExecutionHandle(throwable: Throwable, returnCode: Option[Int] = None, - override val kvPairsToSave: Option[Seq[KvPair]]) extends FailedExecutionHandle { + override val kvPairsToSave: Option[Seq[KvPair]] +) extends FailedExecutionHandle { override val isDone = true override val result = NonRetryableExecution(throwable, returnCode) @@ -49,7 +50,8 @@ final case class FailedNonRetryableExecutionHandle(throwable: Throwable, final case class FailedRetryableExecutionHandle(throwable: Throwable, returnCode: Option[Int] = None, - override val kvPairsToSave: Option[Seq[KvPair]]) extends FailedExecutionHandle { + override val kvPairsToSave: Option[Seq[KvPair]] +) extends FailedExecutionHandle { override val isDone = true override val result = RetryableExecution(throwable, returnCode) diff --git a/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala b/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala index fc8f95e049b..b3c4bcda692 100644 --- a/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala +++ b/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala @@ -16,7 +16,8 @@ final case class SuccessfulExecution(outputs: CallOutputs, returnCode: Int, jobDetritusFiles: Map[String, Path], executionEvents: Seq[ExecutionEvent], - resultsClonedFrom: Option[BackendJobDescriptor] = None) extends ExecutionResult + resultsClonedFrom: Option[BackendJobDescriptor] = None +) extends ExecutionResult /** * A user-requested abort of the command. diff --git a/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala index 8dd9aef3544..e106f06fcd2 100644 --- a/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala +++ b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala @@ -9,61 +9,77 @@ abstract class KnownJobFailureException extends Exception { def stderrPath: Option[Path] } -final case class WrongReturnCode(jobTag: String, returnCode: Int, stderrPath: Option[Path]) extends KnownJobFailureException { - override def getMessage = s"Job $jobTag exited with return code $returnCode which has not been declared as a valid return code. See 'continueOnReturnCode' runtime attribute for more details." +final case class WrongReturnCode(jobTag: String, returnCode: Int, stderrPath: Option[Path]) + extends KnownJobFailureException { + override def getMessage = + s"Job $jobTag exited with return code $returnCode which has not been declared as a valid return code. See 'continueOnReturnCode' runtime attribute for more details." } -final case class ReturnCodeIsNotAnInt(jobTag: String, returnCode: String, stderrPath: Option[Path]) extends KnownJobFailureException { - override def getMessage = { +final case class ReturnCodeIsNotAnInt(jobTag: String, returnCode: String, stderrPath: Option[Path]) + extends KnownJobFailureException { + override def getMessage = if (returnCode.isEmpty) s"The return code file for job $jobTag was empty." else s"Job $jobTag exited with return code $returnCode which couldn't be converted to an Integer." - } } -final case class StderrNonEmpty(jobTag: String, stderrLength: Long, stderrPath: Option[Path]) extends KnownJobFailureException { - override def getMessage = s"stderr for job $jobTag has length $stderrLength and 'failOnStderr' runtime attribute was true." +final case class StderrNonEmpty(jobTag: String, stderrLength: Long, stderrPath: Option[Path]) + extends KnownJobFailureException { + override def getMessage = + s"stderr for job $jobTag has length $stderrLength and 'failOnStderr' runtime attribute was true." } final case class RetryWithMoreMemory(jobTag: String, stderrPath: Option[Path], memoryRetryErrorKeys: Option[List[String]], - logger: LoggingAdapter) extends KnownJobFailureException { + logger: LoggingAdapter +) extends KnownJobFailureException { val errorKeysAsString = memoryRetryErrorKeys match { case None => // this should not occur at this point as one would reach this error class only if Cromwell found one of the // `memory-retry-error-keys` in `stderr` of the task, which is only checked if the `memory-retry-error-keys` // are instantiated in Cromwell config - logger.error(s"Programmer error: found one of the `system.memory-retry-error-keys` in the `stderr` of task but " + - s"didn't find the error keys while generating the exception!") + logger.error( + s"Programmer error: found one of the `system.memory-retry-error-keys` in the `stderr` of task but " + + s"didn't find the error keys while generating the exception!" + ) "" case Some(keys) => keys.mkString(": [", ",", "]") } - override def getMessage = s"stderr for job `$jobTag` contained one of the `memory-retry-error-keys${errorKeysAsString}` specified in " + - s"the Cromwell config. Job might have run out of memory." + override def getMessage = + s"stderr for job `$jobTag` contained one of the `memory-retry-error-keys${errorKeysAsString}` specified in " + + s"the Cromwell config. Job might have run out of memory." } - object RuntimeAttributeValidationFailure { def apply(jobTag: String, runtimeAttributeName: String, - runtimeAttributeValue: Option[WomExpression]): RuntimeAttributeValidationFailure = RuntimeAttributeValidationFailure(jobTag, runtimeAttributeName, runtimeAttributeValue, None) + runtimeAttributeValue: Option[WomExpression] + ): RuntimeAttributeValidationFailure = + RuntimeAttributeValidationFailure(jobTag, runtimeAttributeName, runtimeAttributeValue, None) } final case class RuntimeAttributeValidationFailure private (jobTag: String, runtimeAttributeName: String, runtimeAttributeValue: Option[WomExpression], - stderrPath: Option[Path]) extends KnownJobFailureException { - override def getMessage = s"Task $jobTag has an invalid runtime attribute $runtimeAttributeName = ${runtimeAttributeValue map { _.evaluateValue(Map.empty, NoIoFunctionSet)} getOrElse "!! NOT FOUND !!"}" + stderrPath: Option[Path] +) extends KnownJobFailureException { + override def getMessage = + s"Task $jobTag has an invalid runtime attribute $runtimeAttributeName = ${runtimeAttributeValue map { + _.evaluateValue(Map.empty, NoIoFunctionSet) + } getOrElse "!! NOT FOUND !!"}" } -final case class RuntimeAttributeValidationFailures(throwables: List[RuntimeAttributeValidationFailure]) extends KnownJobFailureException with ThrowableAggregation { +final case class RuntimeAttributeValidationFailures(throwables: List[RuntimeAttributeValidationFailure]) + extends KnownJobFailureException + with ThrowableAggregation { override def exceptionContext = "Runtime validation failed" override val stderrPath: Option[Path] = None } -final case class JobAlreadyFailedInJobStore(jobTag: String, originalErrorMessage: String) extends KnownJobFailureException { +final case class JobAlreadyFailedInJobStore(jobTag: String, originalErrorMessage: String) + extends KnownJobFailureException { override def stderrPath: Option[Path] = None override def getMessage = originalErrorMessage } diff --git a/backend/src/main/scala/cromwell/backend/async/package.scala b/backend/src/main/scala/cromwell/backend/async/package.scala index b496126ef11..1f7037e711e 100644 --- a/backend/src/main/scala/cromwell/backend/async/package.scala +++ b/backend/src/main/scala/cromwell/backend/async/package.scala @@ -2,7 +2,6 @@ package cromwell.backend import scala.concurrent.{ExecutionContext, Future} - package object async { implicit class EnhancedFutureFuture[A](val ffa: Future[Future[A]])(implicit ec: ExecutionContext) { def flatten: Future[A] = ffa flatMap identity diff --git a/backend/src/main/scala/cromwell/backend/backend.scala b/backend/src/main/scala/cromwell/backend/backend.scala index ea413c10367..be486f977f4 100644 --- a/backend/src/main/scala/cromwell/backend/backend.scala +++ b/backend/src/main/scala/cromwell/backend/backend.scala @@ -26,9 +26,7 @@ import scala.util.Try /** * For uniquely identifying a job which has been or will be sent to the backend. */ -case class BackendJobDescriptorKey(call: CommandCallNode, - index: Option[Int], - attempt: Int) extends CallKey { +case class BackendJobDescriptorKey(call: CommandCallNode, index: Option[Int], attempt: Int) extends CallKey { def node = call private val indexString = index map { _.toString } getOrElse "NA" lazy val tag = s"${call.fullyQualifiedName}:$indexString:$attempt" @@ -44,15 +42,19 @@ final case class BackendJobDescriptor(workflowDescriptor: BackendWorkflowDescrip evaluatedTaskInputs: WomEvaluatedCallInputs, maybeCallCachingEligible: MaybeCallCachingEligible, dockerSize: Option[DockerSize], - prefetchedKvStoreEntries: Map[String, KvResponse]) { + prefetchedKvStoreEntries: Map[String, KvResponse] +) { val fullyQualifiedInputs: Map[String, WomValue] = evaluatedTaskInputs map { case (declaration, value) => key.call.identifier.combine(declaration.name).fullyQualifiedName.value -> value } - def findInputFilesByParameterMeta(filter: MetaValueElement => Boolean): Set[WomFile] = evaluatedTaskInputs.collect { - case (declaration, value) if declaration.parameterMeta.exists(filter) => findFiles(value) - }.flatten.toSet + def findInputFilesByParameterMeta(filter: MetaValueElement => Boolean): Set[WomFile] = evaluatedTaskInputs + .collect { + case (declaration, value) if declaration.parameterMeta.exists(filter) => findFiles(value) + } + .flatten + .toSet def findFiles(v: WomValue): Set[WomFile] = v match { case value: WomFile => Set(value) @@ -79,11 +81,13 @@ case class BackendWorkflowDescriptor(id: WorkflowId, customLabels: Labels, hogGroup: HogGroup, breadCrumbs: List[BackendJobBreadCrumb], - outputRuntimeExtractor: Option[WomOutputRuntimeExtractor]) { + outputRuntimeExtractor: Option[WomOutputRuntimeExtractor] +) { val rootWorkflow = breadCrumbs.headOption.map(_.callable).getOrElse(callable) val possiblyNotRootWorkflowId = id.toPossiblyNotRoot val rootWorkflowId = breadCrumbs.headOption.map(_.id).getOrElse(id).toRoot + val possibleParentWorkflowId = breadCrumbs.lastOption.map(_.id) override def toString: String = s"[BackendWorkflowDescriptor id=${id.shortString} workflowName=${callable.name}]" def getWorkflowOption(key: WorkflowOption) = workflowOptions.get(key).toOption @@ -99,33 +103,31 @@ case class BackendConfigurationDescriptor(backendConfig: Config, globalConfig: C Option(backendConfig.getConfig("default-runtime-attributes")) else None - + // So it can be overridden in tests - private [backend] lazy val cromwellFileSystems = CromwellFileSystems.instance + private[backend] lazy val cromwellFileSystems = CromwellFileSystems.instance - lazy val configuredPathBuilderFactories: Map[String, PathBuilderFactory] = { + lazy val configuredPathBuilderFactories: Map[String, PathBuilderFactory] = cromwellFileSystems.factoriesFromConfig(backendConfig).unsafe("Failed to instantiate backend filesystem") - } - private lazy val configuredFactoriesWithDefault = if (configuredPathBuilderFactories.values.exists(_ == DefaultPathBuilderFactory)) { - configuredPathBuilderFactories - } else configuredPathBuilderFactories + DefaultPathBuilderFactory.tuple + private lazy val configuredFactoriesWithDefault = + if (configuredPathBuilderFactories.values.exists(_ == DefaultPathBuilderFactory)) { + configuredPathBuilderFactories + } else configuredPathBuilderFactories + DefaultPathBuilderFactory.tuple /** * Creates path builders using only the configured factories. */ - def pathBuilders(workflowOptions: WorkflowOptions)(implicit as: ActorSystem) = { + def pathBuilders(workflowOptions: WorkflowOptions)(implicit as: ActorSystem) = PathBuilderFactory.instantiatePathBuilders(configuredPathBuilderFactories.values.toList, workflowOptions) - } /** * Creates path builders using only the configured factories + the default factory */ - def pathBuildersWithDefault(workflowOptions: WorkflowOptions)(implicit as: ActorSystem) = { + def pathBuildersWithDefault(workflowOptions: WorkflowOptions)(implicit as: ActorSystem) = PathBuilderFactory.instantiatePathBuilders(configuredFactoriesWithDefault.values.toList, workflowOptions) - } - lazy val slowJobWarningAfter = backendConfig.as[Option[FiniteDuration]](path="slow-job-warning-time") + lazy val slowJobWarningAfter = backendConfig.as[Option[FiniteDuration]](path = "slow-job-warning-time") } object CommonBackendConfigurationAttributes { @@ -146,10 +148,33 @@ object CommonBackendConfigurationAttributes { "dockerhub.token", "dockerhub.auth", "dockerhub.key-name", - "name-for-call-caching-purposes", + "name-for-call-caching-purposes" ) } final case class AttemptedLookupResult(name: String, value: Try[WomValue]) { def toPair = name -> value } + +sealed trait Platform { + def runtimeKey: String +} + +object Platform { + def all: Seq[Platform] = Seq(Gcp, Azure, Aws) + + def apply(str: String): Option[Platform] = + all.find(_.runtimeKey == str) +} + +object Gcp extends Platform { + override def runtimeKey: String = "gcp" +} + +object Azure extends Platform { + override def runtimeKey: String = "azure" +} + +object Aws extends Platform { + override def runtimeKey: String = "aws" +} diff --git a/backend/src/main/scala/cromwell/backend/dummy/DummyAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/dummy/DummyAsyncExecutionActor.scala index 5c491c5a741..b84843fea5e 100644 --- a/backend/src/main/scala/cromwell/backend/dummy/DummyAsyncExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/dummy/DummyAsyncExecutionActor.scala @@ -9,7 +9,12 @@ import cats.implicits._ import common.exception.AggregatedMessageException import common.validation.ErrorOr.ErrorOr import cromwell.backend.BackendJobLifecycleActor -import cromwell.backend.async.{ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, SuccessfulExecutionHandle} +import cromwell.backend.async.{ + ExecutionHandle, + FailedNonRetryableExecutionHandle, + PendingExecutionHandle, + SuccessfulExecutionHandle +} import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncExecutionActorParams, StandardAsyncJob} import cromwell.core.CallOutputs import cromwell.core.retry.SimpleExponentialBackoff @@ -22,12 +27,13 @@ import scala.concurrent.Future import scala.concurrent.duration._ class DummyAsyncExecutionActor(override val standardParams: StandardAsyncExecutionActorParams) - extends BackendJobLifecycleActor + extends BackendJobLifecycleActor with StandardAsyncExecutionActor with CromwellInstrumentation { /** The type of the run info when a job is started. */ override type StandardAsyncRunInfo = String + /** The type of the run status returned during each poll. */ override type StandardAsyncRunState = String @@ -46,14 +52,17 @@ class DummyAsyncExecutionActor(override val standardParams: StandardAsyncExecuti override def dockerImageUsed: Option[String] = None - override def pollBackOff: SimpleExponentialBackoff = SimpleExponentialBackoff(initialInterval = 1.second, maxInterval = 300.seconds, multiplier = 1.1) + override def pollBackOff: SimpleExponentialBackoff = + SimpleExponentialBackoff(initialInterval = 1.second, maxInterval = 300.seconds, multiplier = 1.1) - override def executeOrRecoverBackOff: SimpleExponentialBackoff = SimpleExponentialBackoff(initialInterval = 1.second, maxInterval = 300.seconds, multiplier = 1.1) + override def executeOrRecoverBackOff: SimpleExponentialBackoff = + SimpleExponentialBackoff(initialInterval = 1.second, maxInterval = 300.seconds, multiplier = 1.1) override val logJobIds: Boolean = false val singletonActor = standardParams.backendSingletonActorOption.getOrElse( - throw new RuntimeException("Dummy Backend actor cannot exist without its singleton actor")) + throw new RuntimeException("Dummy Backend actor cannot exist without its singleton actor") + ) var finishTime: Option[OffsetDateTime] = None @@ -71,46 +80,54 @@ class DummyAsyncExecutionActor(override val standardParams: StandardAsyncExecuti ) } - override def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle): Future[String] = { + override def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle): Future[String] = finishTime match { - case Some(ft) if (ft.isBefore(OffsetDateTime.now)) => Future.successful("done") + case Some(ft) if ft.isBefore(OffsetDateTime.now) => Future.successful("done") case Some(_) => Future.successful("running") - case None => Future.failed(new Exception("Dummy backend polling for status before finishTime is established(!!?)")) + case None => + Future.failed(new Exception("Dummy backend polling for status before finishTime is established(!!?)")) } - } - - override def handlePollSuccess(oldHandle: StandardAsyncPendingExecutionHandle, state: String): Future[ExecutionHandle] = { - + override def handlePollSuccess(oldHandle: StandardAsyncPendingExecutionHandle, + state: String + ): Future[ExecutionHandle] = if (state == "done") { increment(NonEmptyList("jobs", List("dummy", "executing", "done"))) singletonActor ! DummySingletonActor.MinusOne - val outputsValidation: ErrorOr[Map[OutputPort, WomValue]] = jobDescriptor.taskCall.outputPorts.toList.traverse { - case expressionBasedOutputPort: ExpressionBasedOutputPort => - expressionBasedOutputPort.expression.evaluateValue(Map.empty, NoIoFunctionSet).map(expressionBasedOutputPort -> _) - case other => s"Unknown output port type for Dummy backend output evaluator: ${other.getClass.getSimpleName}".invalidNel - }.map(_.toMap) + val outputsValidation: ErrorOr[Map[OutputPort, WomValue]] = jobDescriptor.taskCall.outputPorts.toList + .traverse { + case expressionBasedOutputPort: ExpressionBasedOutputPort => + expressionBasedOutputPort.expression + .evaluateValue(Map.empty, NoIoFunctionSet) + .map(expressionBasedOutputPort -> _) + case other => + s"Unknown output port type for Dummy backend output evaluator: ${other.getClass.getSimpleName}".invalidNel + } + .map(_.toMap) outputsValidation match { case Valid(outputs) => - Future.successful(SuccessfulExecutionHandle( - outputs = CallOutputs(outputs.toMap), - returnCode = 0, - jobDetritusFiles = Map.empty, - executionEvents = Seq.empty, - resultsClonedFrom = None - )) + Future.successful( + SuccessfulExecutionHandle( + outputs = CallOutputs(outputs.toMap), + returnCode = 0, + jobDetritusFiles = Map.empty, + executionEvents = Seq.empty, + resultsClonedFrom = None + ) + ) case Invalid(errors) => - Future.successful(FailedNonRetryableExecutionHandle( - throwable = AggregatedMessageException("Evaluate outputs from dummy job", errors.toList), - returnCode = None, - kvPairsToSave = None - )) + Future.successful( + FailedNonRetryableExecutionHandle( + throwable = AggregatedMessageException("Evaluate outputs from dummy job", errors.toList), + returnCode = None, + kvPairsToSave = None + ) + ) } - } - else if (state == "running") { + } else if (state == "running") { Future.successful( PendingExecutionHandle[StandardAsyncJob, StandardAsyncRunInfo, StandardAsyncRunState]( jobDescriptor = jobDescriptor, @@ -119,9 +136,7 @@ class DummyAsyncExecutionActor(override val standardParams: StandardAsyncExecuti previousState = Option(state) ) ) - } - else { + } else { Future.failed(new Exception(s"Unexpected Dummy state in handlePollSuccess: $state")) } - } } diff --git a/backend/src/main/scala/cromwell/backend/dummy/DummyInitializationActor.scala b/backend/src/main/scala/cromwell/backend/dummy/DummyInitializationActor.scala index 34fd2e8bc51..53a310d1dd8 100644 --- a/backend/src/main/scala/cromwell/backend/dummy/DummyInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/dummy/DummyInitializationActor.scala @@ -2,16 +2,22 @@ package cromwell.backend.dummy import cats.syntax.validated._ import common.validation.ErrorOr.ErrorOr -import cromwell.backend.standard.{StandardInitializationActor, StandardInitializationActorParams, StandardValidatedRuntimeAttributesBuilder} +import cromwell.backend.standard.{ + StandardInitializationActor, + StandardInitializationActorParams, + StandardValidatedRuntimeAttributesBuilder +} import cromwell.backend.validation.RuntimeAttributesValidation import wom.expression.WomExpression import wom.types.{WomStringType, WomType} import wom.values.{WomString, WomValue} class DummyInitializationActor(pipelinesParams: StandardInitializationActorParams) - extends StandardInitializationActor(pipelinesParams) { + extends StandardInitializationActor(pipelinesParams) { - override protected lazy val runtimeAttributeValidators: Map[String, Option[WomExpression] => Boolean] = Map("backend" -> { _ => true } ) + override protected lazy val runtimeAttributeValidators: Map[String, Option[WomExpression] => Boolean] = Map( + "backend" -> { _ => true } + ) // Specific validator for "backend" to let me specify it in test cases (to avoid accidentally submitting the workflow to real backends!) val backendAttributeValidation: RuntimeAttributesValidation[String] = new RuntimeAttributesValidation[String] { @@ -25,5 +31,6 @@ class DummyInitializationActor(pipelinesParams: StandardInitializationActorParam } } - override def runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder = super.runtimeAttributesBuilder.withValidation(backendAttributeValidation) + override def runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder = + super.runtimeAttributesBuilder.withValidation(backendAttributeValidation) } diff --git a/backend/src/main/scala/cromwell/backend/dummy/DummyLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/dummy/DummyLifecycleActorFactory.scala index ee959513a36..492de4970a9 100644 --- a/backend/src/main/scala/cromwell/backend/dummy/DummyLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/dummy/DummyLifecycleActorFactory.scala @@ -3,9 +3,15 @@ package cromwell.backend.dummy import akka.actor.{ActorRef, Props} import cromwell.backend.BackendConfigurationDescriptor import cromwell.backend.standard.callcaching.{StandardCacheHitCopyingActor, StandardFileHashingActor} -import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardInitializationActor, StandardLifecycleActorFactory} +import cromwell.backend.standard.{ + StandardAsyncExecutionActor, + StandardInitializationActor, + StandardLifecycleActorFactory +} -class DummyLifecycleActorFactory(override val name: String, override val configurationDescriptor: BackendConfigurationDescriptor) extends StandardLifecycleActorFactory { +class DummyLifecycleActorFactory(override val name: String, + override val configurationDescriptor: BackendConfigurationDescriptor +) extends StandardLifecycleActorFactory { /** * @return the key to use for storing and looking up the job id. @@ -23,8 +29,11 @@ class DummyLifecycleActorFactory(override val name: String, override val configu // Don't hash files override lazy val fileHashingActorClassOption: Option[Class[_ <: StandardFileHashingActor]] = None - override def backendSingletonActorProps(serviceRegistryActor: ActorRef): Option[Props] = Option(Props(new DummySingletonActor())) + override def backendSingletonActorProps(serviceRegistryActor: ActorRef): Option[Props] = Option( + Props(new DummySingletonActor()) + ) - override lazy val initializationActorClass: Class[_ <: StandardInitializationActor] = classOf[DummyInitializationActor] + override lazy val initializationActorClass: Class[_ <: StandardInitializationActor] = + classOf[DummyInitializationActor] } diff --git a/backend/src/main/scala/cromwell/backend/dummy/DummySingletonActor.scala b/backend/src/main/scala/cromwell/backend/dummy/DummySingletonActor.scala index 35c689ae2b4..4ea2cf7951c 100644 --- a/backend/src/main/scala/cromwell/backend/dummy/DummySingletonActor.scala +++ b/backend/src/main/scala/cromwell/backend/dummy/DummySingletonActor.scala @@ -22,7 +22,7 @@ final class DummySingletonActor() extends Actor with StrictLogging { case PlusOne => count = count + 1 case MinusOne => count = count - 1 case PrintCount => - if(countHistory.lastOption.exists(_._2 != count)) { + if (countHistory.lastOption.exists(_._2 != count)) { countHistory = countHistory :+ (OffsetDateTime.now() -> count) logger.info("The current count is now: " + count) if (count == 0) { @@ -52,7 +52,7 @@ final class DummySingletonActor() extends Actor with StrictLogging { bw.close() } - context.system.scheduler.schedule(10.seconds, 1.second) { self ! PrintCount } + context.system.scheduler.schedule(10.seconds, 1.second)(self ! PrintCount) } object DummySingletonActor { @@ -60,4 +60,3 @@ object DummySingletonActor { case object MinusOne case object PrintCount } - diff --git a/backend/src/main/scala/cromwell/backend/io/DirectoryFunctions.scala b/backend/src/main/scala/cromwell/backend/io/DirectoryFunctions.scala index 7b68e2bf723..84d45d1ee61 100644 --- a/backend/src/main/scala/cromwell/backend/io/DirectoryFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/io/DirectoryFunctions.scala @@ -11,7 +11,14 @@ import cromwell.core.path.{Path, PathFactory} import wom.expression.IoFunctionSet.{IoDirectory, IoElement, IoFile} import wom.expression.{IoFunctionSet, IoFunctionSetAdapter} import wom.graph.CommandCallNode -import wom.values.{WomFile, WomGlobFile, WomMaybeListedDirectory, WomMaybePopulatedFile, WomSingleFile, WomUnlistedDirectory} +import wom.values.{ + WomFile, + WomGlobFile, + WomMaybeListedDirectory, + WomMaybePopulatedFile, + WomSingleFile, + WomUnlistedDirectory +} import scala.concurrent.Future import scala.util.Try @@ -21,13 +28,18 @@ trait DirectoryFunctions extends IoFunctionSet with PathFactory with AsyncIoFunc private lazy val evaluateFileFunctions = new IoFunctionSetAdapter(this) with FileEvaluationIoFunctionSet def findDirectoryOutputs(call: CommandCallNode, - jobDescriptor: BackendJobDescriptor): ErrorOr[List[WomUnlistedDirectory]] = { + jobDescriptor: BackendJobDescriptor + ): ErrorOr[List[WomUnlistedDirectory]] = call.callable.outputs.flatTraverse[ErrorOr, WomUnlistedDirectory] { outputDefinition => - outputDefinition.expression.evaluateFiles(jobDescriptor.localInputs, evaluateFileFunctions, outputDefinition.womType) map { - _.toList.flatMap(_.file.flattenFiles) collect { case unlistedDirectory: WomUnlistedDirectory => unlistedDirectory } + outputDefinition.expression.evaluateFiles(jobDescriptor.localInputs, + evaluateFileFunctions, + outputDefinition.womType + ) map { + _.toList.flatMap(_.file.flattenFiles) collect { case unlistedDirectory: WomUnlistedDirectory => + unlistedDirectory + } } } - } override def isDirectory(path: String) = asyncIo.isDirectory(buildPath(path)) @@ -39,7 +51,7 @@ trait DirectoryFunctions extends IoFunctionSet with PathFactory with AsyncIoFunc * implementation which lists files and directories children. What we need is the unix behavior, even for cloud filesystems. * 3) It uses the isDirectory function directly on the path, which cannot be trusted for GCS paths. It should use asyncIo.isDirectory instead. */ - override def listDirectory(path: String)(visited: Vector[String] = Vector.empty): Future[Iterator[IoElement]] = { + override def listDirectory(path: String)(visited: Vector[String] = Vector.empty): Future[Iterator[IoElement]] = Future.fromTry(Try { val visitedPaths = visited.map(buildPath) val cromwellPath = buildPath(path.ensureSlashed) @@ -47,21 +59,21 @@ trait DirectoryFunctions extends IoFunctionSet with PathFactory with AsyncIoFunc // To prevent infinite recursion through symbolic links make sure we don't visit the same directory twice def hasBeenVisited(other: Path) = visitedPaths.exists(_.isSameFileAs(other)) - cromwellPath.list.collect({ - case directory if directory.isDirectory && - !cromwellPath.isSamePathAs(directory) && - !hasBeenVisited(directory) => IoDirectory(directory.pathAsString) + cromwellPath.list.collect { + case directory + if directory.isDirectory && + !cromwellPath.isSamePathAs(directory) && + !hasBeenVisited(directory) => + IoDirectory(directory.pathAsString) case file => IoFile(file.pathAsString) - }) + } }) - } - override def listAllFilesUnderDirectory(dirPath: String): Future[Seq[String]] = { + override def listAllFilesUnderDirectory(dirPath: String): Future[Seq[String]] = temporaryImplListPaths(dirPath) - } // TODO: WOM: WOMFILE: This will likely use a Tuple2(tar file, dir list file) for each dirPath. - private final def temporaryImplListPaths(dirPath: String): Future[Seq[String]] = { + final private def temporaryImplListPaths(dirPath: String): Future[Seq[String]] = { val errorOrPaths = for { dir <- validate(buildPath(dirPath.ensureSlashed)) files <- listFiles(dir) @@ -74,7 +86,7 @@ object DirectoryFunctions { def listFiles(path: Path): ErrorOr[List[Path]] = path.listRecursively.filterNot(_.isDirectory).toList.validNel def listWomSingleFiles(womFile: WomFile, pathFactory: PathFactory): ErrorOr[List[WomSingleFile]] = { - def listWomSingleFiles(womFile: WomFile): ErrorOr[List[WomSingleFile]] = { + def listWomSingleFiles(womFile: WomFile): ErrorOr[List[WomSingleFile]] = womFile match { case womSingleFile: WomSingleFile => List(womSingleFile).valid @@ -99,7 +111,6 @@ object DirectoryFunctions { case _: WomGlobFile => s"Unexpected glob / unable to list glob files at this time: $womFile".invalidNel } - } listWomSingleFiles(womFile) } diff --git a/backend/src/main/scala/cromwell/backend/io/FileEvaluationIoFunctionSet.scala b/backend/src/main/scala/cromwell/backend/io/FileEvaluationIoFunctionSet.scala index f46ff9ce9e2..0408c590cb2 100644 --- a/backend/src/main/scala/cromwell/backend/io/FileEvaluationIoFunctionSet.scala +++ b/backend/src/main/scala/cromwell/backend/io/FileEvaluationIoFunctionSet.scala @@ -5,5 +5,6 @@ import wom.expression.IoFunctionSet import scala.concurrent.Future trait FileEvaluationIoFunctionSet { this: IoFunctionSet => - override def glob(pattern: String) = Future.failed(new IllegalStateException("Cannot perform globing while evaluating files")) + override def glob(pattern: String) = + Future.failed(new IllegalStateException("Cannot perform globing while evaluating files")) } diff --git a/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala b/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala index 680c073bf45..5ed3b4a30ff 100644 --- a/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala @@ -19,7 +19,10 @@ trait GlobFunctions extends IoFunctionSet with AsyncIoFunctions { def findGlobOutputs(call: CommandCallNode, jobDescriptor: BackendJobDescriptor): ErrorOr[List[WomGlobFile]] = { def fromOutputs = call.callable.outputs.flatTraverse[ErrorOr, WomGlobFile] { outputDefinition => - outputDefinition.expression.evaluateFiles(jobDescriptor.localInputs, evaluateFileFunctions, outputDefinition.womType) map { + outputDefinition.expression.evaluateFiles(jobDescriptor.localInputs, + evaluateFileFunctions, + outputDefinition.womType + ) map { _.toList.flatMap(_.file.flattenFiles) collect { case glob: WomGlobFile => glob } } } @@ -40,7 +43,7 @@ trait GlobFunctions extends IoFunctionSet with AsyncIoFunctions { val listFilePath = callContext.root.resolve(s"${globName(pattern)}.list") asyncIo.readLinesAsync(listFilePath.getSymlinkSafePath()) map { lines => lines.toList map { fileName => - (callContext.root / globPatternName / fileName).pathAsString + (callContext.root / globPatternName / fileName).pathAsString } } } diff --git a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala index eb9e9ec31d7..05ad6a56dc1 100644 --- a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala +++ b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala @@ -54,16 +54,14 @@ trait JobPaths { /** * Return a host path corresponding to the specified container path. */ - def hostPathFromContainerPath(string: String): Path = { + def hostPathFromContainerPath(string: String): Path = // No container here, just return a Path of the absolute path to the file. callExecutionRoot.resolve(string.stripPrefix(rootWithSlash)) - } def hostPathFromContainerInputs(string: String): Path = // No container here, just return a Path of the absolute path to the file. callExecutionRoot.resolve(string.stripPrefix(rootWithSlash)) - def scriptFilename: String = "script" def dockerCidFilename: String = "docker_cid" diff --git a/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala index f6d8855f3fa..2fb81a44019 100644 --- a/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala +++ b/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala @@ -6,18 +6,19 @@ import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.core.path.Path object JobPathsWithDocker { - def apply(jobKey: BackendJobDescriptorKey, - workflowDescriptor: BackendWorkflowDescriptor, - config: Config) = { + def apply(jobKey: BackendJobDescriptorKey, workflowDescriptor: BackendWorkflowDescriptor, config: Config) = { val workflowPaths = new WorkflowPathsWithDocker(workflowDescriptor, config, WorkflowPaths.DefaultPathBuilders) new JobPathsWithDocker(workflowPaths, jobKey) } } -case class JobPathsWithDocker private[io] (override val workflowPaths: WorkflowPathsWithDocker, jobKey: BackendJobDescriptorKey, override val isCallCacheCopyAttempt: Boolean = false) extends JobPaths { +case class JobPathsWithDocker private[io] (override val workflowPaths: WorkflowPathsWithDocker, + jobKey: BackendJobDescriptorKey, + override val isCallCacheCopyAttempt: Boolean = false +) extends JobPaths { import JobPaths._ - override lazy val callExecutionRoot = { callRoot.resolve("execution") } + override lazy val callExecutionRoot = callRoot.resolve("execution") override def isDocker: Boolean = true val callDockerRoot = callPathBuilder(workflowPaths.dockerWorkflowRoot, jobKey, isCallCacheCopyAttempt) val callExecutionDockerRoot = callDockerRoot.resolve("execution") @@ -29,34 +30,31 @@ case class JobPathsWithDocker private[io] (override val workflowPaths: WorkflowP override def isInExecution(string: String): Boolean = string.startsWith(callExecutionDockerRootWithSlash) - override def hostPathFromContainerPath(string: String): Path = { + override def hostPathFromContainerPath(string: String): Path = callExecutionRoot.resolve(string.stripPrefix(callExecutionDockerRootWithSlash)) - } - override def hostPathFromContainerInputs(string: String): Path = { val stripped = string.stripPrefix(callInputsDockerRootWithSlash) callInputsRoot.resolve(stripped) } - def toDockerPath(path: Path): Path = { + def toDockerPath(path: Path): Path = path.toAbsolutePath match { case p if p.startsWith(workflowPaths.dockerRoot) => p case p => /* For example: - * - * p = /abs/path/to/cromwell-executions/three-step/f00ba4/call-ps/stdout.txt - * localExecutionRoot = /abs/path/to/cromwell-executions - * subpath = three-step/f00ba4/call-ps/stdout.txt - * - * return value = /root/three-step/f00ba4/call-ps/stdout.txt - * - * TODO: this assumes that p.startsWith(localExecutionRoot) - */ + * + * p = /abs/path/to/cromwell-executions/three-step/f00ba4/call-ps/stdout.txt + * localExecutionRoot = /abs/path/to/cromwell-executions + * subpath = three-step/f00ba4/call-ps/stdout.txt + * + * return value = /root/three-step/f00ba4/call-ps/stdout.txt + * + * TODO: this assumes that p.startsWith(localExecutionRoot) + */ val subpath = p.subpath(workflowPaths.executionRoot.getNameCount, p.getNameCount) workflowPaths.dockerRoot.resolve(subpath) } - } override def forCallCacheCopyAttempts: JobPaths = this.copy(isCallCacheCopyAttempt = true) } diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala index 8c233717898..228285dbd5d 100644 --- a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala +++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala @@ -20,7 +20,8 @@ trait WorkflowPaths extends PathFactory { /** * Path (as a String) of the root directory Cromwell should use for ALL workflows. */ - protected lazy val executionRootString: String = config.as[Option[String]]("root").getOrElse(WorkflowPaths.DefaultExecutionRootString) + protected lazy val executionRootString: String = + config.as[Option[String]]("root").getOrElse(WorkflowPaths.DefaultExecutionRootString) /** * Implementers of this trait might override this to provide an appropriate prefix corresponding to the execution root @@ -51,18 +52,17 @@ trait WorkflowPaths extends PathFactory { def getPath(url: String): Try[Path] = Try(buildPath(url)) // Rebuild potential intermediate call directories in case of a sub workflow - protected def workflowPathBuilder(root: Path): Path = { - workflowDescriptor.breadCrumbs.foldLeft(root)((acc, breadCrumb) => { - breadCrumb.toPath(acc) - }).resolve(workflowDescriptor.callable.name).resolve(workflowDescriptor.id.toString + "/") - } + protected def workflowPathBuilder(root: Path): Path = + workflowDescriptor.breadCrumbs + .foldLeft(root)((acc, breadCrumb) => breadCrumb.toPath(acc)) + .resolve(workflowDescriptor.callable.name) + .resolve(workflowDescriptor.id.toString + "/") lazy val finalCallLogsPath: Option[Path] = workflowDescriptor.getWorkflowOption(FinalCallLogsDir) map getPath map { _.get } - def toJobPaths(jobDescriptor: BackendJobDescriptor): JobPaths = { + def toJobPaths(jobDescriptor: BackendJobDescriptor): JobPaths = toJobPaths(jobDescriptor.key, jobDescriptor.workflowDescriptor) - } /** * Creates job paths using the key and workflow descriptor. @@ -73,11 +73,10 @@ trait WorkflowPaths extends PathFactory { * @param jobWorkflowDescriptor The workflow descriptor for the job. * @return The paths for the job. */ - def toJobPaths(jobKey: BackendJobDescriptorKey, jobWorkflowDescriptor: BackendWorkflowDescriptor): JobPaths = { + def toJobPaths(jobKey: BackendJobDescriptorKey, jobWorkflowDescriptor: BackendWorkflowDescriptor): JobPaths = // If the descriptors are the same, no need to create a new WorkflowPaths if (workflowDescriptor == jobWorkflowDescriptor) toJobPaths(this, jobKey) else toJobPaths(withDescriptor(jobWorkflowDescriptor), jobKey) - } protected def toJobPaths(workflowPaths: WorkflowPaths, jobKey: BackendJobDescriptorKey): JobPaths diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala index 78dc1ed77e3..38ea17fe61c 100644 --- a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala +++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala @@ -9,16 +9,19 @@ object WorkflowPathsWithDocker { val DefaultDockerRoot = "/cromwell-executions" } -final case class WorkflowPathsWithDocker(workflowDescriptor: BackendWorkflowDescriptor, config: Config, pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths { +final case class WorkflowPathsWithDocker(workflowDescriptor: BackendWorkflowDescriptor, + config: Config, + pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders +) extends WorkflowPaths { val dockerRoot: Path = DefaultPathBuilder.get( config.getOrElse[String]("dockerRoot", WorkflowPathsWithDocker.DefaultDockerRoot) ) val dockerWorkflowRoot: Path = workflowPathBuilder(dockerRoot) - override def toJobPaths(workflowPaths: WorkflowPaths, jobKey: BackendJobDescriptorKey): JobPathsWithDocker = { + override def toJobPaths(workflowPaths: WorkflowPaths, jobKey: BackendJobDescriptorKey): JobPathsWithDocker = new JobPathsWithDocker(workflowPaths.asInstanceOf[WorkflowPathsWithDocker], jobKey, isCallCacheCopyAttempt = false) - } - override protected def withDescriptor(workflowDescriptor: BackendWorkflowDescriptor): WorkflowPaths = this.copy(workflowDescriptor = workflowDescriptor) + override protected def withDescriptor(workflowDescriptor: BackendWorkflowDescriptor): WorkflowPaths = + this.copy(workflowDescriptor = workflowDescriptor) } diff --git a/backend/src/main/scala/cromwell/backend/package.scala b/backend/src/main/scala/cromwell/backend/package.scala index 132fcf57887..ca58701641f 100644 --- a/backend/src/main/scala/cromwell/backend/package.scala +++ b/backend/src/main/scala/cromwell/backend/package.scala @@ -1,6 +1,7 @@ package cromwell package object backend { + /** Represents the jobKeys executed by a (potentially sub-) workflow at a given point in time */ type JobExecutionMap = Map[BackendWorkflowDescriptor, List[BackendJobDescriptorKey]] } diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala index c98e429c63d..bbb796220c4 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala @@ -1,7 +1,6 @@ package cromwell.backend.standard import java.io.IOException - import akka.actor.{Actor, ActorLogging, ActorRef} import akka.event.LoggingReceive import cats.implicits._ @@ -12,7 +11,11 @@ import common.util.TryUtil import common.validation.ErrorOr.{ErrorOr, ShortCircuitingFlatMap} import common.validation.IOChecked._ import common.validation.Validation._ -import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobAbortedResponse, JobReconnectionNotSupportedException} +import cromwell.backend.BackendJobExecutionActor.{ + BackendJobExecutionResponse, + JobAbortedResponse, + JobReconnectionNotSupportedException +} import cromwell.backend.BackendLifecycleActor.AbortJobCommand import cromwell.backend.BackendLifecycleActorFactory.{FailedRetryCountKey, MemoryMultiplierKey} import cromwell.backend.OutputEvaluator._ @@ -34,7 +37,7 @@ import net.ceedubs.ficus.Ficus._ import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils import shapeless.Coproduct -import wom.callable.{AdHocValue, CommandTaskDefinition, ContainerizedInputExpression, RuntimeEnvironment} +import wom.callable.{AdHocValue, CommandTaskDefinition, ContainerizedInputExpression} import wom.expression.WomExpression import wom.graph.LocalName import wom.values._ @@ -45,12 +48,12 @@ import scala.concurrent.duration._ import scala.util.{Failure, Success, Try} trait StandardAsyncExecutionActorParams extends StandardJobExecutionActorParams { + /** The promise that will be completed when the async run is complete. */ def completionPromise: Promise[BackendJobExecutionResponse] } -case class DefaultStandardAsyncExecutionActorParams -( +case class DefaultStandardAsyncExecutionActorParams( override val jobIdKey: String, override val serviceRegistryActor: ActorRef, override val ioActor: ActorRef, @@ -62,6 +65,10 @@ case class DefaultStandardAsyncExecutionActorParams override val minimumRuntimeSettings: MinimumRuntimeSettings ) extends StandardAsyncExecutionActorParams +// Typically we want to "executeInSubshell" for encapsulation of bash code. +// Override to `false` when we need the script to set an environment variable in the parent shell. +case class ScriptPreambleData(bashString: String, executeInSubshell: Boolean = true) + /** * An extension of the generic AsyncBackendJobExecutionActor providing a standard abstract implementation of an * asynchronous polling backend. @@ -73,7 +80,11 @@ case class DefaultStandardAsyncExecutionActorParams * as the common behavior among the backends adjusts in unison. */ trait StandardAsyncExecutionActor - extends AsyncBackendJobExecutionActor with StandardCachingActorHelper with AsyncIoActorClient with KvClient with SlowJobWarning { + extends AsyncBackendJobExecutionActor + with StandardCachingActorHelper + with AsyncIoActorClient + with KvClient + with SlowJobWarning { this: Actor with ActorLogging with BackendJobLifecycleActor => override lazy val ioCommandBuilder: IoCommandBuilder = DefaultIoCommandBuilder @@ -97,7 +108,8 @@ trait StandardAsyncExecutionActor def statusEquivalentTo(thiz: StandardAsyncRunState)(that: StandardAsyncRunState): Boolean /** The pending execution handle for each poll. */ - type StandardAsyncPendingExecutionHandle = PendingExecutionHandle[StandardAsyncJob, StandardAsyncRunInfo, StandardAsyncRunState] + type StandardAsyncPendingExecutionHandle = + PendingExecutionHandle[StandardAsyncJob, StandardAsyncRunInfo, StandardAsyncRunState] /** Standard set of parameters passed to the backend. */ def standardParams: StandardAsyncExecutionActorParams @@ -129,7 +141,7 @@ trait StandardAsyncExecutionActor lazy val temporaryDirectory: String = configurationDescriptor.backendConfig.getOrElse( path = "temporary-directory", - default = s"""$$(mkdir -p "${runtimeEnvironment.tempPath}" && echo "${runtimeEnvironment.tempPath}")""" + default = """$(mktemp -d "$PWD"/tmp.XXXXXX)""" ) val logJobIds: Boolean = true @@ -147,39 +159,36 @@ trait StandardAsyncExecutionActor protected def cloudResolveWomFile(womFile: WomFile): WomFile = womFile /** Process files while resolving files that will not be localized to their actual cloud locations. */ - private def mapOrCloudResolve(mapper: WomFile => WomFile): WomValue => Try[WomValue] = { - WomFileMapper.mapWomFiles( - womFile => - if (inputsToNotLocalize.contains(womFile)) { - cloudResolveWomFile(womFile) - } else { - mapper(womFile) - } + private def mapOrCloudResolve(mapper: WomFile => WomFile): WomValue => Try[WomValue] = + WomFileMapper.mapWomFiles(womFile => + if (inputsToNotLocalize.contains(womFile)) { + cloudResolveWomFile(womFile) + } else { + mapper(womFile) + } ) - } /** Process files while ignoring files that will not be localized. */ - private def mapOrNoResolve(mapper: WomFile => WomFile): WomValue => Try[WomValue] = { - WomFileMapper.mapWomFiles( - womFile => - if (inputsToNotLocalize.contains(womFile)) { - womFile - } else { - mapper(womFile) - } + private def mapOrNoResolve(mapper: WomFile => WomFile): WomValue => Try[WomValue] = + WomFileMapper.mapWomFiles(womFile => + if (inputsToNotLocalize.contains(womFile)) { + womFile + } else { + mapper(womFile) + } ) - } /** @see [[Command.instantiate]] */ final def commandLinePreProcessor(inputs: WomEvaluatedCallInputs): Try[WomEvaluatedCallInputs] = { val map = inputs map { case (k, v) => k -> mapOrCloudResolve(preProcessWomFile)(v) } - TryUtil.sequenceMap(map). - recoverWith { - case e => Failure(new IOException(e.getMessage) with CromwellFatalExceptionMarker) - } + TryUtil.sequenceMap(map).recoverWith { case e => + Failure(new IOException(e.getMessage) with CromwellFatalExceptionMarker) + } } - final lazy val localizedInputs: Try[WomEvaluatedCallInputs] = commandLinePreProcessor(jobDescriptor.evaluatedTaskInputs) + final lazy val localizedInputs: Try[WomEvaluatedCallInputs] = commandLinePreProcessor( + jobDescriptor.evaluatedTaskInputs + ) /** * Maps WomFile to a local path, for use in the commandLineValueMapper. @@ -202,47 +211,58 @@ trait StandardAsyncExecutionActor def inputsToNotLocalize: Set[WomFile] = Set.empty /** @see [[Command.instantiate]] */ - final lazy val commandLineValueMapper: WomValue => WomValue = { - womValue => mapOrNoResolve(mapCommandLineWomFile)(womValue).get + final lazy val commandLineValueMapper: WomValue => WomValue = { womValue => + mapOrNoResolve(mapCommandLineWomFile)(womValue).get } /** @see [[Command.instantiate]] */ - final lazy val commandLineJobInputValueMapper: WomValue => WomValue = { - womValue => mapOrNoResolve(mapCommandLineJobInputWomFile)(womValue).get + final lazy val commandLineJobInputValueMapper: WomValue => WomValue = { womValue => + mapOrNoResolve(mapCommandLineJobInputWomFile)(womValue).get } - lazy val jobShell: String = configurationDescriptor.backendConfig.getOrElse("job-shell", - configurationDescriptor.globalConfig.getOrElse("system.job-shell", "/bin/bash")) + lazy val jobShell: String = configurationDescriptor.backendConfig.getOrElse( + "job-shell", + configurationDescriptor.globalConfig.getOrElse("system.job-shell", "/bin/bash") + ) - lazy val abbreviateCommandLength: Int = configurationDescriptor.backendConfig.getOrElse("abbreviate-command-length", - configurationDescriptor.globalConfig.getOrElse("system.abbreviate-command-length", 0)) + lazy val abbreviateCommandLength: Int = configurationDescriptor.backendConfig.getOrElse( + "abbreviate-command-length", + configurationDescriptor.globalConfig.getOrElse("system.abbreviate-command-length", 0) + ) /** * The local path where the command will run. */ lazy val commandDirectory: Path = jobPaths.callExecutionRoot - lazy val memoryRetryErrorKeys: Option[List[String]] = configurationDescriptor.globalConfig.as[Option[List[String]]]("system.memory-retry-error-keys") + lazy val memoryRetryErrorKeys: Option[List[String]] = + configurationDescriptor.globalConfig.as[Option[List[String]]]("system.memory-retry-error-keys") - lazy val memoryRetryFactor: Option[MemoryRetryMultiplierRefined] = { + lazy val memoryRetryFactor: Option[MemoryRetryMultiplierRefined] = jobDescriptor.workflowDescriptor.getWorkflowOption(WorkflowOptions.MemoryRetryMultiplier) flatMap { value: String => Try(value.toDouble) match { - case Success(v) => refineV[MemoryRetryMultiplier](v.toDouble) match { - case Left(e) => - // should not happen, this case should have been screened for and fast-failed during workflow materialization. - log.error(e, s"Programmer error: unexpected failure attempting to read value for workflow option " + - s"'${WorkflowOptions.MemoryRetryMultiplier.name}'. Expected value should be in range 1.0 ≤ n ≤ 99.0") - None - case Right(refined) => Option(refined) - } + case Success(v) => + refineV[MemoryRetryMultiplier](v.toDouble) match { + case Left(e) => + // should not happen, this case should have been screened for and fast-failed during workflow materialization. + log.error( + e, + s"Programmer error: unexpected failure attempting to read value for workflow option " + + s"'${WorkflowOptions.MemoryRetryMultiplier.name}'. Expected value should be in range 1.0 ≤ n ≤ 99.0" + ) + None + case Right(refined) => Option(refined) + } case Failure(e) => // should not happen, this case should have been screened for and fast-failed during workflow materialization. - log.error(e, s"Programmer error: unexpected failure attempting to convert value for workflow option " + - s"'${WorkflowOptions.MemoryRetryMultiplier.name}' to Double.") + log.error( + e, + s"Programmer error: unexpected failure attempting to convert value for workflow option " + + s"'${WorkflowOptions.MemoryRetryMultiplier.name}' to Double." + ) None } } - } lazy val memoryRetryRequested: Boolean = memoryRetryFactor.nonEmpty @@ -302,7 +322,8 @@ trait StandardAsyncExecutionActor val globList = parentDirectory./(s"$globDir.list") val controlFileName = "cromwell_glob_control_file" val absoluteGlobValue = commandDirectory.resolve(globFile.value).pathAsString - val globLinkCommand: String = configurationDescriptor.backendConfig.getAs[String]("glob-link-command") + val globLinkCommand: String = configurationDescriptor.backendConfig + .getAs[String]("glob-link-command") .map("( " + _ + " )") .getOrElse("( ln -L GLOB_PATTERN GLOB_DIRECTORY 2> /dev/null ) || ( ln GLOB_PATTERN GLOB_DIRECTORY )") .replaceAll("GLOB_PATTERN", absoluteGlobValue) @@ -329,7 +350,7 @@ trait StandardAsyncExecutionActor } /** Any custom code that should be run within commandScriptContents before the instantiated command. */ - def scriptPreamble: String = "" + def scriptPreamble: ErrorOr[ScriptPreambleData] = ScriptPreambleData("").valid def cwd: Path = commandDirectory def rcPath: Path = cwd./(jobPaths.returnCodeFilename) @@ -347,13 +368,14 @@ trait StandardAsyncExecutionActor // Absolutize any redirect and overridden paths. All of these files must have absolute paths since the command script // references them outside a (cd "execution dir"; ...) subshell. The default names are known to be relative paths, // the names from redirections may or may not be relative. - private def absolutizeContainerPath(path: String): String = { + private def absolutizeContainerPath(path: String): String = if (path.startsWith(cwd.pathAsString)) path else cwd.resolve(path).pathAsString - } def executionStdin: Option[String] = instantiatedCommand.evaluatedStdinRedirection map absolutizeContainerPath - def executionStdout: String = instantiatedCommand.evaluatedStdoutOverride.getOrElse(jobPaths.defaultStdoutFilename) |> absolutizeContainerPath - def executionStderr: String = instantiatedCommand.evaluatedStderrOverride.getOrElse(jobPaths.defaultStderrFilename) |> absolutizeContainerPath + def executionStdout: String = + instantiatedCommand.evaluatedStdoutOverride.getOrElse(jobPaths.defaultStdoutFilename) |> absolutizeContainerPath + def executionStderr: String = + instantiatedCommand.evaluatedStderrOverride.getOrElse(jobPaths.defaultStderrFilename) |> absolutizeContainerPath /* * Ensures the standard paths are correct w.r.t overridden paths. This is called in two places: when generating the command and @@ -405,9 +427,10 @@ trait StandardAsyncExecutionActor val errorOrGlobFiles: ErrorOr[List[WomGlobFile]] = backendEngineFunctions.findGlobOutputs(call, jobDescriptor) - lazy val environmentVariables = instantiatedCommand.environmentVariables map { case (k, v) => s"""export $k="$v"""" } mkString("", "\n", "\n") + lazy val environmentVariables = instantiatedCommand.environmentVariables map { case (k, v) => + s"""export $k="$v"""" + } mkString ("", "\n", "\n") - val home = jobDescriptor.taskCall.callable.homeOverride.map { _ (runtimeEnvironment) }.getOrElse("$HOME") val shortId = jobDescriptor.workflowDescriptor.id.shortString // Give the out and error FIFO variables names that are unlikely to conflict with anything the user is doing. val (out, err) = (s"out$shortId", s"err$shortId") @@ -419,65 +442,72 @@ trait StandardAsyncExecutionActor // Only adjust the temporary directory permissions if this is executing under Docker. val tmpDirPermissionsAdjustment = if (isDockerRun) s"""chmod 777 "$$tmpDir"""" else "" - val emptyDirectoryFillCommand: String = configurationDescriptor.backendConfig.getAs[String]("empty-dir-fill-command") - .getOrElse( - s"""( - |# add a .file in every empty directory to facilitate directory delocalization on the cloud - |cd $cwd - |find . -type d -exec sh -c '[ -z "$$(ls -A '"'"'{}'"'"')" ] && touch '"'"'{}'"'"'/.file' \\; - |)""".stripMargin) + val emptyDirectoryFillCommand: String = configurationDescriptor.backendConfig + .getAs[String]("empty-dir-fill-command") + .getOrElse(s"""( + |# add a .file in every empty directory to facilitate directory delocalization on the cloud + |cd $cwd + |find . -type d -exec sh -c '[ -z "$$(ls -A '"'"'{}'"'"')" ] && touch '"'"'{}'"'"'/.file' \\; + |)""".stripMargin) + + val errorOrPreamble: ErrorOr[String] = scriptPreamble.map { preambleData => + preambleData.executeInSubshell match { + case true => + s""" + |( + |cd ${cwd.pathAsString} + |${preambleData.bashString} + |) + |""".stripMargin + case false => + s""" + |cd ${cwd.pathAsString} + |${preambleData.bashString} + |""".stripMargin + } + } // The `tee` trickery below is to be able to redirect to known filenames for CWL while also streaming // stdout and stderr for PAPI to periodically upload to cloud storage. // https://stackoverflow.com/questions/692000/how-do-i-write-stderr-to-a-file-while-using-tee-with-a-pipe - (errorOrDirectoryOutputs, errorOrGlobFiles).mapN((directoryOutputs, globFiles) => - s"""|#!$jobShell - |DOCKER_OUTPUT_DIR_LINK - |cd ${cwd.pathAsString} - |tmpDir=$temporaryDirectory - |$tmpDirPermissionsAdjustment - |export _JAVA_OPTIONS=-Djava.io.tmpdir="$$tmpDir" - |export TMPDIR="$$tmpDir" - |export HOME="$home" - |( - |cd ${cwd.pathAsString} - |SCRIPT_PREAMBLE - |) - |$out="$${tmpDir}/out.$$$$" $err="$${tmpDir}/err.$$$$" - |mkfifo "$$$out" "$$$err" - |trap 'rm "$$$out" "$$$err"' EXIT - |touch $stdoutRedirection $stderrRedirection - |tee $stdoutRedirection < "$$$out" & - |tee $stderrRedirection < "$$$err" >&2 & - |( - |cd ${cwd.pathAsString} - |ENVIRONMENT_VARIABLES - |INSTANTIATED_COMMAND - |) $stdinRedirection > "$$$out" 2> "$$$err" - |echo $$? > $rcTmpPath - |$emptyDirectoryFillCommand - |( - |cd ${cwd.pathAsString} - |SCRIPT_EPILOGUE - |${globScripts(globFiles)} - |${directoryScripts(directoryOutputs)} - |) - |mv $rcTmpPath $rcPath - |""".stripMargin - .replace("SCRIPT_PREAMBLE", scriptPreamble) - .replace("ENVIRONMENT_VARIABLES", environmentVariables) - .replace("INSTANTIATED_COMMAND", commandString) - .replace("SCRIPT_EPILOGUE", scriptEpilogue) - .replace("DOCKER_OUTPUT_DIR_LINK", dockerOutputDir)) - } - - def runtimeEnvironmentPathMapper(env: RuntimeEnvironment): RuntimeEnvironment = { - def localize(path: String): String = (WomSingleFile(path) |> commandLineValueMapper).valueString - env.copy(outputPath = env.outputPath |> localize, tempPath = env.tempPath |> localize) - } - - lazy val runtimeEnvironment: RuntimeEnvironment = { - RuntimeEnvironmentBuilder(jobDescriptor.runtimeAttributes, jobPaths)(standardParams.minimumRuntimeSettings) |> runtimeEnvironmentPathMapper + (errorOrDirectoryOutputs, errorOrGlobFiles, errorOrPreamble).mapN((directoryOutputs, globFiles, preamble) => + s"""|#!$jobShell + |DOCKER_OUTPUT_DIR_LINK + |cd ${cwd.pathAsString} + |tmpDir=$temporaryDirectory + |$tmpDirPermissionsAdjustment + |export _JAVA_OPTIONS=-Djava.io.tmpdir="$$tmpDir" + |export TMPDIR="$$tmpDir" + | + |SCRIPT_PREAMBLE + | + |$out="$${tmpDir}/out.$$$$" $err="$${tmpDir}/err.$$$$" + |mkfifo "$$$out" "$$$err" + |trap 'rm "$$$out" "$$$err"' EXIT + |touch $stdoutRedirection $stderrRedirection + |tee $stdoutRedirection < "$$$out" & + |tee $stderrRedirection < "$$$err" >&2 & + |( + |cd ${cwd.pathAsString} + |ENVIRONMENT_VARIABLES + |INSTANTIATED_COMMAND + |) $stdinRedirection > "$$$out" 2> "$$$err" + |echo $$? > $rcTmpPath + |$emptyDirectoryFillCommand + |( + |cd ${cwd.pathAsString} + |SCRIPT_EPILOGUE + |${globScripts(globFiles)} + |${directoryScripts(directoryOutputs)} + |) + |mv $rcTmpPath $rcPath + |""".stripMargin + .replace("SCRIPT_PREAMBLE", preamble) + .replace("ENVIRONMENT_VARIABLES", environmentVariables) + .replace("INSTANTIATED_COMMAND", commandString) + .replace("SCRIPT_EPILOGUE", scriptEpilogue) + .replace("DOCKER_OUTPUT_DIR_LINK", dockerOutputDir) + ) } /** @@ -488,23 +518,21 @@ trait StandardAsyncExecutionActor * relativeLocalizationPath("gs://some/bucket/foo.txt") -> "some/bucket/foo.txt" * etc */ - protected def relativeLocalizationPath(file: WomFile): WomFile = { + protected def relativeLocalizationPath(file: WomFile): WomFile = file.mapFile(value => getPath(value) match { case Success(path) => path.pathWithoutScheme case _ => value } ) - } - protected def fileName(file: WomFile): WomFile = { + protected def fileName(file: WomFile): WomFile = file.mapFile(value => getPath(value) match { case Success(path) => path.name case _ => value } ) - } protected def localizationPath(f: CommandSetupSideEffectFile): WomFile = { val fileTransformer = if (isAdHocFile(f.file)) fileName _ else relativeLocalizationPath _ @@ -520,7 +548,6 @@ trait StandardAsyncExecutionActor * Maybe this should be the other way around: the default implementation is noop and SFS / TES override it ? */ lazy val localizeAdHocValues: List[AdHocValue] => ErrorOr[List[StandardAdHocValue]] = { adHocValues => - // Localize an adhoc file to the callExecutionRoot as needed val localize: (AdHocValue, Path) => Future[LocalizedAdHocValue] = { (adHocValue, file) => val actualName = adHocValue.alternativeName.getOrElse(file.name) @@ -528,17 +555,19 @@ trait StandardAsyncExecutionActor // First check that it's not already there under execution root asyncIo.existsAsync(finalPath) flatMap { // If it's not then copy it - case false => asyncIo.copyAsync(file, finalPath) as { LocalizedAdHocValue(adHocValue, finalPath) } + case false => asyncIo.copyAsync(file, finalPath) as LocalizedAdHocValue(adHocValue, finalPath) case true => Future.successful(LocalizedAdHocValue(adHocValue, finalPath)) } } - adHocValues.traverse[ErrorOr, (AdHocValue, Path)]({ adHocValue => - // Build an actionable Path from the ad hoc file - getPath(adHocValue.womValue.value).toErrorOr.map(adHocValue -> _) - }) + adHocValues + .traverse[ErrorOr, (AdHocValue, Path)] { adHocValue => + // Build an actionable Path from the ad hoc file + getPath(adHocValue.womValue.value).toErrorOr.map(adHocValue -> _) + } // Localize the values if necessary - .map(_.traverse[Future, LocalizedAdHocValue](localize.tupled)).toEither + .map(_.traverse[Future, LocalizedAdHocValue](localize.tupled)) + .toEither // TODO: Asynchronify // This is obviously sad but turning it into a Future has earth-shattering consequences, so synchronizing it for now .flatMap(future => Try(Await.result(future, 1.hour)).toChecked) @@ -558,37 +587,41 @@ trait StandardAsyncExecutionActor val callable = jobDescriptor.taskCall.callable /* - * Try to map the command line values. - * - * May not work as the commandLineValueMapper was originally meant to modify paths in the later stages of command - * line instantiation. However, due to [[AdHocValue]] support the command line instantiation itself currently needs - * to use the commandLineValue mapper. So the commandLineValueMapper is attempted first, and if that fails then - * returns the original womValue. - */ - def tryCommandLineValueMapper(womValue: WomValue): WomValue = { + * Try to map the command line values. + * + * May not work as the commandLineValueMapper was originally meant to modify paths in the later stages of command + * line instantiation. However, due to [[AdHocValue]] support the command line instantiation itself currently needs + * to use the commandLineValue mapper. So the commandLineValueMapper is attempted first, and if that fails then + * returns the original womValue. + */ + def tryCommandLineValueMapper(womValue: WomValue): WomValue = Try(commandLineJobInputValueMapper(womValue)).getOrElse(womValue) - } - val unmappedInputs: Map[String, WomValue] = jobDescriptor.evaluatedTaskInputs.map({ + val unmappedInputs: Map[String, WomValue] = jobDescriptor.evaluatedTaskInputs.map { case (inputDefinition, womValue) => inputDefinition.localName.value -> womValue - }) - - val mappedInputs: Checked[Map[String, WomValue]] = localizedInputs.toErrorOr.map( - _.map({ - case (inputDefinition, value) => inputDefinition.localName.value -> tryCommandLineValueMapper(value) - }) - ).toEither - - val evaluateAndInitialize = (containerizedInputExpression: ContainerizedInputExpression) => for { - mapped <- mappedInputs - evaluated <- containerizedInputExpression.evaluate(unmappedInputs, mapped, backendEngineFunctions).toChecked - initialized <- evaluated.traverse[IOChecked, AdHocValue]({ adHocValue => - adHocValue.womValue.initialize(backendEngineFunctions).map({ - case file: WomFile => adHocValue.copy(womValue = file) - case _ => adHocValue - }) - }).toChecked - } yield initialized + } + + val mappedInputs: Checked[Map[String, WomValue]] = localizedInputs.toErrorOr + .map( + _.map { case (inputDefinition, value) => + inputDefinition.localName.value -> tryCommandLineValueMapper(value) + } + ) + .toEither + + val evaluateAndInitialize = (containerizedInputExpression: ContainerizedInputExpression) => + for { + mapped <- mappedInputs + evaluated <- containerizedInputExpression.evaluate(unmappedInputs, mapped, backendEngineFunctions).toChecked + initialized <- evaluated + .traverse[IOChecked, AdHocValue] { adHocValue => + adHocValue.womValue.initialize(backendEngineFunctions).map { + case file: WomFile => adHocValue.copy(womValue = file) + case _ => adHocValue + } + } + .toChecked + } yield initialized callable.adHocFileCreation.toList .flatTraverse[ErrorOr, AdHocValue](evaluateAndInitialize.andThen(_.toValidated)) @@ -598,9 +631,10 @@ trait StandardAsyncExecutionActor .flatMap(localizeAdHocValues.andThen(_.toEither)) .toValidated - protected def asAdHocFile(womFile: WomFile): Option[AdHocValue] = evaluatedAdHocFiles map { _.find({ - case AdHocValue(file, _, _) => file.value == womFile.value - }) + protected def asAdHocFile(womFile: WomFile): Option[AdHocValue] = evaluatedAdHocFiles map { + _.find { case AdHocValue(file, _, _) => + file.value == womFile.value + } } getOrElse None protected def isAdHocFile(womFile: WomFile): Boolean = asAdHocFile(womFile).isDefined @@ -610,18 +644,22 @@ trait StandardAsyncExecutionActor val callable = jobDescriptor.taskCall.callable // Replace input files with the ad hoc updated version - def adHocFilePreProcessor(in: WomEvaluatedCallInputs): Try[WomEvaluatedCallInputs] = { + def adHocFilePreProcessor(in: WomEvaluatedCallInputs): Try[WomEvaluatedCallInputs] = localizedAdHocValues.toTry("Error evaluating ad hoc files") map { adHocFiles => - in map { - case (inputDefinition, originalWomValue) => - inputDefinition -> adHocFiles.collectFirst({ - case AsAdHocValue(AdHocValue(originalWomFile, _, Some(inputName))) if inputName == inputDefinition.localName.value => originalWomFile - case AsLocalizedAdHocValue(LocalizedAdHocValue(AdHocValue(originalWomFile, _, Some(inputName)), localizedPath)) if inputName == inputDefinition.localName.value => + in map { case (inputDefinition, originalWomValue) => + inputDefinition -> adHocFiles + .collectFirst { + case AsAdHocValue(AdHocValue(originalWomFile, _, Some(inputName))) + if inputName == inputDefinition.localName.value => + originalWomFile + case AsLocalizedAdHocValue( + LocalizedAdHocValue(AdHocValue(originalWomFile, _, Some(inputName)), localizedPath) + ) if inputName == inputDefinition.localName.value => originalWomFile.mapFile(_ => localizedPath.pathAsString) - }).getOrElse(originalWomValue) + } + .getOrElse(originalWomValue) } } - } // Gets the inputs that will be mutated by instantiating the command. val mutatingPreProcessor: WomEvaluatedCallInputs => Try[WomEvaluatedCallInputs] = { _ => @@ -635,16 +673,19 @@ trait StandardAsyncExecutionActor jobDescriptor, backendEngineFunctions, mutatingPreProcessor, - commandLineValueMapper, - runtimeEnvironment + commandLineValueMapper ) - def makeStringKeyedMap(list: List[(LocalName, WomValue)]): Map[String, WomValue] = list.toMap map { case (k, v) => k.value -> v } + def makeStringKeyedMap(list: List[(LocalName, WomValue)]): Map[String, WomValue] = list.toMap map { case (k, v) => + k.value -> v + } val command = instantiatedCommandValidation flatMap { instantiatedCommand => val valueMappedPreprocessedInputs = instantiatedCommand.valueMappedPreprocessedInputs |> makeStringKeyedMap - val adHocFileCreationSideEffectFiles: ErrorOr[List[CommandSetupSideEffectFile]] = localizedAdHocValues map { _.map(adHocValueToCommandSetupSideEffectFile) } + val adHocFileCreationSideEffectFiles: ErrorOr[List[CommandSetupSideEffectFile]] = localizedAdHocValues map { + _.map(adHocValueToCommandSetupSideEffectFile) + } def evaluateEnvironmentExpression(nameAndExpression: (String, WomExpression)): ErrorOr[(String, String)] = { val (name, expression) = nameAndExpression @@ -655,16 +696,25 @@ trait StandardAsyncExecutionActor // Build a list of functions from a CommandTaskDefinition to an Option[WomExpression] representing a possible // redirection or override of the filename of a redirection. Evaluate that expression if present and stringify. - val List(stdinRedirect, stdoutOverride, stderrOverride) = List[CommandTaskDefinition => Option[WomExpression]]( - _.stdinRedirection, _.stdoutOverride, _.stderrOverride) map { - _.apply(callable).traverse[ErrorOr, String] { _.evaluateValue(valueMappedPreprocessedInputs, backendEngineFunctions) map { _.valueString} } - } + val List(stdinRedirect, stdoutOverride, stderrOverride) = + List[CommandTaskDefinition => Option[WomExpression]](_.stdinRedirection, + _.stdoutOverride, + _.stderrOverride + ) map { + _.apply(callable).traverse[ErrorOr, String] { + _.evaluateValue(valueMappedPreprocessedInputs, backendEngineFunctions) map { _.valueString } + } + } (adHocFileCreationSideEffectFiles, environmentVariables, stdinRedirect, stdoutOverride, stderrOverride) mapN { (adHocFiles, env, in, out, err) => instantiatedCommand.copy( - createdFiles = instantiatedCommand.createdFiles ++ adHocFiles, environmentVariables = env.toMap, - evaluatedStdinRedirection = in, evaluatedStdoutOverride = out, evaluatedStderrOverride = err) + createdFiles = instantiatedCommand.createdFiles ++ adHocFiles, + environmentVariables = env.toMap, + evaluatedStdinRedirection = in, + evaluatedStdoutOverride = out, + evaluatedStderrOverride = err + ) }: ErrorOr[InstantiatedCommand] } @@ -681,11 +731,10 @@ trait StandardAsyncExecutionActor * * If the `command` errors for some reason, put a "-1" into the rc file. */ - def redirectOutputs(command: String): String = { + def redirectOutputs(command: String): String = // > 128 is the cutoff for signal-induced process deaths such as might be observed with abort. // http://www.tldp.org/LDP/abs/html/exitcodes.html s"""$command < /dev/null || { rc=$$?; if [ "$$rc" -gt "128" ]; then echo $$rc; else echo -1; fi } > ${jobPaths.returnCode}""" - } /** A tag that may be used for logging. */ lazy val tag = s"${this.getClass.getSimpleName} [UUID(${workflowIdForLogging.shortString}):${jobDescriptor.key.tag}]" @@ -695,42 +744,47 @@ trait StandardAsyncExecutionActor * * @return True if a non-empty `remoteStdErrPath` should fail the job. */ - lazy val failOnStdErr: Boolean = RuntimeAttributesValidation.extract( - FailOnStderrValidation.instance, validatedRuntimeAttributes) + lazy val failOnStdErr: Boolean = + RuntimeAttributesValidation.extract(FailOnStderrValidation.instance, validatedRuntimeAttributes) /** - * Returns the behavior for continuing on the return code, obtained by converting `returnCodeContents` to an Int. - * - * @return the behavior for continuing on the return code. - */ - lazy val continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract( - ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) + * Returns the behavior for continuing on the return code, obtained by converting `returnCodeContents` to an Int. + * + * @return the behavior for continuing on the return code. + */ + lazy val continueOnReturnCode: ContinueOnReturnCode = + RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) /** * Returns the max number of times that a failed job should be retried, obtained by converting `maxRetries` to an Int. */ - lazy val maxRetries: Int = RuntimeAttributesValidation.extract( - MaxRetriesValidation.instance, validatedRuntimeAttributes) + lazy val maxRetries: Int = + RuntimeAttributesValidation.extract(MaxRetriesValidation.instance, validatedRuntimeAttributes) - - lazy val previousFailedRetries: Int = jobDescriptor.prefetchedKvStoreEntries.get(BackendLifecycleActorFactory.FailedRetryCountKey) match { - case Some(KvPair(_,v)) => v.toInt - case _ => 0 - } + lazy val previousFailedRetries: Int = + jobDescriptor.prefetchedKvStoreEntries.get(BackendLifecycleActorFactory.FailedRetryCountKey) match { + case Some(KvPair(_, v)) => v.toInt + case _ => 0 + } /** * Returns the memory multiplier for previous attempt if available */ - lazy val previousMemoryMultiplier: Option[Double] = jobDescriptor.prefetchedKvStoreEntries.get(BackendLifecycleActorFactory.MemoryMultiplierKey) match { - case Some(KvPair(_,v)) => Try(v.toDouble) match { - case Success(m) => Option(m) - case Failure(e) => - // should not happen as Cromwell itself had written the value as a Double - log.error(e, s"Programmer error: unexpected failure attempting to convert value of MemoryMultiplierKey from JOB_KEY_VALUE_ENTRY table to Double.") - None + lazy val previousMemoryMultiplier: Option[Double] = + jobDescriptor.prefetchedKvStoreEntries.get(BackendLifecycleActorFactory.MemoryMultiplierKey) match { + case Some(KvPair(_, v)) => + Try(v.toDouble) match { + case Success(m) => Option(m) + case Failure(e) => + // should not happen as Cromwell itself had written the value as a Double + log.error( + e, + s"Programmer error: unexpected failure attempting to convert value of MemoryMultiplierKey from JOB_KEY_VALUE_ENTRY table to Double." + ) + None + } + case _ => None } - case _ => None - } /** * Execute the job specified in the params. Should return a `StandardAsyncPendingExecutionHandle`, or a @@ -738,9 +792,8 @@ trait StandardAsyncExecutionActor * * @return the execution handle for the job. */ - def execute(): ExecutionHandle = { + def execute(): ExecutionHandle = throw new UnsupportedOperationException(s"Neither execute() nor executeAsync() implemented by $getClass") - } /** * Async execute the job specified in the params. Should return a `StandardAsyncPendingExecutionHandle`, or a @@ -778,7 +831,8 @@ trait StandardAsyncExecutionActor * @param jobId The previously recorded job id. * @return the execution handle for the job. */ - def reconnectToAbortAsync(jobId: StandardAsyncJob): Future[ExecutionHandle] = Future.failed(JobReconnectionNotSupportedException(jobDescriptor.key)) + def reconnectToAbortAsync(jobId: StandardAsyncJob): Future[ExecutionHandle] = + Future.failed(JobReconnectionNotSupportedException(jobDescriptor.key)) /** * This is in spirit similar to recover except it does not defaults back to running the job if not implemented. @@ -790,7 +844,8 @@ trait StandardAsyncExecutionActor * @param jobId The previously recorded job id. * @return the execution handle for the job. */ - def reconnectAsync(jobId: StandardAsyncJob): Future[ExecutionHandle] = Future.failed(JobReconnectionNotSupportedException(jobDescriptor.key)) + def reconnectAsync(jobId: StandardAsyncJob): Future[ExecutionHandle] = + Future.failed(JobReconnectionNotSupportedException(jobDescriptor.key)) /** * Returns the run status for the job. @@ -798,9 +853,8 @@ trait StandardAsyncExecutionActor * @param handle The handle of the running job. * @return The status of the job. */ - def pollStatus(handle: StandardAsyncPendingExecutionHandle): StandardAsyncRunState = { + def pollStatus(handle: StandardAsyncPendingExecutionHandle): StandardAsyncRunState = throw new UnsupportedOperationException(s"Neither pollStatus nor pollStatusAsync implemented by $getClass") - } /** * Returns the async run status for the job. @@ -808,7 +862,8 @@ trait StandardAsyncExecutionActor * @param handle The handle of the running job. * @return The status of the job. */ - def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle): Future[StandardAsyncRunState] = Future.fromTry(Try(pollStatus(handle))) + def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle): Future[StandardAsyncRunState] = + Future.fromTry(Try(pollStatus(handle))) /** * Adds custom behavior invoked when polling fails due to some exception. By default adds nothing. @@ -819,9 +874,8 @@ trait StandardAsyncExecutionActor * * @return A partial function handler for exceptions after polling. */ - def customPollStatusFailure: PartialFunction[(ExecutionHandle, Exception), ExecutionHandle] = { + def customPollStatusFailure: PartialFunction[(ExecutionHandle, Exception), ExecutionHandle] = PartialFunction.empty - } /** * Returns true when a job is complete, either successfully or unsuccessfully. @@ -892,13 +946,12 @@ trait StandardAsyncExecutionActor * * By default handles the behavior of `requestsAbortAndDiesImmediately`. */ - def postAbort(): Unit = { + def postAbort(): Unit = if (requestsAbortAndDiesImmediately) { tellMetadata(Map(CallMetadataKeys.BackendStatus -> "Aborted")) context.parent ! JobAbortedResponse(jobDescriptor.key) context.stop(self) } - } /** * Output value mapper. @@ -906,19 +959,17 @@ trait StandardAsyncExecutionActor * @param womValue The original WOM value. * @return The Try wrapped and mapped WOM value. */ - final def outputValueMapper(womValue: WomValue): Try[WomValue] = { + final def outputValueMapper(womValue: WomValue): Try[WomValue] = WomFileMapper.mapWomFiles(mapOutputWomFile)(womValue) - } /** * Used to convert to output paths. * */ def mapOutputWomFile(womFile: WomFile): WomFile = - womFile.mapFile{ - path => - val pathFromContainerInputs = jobPaths.hostPathFromContainerInputs(path) - pathFromContainerInputs.toAbsolutePath.toString + womFile.mapFile { path => + val pathFromContainerInputs = jobPaths.hostPathFromContainerInputs(path) + pathFromContainerInputs.toAbsolutePath.toString } /** @@ -928,9 +979,8 @@ trait StandardAsyncExecutionActor * * @return A Try wrapping evaluated outputs. */ - def evaluateOutputs()(implicit ec: ExecutionContext): Future[EvaluatedJobOutputs] = { + def evaluateOutputs()(implicit ec: ExecutionContext): Future[EvaluatedJobOutputs] = OutputEvaluator.evaluateOutputs(jobDescriptor, backendEngineFunctions, outputValueMapper) - } /** * Tests whether an attempted result of evaluateOutputs should possibly be retried. @@ -946,7 +996,7 @@ trait StandardAsyncExecutionActor * @param exception The exception, possibly an CromwellAggregatedException. * @return True if evaluateOutputs should be retried later. */ - final def retryEvaluateOutputsAggregated(exception: Exception): Boolean = { + final def retryEvaluateOutputsAggregated(exception: Exception): Boolean = exception match { case aggregated: CromwellAggregatedException => aggregated.throwables.collectFirst { @@ -954,7 +1004,6 @@ trait StandardAsyncExecutionActor }.isDefined case _ => retryEvaluateOutputs(exception) } - } /** * Tests whether an attempted result of evaluateOutputs should possibly be retried. @@ -980,7 +1029,8 @@ trait StandardAsyncExecutionActor */ def handleExecutionSuccess(runStatus: StandardAsyncRunState, handle: StandardAsyncPendingExecutionHandle, - returnCode: Int)(implicit ec: ExecutionContext): Future[ExecutionHandle] = { + returnCode: Int + )(implicit ec: ExecutionContext): Future[ExecutionHandle] = evaluateOutputs() map { case ValidJobOutputs(outputs) => // Need to make sure the paths are up to date before sending the detritus back in the response @@ -997,7 +1047,6 @@ trait StandardAsyncExecutionActor handle case JobOutputsEvaluationException(ex) => FailedNonRetryableExecutionHandle(ex, kvPairsToSave = None) } - } /** * Process an unsuccessful run, as interpreted by `handleExecutionFailure`. @@ -1005,18 +1054,18 @@ trait StandardAsyncExecutionActor * @return The execution handle. */ def retryElseFail(backendExecutionStatus: Future[ExecutionHandle], - retryWithMoreMemory: Boolean = false): Future[ExecutionHandle] = { - + retryWithMoreMemory: Boolean = false + ): Future[ExecutionHandle] = backendExecutionStatus flatMap { case failedRetryableOrNonRetryable: FailedExecutionHandle => - val kvsFromPreviousAttempt = jobDescriptor.prefetchedKvStoreEntries.collect { case (key: String, kvPair: KvPair) => key -> kvPair } val kvsForNextAttempt = failedRetryableOrNonRetryable.kvPairsToSave match { - case Some(kvPairs) => kvPairs.map { - case kvPair@KvPair(ScopedKey(_, _, key), _) => key -> kvPair - }.toMap + case Some(kvPairs) => + kvPairs.map { case kvPair @ KvPair(ScopedKey(_, _, key), _) => + key -> kvPair + }.toMap case None => Map.empty[String, KvPair] } @@ -1026,23 +1075,35 @@ trait StandardAsyncExecutionActor (retryWithMoreMemory, memoryRetryFactor, previousMemoryMultiplier) match { case (true, Some(retryFactor), Some(previousMultiplier)) => val nextMemoryMultiplier = previousMultiplier * retryFactor.value - saveAttrsAndRetry(failed, kvsFromPreviousAttempt, kvsForNextAttempt, incFailedCount = true, Option(nextMemoryMultiplier)) + saveAttrsAndRetry(failed, + kvsFromPreviousAttempt, + kvsForNextAttempt, + incFailedCount = true, + Option(nextMemoryMultiplier) + ) case (true, Some(retryFactor), None) => - saveAttrsAndRetry(failed, kvsFromPreviousAttempt, kvsForNextAttempt, incFailedCount = true, Option(retryFactor.value)) - case (_, _, _) => saveAttrsAndRetry(failed, kvsFromPreviousAttempt, kvsForNextAttempt, incFailedCount = true) + saveAttrsAndRetry(failed, + kvsFromPreviousAttempt, + kvsForNextAttempt, + incFailedCount = true, + Option(retryFactor.value) + ) + case (_, _, _) => + saveAttrsAndRetry(failed, kvsFromPreviousAttempt, kvsForNextAttempt, incFailedCount = true) } case failedNonRetryable: FailedNonRetryableExecutionHandle => Future.successful(failedNonRetryable) - case failedRetryable: FailedRetryableExecutionHandle => saveAttrsAndRetry(failedRetryable, kvsFromPreviousAttempt, kvsForNextAttempt, incFailedCount = false) + case failedRetryable: FailedRetryableExecutionHandle => + saveAttrsAndRetry(failedRetryable, kvsFromPreviousAttempt, kvsForNextAttempt, incFailedCount = false) } case _ => backendExecutionStatus } - } private def saveAttrsAndRetry(failedExecHandle: FailedExecutionHandle, kvPrev: Map[String, KvPair], kvNext: Map[String, KvPair], incFailedCount: Boolean, - nextMemoryMultiplier: Option[Double] = None): Future[FailedRetryableExecutionHandle] = { + nextMemoryMultiplier: Option[Double] = None + ): Future[FailedRetryableExecutionHandle] = failedExecHandle match { case failedNonRetryable: FailedNonRetryableExecutionHandle => saveKvPairsForNextAttempt(kvPrev, kvNext, incFailedCount, nextMemoryMultiplier) map { _ => @@ -1051,7 +1112,6 @@ trait StandardAsyncExecutionActor case failedRetryable: FailedRetryableExecutionHandle => saveKvPairsForNextAttempt(kvPrev, kvNext, incFailedCount, nextMemoryMultiplier) map (_ => failedRetryable) } - } /** * Merge key-value pairs from previous job execution attempt with incoming pairs from current attempt, which has just @@ -1062,8 +1122,10 @@ trait StandardAsyncExecutionActor private def saveKvPairsForNextAttempt(kvsFromPreviousAttempt: Map[String, KvPair], kvsForNextAttempt: Map[String, KvPair], incrementFailedRetryCount: Boolean, - nextMemoryMultiplierOption: Option[Double]): Future[Seq[KvResponse]] = { - val nextKvJobKey = KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt + 1) + nextMemoryMultiplierOption: Option[Double] + ): Future[Seq[KvResponse]] = { + val nextKvJobKey = + KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt + 1) def getNextKvPair[A](key: String, value: String): Map[String, KvPair] = { val nextScopedKey = ScopedKey(jobDescriptor.workflowDescriptor.id, nextKvJobKey, key) @@ -1071,7 +1133,8 @@ trait StandardAsyncExecutionActor Map(key -> nextKvPair) } - val kvsFromPreviousAttemptUpd = kvsFromPreviousAttempt.view.mapValues(kvPair => kvPair.copy(key = kvPair.key.copy(jobKey = nextKvJobKey))) + val kvsFromPreviousAttemptUpd = + kvsFromPreviousAttempt.view.mapValues(kvPair => kvPair.copy(key = kvPair.key.copy(jobKey = nextKvJobKey))) val failedRetryCountKvPair: Map[String, KvPair] = if (incrementFailedRetryCount) getNextKvPair(FailedRetryCountKey, (previousFailedRetries + 1).toString) @@ -1089,8 +1152,10 @@ trait StandardAsyncExecutionActor if (failures.isEmpty) { respSeq } else { - throw new RuntimeException("Failed to save one or more job execution attributes to the database between " + - "attempts:\n " + failures.mkString("\n")) + throw new RuntimeException( + "Failed to save one or more job execution attributes to the database between " + + "attempts:\n " + failures.mkString("\n") + ) } } } @@ -1101,8 +1166,7 @@ trait StandardAsyncExecutionActor * @param runStatus The run status. * @return The execution handle. */ - def handleExecutionFailure(runStatus: StandardAsyncRunState, - returnCode: Option[Int]): Future[ExecutionHandle] = { + def handleExecutionFailure(runStatus: StandardAsyncRunState, returnCode: Option[Int]): Future[ExecutionHandle] = { val exception = new RuntimeException(s"Task ${jobDescriptor.key.tag} failed for unknown reason: $runStatus") Future.successful(FailedNonRetryableExecutionHandle(exception, returnCode, None)) } @@ -1131,30 +1195,33 @@ trait StandardAsyncExecutionActor } override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext): Future[ExecutionHandle] = { - val executeOrRecoverFuture = { + val executeOrRecoverFuture = mode match { - case Reconnect(jobId: StandardAsyncJob@unchecked) => reconnectAsync(jobId) - case ReconnectToAbort(jobId: StandardAsyncJob@unchecked) => reconnectToAbortAsync(jobId) - case Recover(jobId: StandardAsyncJob@unchecked) => recoverAsync(jobId) + case Reconnect(jobId: StandardAsyncJob @unchecked) => reconnectAsync(jobId) + case ReconnectToAbort(jobId: StandardAsyncJob @unchecked) => reconnectToAbortAsync(jobId) + case Recover(jobId: StandardAsyncJob @unchecked) => recoverAsync(jobId) case _ => tellMetadata(startMetadataKeyValues) executeAsync() } - } - executeOrRecoverFuture flatMap executeOrRecoverSuccess recoverWith { - case throwable: Throwable => Future failed { + executeOrRecoverFuture flatMap executeOrRecoverSuccess recoverWith { case throwable: Throwable => + Future failed { jobLogger.error(s"Error attempting to $mode", throwable) throwable } } } - private def executeOrRecoverSuccess(executionHandle: ExecutionHandle): Future[ExecutionHandle] = { + private def executeOrRecoverSuccess(executionHandle: ExecutionHandle): Future[ExecutionHandle] = executionHandle match { - case handle: PendingExecutionHandle[StandardAsyncJob@unchecked, StandardAsyncRunInfo@unchecked, StandardAsyncRunState@unchecked] => - - configurationDescriptor.slowJobWarningAfter foreach { duration => self ! WarnAboutSlownessAfter(handle.pendingJob.jobId, duration) } + case handle: PendingExecutionHandle[StandardAsyncJob @unchecked, + StandardAsyncRunInfo @unchecked, + StandardAsyncRunState @unchecked + ] => + configurationDescriptor.slowJobWarningAfter foreach { duration => + self ! WarnAboutSlownessAfter(handle.pendingJob.jobId, duration) + } tellKvJobId(handle.pendingJob) map { _ => if (logJobIds) jobLogger.info(s"job id: ${handle.pendingJob.jobId}") @@ -1164,34 +1231,31 @@ trait StandardAsyncExecutionActor the prior runnable to the thread pool this actor doesn't know the job id for aborting. These runnables are queued up and may still be run by the thread pool anytime in the future. Issue #1218 may address this inconsistency at a later time. For now, just go back and check if we missed the abort command. - */ + */ self ! CheckMissedAbort(handle.pendingJob) executionHandle } case _ => Future.successful(executionHandle) } - } - override def poll(previous: ExecutionHandle)(implicit ec: ExecutionContext): Future[ExecutionHandle] = { + override def poll(previous: ExecutionHandle)(implicit ec: ExecutionContext): Future[ExecutionHandle] = previous match { - case handle: PendingExecutionHandle[ - StandardAsyncJob@unchecked, StandardAsyncRunInfo@unchecked, StandardAsyncRunState@unchecked] => - + case handle: PendingExecutionHandle[StandardAsyncJob @unchecked, + StandardAsyncRunInfo @unchecked, + StandardAsyncRunState @unchecked + ] => jobLogger.debug(s"$tag Polling Job ${handle.pendingJob}") - pollStatusAsync(handle) flatMap { - backendRunStatus => - self ! WarnAboutSlownessIfNecessary - handlePollSuccess(handle, backendRunStatus) - } recover { - case throwable => - handlePollFailure(handle, throwable) + pollStatusAsync(handle) flatMap { backendRunStatus => + self ! WarnAboutSlownessIfNecessary + handlePollSuccess(handle, backendRunStatus) + } recover { case throwable => + handlePollFailure(handle, throwable) } case successful: SuccessfulExecutionHandle => Future.successful(successful) case failed: FailedNonRetryableExecutionHandle => Future.successful(failed) case failedRetryable: FailedRetryableExecutionHandle => Future.successful(failedRetryable) case badHandle => Future.failed(new IllegalArgumentException(s"Unexpected execution handle: $badHandle")) } - } /** * Process a poll success. @@ -1201,7 +1265,8 @@ trait StandardAsyncExecutionActor * @return The updated execution handle. */ def handlePollSuccess(oldHandle: StandardAsyncPendingExecutionHandle, - state: StandardAsyncRunState): Future[ExecutionHandle] = { + state: StandardAsyncRunState + ): Future[ExecutionHandle] = { val previousState = oldHandle.previousState if (!(previousState exists statusEquivalentTo(state))) { // If this is the first time checking the status, we log the transition as '-' to 'currentStatus'. Otherwise just use @@ -1217,7 +1282,10 @@ trait StandardAsyncExecutionActor val metadata = getTerminalMetadata(state) tellMetadata(metadata) handleExecutionResult(state, oldHandle) - case s => Future.successful(oldHandle.copy(previousState = Option(s))) // Copy the current handle with updated previous status. + case s => + Future.successful( + oldHandle.copy(previousState = Option(s)) + ) // Copy the current handle with updated previous status. } } @@ -1228,8 +1296,7 @@ trait StandardAsyncExecutionActor * @param throwable The cause of the polling failure. * @return The updated execution handle. */ - def handlePollFailure(oldHandle: StandardAsyncPendingExecutionHandle, - throwable: Throwable): ExecutionHandle = { + def handlePollFailure(oldHandle: StandardAsyncPendingExecutionHandle, throwable: Throwable): ExecutionHandle = throwable match { case exception: Exception => val handler: PartialFunction[(ExecutionHandle, Exception), ExecutionHandle] = @@ -1240,7 +1307,9 @@ trait StandardAsyncExecutionActor FailedNonRetryableExecutionHandle(exception, kvPairsToSave = None) case (handle: ExecutionHandle, exception: Exception) => // Log exceptions and return the original handle to try again. - jobLogger.warn(s"Caught non-fatal ${exception.getClass.getSimpleName} exception trying to poll, retrying", exception) + jobLogger.warn(s"Caught non-fatal ${exception.getClass.getSimpleName} exception trying to poll, retrying", + exception + ) handle } handler((oldHandle, exception)) @@ -1249,7 +1318,6 @@ trait StandardAsyncExecutionActor // Someone has subclassed or instantiated Throwable directly. Kill the job. They should be using an Exception. FailedNonRetryableExecutionHandle(throwable, kvPairsToSave = None) } - } /** * Process an execution result. @@ -1259,33 +1327,35 @@ trait StandardAsyncExecutionActor * @return The updated execution handle. */ def handleExecutionResult(status: StandardAsyncRunState, - oldHandle: StandardAsyncPendingExecutionHandle): Future[ExecutionHandle] = { + oldHandle: StandardAsyncPendingExecutionHandle + ): Future[ExecutionHandle] = { // Returns true if the task has written an RC file that indicates OOM, false otherwise def memoryRetryRC: Future[Boolean] = { - def returnCodeAsBoolean(codeAsOption: Option[String]): Boolean = { + def returnCodeAsBoolean(codeAsOption: Option[String]): Boolean = codeAsOption match { case Some(codeAsString) => Try(codeAsString.trim.toInt) match { - case Success(code) => code match { - case StderrContainsRetryKeysCode => true - case _ => false - } + case Success(code) => + code match { + case StderrContainsRetryKeysCode => true + case _ => false + } case Failure(e) => - log.error(s"'CheckingForMemoryRetry' action exited with code '$codeAsString' which couldn't be " + - s"converted to an Integer. Task will not be retried with more memory. Error: ${ExceptionUtils.getMessage(e)}") + log.error( + s"'CheckingForMemoryRetry' action exited with code '$codeAsString' which couldn't be " + + s"converted to an Integer. Task will not be retried with more memory. Error: ${ExceptionUtils.getMessage(e)}" + ) false } case None => false } - } - def readMemoryRetryRCFile(fileExists: Boolean): Future[Option[String]] = { + def readMemoryRetryRCFile(fileExists: Boolean): Future[Option[String]] = if (fileExists) asyncIo.contentAsStringAsync(jobPaths.memoryRetryRC, None, failOnOverflow = false).map(Option(_)) else Future.successful(None) - } for { fileExists <- asyncIo.existsAsync(jobPaths.memoryRetryRC) @@ -1305,47 +1375,73 @@ trait StandardAsyncExecutionActor outOfMemoryDetected <- memoryRetryRC } yield (stderrSize, returnCodeAsString, outOfMemoryDetected) - stderrSizeAndReturnCodeAndMemoryRetry flatMap { - case (stderrSize, returnCodeAsString, outOfMemoryDetected) => - val tryReturnCodeAsInt = Try(returnCodeAsString.trim.toInt) - - if (isDone(status)) { - tryReturnCodeAsInt match { - case Success(returnCodeAsInt) if failOnStdErr && stderrSize.intValue > 0 => - val executionHandle = Future.successful(FailedNonRetryableExecutionHandle(StderrNonEmpty(jobDescriptor.key.tag, stderrSize, stderrAsOption), Option(returnCodeAsInt), None)) - retryElseFail(executionHandle) - case Success(returnCodeAsInt) if continueOnReturnCode.continueFor(returnCodeAsInt) => - handleExecutionSuccess(status, oldHandle, returnCodeAsInt) - // It's important that we check retryWithMoreMemory case before isAbort. RC could be 137 in either case; - // if it was caused by OOM killer, want to handle as OOM and not job abort. - case Success(returnCodeAsInt) if outOfMemoryDetected && memoryRetryRequested => - val executionHandle = Future.successful(FailedNonRetryableExecutionHandle(RetryWithMoreMemory(jobDescriptor.key.tag, stderrAsOption, memoryRetryErrorKeys, log), Option(returnCodeAsInt), None)) - retryElseFail(executionHandle, outOfMemoryDetected) - case Success(returnCodeAsInt) if isAbort(returnCodeAsInt) => - Future.successful(AbortedExecutionHandle) - case Success(returnCodeAsInt) => - val executionHandle = Future.successful(FailedNonRetryableExecutionHandle(WrongReturnCode(jobDescriptor.key.tag, returnCodeAsInt, stderrAsOption), Option(returnCodeAsInt), None)) - retryElseFail(executionHandle) - case Failure(_) => - Future.successful(FailedNonRetryableExecutionHandle(ReturnCodeIsNotAnInt(jobDescriptor.key.tag, returnCodeAsString, stderrAsOption), kvPairsToSave = None)) - } - } else { - tryReturnCodeAsInt match { - case Success(returnCodeAsInt) if outOfMemoryDetected && memoryRetryRequested && !continueOnReturnCode.continueFor(returnCodeAsInt) => - val executionHandle = Future.successful(FailedNonRetryableExecutionHandle(RetryWithMoreMemory(jobDescriptor.key.tag, stderrAsOption, memoryRetryErrorKeys, log), Option(returnCodeAsInt), None)) - retryElseFail(executionHandle, outOfMemoryDetected) - case _ => - val failureStatus = handleExecutionFailure(status, tryReturnCodeAsInt.toOption) - retryElseFail(failureStatus) - } + stderrSizeAndReturnCodeAndMemoryRetry flatMap { case (stderrSize, returnCodeAsString, outOfMemoryDetected) => + val tryReturnCodeAsInt = Try(returnCodeAsString.trim.toInt) + + if (isDone(status)) { + tryReturnCodeAsInt match { + case Success(returnCodeAsInt) if failOnStdErr && stderrSize.intValue > 0 => + val executionHandle = Future.successful( + FailedNonRetryableExecutionHandle(StderrNonEmpty(jobDescriptor.key.tag, stderrSize, stderrAsOption), + Option(returnCodeAsInt), + None + ) + ) + retryElseFail(executionHandle) + case Success(returnCodeAsInt) if continueOnReturnCode.continueFor(returnCodeAsInt) => + handleExecutionSuccess(status, oldHandle, returnCodeAsInt) + // It's important that we check retryWithMoreMemory case before isAbort. RC could be 137 in either case; + // if it was caused by OOM killer, want to handle as OOM and not job abort. + case Success(returnCodeAsInt) if outOfMemoryDetected && memoryRetryRequested => + val executionHandle = Future.successful( + FailedNonRetryableExecutionHandle( + RetryWithMoreMemory(jobDescriptor.key.tag, stderrAsOption, memoryRetryErrorKeys, log), + Option(returnCodeAsInt), + None + ) + ) + retryElseFail(executionHandle, outOfMemoryDetected) + case Success(returnCodeAsInt) if isAbort(returnCodeAsInt) => + Future.successful(AbortedExecutionHandle) + case Success(returnCodeAsInt) => + val executionHandle = Future.successful( + FailedNonRetryableExecutionHandle(WrongReturnCode(jobDescriptor.key.tag, returnCodeAsInt, stderrAsOption), + Option(returnCodeAsInt), + None + ) + ) + retryElseFail(executionHandle) + case Failure(_) => + Future.successful( + FailedNonRetryableExecutionHandle( + ReturnCodeIsNotAnInt(jobDescriptor.key.tag, returnCodeAsString, stderrAsOption), + kvPairsToSave = None + ) + ) } - } recoverWith { - case exception => - if (isDone(status)) Future.successful(FailedNonRetryableExecutionHandle(exception, kvPairsToSave = None)) - else { - val failureStatus = handleExecutionFailure(status, None) - retryElseFail(failureStatus) + } else { + tryReturnCodeAsInt match { + case Success(returnCodeAsInt) + if outOfMemoryDetected && memoryRetryRequested && !continueOnReturnCode.continueFor(returnCodeAsInt) => + val executionHandle = Future.successful( + FailedNonRetryableExecutionHandle( + RetryWithMoreMemory(jobDescriptor.key.tag, stderrAsOption, memoryRetryErrorKeys, log), + Option(returnCodeAsInt), + None + ) + ) + retryElseFail(executionHandle, outOfMemoryDetected) + case _ => + val failureStatus = handleExecutionFailure(status, tryReturnCodeAsInt.toOption) + retryElseFail(failureStatus) } + } + } recoverWith { case exception => + if (isDone(status)) Future.successful(FailedNonRetryableExecutionHandle(exception, kvPairsToSave = None)) + else { + val failureStatus = handleExecutionFailure(status, None) + retryElseFail(failureStatus) + } } } @@ -1374,7 +1470,7 @@ trait StandardAsyncExecutionActor serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) } - override protected implicit lazy val ec: ExecutionContextExecutor = context.dispatcher + implicit override protected lazy val ec: ExecutionContextExecutor = context.dispatcher } /** diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala index a64c6a5439c..9d049ee8fff 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala @@ -48,16 +48,16 @@ trait StandardCachingActorHelper extends JobCachingActorHelper { lazy val call: CommandCallNode = jobDescriptor.key.call - lazy val standardInitializationData: StandardInitializationData = BackendInitializationData. - as[StandardInitializationData](backendInitializationDataOption) + lazy val standardInitializationData: StandardInitializationData = + BackendInitializationData.as[StandardInitializationData](backendInitializationDataOption) lazy val validatedRuntimeAttributes: ValidatedRuntimeAttributes = { val builder = standardInitializationData.runtimeAttributesBuilder builder.build(jobDescriptor.runtimeAttributes, jobLogger) } - lazy val isDockerRun: Boolean = RuntimeAttributesValidation.extractOption( - DockerValidation.instance, validatedRuntimeAttributes).isDefined + lazy val isDockerRun: Boolean = + RuntimeAttributesValidation.extractOption(DockerValidation.instance, validatedRuntimeAttributes).isDefined /** * Returns the paths to the job. diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala b/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala index 7782c2da901..ac4c39fc8d7 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala @@ -24,11 +24,15 @@ case class DefaultStandardExpressionFunctionsParams(override val pathBuilders: P override val callContext: CallContext, override val ioActorProxy: ActorRef, override val executionContext: ExecutionContext - ) extends StandardExpressionFunctionsParams +) extends StandardExpressionFunctionsParams // TODO: Once we figure out premapping and postmapping, maybe we can standardize that behavior. Currently that's the most important feature that subclasses override. class StandardExpressionFunctions(val standardParams: StandardExpressionFunctionsParams) - extends GlobFunctions with DirectoryFunctions with ReadLikeFunctions with WriteFunctions with CallCorePathFunctions { + extends GlobFunctions + with DirectoryFunctions + with ReadLikeFunctions + with WriteFunctions + with CallCorePathFunctions { override lazy val ec = standardParams.executionContext @@ -41,6 +45,6 @@ class StandardExpressionFunctions(val standardParams: StandardExpressionFunction val callContext: CallContext = standardParams.callContext val writeDirectory: Path = callContext.root - + val isDocker: Boolean = callContext.isDocker } diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala index 22eb6763b53..4bac258312e 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala @@ -24,8 +24,7 @@ trait StandardFinalizationActorParams { def configurationDescriptor: BackendConfigurationDescriptor } -case class DefaultStandardFinalizationActorParams -( +case class DefaultStandardFinalizationActorParams( workflowDescriptor: BackendWorkflowDescriptor, calls: Set[CommandCallNode], jobExecutionMap: JobExecutionMap, @@ -42,7 +41,7 @@ case class DefaultStandardFinalizationActorParams * @param standardParams Standard parameters. */ class StandardFinalizationActor(val standardParams: StandardFinalizationActorParams) - extends BackendWorkflowFinalizationActor { + extends BackendWorkflowFinalizationActor { override lazy val workflowDescriptor: BackendWorkflowDescriptor = standardParams.workflowDescriptor override lazy val calls: Set[CommandCallNode] = standardParams.calls @@ -59,7 +58,7 @@ class StandardFinalizationActor(val standardParams: StandardFinalizationActorPar override def afterAll(): Future[Unit] = copyCallLogs() - lazy val logPaths: Seq[Path] = { + lazy val logPaths: Seq[Path] = for { actualWorkflowPath <- workflowPaths.toSeq (backendWorkflowDescriptor, keys) <- jobExecutionMap.toSeq @@ -67,9 +66,8 @@ class StandardFinalizationActor(val standardParams: StandardFinalizationActorPar jobPaths = actualWorkflowPath.toJobPaths(key, backendWorkflowDescriptor) logPath <- jobPaths.logPaths.values } yield logPath - } - protected def copyCallLogs(): Future[Unit] = { + protected def copyCallLogs(): Future[Unit] = /* NOTE: Only using one thread pool slot here to upload all the files for all the calls. Using the io-dispatcher defined in application.conf because this might take a while. @@ -77,21 +75,18 @@ class StandardFinalizationActor(val standardParams: StandardFinalizationActorPar pool for parallel uploads. Measure and optimize as necessary. Will likely need retry code at some level as well. - */ + */ workflowPaths match { case Some(paths) => Future(paths.finalCallLogsPath foreach copyCallLogs)(ioExecutionContext) case _ => Future.successful(()) } - } - private def copyCallLogs(callLogsPath: Path): Unit = { + private def copyCallLogs(callLogsPath: Path): Unit = copyLogs(callLogsPath, logPaths) - } - private def copyLogs(callLogsDirPath: Path, logPaths: Seq[Path]): Unit = { + private def copyLogs(callLogsDirPath: Path, logPaths: Seq[Path]): Unit = workflowPaths match { case Some(paths) => logPaths.foreach(PathCopier.copy(paths.executionRoot, _, callLogsDirPath)) case None => } - } } diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala index 95e898d6711..c4adba3cd59 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala @@ -4,7 +4,12 @@ import akka.actor.ActorRef import cromwell.backend.io.WorkflowPaths import cromwell.backend.validation.RuntimeAttributesDefault import cromwell.backend.wfs.WorkflowPathBuilder -import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor} +import cromwell.backend.{ + BackendConfigurationDescriptor, + BackendInitializationData, + BackendWorkflowDescriptor, + BackendWorkflowInitializationActor +} import cromwell.core.WorkflowOptions import cromwell.core.path.PathBuilder import wom.expression.WomExpression @@ -24,8 +29,7 @@ trait StandardInitializationActorParams { def configurationDescriptor: BackendConfigurationDescriptor } -case class DefaultInitializationActorParams -( +case class DefaultInitializationActorParams( workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[CommandCallNode], @@ -42,7 +46,7 @@ case class DefaultInitializationActorParams * @param standardParams Standard parameters */ class StandardInitializationActor(val standardParams: StandardInitializationActorParams) - extends BackendWorkflowInitializationActor { + extends BackendWorkflowInitializationActor { implicit protected val system = context.system @@ -50,16 +54,18 @@ class StandardInitializationActor(val standardParams: StandardInitializationActo override lazy val calls: Set[CommandCallNode] = standardParams.calls - override def beforeAll(): Future[Option[BackendInitializationData]] = { + override def beforeAll(): Future[Option[BackendInitializationData]] = initializationData map Option.apply - } lazy val initializationData: Future[StandardInitializationData] = - workflowPaths map { new StandardInitializationData(_, runtimeAttributesBuilder, classOf[StandardExpressionFunctions]) } + workflowPaths map { + new StandardInitializationData(_, runtimeAttributesBuilder, classOf[StandardExpressionFunctions]) + } lazy val expressionFunctions: Class[_ <: StandardExpressionFunctions] = classOf[StandardExpressionFunctions] - lazy val pathBuilders: Future[List[PathBuilder]] = standardParams.configurationDescriptor.pathBuilders(workflowDescriptor.workflowOptions) + lazy val pathBuilders: Future[List[PathBuilder]] = + standardParams.configurationDescriptor.pathBuilders(workflowDescriptor.workflowOptions) lazy val workflowPaths: Future[WorkflowPaths] = pathBuilders map { WorkflowPathBuilder.workflowPaths(configurationDescriptor, workflowDescriptor, _) } @@ -74,13 +80,11 @@ class StandardInitializationActor(val standardParams: StandardInitializationActo def runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder = StandardValidatedRuntimeAttributesBuilder.default(configurationDescriptor.backendRuntimeAttributesConfig) - override protected lazy val runtimeAttributeValidators: Map[String, (Option[WomExpression]) => Boolean] = { + override protected lazy val runtimeAttributeValidators: Map[String, (Option[WomExpression]) => Boolean] = runtimeAttributesBuilder.validatorMap - } - override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WomValue]] = { + override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WomValue]] = RuntimeAttributesDefault.workflowOptionsDefault(options, runtimeAttributesBuilder.coercionMap) - } def validateWorkflowOptions(): Try[Unit] = Success(()) @@ -93,19 +97,19 @@ class StandardInitializationActor(val standardParams: StandardInitializationActo val notSupportedAttrString = notSupportedAttributes mkString ", " workflowLogger.warn( s"Key/s [$notSupportedAttrString] is/are not supported by backend. " + - s"Unsupported attributes will not be part of job executions.") + s"Unsupported attributes will not be part of job executions." + ) } } } - override def validate(): Future[Unit] = { + override def validate(): Future[Unit] = Future.fromTry( for { _ <- validateWorkflowOptions() _ <- checkForUnsupportedRuntimeAttributes() } yield () ) - } override protected lazy val workflowDescriptor: BackendWorkflowDescriptor = standardParams.workflowDescriptor diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala index e2618818e50..d317af2ada2 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala @@ -6,8 +6,7 @@ import cromwell.backend.io.{JobPaths, WorkflowPaths} import scala.concurrent.ExecutionContext -class StandardInitializationData -( +class StandardInitializationData( val workflowPaths: WorkflowPaths, val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder, val standardExpressionFunctionsClass: Class[_ <: StandardExpressionFunctions] @@ -17,7 +16,10 @@ class StandardInitializationData private lazy val standardExpressionFunctionsConstructor = standardExpressionFunctionsClass.getConstructor(classOf[StandardExpressionFunctionsParams]) - def expressionFunctions(jobPaths: JobPaths, ioActorProxy: ActorRef, ec: ExecutionContext): StandardExpressionFunctions = { + def expressionFunctions(jobPaths: JobPaths, + ioActorProxy: ActorRef, + ec: ExecutionContext + ): StandardExpressionFunctions = { val pathBuilders = jobPaths.workflowPaths.pathBuilders val standardParams = DefaultStandardExpressionFunctionsParams(pathBuilders, jobPaths.callContext, ioActorProxy, ec) standardExpressionFunctionsConstructor.newInstance(standardParams) diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala b/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala index ac670617bb9..a595f589647 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala @@ -1,12 +1,18 @@ package cromwell.backend.standard import akka.actor.ActorRef -import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, MinimumRuntimeSettings} +import cromwell.backend.{ + BackendConfigurationDescriptor, + BackendInitializationData, + BackendJobDescriptor, + MinimumRuntimeSettings +} /** * Base trait for params passed to both the sync and async backend actors. */ trait StandardJobExecutionActorParams { + /** The service registry actor for key/value and metadata. */ def serviceRegistryActor: ActorRef diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala index 64f79f5ad7d..aec03977ad9 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala @@ -16,6 +16,7 @@ import scala.concurrent.ExecutionContext * May be extended for using the standard sync/async backend pattern. */ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory { + /** * Config values for the backend, and a pointer to the global config. * @@ -59,41 +60,65 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory { * * @return the cache hit copying class. */ - lazy val cacheHitCopyingActorClassOption: Option[Class[_ <: StandardCacheHitCopyingActor]] = Option(classOf[DefaultStandardCacheHitCopyingActor]) + lazy val cacheHitCopyingActorClassOption: Option[Class[_ <: StandardCacheHitCopyingActor]] = Option( + classOf[DefaultStandardCacheHitCopyingActor] + ) /** * Returns the cache hit copying class. * * @return the cache hit copying class. */ - lazy val fileHashingActorClassOption: Option[Class[_ <: StandardFileHashingActor]] = Option(classOf[DefaultStandardFileHashingActor]) + lazy val fileHashingActorClassOption: Option[Class[_ <: StandardFileHashingActor]] = Option( + classOf[DefaultStandardFileHashingActor] + ) /** * Returns the finalization class. * * @return the finalization class. */ - lazy val finalizationActorClassOption: Option[Class[_ <: StandardFinalizationActor]] = Option(classOf[StandardFinalizationActor]) + lazy val finalizationActorClassOption: Option[Class[_ <: StandardFinalizationActor]] = Option( + classOf[StandardFinalizationActor] + ) - override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[CommandCallNode], - serviceRegistryActor: ActorRef, restart: Boolean): Option[Props] = { + override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, + ioActor: ActorRef, + calls: Set[CommandCallNode], + serviceRegistryActor: ActorRef, + restart: Boolean + ): Option[Props] = { val params = workflowInitializationActorParams(workflowDescriptor, ioActor, calls, serviceRegistryActor, restart) val props = Props(initializationActorClass, params).withDispatcher(Dispatcher.BackendDispatcher) Option(props) } - def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[CommandCallNode], - serviceRegistryActor: ActorRef, restarting: Boolean): StandardInitializationActorParams = { - DefaultInitializationActorParams(workflowDescriptor, ioActor, calls, serviceRegistryActor, configurationDescriptor, restarting) - } + def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, + ioActor: ActorRef, + calls: Set[CommandCallNode], + serviceRegistryActor: ActorRef, + restarting: Boolean + ): StandardInitializationActorParams = + DefaultInitializationActorParams(workflowDescriptor, + ioActor, + calls, + serviceRegistryActor, + configurationDescriptor, + restarting + ) override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationDataOption: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef, - backendSingletonActorOption: Option[ActorRef]): Props = { - val params = jobExecutionActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor, - ioActor, backendSingletonActorOption) + backendSingletonActorOption: Option[ActorRef] + ): Props = { + val params = jobExecutionActorParams(jobDescriptor, + initializationDataOption, + serviceRegistryActor, + ioActor, + backendSingletonActorOption + ) Props(new StandardSyncExecutionActor(params)).withDispatcher(Dispatcher.BackendDispatcher) } @@ -101,25 +126,35 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory { initializationDataOption: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef, - backendSingletonActorOption: Option[ActorRef]): StandardSyncExecutionActorParams = { - DefaultStandardSyncExecutionActorParams(jobIdKey, serviceRegistryActor, ioActor, jobDescriptor, configurationDescriptor, - initializationDataOption, backendSingletonActorOption, asyncExecutionActorClass, MinimumRuntimeSettings()) - } + backendSingletonActorOption: Option[ActorRef] + ): StandardSyncExecutionActorParams = + DefaultStandardSyncExecutionActorParams( + jobIdKey, + serviceRegistryActor, + ioActor, + jobDescriptor, + configurationDescriptor, + initializationDataOption, + backendSingletonActorOption, + asyncExecutionActorClass, + MinimumRuntimeSettings() + ) - override def fileHashingActorProps: - Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Option[ActorRef]) => Props] = { - fileHashingActorClassOption map { - standardFileHashingActor => fileHashingActorInner(standardFileHashingActor) _ + override def fileHashingActorProps + : Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Option[ActorRef]) => Props] = + fileHashingActorClassOption map { standardFileHashingActor => + fileHashingActorInner(standardFileHashingActor) _ } - } - - def fileHashingActorInner(standardFileHashingActor: Class[_ <: StandardFileHashingActor]) - (jobDescriptor: BackendJobDescriptor, - initializationDataOption: Option[BackendInitializationData], - serviceRegistryActor: ActorRef, - ioActor: ActorRef, - fileHashCacheActor: Option[ActorRef]): Props = { - val params = fileHashingActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor, fileHashCacheActor) + + def fileHashingActorInner(standardFileHashingActor: Class[_ <: StandardFileHashingActor])( + jobDescriptor: BackendJobDescriptor, + initializationDataOption: Option[BackendInitializationData], + serviceRegistryActor: ActorRef, + ioActor: ActorRef, + fileHashCacheActor: Option[ActorRef] + ): Props = { + val params = + fileHashingActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor, fileHashCacheActor) Props(standardFileHashingActor, params).withDispatcher(BackendDispatcher) } @@ -127,26 +162,38 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory { initializationDataOption: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef, - fileHashCacheActor: Option[ActorRef]): StandardFileHashingActorParams = { - DefaultStandardFileHashingActorParams( - jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor, configurationDescriptor, fileHashCacheActor) - } + fileHashCacheActor: Option[ActorRef] + ): StandardFileHashingActorParams = + DefaultStandardFileHashingActorParams(jobDescriptor, + initializationDataOption, + serviceRegistryActor, + ioActor, + configurationDescriptor, + fileHashCacheActor + ) - override def cacheHitCopyingActorProps: - Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Int, Option[BlacklistCache]) => Props] = { - cacheHitCopyingActorClassOption map { - standardCacheHitCopyingActor => cacheHitCopyingActorInner(standardCacheHitCopyingActor) _ + override def cacheHitCopyingActorProps: Option[ + (BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Int, Option[BlacklistCache]) => Props + ] = + cacheHitCopyingActorClassOption map { standardCacheHitCopyingActor => + cacheHitCopyingActorInner(standardCacheHitCopyingActor) _ } - } - def cacheHitCopyingActorInner(standardCacheHitCopyingActor: Class[_ <: StandardCacheHitCopyingActor]) - (jobDescriptor: BackendJobDescriptor, - initializationDataOption: Option[BackendInitializationData], - serviceRegistryActor: ActorRef, - ioActor: ActorRef, - cacheCopyAttempt: Int, - blacklistCache: Option[BlacklistCache]): Props = { - val params = cacheHitCopyingActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor, cacheCopyAttempt, blacklistCache) + def cacheHitCopyingActorInner(standardCacheHitCopyingActor: Class[_ <: StandardCacheHitCopyingActor])( + jobDescriptor: BackendJobDescriptor, + initializationDataOption: Option[BackendInitializationData], + serviceRegistryActor: ActorRef, + ioActor: ActorRef, + cacheCopyAttempt: Int, + blacklistCache: Option[BlacklistCache] + ): Props = { + val params = cacheHitCopyingActorParams(jobDescriptor, + initializationDataOption, + serviceRegistryActor, + ioActor, + cacheCopyAttempt, + blacklistCache + ) Props(standardCacheHitCopyingActor, params).withDispatcher(BackendDispatcher) } @@ -155,71 +202,94 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory { serviceRegistryActor: ActorRef, ioActor: ActorRef, cacheCopyAttempt: Int, - blacklistCache: Option[BlacklistCache]): StandardCacheHitCopyingActorParams = { - DefaultStandardCacheHitCopyingActorParams( - jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor, configurationDescriptor, cacheCopyAttempt, blacklistCache) - } + blacklistCache: Option[BlacklistCache] + ): StandardCacheHitCopyingActorParams = + DefaultStandardCacheHitCopyingActorParams(jobDescriptor, + initializationDataOption, + serviceRegistryActor, + ioActor, + configurationDescriptor, + cacheCopyAttempt, + blacklistCache + ) - override def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[CommandCallNode], - jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, - initializationData: Option[BackendInitializationData]): Option[Props] = { + override def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, + ioActor: ActorRef, + calls: Set[CommandCallNode], + jobExecutionMap: JobExecutionMap, + workflowOutputs: CallOutputs, + initializationData: Option[BackendInitializationData] + ): Option[Props] = finalizationActorClassOption map { finalizationActorClass => - val params = workflowFinalizationActorParams(workflowDescriptor, ioActor, calls, jobExecutionMap, workflowOutputs, - initializationData) + val params = workflowFinalizationActorParams(workflowDescriptor, + ioActor, + calls, + jobExecutionMap, + workflowOutputs, + initializationData + ) Props(finalizationActorClass, params).withDispatcher(BackendDispatcher) } - } - def workflowFinalizationActorParams(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[CommandCallNode], - jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, - initializationDataOption: Option[BackendInitializationData]): - StandardFinalizationActorParams = { - DefaultStandardFinalizationActorParams(workflowDescriptor, calls, jobExecutionMap, workflowOutputs, - initializationDataOption, configurationDescriptor) - } + def workflowFinalizationActorParams(workflowDescriptor: BackendWorkflowDescriptor, + ioActor: ActorRef, + calls: Set[CommandCallNode], + jobExecutionMap: JobExecutionMap, + workflowOutputs: CallOutputs, + initializationDataOption: Option[BackendInitializationData] + ): StandardFinalizationActorParams = + DefaultStandardFinalizationActorParams(workflowDescriptor, + calls, + jobExecutionMap, + workflowOutputs, + initializationDataOption, + configurationDescriptor + ) override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor, jobKey: BackendJobDescriptorKey, initializationDataOption: Option[BackendInitializationData], ioActorProxy: ActorRef, - ec: ExecutionContext): - IoFunctionSet = { + ec: ExecutionContext + ): IoFunctionSet = { val standardInitializationData = BackendInitializationData.as[StandardInitializationData](initializationDataOption) val jobPaths = standardInitializationData.workflowPaths.toJobPaths(jobKey, workflowDescriptor) standardInitializationData.expressionFunctions(jobPaths, ioActorProxy, ec) } - + override def pathBuilders(initializationDataOption: Option[BackendInitializationData]) = { val standardInitializationData = BackendInitializationData.as[StandardInitializationData](initializationDataOption) standardInitializationData.workflowPaths.pathBuilders - } + } - override def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, - initializationData: Option[BackendInitializationData]): Path = { + override def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, + backendConfig: Config, + initializationData: Option[BackendInitializationData] + ): Path = initializationData match { case Some(data) => data.asInstanceOf[StandardInitializationData].workflowPaths.executionRoot case None => super.getExecutionRootPath(workflowDescriptor, backendConfig, initializationData) } - } - override def getWorkflowExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, - initializationData: Option[BackendInitializationData]): Path = { + override def getWorkflowExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, + backendConfig: Config, + initializationData: Option[BackendInitializationData] + ): Path = initializationData match { case Some(data) => data.asInstanceOf[StandardInitializationData].workflowPaths.workflowRoot case None => super.getWorkflowExecutionRootPath(workflowDescriptor, backendConfig, initializationData) } - } - override def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]): Set[RuntimeAttributeDefinition] = { - val initializationData = BackendInitializationData. - as[StandardInitializationData](initializationDataOption) + override def runtimeAttributeDefinitions( + initializationDataOption: Option[BackendInitializationData] + ): Set[RuntimeAttributeDefinition] = { + val initializationData = BackendInitializationData.as[StandardInitializationData](initializationDataOption) initializationData.runtimeAttributesBuilder.definitions.toSet } override def dockerHashCredentials(workflowDescriptor: BackendWorkflowDescriptor, - initializationDataOption: Option[BackendInitializationData], - ): List[Any] = { + initializationDataOption: Option[BackendInitializationData] + ): List[Any] = BackendDockerConfiguration.build(configurationDescriptor.backendConfig).dockerCredentials.toList - } } diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala index 99e81fbf8b2..794602887e4 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala @@ -13,12 +13,12 @@ import scala.concurrent.{Future, Promise} import scala.util.control.NoStackTrace trait StandardSyncExecutionActorParams extends StandardJobExecutionActorParams { + /** The class for creating an async backend. */ def asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor] } -case class DefaultStandardSyncExecutionActorParams -( +case class DefaultStandardSyncExecutionActorParams( override val jobIdKey: String, override val serviceRegistryActor: ActorRef, override val ioActor: ActorRef, @@ -52,7 +52,7 @@ case class DefaultStandardSyncExecutionActorParams * - Asynchronous actor completes the promise with a success or failure. */ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorParams) - extends BackendJobExecutionActor { + extends BackendJobExecutionActor { override val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor override val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor @@ -61,10 +61,9 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP context.become(startup orElse receive) - private def startup: Receive = { - case AbortJobCommand => - context.parent ! JobAbortedResponse(jobDescriptor.key) - context.stop(self) + private def startup: Receive = { case AbortJobCommand => + context.parent ! JobAbortedResponse(jobDescriptor.key) + context.stop(self) } private def running(executor: ActorRef): Receive = { @@ -78,7 +77,7 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP completionPromise.tryFailure(e) throw new RuntimeException(s"Failure attempting to look up job id for key ${jobDescriptor.key}", e) } - + private def recovering(executor: ActorRef): Receive = running(executor).orElse { case KvPair(key, jobId) if key.key == jobIdKey => // Successful operation ID lookup. @@ -131,20 +130,17 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP serviceRegistryActor ! kvGet completionPromise.future } - - override def recover: Future[BackendJobExecutionResponse] = { + + override def recover: Future[BackendJobExecutionResponse] = onRestart(recovering) - } - - override def reconnectToAborting: Future[BackendJobExecutionResponse] = { + + override def reconnectToAborting: Future[BackendJobExecutionResponse] = onRestart(reconnectingToAbort) - } - override def reconnect: Future[BackendJobExecutionResponse] = { + override def reconnect: Future[BackendJobExecutionResponse] = onRestart(reconnecting) - } - def createAsyncParams(): StandardAsyncExecutionActorParams = { + def createAsyncParams(): StandardAsyncExecutionActorParams = DefaultStandardAsyncExecutionActorParams( standardParams.jobIdKey, standardParams.serviceRegistryActor, @@ -156,16 +152,14 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP completionPromise, standardParams.minimumRuntimeSettings ) - } def createAsyncProps(): Props = { val asyncParams = createAsyncParams() Props(standardParams.asyncJobExecutionActorClass, asyncParams) } - def createAsyncRefName(): String = { + def createAsyncRefName(): String = standardParams.asyncJobExecutionActorClass.getSimpleName - } def createAsyncRef(): ActorRef = { val props = createAsyncProps().withDispatcher(Dispatcher.BackendDispatcher) @@ -173,16 +167,18 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP context.actorOf(props, name) } - override def abort(): Unit = { + override def abort(): Unit = throw new UnsupportedOperationException("Abort is implemented via a custom receive of the message AbortJobCommand.") - } // Supervision strategy: if the async actor throws an exception, stop the actor and fail the job. - def jobFailingDecider: Decider = { - case exception: Exception => - completionPromise.tryFailure( - new RuntimeException(s"${createAsyncRefName()} failed and didn't catch its exception. This condition has been handled and the job will be marked as failed.", exception) with NoStackTrace) - Stop + def jobFailingDecider: Decider = { case exception: Exception => + completionPromise.tryFailure( + new RuntimeException( + s"${createAsyncRefName()} failed and didn't catch its exception. This condition has been handled and the job will be marked as failed.", + exception + ) with NoStackTrace + ) + Stop } override val supervisorStrategy: OneForOneStrategy = OneForOneStrategy()(jobFailingDecider) diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala index 4160d9a8522..77f890b4a19 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala @@ -17,8 +17,7 @@ import cromwell.backend.validation._ */ object StandardValidatedRuntimeAttributesBuilder { - private case class StandardValidatedRuntimeAttributesBuilderImpl - ( + private case class StandardValidatedRuntimeAttributesBuilderImpl( override val requiredValidations: Seq[RuntimeAttributesValidation[_]], override val customValidations: Seq[RuntimeAttributesValidation[_]] ) extends StandardValidatedRuntimeAttributesBuilder @@ -41,8 +40,8 @@ object StandardValidatedRuntimeAttributesBuilder { } private def withValidations(builder: StandardValidatedRuntimeAttributesBuilder, - customValidations: Seq[RuntimeAttributesValidation[_]]): - StandardValidatedRuntimeAttributesBuilder = { + customValidations: Seq[RuntimeAttributesValidation[_]] + ): StandardValidatedRuntimeAttributesBuilder = { val required = builder.requiredValidations val custom = builder.customValidations ++ customValidations StandardValidatedRuntimeAttributesBuilderImpl(custom, required) @@ -50,19 +49,18 @@ object StandardValidatedRuntimeAttributesBuilder { } sealed trait StandardValidatedRuntimeAttributesBuilder extends ValidatedRuntimeAttributesBuilder { + /** * Returns a new builder with the additional validation(s). * * @param validation Additional validation. * @return New builder with the validation. */ - final def withValidation(validation: RuntimeAttributesValidation[_]*): - StandardValidatedRuntimeAttributesBuilder = { + final def withValidation(validation: RuntimeAttributesValidation[_]*): StandardValidatedRuntimeAttributesBuilder = StandardValidatedRuntimeAttributesBuilder.withValidations(this, validation) - } /** Returns all the validations, those required for the standard backend, plus custom addons for the subclass. */ - override final lazy val validations: Seq[RuntimeAttributesValidation[_]] = requiredValidations ++ customValidations + final override lazy val validations: Seq[RuntimeAttributesValidation[_]] = requiredValidations ++ customValidations private[standard] def requiredValidations: Seq[RuntimeAttributesValidation[_]] diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/BlacklistCache.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/BlacklistCache.scala index ff3248123ff..fa5207c99c6 100644 --- a/backend/src/main/scala/cromwell/backend/standard/callcaching/BlacklistCache.scala +++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/BlacklistCache.scala @@ -11,19 +11,20 @@ case object UntestedCacheResult extends BlacklistStatus sealed abstract class BlacklistCache(bucketCacheConfig: CacheConfig, hitCacheConfig: CacheConfig, - val name: Option[String]) { + val name: Option[String] +) { val bucketCache = { // Queries to the bucket blacklist cache return UntestedCacheResult by default. val unknownLoader = new CacheLoader[String, BlacklistStatus]() { override def load(key: String): BlacklistStatus = UntestedCacheResult } - CacheBuilder. - newBuilder(). - concurrencyLevel(bucketCacheConfig.concurrency). - maximumSize(bucketCacheConfig.size). - expireAfterWrite(bucketCacheConfig.ttl.length, bucketCacheConfig.ttl.unit). - build[String, BlacklistStatus](unknownLoader) + CacheBuilder + .newBuilder() + .concurrencyLevel(bucketCacheConfig.concurrency) + .maximumSize(bucketCacheConfig.size) + .expireAfterWrite(bucketCacheConfig.ttl.length, bucketCacheConfig.ttl.unit) + .build[String, BlacklistStatus](unknownLoader) } val hitCache = { @@ -32,12 +33,12 @@ sealed abstract class BlacklistCache(bucketCacheConfig: CacheConfig, override def load(key: CallCachingEntryId): BlacklistStatus = UntestedCacheResult } - CacheBuilder. - newBuilder(). - concurrencyLevel(hitCacheConfig.concurrency). - maximumSize(hitCacheConfig.size). - expireAfterWrite(hitCacheConfig.ttl.length, hitCacheConfig.ttl.unit). - build[CallCachingEntryId, BlacklistStatus](unknownLoader) + CacheBuilder + .newBuilder() + .concurrencyLevel(hitCacheConfig.concurrency) + .maximumSize(hitCacheConfig.size) + .expireAfterWrite(hitCacheConfig.ttl.length, hitCacheConfig.ttl.unit) + .build[CallCachingEntryId, BlacklistStatus](unknownLoader) } def getBlacklistStatus(hit: CallCachingEntryId): BlacklistStatus = hitCache.get(hit) @@ -53,8 +54,8 @@ sealed abstract class BlacklistCache(bucketCacheConfig: CacheConfig, def whitelist(bucket: String): Unit = bucketCache.put(bucket, GoodCacheResult) } -class RootWorkflowBlacklistCache(bucketCacheConfig: CacheConfig, hitCacheConfig: CacheConfig) extends - BlacklistCache(bucketCacheConfig = bucketCacheConfig, hitCacheConfig = hitCacheConfig, name = None) +class RootWorkflowBlacklistCache(bucketCacheConfig: CacheConfig, hitCacheConfig: CacheConfig) + extends BlacklistCache(bucketCacheConfig = bucketCacheConfig, hitCacheConfig = hitCacheConfig, name = None) -class GroupingBlacklistCache(bucketCacheConfig: CacheConfig, hitCacheConfig: CacheConfig, val group: String) extends - BlacklistCache(bucketCacheConfig = bucketCacheConfig, hitCacheConfig = hitCacheConfig, name = Option(group)) +class GroupingBlacklistCache(bucketCacheConfig: CacheConfig, hitCacheConfig: CacheConfig, val group: String) + extends BlacklistCache(bucketCacheConfig = bucketCacheConfig, hitCacheConfig = hitCacheConfig, name = Option(group)) diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManager.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManager.scala index a22631aeeb4..f1c6a988021 100644 --- a/backend/src/main/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManager.scala +++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManager.scala @@ -47,7 +47,7 @@ class CallCachingBlacklistManager(rootConfig: Config, logger: LoggingAdapter) { import CallCachingBlacklistManager.Defaults.Groupings._ for { _ <- blacklistGroupingWorkflowOptionKey - groupingsOption = rootConfig.as[Option[Config]] ("call-caching.blacklist-cache.groupings") + groupingsOption = rootConfig.as[Option[Config]]("call-caching.blacklist-cache.groupings") conf = CacheConfig.config(groupingsOption, defaultConcurrency = Concurrency, defaultSize = Size, defaultTtl = Ttl) } yield conf } @@ -74,7 +74,10 @@ class CallCachingBlacklistManager(rootConfig: Config, logger: LoggingAdapter) { // If configuration allows, build a cache of blacklist groupings to BlacklistCaches. private val blacklistGroupingsCache: Option[LoadingCache[String, BlacklistCache]] = { - def buildBlacklistGroupingsCache(groupingConfig: CacheConfig, bucketConfig: CacheConfig, hitConfig: CacheConfig): LoadingCache[String, BlacklistCache] = { + def buildBlacklistGroupingsCache(groupingConfig: CacheConfig, + bucketConfig: CacheConfig, + hitConfig: CacheConfig + ): LoadingCache[String, BlacklistCache] = { val emptyBlacklistCacheLoader = new CacheLoader[String, BlacklistCache]() { override def load(key: String): BlacklistCache = new GroupingBlacklistCache( bucketCacheConfig = bucketConfig, @@ -83,12 +86,12 @@ class CallCachingBlacklistManager(rootConfig: Config, logger: LoggingAdapter) { ) } - CacheBuilder. - newBuilder(). - concurrencyLevel(groupingConfig.concurrency). - maximumSize(groupingConfig.size). - expireAfterWrite(groupingConfig.ttl.length, groupingConfig.ttl.unit). - build[String, BlacklistCache](emptyBlacklistCacheLoader) + CacheBuilder + .newBuilder() + .concurrencyLevel(groupingConfig.concurrency) + .maximumSize(groupingConfig.size) + .expireAfterWrite(groupingConfig.ttl.length, groupingConfig.ttl.unit) + .build[String, BlacklistCache](emptyBlacklistCacheLoader) } for { @@ -121,8 +124,13 @@ class CallCachingBlacklistManager(rootConfig: Config, logger: LoggingAdapter) { val maybeCache = groupBlacklistCache orElse rootWorkflowBlacklistCache maybeCache collect { case group: GroupingBlacklistCache => - logger.info("Workflow {} using group blacklist cache '{}' containing blacklist status for {} hits and {} buckets.", - workflow.id, group.group, group.hitCache.size(), group.bucketCache.size()) + logger.info( + "Workflow {} using group blacklist cache '{}' containing blacklist status for {} hits and {} buckets.", + workflow.id, + group.group, + group.hitCache.size(), + group.bucketCache.size() + ) case _: RootWorkflowBlacklistCache => logger.info("Workflow {} using root workflow blacklist cache.", workflow.id) } diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/CopyingActorBlacklistCacheSupport.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/CopyingActorBlacklistCacheSupport.scala index 8ad88ae4f49..d90bba7ee90 100644 --- a/backend/src/main/scala/cromwell/backend/standard/callcaching/CopyingActorBlacklistCacheSupport.scala +++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/CopyingActorBlacklistCacheSupport.scala @@ -4,7 +4,6 @@ import cromwell.backend.BackendCacheHitCopyingActor.CopyOutputsCommand import cromwell.core.io.{IoCommand, IoCopyCommand} import cromwell.services.CallCaching.CallCachingEntryId - object CopyingActorBlacklistCacheSupport { trait HasFormatting { def metricFormat: String = getClass.getName.toLowerCase.split('$').last @@ -52,13 +51,12 @@ trait CopyingActorBlacklistCacheSupport { } def publishBlacklistMetric(verb: Verb, entityType: EntityType, value: BlacklistStatus): Unit = { - val metricPath = NonEmptyList.of( - "job", - "callcaching", "blacklist", verb.metricFormat, entityType.metricFormat, value.toString) + val metricPath = + NonEmptyList.of("job", "callcaching", "blacklist", verb.metricFormat, entityType.metricFormat, value.toString) increment(metricPath) } - def blacklistAndMetricHit(blacklistCache: BlacklistCache, hit: CallCachingEntryId): Unit = { + def blacklistAndMetricHit(blacklistCache: BlacklistCache, hit: CallCachingEntryId): Unit = blacklistCache.getBlacklistStatus(hit) match { case UntestedCacheResult => blacklistCache.blacklist(hit) @@ -71,13 +69,13 @@ trait CopyingActorBlacklistCacheSupport { // mark the hit as BadCacheResult and log this strangeness. log.warning( "Cache hit {} found in GoodCacheResult blacklist state, but cache hit copying has failed for permissions reasons. Overwriting status to BadCacheResult state.", - hit.id) + hit.id + ) blacklistCache.blacklist(hit) publishBlacklistMetric(Write, Hit, value = BadCacheResult) } - } - def blacklistAndMetricBucket(blacklistCache: BlacklistCache, bucket: String): Unit = { + def blacklistAndMetricBucket(blacklistCache: BlacklistCache, bucket: String): Unit = blacklistCache.getBlacklistStatus(bucket) match { case UntestedCacheResult => blacklistCache.blacklist(bucket) @@ -90,13 +88,13 @@ trait CopyingActorBlacklistCacheSupport { // mark the bucket as BadCacheResult and log this strangeness. log.warning( "Bucket {} found in GoodCacheResult blacklist state, but cache hit copying has failed for permissions reasons. Overwriting status to BadCacheResult state.", - bucket) + bucket + ) blacklistCache.blacklist(bucket) publishBlacklistMetric(Write, Bucket, value = BadCacheResult) } - } - def whitelistAndMetricHit(blacklistCache: BlacklistCache, hit: CallCachingEntryId): Unit = { + def whitelistAndMetricHit(blacklistCache: BlacklistCache, hit: CallCachingEntryId): Unit = blacklistCache.getBlacklistStatus(hit) match { case UntestedCacheResult => blacklistCache.whitelist(hit) @@ -107,11 +105,11 @@ trait CopyingActorBlacklistCacheSupport { // Don't overwrite this to GoodCacheResult, hopefully there are less weird cache hits out there. log.warning( "Cache hit {} found in BadCacheResult blacklist state, not overwriting to GoodCacheResult despite successful copy.", - hit.id) + hit.id + ) } - } - def whitelistAndMetricBucket(blacklistCache: BlacklistCache, bucket: String): Unit = { + def whitelistAndMetricBucket(blacklistCache: BlacklistCache, bucket: String): Unit = blacklistCache.getBlacklistStatus(bucket) match { case UntestedCacheResult => blacklistCache.whitelist(bucket) @@ -122,11 +120,11 @@ trait CopyingActorBlacklistCacheSupport { // of a successful copy. Don't overwrite this to GoodCacheResult, hopefully there are less weird cache hits out there. log.warning( "Bucket {} found in BadCacheResult blacklist state, not overwriting to GoodCacheResult despite successful copy.", - bucket) + bucket + ) } - } - def publishBlacklistReadMetrics(command: CopyOutputsCommand, cacheHit: CallCachingEntryId, cacheReadType: Product) = { + def publishBlacklistReadMetrics(command: CopyOutputsCommand, cacheHit: CallCachingEntryId, cacheReadType: Product) = for { c <- standardParams.blacklistCache hitBlacklistStatus = c.getBlacklistStatus(cacheHit) @@ -139,7 +137,6 @@ trait CopyingActorBlacklistCacheSupport { bucketBlacklistStatus = c.getBlacklistStatus(prefix) _ = publishBlacklistMetric(Read, Bucket, bucketBlacklistStatus) } yield () - } def isSourceBlacklisted(command: CopyOutputsCommand): Boolean = { val path = sourcePathFromCopyOutputsCommand(command) @@ -150,10 +147,9 @@ trait CopyingActorBlacklistCacheSupport { } yield value == BadCacheResult).getOrElse(false) } - def isSourceBlacklisted(hit: CallCachingEntryId): Boolean = { + def isSourceBlacklisted(hit: CallCachingEntryId): Boolean = (for { cache <- standardParams.blacklistCache value = cache.getBlacklistStatus(hit) } yield value == BadCacheResult).getOrElse(false) - } } diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/RootWorkflowFileHashCacheActor.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/RootWorkflowFileHashCacheActor.scala index c3d1452660e..3ab77bdf778 100644 --- a/backend/src/main/scala/cromwell/backend/standard/callcaching/RootWorkflowFileHashCacheActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/RootWorkflowFileHashCacheActor.scala @@ -11,8 +11,10 @@ import cromwell.core.WorkflowId import cromwell.core.actor.RobustClientHelper.RequestTimeout import cromwell.core.io._ - -class RootWorkflowFileHashCacheActor private[callcaching](override val ioActor: ActorRef, workflowId: WorkflowId) extends Actor with ActorLogging with IoClientHelper { +class RootWorkflowFileHashCacheActor private[callcaching] (override val ioActor: ActorRef, workflowId: WorkflowId) + extends Actor + with ActorLogging + with IoClientHelper { case class FileHashRequester(replyTo: ActorRef, fileHashContext: FileHashContext, ioCommand: IoCommand[_]) sealed trait FileHashValue @@ -25,8 +27,9 @@ class RootWorkflowFileHashCacheActor private[callcaching](override val ioActor: // Hashing failed. case class FileHashFailure(error: String) extends FileHashValue - val cache: LoadingCache[String, FileHashValue] = CacheBuilder.newBuilder().build( - new CacheLoader[String, FileHashValue] { + val cache: LoadingCache[String, FileHashValue] = CacheBuilder + .newBuilder() + .build(new CacheLoader[String, FileHashValue] { override def load(key: String): FileHashValue = FileHashValueNotRequested }) @@ -66,44 +69,49 @@ class RootWorkflowFileHashCacheActor private[callcaching](override val ioActor: requesters foreach { case FileHashRequester(replyTo, fileHashContext, ioCommand) => replyTo ! Tuple2(fileHashContext, IoFailure(ioCommand, failure.failure)) } - cache.put(hashContext.file, FileHashFailure(s"Error hashing file '${hashContext.file}': ${failure.failure.getMessage}")) + cache.put(hashContext.file, + FileHashFailure(s"Error hashing file '${hashContext.file}': ${failure.failure.getMessage}") + ) } case other => log.warning(s"Root workflow file hash caching actor received unexpected message: $other") } // Invoke the supplied block on the happy path, handle unexpected states for IoSuccess and IoFailure with common code. - private def handleHashResult(ioAck: IoAck[_], fileHashContext: FileHashContext) - (notifyRequestersAndCacheValue: List[FileHashRequester] => Unit): Unit = { + private def handleHashResult(ioAck: IoAck[_], fileHashContext: FileHashContext)( + notifyRequestersAndCacheValue: List[FileHashRequester] => Unit + ): Unit = cache.get(fileHashContext.file) match { case FileHashValueRequested(requesters) => notifyRequestersAndCacheValue(requesters.toList) case FileHashValueNotRequested => log.info(msgIoAckWithNoRequesters.format(fileHashContext.file)) notifyRequestersAndCacheValue(List.empty[FileHashRequester]) case _ => - // IoAck arrived when hash result is already saved in cache. This is a result of benign race condition. - // No further action is required. + // IoAck arrived when hash result is already saved in cache. This is a result of benign race condition. + // No further action is required. } - } - override protected def onTimeout(message: Any, to: ActorRef): Unit = { + override protected def onTimeout(message: Any, to: ActorRef): Unit = message match { case (fileHashContext: FileHashContext, _) => // Send this message to all requestors. cache.get(fileHashContext.file) match { case FileHashValueRequested(requesters) => - requesters.toList foreach { case FileHashRequester(replyTo, requestContext, ioCommand) => replyTo ! RequestTimeout(Tuple2(requestContext, ioCommand), replyTo) } + requesters.toList foreach { case FileHashRequester(replyTo, requestContext, ioCommand) => + replyTo ! RequestTimeout(Tuple2(requestContext, ioCommand), replyTo) + } // Allow for the possibility of trying again on a timeout. cache.put(fileHashContext.file, FileHashValueNotRequested) case FileHashValueNotRequested => - // Due to race condition, timeout came after the actual response. This is fine and no further action required. + // Due to race condition, timeout came after the actual response. This is fine and no further action required. case v => log.info(msgTimeoutAfterIoAck.format(v, fileHashContext.file)) } case other => - log.error(s"Programmer error! Root workflow file hash caching actor received unexpected timeout message: $other") + log.error( + s"Programmer error! Root workflow file hash caching actor received unexpected timeout message: $other" + ) } - } override def preRestart(reason: Throwable, message: Option[Any]): Unit = { log.error(reason, s"RootWorkflowFileHashCacheActor for workflow '$workflowId' is unexpectedly being restarted") @@ -121,5 +129,7 @@ object RootWorkflowFileHashCacheActor { case class IoHashCommandWithContext(ioHashCommand: IoHashCommand, fileHashContext: FileHashContext) - def props(ioActor: ActorRef, workflowId: WorkflowId): Props = Props(new RootWorkflowFileHashCacheActor(ioActor, workflowId)) + def props(ioActor: ActorRef, workflowId: WorkflowId): Props = Props( + new RootWorkflowFileHashCacheActor(ioActor, workflowId) + ) } diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardCacheHitCopyingActor.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardCacheHitCopyingActor.scala index f1662db858b..3c052f69f8e 100644 --- a/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardCacheHitCopyingActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardCacheHitCopyingActor.scala @@ -9,7 +9,12 @@ import cromwell.backend.io.JobPaths import cromwell.backend.standard.StandardCachingActorHelper import cromwell.backend.standard.callcaching.CopyingActorBlacklistCacheSupport._ import cromwell.backend.standard.callcaching.StandardCacheHitCopyingActor._ -import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, MetricableCacheCopyErrorCategory} +import cromwell.backend.{ + BackendConfigurationDescriptor, + BackendInitializationData, + BackendJobDescriptor, + MetricableCacheCopyErrorCategory +} import cromwell.core.CallOutputs import cromwell.core.io._ import cromwell.core.logging.JobLogging @@ -46,8 +51,7 @@ trait StandardCacheHitCopyingActorParams { } /** A default implementation of the cache hit copying params. */ -case class DefaultStandardCacheHitCopyingActorParams -( +case class DefaultStandardCacheHitCopyingActorParams( override val jobDescriptor: BackendJobDescriptor, override val backendInitializationDataOption: Option[BackendInitializationData], override val serviceRegistryActor: ActorRef, @@ -81,32 +85,33 @@ object StandardCacheHitCopyingActor { newDetritus: DetritusMap, cacheHit: CallCachingEntryId, returnCode: Option[Int] - ) { + ) { /** * Removes the command from commandsToWaitFor * returns a pair of the new state data and CommandSetState giving information about what to do next */ - def commandComplete(command: IoCommand[_]): (StandardCacheHitCopyingActorData, CommandSetState) = commandsToWaitFor match { - // If everything was already done send back current data and AllCommandsDone - case Nil => (this, AllCommandsDone) - case lastSubset :: Nil => - val updatedSubset = lastSubset - command - // If the last subset is now empty, we're done - if (updatedSubset.isEmpty) (this.copy(commandsToWaitFor = List.empty), AllCommandsDone) - // otherwise update commandsToWaitFor and keep waiting - else (this.copy(commandsToWaitFor = List(updatedSubset)), StillWaiting) - case currentSubset :: otherSubsets => - val updatedSubset = currentSubset - command - // This subset is done but there are other ones, remove it from commandsToWaitFor and return the next round of commands - if (updatedSubset.isEmpty) (this.copy(commandsToWaitFor = otherSubsets), NextSubSet(otherSubsets.head)) - // otherwise update the head subset and keep waiting - else (this.copy(commandsToWaitFor = List(updatedSubset) ++ otherSubsets), StillWaiting) - } + def commandComplete(command: IoCommand[_]): (StandardCacheHitCopyingActorData, CommandSetState) = + commandsToWaitFor match { + // If everything was already done send back current data and AllCommandsDone + case Nil => (this, AllCommandsDone) + case lastSubset :: Nil => + val updatedSubset = lastSubset - command + // If the last subset is now empty, we're done + if (updatedSubset.isEmpty) (this.copy(commandsToWaitFor = List.empty), AllCommandsDone) + // otherwise update commandsToWaitFor and keep waiting + else (this.copy(commandsToWaitFor = List(updatedSubset)), StillWaiting) + case currentSubset :: otherSubsets => + val updatedSubset = currentSubset - command + // This subset is done but there are other ones, remove it from commandsToWaitFor and return the next round of commands + if (updatedSubset.isEmpty) (this.copy(commandsToWaitFor = otherSubsets), NextSubSet(otherSubsets.head)) + // otherwise update the head subset and keep waiting + else (this.copy(commandsToWaitFor = List(updatedSubset) ++ otherSubsets), StillWaiting) + } } // Internal ADT to keep track of command set states - private[callcaching] sealed trait CommandSetState + sealed private[callcaching] trait CommandSetState private[callcaching] case object StillWaiting extends CommandSetState private[callcaching] case object AllCommandsDone extends CommandSetState private[callcaching] case class NextSubSet(commands: Set[IoCommand[_]]) extends CommandSetState @@ -114,17 +119,23 @@ object StandardCacheHitCopyingActor { private val BucketRegex: Regex = "^gs://([^/]+).*".r } -class DefaultStandardCacheHitCopyingActor(standardParams: StandardCacheHitCopyingActorParams) extends StandardCacheHitCopyingActor(standardParams) +class DefaultStandardCacheHitCopyingActor(standardParams: StandardCacheHitCopyingActorParams) + extends StandardCacheHitCopyingActor(standardParams) /** * Standard implementation of a BackendCacheHitCopyingActor. */ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHitCopyingActorParams) - extends FSM[StandardCacheHitCopyingActorState, Option[StandardCacheHitCopyingActorData]] - with JobLogging with StandardCachingActorHelper with IoClientHelper with CromwellInstrumentationActor with CopyingActorBlacklistCacheSupport { + extends FSM[StandardCacheHitCopyingActorState, Option[StandardCacheHitCopyingActorData]] + with JobLogging + with StandardCachingActorHelper + with IoClientHelper + with CromwellInstrumentationActor + with CopyingActorBlacklistCacheSupport { override lazy val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor - override lazy val backendInitializationDataOption: Option[BackendInitializationData] = standardParams.backendInitializationDataOption + override lazy val backendInitializationDataOption: Option[BackendInitializationData] = + standardParams.backendInitializationDataOption override lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor override lazy val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor protected val commandBuilder: IoCommandBuilder = DefaultIoCommandBuilder @@ -142,78 +153,78 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit /** Override this method if you want to provide an alternative way to duplicate files than copying them. */ protected def duplicate(copyPairs: Set[PathPair]): Option[Try[Unit]] = None - when(Idle) { - case Event(command @ CopyOutputsCommand(simpletons, jobDetritus, cacheHit, returnCode), None) => - val (nextState, cacheReadType) = - if (isSourceBlacklisted(cacheHit)) { - // We don't want to log this because blacklisting is a common and expected occurrence. - (failAndStop(BlacklistSkip(MetricableCacheCopyErrorCategory.HitBlacklisted)), ReadHitOnly) - } else if (isSourceBlacklisted(command)) { - // We don't want to log this because blacklisting is a common and expected occurrence. - (failAndStop(BlacklistSkip(MetricableCacheCopyErrorCategory.BucketBlacklisted)), ReadHitAndBucket) - } else { - // Try to make a Path of the callRootPath from the detritus - val next = lookupSourceCallRootPath(jobDetritus) match { - case Success(sourceCallRootPath) => - - // process simpletons and detritus to get updated paths and corresponding IoCommands - val processed = for { - (destinationCallOutputs, simpletonIoCommands) <- processSimpletons(simpletons, sourceCallRootPath) - (destinationDetritus, detritusIoCommands) <- processDetritus(jobDetritus) - } yield (destinationCallOutputs, destinationDetritus, simpletonIoCommands ++ detritusIoCommands) - - processed match { - case Success((destinationCallOutputs, destinationDetritus, detritusAndOutputsIoCommands)) => - duplicate(ioCommandsToCopyPairs(detritusAndOutputsIoCommands)) match { - // Use the duplicate override if exists - case Some(Success(_)) => succeedAndStop(returnCode, destinationCallOutputs, destinationDetritus) - case Some(Failure(failure)) => - // Something went wrong in the custom duplication code. We consider this loggable because it's most likely a user-permission error: - failAndStop(CopyAttemptError(failure)) - // Otherwise send the first round of IoCommands (file outputs and detritus) if any - case None if detritusAndOutputsIoCommands.nonEmpty => - detritusAndOutputsIoCommands foreach sendIoCommand - - // Add potential additional commands to the list - val additionalCommandsTry = - additionalIoCommands( - sourceCallRootPath = sourceCallRootPath, - originalSimpletons = simpletons, - newOutputs = destinationCallOutputs, - originalDetritus = jobDetritus, - newDetritus = destinationDetritus, - ) - additionalCommandsTry match { - case Success(additionalCommands) => - val allCommands = List(detritusAndOutputsIoCommands) ++ additionalCommands - goto(WaitingForIoResponses) using - Option(StandardCacheHitCopyingActorData( + when(Idle) { case Event(command @ CopyOutputsCommand(simpletons, jobDetritus, cacheHit, returnCode), None) => + val (nextState, cacheReadType) = + if (isSourceBlacklisted(cacheHit)) { + // We don't want to log this because blacklisting is a common and expected occurrence. + (failAndStop(BlacklistSkip(MetricableCacheCopyErrorCategory.HitBlacklisted)), ReadHitOnly) + } else if (isSourceBlacklisted(command)) { + // We don't want to log this because blacklisting is a common and expected occurrence. + (failAndStop(BlacklistSkip(MetricableCacheCopyErrorCategory.BucketBlacklisted)), ReadHitAndBucket) + } else { + // Try to make a Path of the callRootPath from the detritus + val next = lookupSourceCallRootPath(jobDetritus) match { + case Success(sourceCallRootPath) => + // process simpletons and detritus to get updated paths and corresponding IoCommands + val processed = for { + (destinationCallOutputs, simpletonIoCommands) <- processSimpletons(simpletons, sourceCallRootPath) + (destinationDetritus, detritusIoCommands) <- processDetritus(jobDetritus) + } yield (destinationCallOutputs, destinationDetritus, simpletonIoCommands ++ detritusIoCommands) + + processed match { + case Success((destinationCallOutputs, destinationDetritus, detritusAndOutputsIoCommands)) => + duplicate(ioCommandsToCopyPairs(detritusAndOutputsIoCommands)) match { + // Use the duplicate override if exists + case Some(Success(_)) => succeedAndStop(returnCode, destinationCallOutputs, destinationDetritus) + case Some(Failure(failure)) => + // Something went wrong in the custom duplication code. We consider this loggable because it's most likely a user-permission error: + failAndStop(CopyAttemptError(failure)) + // Otherwise send the first round of IoCommands (file outputs and detritus) if any + case None if detritusAndOutputsIoCommands.nonEmpty => + detritusAndOutputsIoCommands foreach sendIoCommand + + // Add potential additional commands to the list + val additionalCommandsTry = + additionalIoCommands( + sourceCallRootPath = sourceCallRootPath, + originalSimpletons = simpletons, + newOutputs = destinationCallOutputs, + originalDetritus = jobDetritus, + newDetritus = destinationDetritus + ) + additionalCommandsTry match { + case Success(additionalCommands) => + val allCommands = List(detritusAndOutputsIoCommands) ++ additionalCommands + goto(WaitingForIoResponses) using + Option( + StandardCacheHitCopyingActorData( commandsToWaitFor = allCommands, newJobOutputs = destinationCallOutputs, newDetritus = destinationDetritus, cacheHit = cacheHit, - returnCode = returnCode, - )) - // Something went wrong in generating duplication commands. - // We consider this a loggable error because we don't expect this to happen: - case Failure(failure) => failAndStop(CopyAttemptError(failure)) - } - case _ => succeedAndStop(returnCode, destinationCallOutputs, destinationDetritus) - } - - // Something went wrong in generating duplication commands. We consider this loggable error because we don't expect this to happen: - case Failure(failure) => failAndStop(CopyAttemptError(failure)) - } - - // Something went wrong in looking up the call root... loggable because we don't expect this to happen: - case Failure(failure) => failAndStop(CopyAttemptError(failure)) - } - (next, ReadHitAndBucket) + returnCode = returnCode + ) + ) + // Something went wrong in generating duplication commands. + // We consider this a loggable error because we don't expect this to happen: + case Failure(failure) => failAndStop(CopyAttemptError(failure)) + } + case _ => succeedAndStop(returnCode, destinationCallOutputs, destinationDetritus) + } + + // Something went wrong in generating duplication commands. We consider this loggable error because we don't expect this to happen: + case Failure(failure) => failAndStop(CopyAttemptError(failure)) + } + + // Something went wrong in looking up the call root... loggable because we don't expect this to happen: + case Failure(failure) => failAndStop(CopyAttemptError(failure)) } + (next, ReadHitAndBucket) + } - publishBlacklistReadMetrics(command, cacheHit, cacheReadType) + publishBlacklistReadMetrics(command, cacheHit, cacheReadType) - nextState + nextState } when(WaitingForIoResponses) { @@ -293,7 +304,7 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit _ = blacklistAndMetricHit(cache, data.cacheHit) prefix <- extractBlacklistPrefix(path) _ = blacklistAndMetricBucket(cache, prefix) - } yield() + } yield () andThen } @@ -309,8 +320,18 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit def succeedAndStop(returnCode: Option[Int], copiedJobOutputs: CallOutputs, detritusMap: DetritusMap): State = { import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter - serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), startMetadataKeyValues) - context.parent ! JobSucceededResponse(jobDescriptor.key, returnCode, copiedJobOutputs, Option(detritusMap), Seq.empty, None, resultGenerationMode = CallCached) + serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, + Option(jobDescriptor.key), + startMetadataKeyValues + ) + context.parent ! JobSucceededResponse(jobDescriptor.key, + returnCode, + copiedJobOutputs, + Option(detritusMap), + Seq.empty, + None, + resultGenerationMode = CallCached + ) context stop self stay() } @@ -323,7 +344,10 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit /** If there are no responses pending this behaves like `failAndStop`, otherwise this goes to `FailedState` and waits * for all the pending responses to come back before stopping. */ - def failAndAwaitPendingResponses(failure: CacheCopyFailure, command: IoCommand[_], data: StandardCacheHitCopyingActorData): State = { + def failAndAwaitPendingResponses(failure: CacheCopyFailure, + command: IoCommand[_], + data: StandardCacheHitCopyingActorData + ): State = { context.parent ! CopyingOutputsFailedResponse(jobDescriptor.key, standardParams.cacheCopyAttempt, failure) val (newData, commandState) = data.commandComplete(command) @@ -344,12 +368,16 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit stay() } - protected def lookupSourceCallRootPath(sourceJobDetritusFiles: Map[String, String]): Try[Path] = { + protected def lookupSourceCallRootPath(sourceJobDetritusFiles: Map[String, String]): Try[Path] = sourceJobDetritusFiles.get(JobPaths.CallRootPathKey) match { case Some(source) => getPath(source) - case None => Failure(new RuntimeException(s"${JobPaths.CallRootPathKey} wasn't found for call ${jobDescriptor.taskCall.fullyQualifiedName}")) + case None => + Failure( + new RuntimeException( + s"${JobPaths.CallRootPathKey} wasn't found for call ${jobDescriptor.taskCall.fullyQualifiedName}" + ) + ) } - } private def ioCommandsToCopyPairs(commands: Set[IoCommand[_]]): Set[PathPair] = commands collect { case copyCommand: IoCopyCommand => copyCommand.source -> copyCommand.destination @@ -358,18 +386,22 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit /** * Returns a pair of the list of simpletons with copied paths, and copy commands necessary to perform those copies. */ - protected def processSimpletons(womValueSimpletons: Seq[WomValueSimpleton], sourceCallRootPath: Path): Try[(CallOutputs, Set[IoCommand[_]])] = Try { - val (destinationSimpletons, ioCommands): (List[WomValueSimpleton], Set[IoCommand[_]]) = womValueSimpletons.toList.foldMap({ - case WomValueSimpleton(key, wdlFile: WomSingleFile) => - val sourcePath = getPath(wdlFile.value).get - val destinationPath = PathCopier.getDestinationFilePath(sourceCallRootPath, sourcePath, destinationCallRootPath) - - val destinationSimpleton = WomValueSimpleton(key, WomSingleFile(destinationPath.pathAsString)) - - // PROD-444: Keep It Short and Simple: Throw on the first error and let the outer Try catch-and-re-wrap - List(destinationSimpleton) -> Set(commandBuilder.copyCommand(sourcePath, destinationPath).get) - case nonFileSimpleton => (List(nonFileSimpleton), Set.empty[IoCommand[_]]) - }) + protected def processSimpletons(womValueSimpletons: Seq[WomValueSimpleton], + sourceCallRootPath: Path + ): Try[(CallOutputs, Set[IoCommand[_]])] = Try { + val (destinationSimpletons, ioCommands): (List[WomValueSimpleton], Set[IoCommand[_]]) = + womValueSimpletons.toList.foldMap { + case WomValueSimpleton(key, wdlFile: WomSingleFile) => + val sourcePath = getPath(wdlFile.value).get + val destinationPath = + PathCopier.getDestinationFilePath(sourceCallRootPath, sourcePath, destinationCallRootPath) + + val destinationSimpleton = WomValueSimpleton(key, WomSingleFile(destinationPath.pathAsString)) + + // PROD-444: Keep It Short and Simple: Throw on the first error and let the outer Try catch-and-re-wrap + List(destinationSimpleton) -> Set(commandBuilder.copyCommand(sourcePath, destinationPath).get) + case nonFileSimpleton => (List(nonFileSimpleton), Set.empty[IoCommand[_]]) + } (WomValueBuilder.toJobOutputs(jobDescriptor.taskCall.outputPorts, destinationSimpletons), ioCommands) } @@ -377,7 +409,7 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit /** * Returns the file (and ONLY the file detritus) intersection between the cache hit and this call. */ - protected final def detritusFileKeys(sourceJobDetritusFiles: Map[String, String]): Set[String] = { + final protected def detritusFileKeys(sourceJobDetritusFiles: Map[String, String]): Set[String] = { val sourceKeys = sourceJobDetritusFiles.keySet val destinationKeys = destinationJobDetritusPaths.keySet sourceKeys.intersect(destinationKeys).filterNot(_ == JobPaths.CallRootPathKey) @@ -386,21 +418,22 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit /** * Returns a pair of the detritus with copied paths, and copy commands necessary to perform those copies. */ - protected def processDetritus(sourceJobDetritusFiles: Map[String, String]): Try[(Map[String, Path], Set[IoCommand[_]])] = Try { + protected def processDetritus( + sourceJobDetritusFiles: Map[String, String] + ): Try[(Map[String, Path], Set[IoCommand[_]])] = Try { val fileKeys = detritusFileKeys(sourceJobDetritusFiles) val zero = (Map.empty[String, Path], Set.empty[IoCommand[_]]) - val (destinationDetritus, ioCommands) = fileKeys.foldLeft(zero)({ - case ((detrituses, commands), detritus) => - val sourcePath = getPath(sourceJobDetritusFiles(detritus)).get - val destinationPath = destinationJobDetritusPaths(detritus) + val (destinationDetritus, ioCommands) = fileKeys.foldLeft(zero) { case ((detrituses, commands), detritus) => + val sourcePath = getPath(sourceJobDetritusFiles(detritus)).get + val destinationPath = destinationJobDetritusPaths(detritus) - val newDetrituses = detrituses + (detritus -> destinationPath) + val newDetrituses = detrituses + (detritus -> destinationPath) // PROD-444: Keep It Short and Simple: Throw on the first error and let the outer Try catch-and-re-wrap (newDetrituses, commands + commandBuilder.copyCommand(sourcePath, destinationPath).get) - }) + } (destinationDetritus + (JobPaths.CallRootPathKey -> destinationCallRootPath), ioCommands) } @@ -412,13 +445,16 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit protected def additionalIoCommands(sourceCallRootPath: Path, originalSimpletons: Seq[WomValueSimpleton], newOutputs: CallOutputs, - originalDetritus: Map[String, String], - newDetritus: Map[String, Path]): Try[List[Set[IoCommand[_]]]] = Success(Nil) + originalDetritus: Map[String, String], + newDetritus: Map[String, Path] + ): Try[List[Set[IoCommand[_]]]] = Success(Nil) override protected def onTimeout(message: Any, to: ActorRef): Unit = { val exceptionMessage = message match { - case copyCommand: IoCopyCommand => s"The Cache hit copying actor timed out waiting for a response to copy ${copyCommand.source.pathAsString} to ${copyCommand.destination.pathAsString}" - case touchCommand: IoTouchCommand => s"The Cache hit copying actor timed out waiting for a response to touch ${touchCommand.file.pathAsString}" + case copyCommand: IoCopyCommand => + s"The Cache hit copying actor timed out waiting for a response to copy ${copyCommand.source.pathAsString} to ${copyCommand.destination.pathAsString}" + case touchCommand: IoTouchCommand => + s"The Cache hit copying actor timed out waiting for a response to touch ${touchCommand.file.pathAsString}" case other => s"The Cache hit copying actor timed out waiting for an unknown I/O operation: $other" } diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardFileHashingActor.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardFileHashingActor.scala index 78fdc078cb9..67199a7a062 100644 --- a/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardFileHashingActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardFileHashingActor.scala @@ -34,8 +34,7 @@ trait StandardFileHashingActorParams { } /** A default implementation of the cache hit copying params. */ -case class DefaultStandardFileHashingActorParams -( +case class DefaultStandardFileHashingActorParams( override val jobDescriptor: BackendJobDescriptor, override val backendInitializationDataOption: Option[BackendInitializationData], override val serviceRegistryActor: ActorRef, @@ -46,7 +45,8 @@ case class DefaultStandardFileHashingActorParams case class FileHashContext(hashKey: HashKey, file: String) -class DefaultStandardFileHashingActor(standardParams: StandardFileHashingActorParams) extends StandardFileHashingActor(standardParams) { +class DefaultStandardFileHashingActor(standardParams: StandardFileHashingActorParams) + extends StandardFileHashingActor(standardParams) { override val ioCommandBuilder: IoCommandBuilder = DefaultIoCommandBuilder } @@ -54,14 +54,20 @@ object StandardFileHashingActor { case class FileHashingFunction(work: (SingleFileHashRequest, LoggingAdapter) => Try[String]) sealed trait BackendSpecificHasherCommand { def jobKey: JobKey } - final case class SingleFileHashRequest(jobKey: JobKey, hashKey: HashKey, file: WomFile, initializationData: Option[BackendInitializationData]) extends BackendSpecificHasherCommand + final case class SingleFileHashRequest(jobKey: JobKey, + hashKey: HashKey, + file: WomFile, + initializationData: Option[BackendInitializationData] + ) extends BackendSpecificHasherCommand sealed trait BackendSpecificHasherResponse extends SuccessfulHashResultMessage - case class FileHashResponse(hashResult: HashResult) extends BackendSpecificHasherResponse { override def hashes = Set(hashResult) } + case class FileHashResponse(hashResult: HashResult) extends BackendSpecificHasherResponse { + override def hashes = Set(hashResult) + } } abstract class StandardFileHashingActor(standardParams: StandardFileHashingActorParams) - extends Actor + extends Actor with ActorLogging with JobLogging with IoClientHelper @@ -69,19 +75,21 @@ abstract class StandardFileHashingActor(standardParams: StandardFileHashingActor with Timers { override lazy val ioActor: ActorRef = standardParams.ioActor override lazy val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor - override lazy val backendInitializationDataOption: Option[BackendInitializationData] = standardParams.backendInitializationDataOption + override lazy val backendInitializationDataOption: Option[BackendInitializationData] = + standardParams.backendInitializationDataOption override lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor override lazy val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor protected def ioCommandBuilder: IoCommandBuilder = DefaultIoCommandBuilder def customHashStrategy(fileRequest: SingleFileHashRequest): Option[Try[String]] = None - + def fileHashingReceive: Receive = { // Hash Request case fileRequest: SingleFileHashRequest => customHashStrategy(fileRequest) match { - case Some(Success(result)) => context.parent ! FileHashResponse(HashResult(fileRequest.hashKey, HashValue(result))) + case Some(Success(result)) => + context.parent ! FileHashResponse(HashResult(fileRequest.hashKey, HashValue(result))) case Some(Failure(failure)) => context.parent ! HashingFailedMessage(fileRequest.file.value, failure) case None => asyncHashing(fileRequest, context.parent) } @@ -93,7 +101,7 @@ abstract class StandardFileHashingActor(standardParams: StandardFileHashingActor case (fileHashRequest: FileHashContext, IoSuccess(_, other)) => context.parent ! HashingFailedMessage( fileHashRequest.file, - new Exception(s"Hash function supposedly succeeded but responded with '$other' instead of a string hash"), + new Exception(s"Hash function supposedly succeeded but responded with '$other' instead of a string hash") ) // Hash Failure @@ -124,9 +132,9 @@ abstract class StandardFileHashingActor(standardParams: StandardFileHashingActor } } - override def receive: Receive = ioReceive orElse fileHashingReceive + override def receive: Receive = ioReceive orElse fileHashingReceive - override protected def onTimeout(message: Any, to: ActorRef): Unit = { + override protected def onTimeout(message: Any, to: ActorRef): Unit = message match { case (_, ioHashCommand: IoHashCommand) => val fileAsString = ioHashCommand.file.pathAsString @@ -137,5 +145,4 @@ abstract class StandardFileHashingActor(standardParams: StandardFileHashingActor log.warning(s"Async File hashing actor received unexpected timeout message: $other") context.parent ! HashingServiceUnvailable } - } } diff --git a/backend/src/main/scala/cromwell/backend/standard/package.scala b/backend/src/main/scala/cromwell/backend/standard/package.scala index 089efd4f614..b25d74ec8e7 100644 --- a/backend/src/main/scala/cromwell/backend/standard/package.scala +++ b/backend/src/main/scala/cromwell/backend/standard/package.scala @@ -12,7 +12,7 @@ package object standard { def unapply(arg: StandardAdHocValue): Option[LocalizedAdHocValue] = arg.select[LocalizedAdHocValue] } } - + // This is used to represent an AdHocValue that might have been localized type StandardAdHocValue = AdHocValue :+: LocalizedAdHocValue :+: CNil } diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala index 06bc3d56c0a..c314a660122 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala @@ -10,18 +10,18 @@ object ContinueOnReturnCode { * Decides if a call/job continues upon a specific return code. */ sealed trait ContinueOnReturnCode { + /** * Returns true if the call is a success based on the return code. * * @param returnCode Return code from the process / script. * @return True if the call is a success. */ - final def continueFor(returnCode: Int): Boolean = { + final def continueFor(returnCode: Int): Boolean = this match { case ContinueOnReturnCodeFlag(continue) => continue || returnCode == 0 case ContinueOnReturnCodeSet(returnCodes) => returnCodes.contains(returnCode) } - } } /** diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala index aaa2754f3e1..7573909eb69 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala @@ -3,8 +3,8 @@ package cromwell.backend.validation import cats.data.Validated.{Invalid, Valid} import cats.implicits._ import com.typesafe.config.Config -import cromwell.backend.validation.RuntimeAttributesValidation._ import common.validation.ErrorOr._ +import cromwell.backend.validation.RuntimeAttributesValidation.validateInt import wom.RuntimeAttributesKeys import wom.types._ import wom.values._ @@ -23,10 +23,14 @@ import scala.util.Try * `default` a validation with the default value specified by the reference.conf file. */ object ContinueOnReturnCodeValidation { - lazy val instance: RuntimeAttributesValidation[ContinueOnReturnCode] = new ContinueOnReturnCodeValidation - def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ContinueOnReturnCode] = instance.withDefault( - configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0)) - def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(runtimeConfig) + lazy val instance: RuntimeAttributesValidation[ContinueOnReturnCode] = + new ContinueOnReturnCodeValidation + def default( + runtimeConfig: Option[Config] + ): RuntimeAttributesValidation[ContinueOnReturnCode] = + instance.withDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0)) + def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = + instance.configDefaultWomValue(runtimeConfig) } class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[ContinueOnReturnCode] { @@ -38,30 +42,34 @@ class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[Continu override def validateValue: PartialFunction[WomValue, ErrorOr[ContinueOnReturnCode]] = { case WomBoolean(value) => ContinueOnReturnCodeFlag(value).validNel case WomString(value) if Try(value.toBoolean).isSuccess => ContinueOnReturnCodeFlag(value.toBoolean).validNel + case WomString(value) if value.equals("*") => ContinueOnReturnCodeFlag(true).validNel case WomString(value) if Try(value.toInt).isSuccess => ContinueOnReturnCodeSet(Set(value.toInt)).validNel case WomInteger(value) => ContinueOnReturnCodeSet(Set(value)).validNel - case value@WomArray(_, seq) => + case value @ WomArray(_, seq) => val errorOrInts: ErrorOr[List[Int]] = (seq.toList map validateInt).sequence[ErrorOr, Int] errorOrInts match { case Valid(ints) => ContinueOnReturnCodeSet(ints.toSet).validNel case Invalid(_) => invalidValueFailure(value) } + case value => invalidValueFailure(value) } override def validateExpression: PartialFunction[WomValue, Boolean] = { case WomBoolean(_) => true case WomString(value) if Try(value.toInt).isSuccess => true case WomString(value) if Try(value.toBoolean).isSuccess => true + case WomString(value) if value.equals("*") => true case WomInteger(_) => true case WomArray(WomArrayType(WomStringType), elements) => - elements forall { - value => Try(value.valueString.toInt).isSuccess + elements forall { value => + Try(value.valueString.toInt).isSuccess } case WomArray(WomArrayType(WomIntegerType), _) => true + case _ => false } - override protected def missingValueMessage: String = s"Expecting $key" + - " runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + override protected def missingValueMessage: String = "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." override def usedInCallCaching: Boolean = true } diff --git a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala index 0cc06a7dfa4..0038290010c 100644 --- a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala @@ -24,10 +24,6 @@ import wom.values.{WomInteger, WomValue} object CpuValidation { lazy val instance: RuntimeAttributesValidation[Int Refined Positive] = new CpuValidation(CpuKey) lazy val optional: OptionalRuntimeAttributesValidation[Int Refined Positive] = instance.optional - lazy val instanceMin: RuntimeAttributesValidation[Int Refined Positive] = new CpuValidation(CpuMinKey) - lazy val optionalMin: OptionalRuntimeAttributesValidation[Int Refined Positive] = instanceMin.optional - lazy val instanceMax: RuntimeAttributesValidation[Int Refined Positive] = new CpuValidation(CpuMaxKey) - lazy val optionalMax: OptionalRuntimeAttributesValidation[Int Refined Positive] = instanceMax.optional lazy val defaultMin: WomValue = WomInteger(1) def configDefaultWomValue(config: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(config) diff --git a/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala b/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala index b6c3fe0112b..eb8afc17c12 100644 --- a/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala @@ -26,7 +26,7 @@ class DockerValidation extends StringRuntimeAttributesValidation(RuntimeAttribut override protected def invalidValueMessage(value: WomValue): String = super.missingValueMessage // NOTE: Docker's current test specs don't like WdlInteger, etc. auto converted to WdlString. - override protected def validateValue: PartialFunction[WomValue, ErrorOr[String]] = { - case WomString(value) => value.validNel + override protected def validateValue: PartialFunction[WomValue, ErrorOr[String]] = { case WomString(value) => + value.validNel } } diff --git a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala index 8ac39e50d16..7b55f9657fa 100644 --- a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala @@ -18,9 +18,10 @@ import wom.values._ object FailOnStderrValidation { lazy val instance: RuntimeAttributesValidation[Boolean] = new FailOnStderrValidation - def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Boolean] = instance.withDefault( - configDefaultWdlValue(runtimeConfig) getOrElse WomBoolean(false)) - def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(runtimeConfig) + def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Boolean] = + instance.withDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomBoolean(false)) + def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = + instance.configDefaultWomValue(runtimeConfig) } class FailOnStderrValidation extends BooleanRuntimeAttributesValidation(RuntimeAttributesKeys.FailOnStderrKey) { diff --git a/backend/src/main/scala/cromwell/backend/validation/InformationValidation.scala b/backend/src/main/scala/cromwell/backend/validation/InformationValidation.scala index 39b9f0b3031..e8c9dd1d7b3 100644 --- a/backend/src/main/scala/cromwell/backend/validation/InformationValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/InformationValidation.scala @@ -25,18 +25,32 @@ import scala.util.{Failure, Success} * `withDefault` can be used to create a validation that defaults to a particular size. */ object InformationValidation { - def instance(attributeName: String = RuntimeAttributesKeys.MemoryKey, defaultUnit: MemoryUnit, allowZero: Boolean = false): RuntimeAttributesValidation[MemorySize] = + def instance(attributeName: String = RuntimeAttributesKeys.MemoryKey, + defaultUnit: MemoryUnit, + allowZero: Boolean = false + ): RuntimeAttributesValidation[MemorySize] = new InformationValidation(attributeName, defaultUnit, allowZero) - def optional(attributeName: String = RuntimeAttributesKeys.MemoryKey, defaultUnit: MemoryUnit, allowZero: Boolean = false): OptionalRuntimeAttributesValidation[MemorySize] = + def optional(attributeName: String = RuntimeAttributesKeys.MemoryKey, + defaultUnit: MemoryUnit, + allowZero: Boolean = false + ): OptionalRuntimeAttributesValidation[MemorySize] = instance(attributeName, defaultUnit, allowZero).optional - def configDefaultString(attributeName: String = RuntimeAttributesKeys.MemoryKey, config: Option[Config], defaultUnit: MemoryUnit, allowZero: Boolean = false): Option[String] = + def configDefaultString(attributeName: String = RuntimeAttributesKeys.MemoryKey, + config: Option[Config], + defaultUnit: MemoryUnit, + allowZero: Boolean = false + ): Option[String] = instance(attributeName, defaultUnit, allowZero).configDefaultValue(config) - def withDefaultMemory(attributeName: String = RuntimeAttributesKeys.MemoryKey, memorySize: String, defaultUnit: MemoryUnit, allowZero: Boolean = false): RuntimeAttributesValidation[MemorySize] = { + def withDefaultMemory(attributeName: String = RuntimeAttributesKeys.MemoryKey, + memorySize: String, + defaultUnit: MemoryUnit, + allowZero: Boolean = false + ): RuntimeAttributesValidation[MemorySize] = MemorySize.parse(memorySize) match { case Success(memory) => instance(attributeName, defaultUnit, allowZero).withDefault(WomLong(memory.bytes.toLong)) - case Failure(_) => instance(attributeName, defaultUnit, allowZero).withDefault(BadDefaultAttribute(WomString(memorySize.toString))) + case Failure(_) => + instance(attributeName, defaultUnit, allowZero).withDefault(BadDefaultAttribute(WomString(memorySize.toString))) } - } private[validation] val wrongAmountFormat = "Expecting %s runtime attribute value greater than 0 but got %s" @@ -44,39 +58,56 @@ object InformationValidation { "Expecting %s runtime attribute to be an Integer or String with format '8 GB'." + " Exception: %s" - private[validation] def validateString(attributeName: String, wdlString: WomString, allowZero: Boolean): ErrorOr[MemorySize] = + private[validation] def validateString(attributeName: String, + wdlString: WomString, + allowZero: Boolean + ): ErrorOr[MemorySize] = validateString(attributeName, wdlString.value, allowZero) - private[validation] def validateString(attributeName: String, value: String, allowZero: Boolean): ErrorOr[MemorySize] = { + private[validation] def validateString(attributeName: String, + value: String, + allowZero: Boolean + ): ErrorOr[MemorySize] = MemorySize.parse(value) match { - case scala.util.Success(memorySize: MemorySize) if memorySize.amount > 0 || (memorySize.amount == 0 && allowZero) => + case scala.util.Success(memorySize: MemorySize) + if memorySize.amount > 0 || (memorySize.amount == 0 && allowZero) => memorySize.to(MemoryUnit.GB).validNel case scala.util.Success(memorySize: MemorySize) => wrongAmountFormat.format(attributeName, memorySize.amount).invalidNel case scala.util.Failure(throwable) => wrongTypeFormat.format(attributeName, throwable.getMessage).invalidNel } - } - private[validation] def validateInteger(attributeName: String, wdlInteger: WomInteger, defaultUnit: MemoryUnit, allowZero: Boolean): ErrorOr[MemorySize] = + private[validation] def validateInteger(attributeName: String, + wdlInteger: WomInteger, + defaultUnit: MemoryUnit, + allowZero: Boolean + ): ErrorOr[MemorySize] = validateInteger(attributeName, wdlInteger.value, defaultUnit, allowZero) - private[validation] def validateInteger(attributeName: String, value: Int, defaultUnit: MemoryUnit, allowZero: Boolean): ErrorOr[MemorySize] = { + private[validation] def validateInteger(attributeName: String, + value: Int, + defaultUnit: MemoryUnit, + allowZero: Boolean + ): ErrorOr[MemorySize] = if (value < 0 || (value == 0 && !allowZero)) wrongAmountFormat.format(attributeName, value).invalidNel else MemorySize(value.toDouble, defaultUnit).to(MemoryUnit.GB).validNel - } - def validateLong(attributeName: String, value: Long, defaultUnit: MemoryUnit, allowZero: Boolean): ErrorOr[MemorySize] = { + def validateLong(attributeName: String, + value: Long, + defaultUnit: MemoryUnit, + allowZero: Boolean + ): ErrorOr[MemorySize] = if (value < 0 || (value == 0 && !allowZero)) wrongAmountFormat.format(attributeName, value).invalidNel else MemorySize(value.toDouble, defaultUnit).to(MemoryUnit.GB).validNel - } } -class InformationValidation(attributeName: String, defaultUnit: MemoryUnit, allowZero: Boolean = false) extends RuntimeAttributesValidation[MemorySize] { +class InformationValidation(attributeName: String, defaultUnit: MemoryUnit, allowZero: Boolean = false) + extends RuntimeAttributesValidation[MemorySize] { import InformationValidation._ diff --git a/backend/src/main/scala/cromwell/backend/validation/MaxRetriesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/MaxRetriesValidation.scala index fdfd6467a85..f2c9ba9522e 100644 --- a/backend/src/main/scala/cromwell/backend/validation/MaxRetriesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/MaxRetriesValidation.scala @@ -21,8 +21,8 @@ object MaxRetriesValidation { lazy val instance: RuntimeAttributesValidation[Int] = new MaxRetriesValidation(MaxRetriesKey) lazy val optional: OptionalRuntimeAttributesValidation[Int] = instance.optional - def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = instance.withDefault( - configDefaultWomValue(runtimeConfig) getOrElse WomInteger(0)) + def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = + instance.withDefault(configDefaultWomValue(runtimeConfig) getOrElse WomInteger(0)) def configDefaultWomValue(config: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(config) } diff --git a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala index 32be28026f0..299e043ec8f 100644 --- a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala @@ -25,16 +25,22 @@ import scala.util.{Failure, Success} object MemoryValidation { def instance(attributeName: String = RuntimeAttributesKeys.MemoryKey): RuntimeAttributesValidation[MemorySize] = new MemoryValidation(attributeName) - def optional(attributeName: String = RuntimeAttributesKeys.MemoryKey): OptionalRuntimeAttributesValidation[MemorySize] = + def optional( + attributeName: String = RuntimeAttributesKeys.MemoryKey + ): OptionalRuntimeAttributesValidation[MemorySize] = instance(attributeName).optional - def configDefaultString(attributeName: String = RuntimeAttributesKeys.MemoryKey, config: Option[Config]): Option[String] = + def configDefaultString(attributeName: String = RuntimeAttributesKeys.MemoryKey, + config: Option[Config] + ): Option[String] = instance(attributeName).configDefaultValue(config) - def withDefaultMemory(attributeName: String = RuntimeAttributesKeys.MemoryKey, memorySize: String): RuntimeAttributesValidation[MemorySize] = { + def withDefaultMemory(attributeName: String = RuntimeAttributesKeys.MemoryKey, + memorySize: String + ): RuntimeAttributesValidation[MemorySize] = MemorySize.parse(memorySize) match { case Success(memory) => instance(attributeName).withDefault(WomLong(memory.bytes.toLong)) case Failure(_) => instance(attributeName).withDefault(BadDefaultAttribute(WomString(memorySize.toString))) } - } } -class MemoryValidation(attributeName: String = RuntimeAttributesKeys.MemoryKey) extends InformationValidation(attributeName, MemoryUnit.Bytes) +class MemoryValidation(attributeName: String = RuntimeAttributesKeys.MemoryKey) + extends InformationValidation(attributeName, MemoryUnit.Bytes) diff --git a/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala index d4dc78f83b2..af41bf8ad52 100644 --- a/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala @@ -44,24 +44,24 @@ sealed trait PrimitiveRuntimeAttributesValidation[A, B <: WomPrimitive] extends protected def validateCoercedValue(womValue: B): ErrorOr[A] } -class BooleanRuntimeAttributesValidation(override val key: String) extends - PrimitiveRuntimeAttributesValidation[Boolean, WomBoolean] { +class BooleanRuntimeAttributesValidation(override val key: String) + extends PrimitiveRuntimeAttributesValidation[Boolean, WomBoolean] { override val womType = WomBooleanType override protected def validateCoercedValue(womValue: WomBoolean): ErrorOr[Boolean] = womValue.value.validNel } -class FloatRuntimeAttributesValidation(override val key: String) extends - PrimitiveRuntimeAttributesValidation[Double, WomFloat] { +class FloatRuntimeAttributesValidation(override val key: String) + extends PrimitiveRuntimeAttributesValidation[Double, WomFloat] { override val womType = WomFloatType override protected def validateCoercedValue(womValue: WomFloat): ErrorOr[Double] = womValue.value.validNel } -class IntRuntimeAttributesValidation(override val key: String) extends - PrimitiveRuntimeAttributesValidation[Int, WomInteger] { +class IntRuntimeAttributesValidation(override val key: String) + extends PrimitiveRuntimeAttributesValidation[Int, WomInteger] { override val womType = WomIntegerType @@ -70,18 +70,19 @@ class IntRuntimeAttributesValidation(override val key: String) extends override protected def typeString: String = "an Integer" } -class PositiveIntRuntimeAttributesValidation(override val key: String) extends - PrimitiveRuntimeAttributesValidation[Int Refined Positive, WomInteger] { +class PositiveIntRuntimeAttributesValidation(override val key: String) + extends PrimitiveRuntimeAttributesValidation[Int Refined Positive, WomInteger] { override val womType = WomIntegerType - override protected def validateCoercedValue(womValue: WomInteger): ErrorOr[Int Refined Positive] = refineV[Positive](womValue.value).leftMap(NonEmptyList.one).toValidated + override protected def validateCoercedValue(womValue: WomInteger): ErrorOr[Int Refined Positive] = + refineV[Positive](womValue.value).leftMap(NonEmptyList.one).toValidated override protected def typeString: String = "an Integer" } -class StringRuntimeAttributesValidation(override val key: String) extends - PrimitiveRuntimeAttributesValidation[String, WomString] { +class StringRuntimeAttributesValidation(override val key: String) + extends PrimitiveRuntimeAttributesValidation[String, WomString] { override val womType = WomStringType diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala index d46bc7a66d5..706cc1336e8 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala @@ -12,29 +12,32 @@ import scala.util.{Failure, Try} object RuntimeAttributesDefault { - def workflowOptionsDefault(options: WorkflowOptions, mapping: Map[String, Iterable[WomType]]): - Try[Map[String, WomValue]] = { + def workflowOptionsDefault(options: WorkflowOptions, + mapping: Map[String, Iterable[WomType]] + ): Try[Map[String, WomValue]] = options.defaultRuntimeOptions flatMap { attrs => - TryUtil.sequenceMap(attrs collect { - case (k, v) if mapping.contains(k) => - val maybeTriedValue = mapping(k) map { _.coerceRawValue(v) } find { _.isSuccess } getOrElse { - Failure(new RuntimeException(s"Could not parse JsonValue $v to valid WomValue for runtime attribute $k")) - } - k -> maybeTriedValue - }, "Failed to coerce default runtime options") - } recover { - case _: OptionNotFoundException => Map.empty[String, WomValue] + TryUtil.sequenceMap( + attrs collect { + case (k, v) if mapping.contains(k) => + val maybeTriedValue = mapping(k) map { _.coerceRawValue(v) } find { _.isSuccess } getOrElse { + Failure(new RuntimeException(s"Could not parse JsonValue $v to valid WomValue for runtime attribute $k")) + } + k -> maybeTriedValue + }, + "Failed to coerce default runtime options" + ) + } recover { case _: OptionNotFoundException => + Map.empty[String, WomValue] } - } /** * Traverse defaultsList in order, and for each of them add the missing (and only missing) runtime attributes. */ - def withDefaults(attrs: EvaluatedRuntimeAttributes, defaultsList: List[EvaluatedRuntimeAttributes]): EvaluatedRuntimeAttributes = { - defaultsList.foldLeft(attrs)((acc, default) => { - acc ++ default.view.filterKeys(!acc.keySet.contains(_)) - }) - } + def withDefaults(attrs: EvaluatedRuntimeAttributes, + defaultsList: List[EvaluatedRuntimeAttributes] + ): EvaluatedRuntimeAttributes = + defaultsList.foldLeft(attrs)((acc, default) => acc ++ default.view.filterKeys(!acc.keySet.contains(_))) - def noValueFoundFor[A](attribute: String): ValidatedNel[String, A] = s"Can't find an attribute value for key $attribute".invalidNel + def noValueFoundFor[A](attribute: String): ValidatedNel[String, A] = + s"Can't find an attribute value for key $attribute".invalidNel } diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala index 65f2119a64b..5fa52dac53a 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala @@ -26,61 +26,56 @@ object RuntimeAttributesValidation { if (unrecognized.nonEmpty) logger.warn(s"Unrecognized runtime attribute keys: $unrecognized") } - def validateDocker(docker: Option[WomValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = { + def validateDocker(docker: Option[WomValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = validateWithValidation(docker, DockerValidation.instance.optional, onMissingKey) - } - def validateFailOnStderr(value: Option[WomValue], onMissingKey: => ErrorOr[Boolean]): ErrorOr[Boolean] = { + def validateFailOnStderr(value: Option[WomValue], onMissingKey: => ErrorOr[Boolean]): ErrorOr[Boolean] = validateWithValidation(value, FailOnStderrValidation.instance, onMissingKey) - } def validateContinueOnReturnCode(value: Option[WomValue], - onMissingKey: => ErrorOr[ContinueOnReturnCode]): ErrorOr[ContinueOnReturnCode] = { + onMissingKey: => ErrorOr[ContinueOnReturnCode] + ): ErrorOr[ContinueOnReturnCode] = validateWithValidation(value, ContinueOnReturnCodeValidation.instance, onMissingKey) - } - def validateMemory(value: Option[WomValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] = { + def validateMemory(value: Option[WomValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] = validateWithValidation(value, MemoryValidation.instance(), onMissingKey) - } - def validateCpu(cpu: Option[WomValue], onMissingKey: => ErrorOr[Int Refined Positive]): ErrorOr[Int Refined Positive] = { + def validateCpu(cpu: Option[WomValue], + onMissingKey: => ErrorOr[Int Refined Positive] + ): ErrorOr[Int Refined Positive] = validateWithValidation(cpu, CpuValidation.instance, onMissingKey) - } - def validateMaxRetries(maxRetries: Option[WomValue], onMissingKey: => ErrorOr[Int]): ErrorOr[Int] = { + def validateMaxRetries(maxRetries: Option[WomValue], onMissingKey: => ErrorOr[Int]): ErrorOr[Int] = validateWithValidation(maxRetries, MaxRetriesValidation.instance, onMissingKey) - } private def validateWithValidation[T](valueOption: Option[WomValue], validation: RuntimeAttributesValidation[T], - onMissingValue: => ErrorOr[T]): ErrorOr[T] = { + onMissingValue: => ErrorOr[T] + ): ErrorOr[T] = valueOption match { case Some(value) => validation.validateValue.applyOrElse(value, (_: Any) => validation.invalidValueFailure(value)) case None => onMissingValue } - } - def validateInt(value: WomValue): ErrorOr[Int] = { + def validateInt(value: WomValue): ErrorOr[Int] = WomIntegerType.coerceRawValue(value) match { case scala.util.Success(WomInteger(i)) => i.intValue.validNel case _ => s"Could not coerce ${value.valueString} into an integer".invalidNel } - } - def validateBoolean(value: WomValue): ErrorOr[Boolean] = { + def validateBoolean(value: WomValue): ErrorOr[Boolean] = WomBooleanType.coerceRawValue(value) match { case scala.util.Success(WomBoolean(b)) => b.booleanValue.validNel case _ => s"Could not coerce ${value.valueString} into a boolean".invalidNel } - } - def parseMemoryString(k: String, s: WomString): ErrorOr[MemorySize] = { + def parseMemoryString(k: String, s: WomString): ErrorOr[MemorySize] = InformationValidation.validateString(k, s, allowZero = false) - } def withDefault[ValidatedType](validation: RuntimeAttributesValidation[ValidatedType], - default: WomValue): RuntimeAttributesValidation[ValidatedType] = { + default: WomValue + ): RuntimeAttributesValidation[ValidatedType] = new RuntimeAttributesValidation[ValidatedType] { override def key: String = validation.key @@ -101,10 +96,10 @@ object RuntimeAttributesValidation { override protected def staticDefaultOption = Option(default) } - } def withUsedInCallCaching[ValidatedType](validation: RuntimeAttributesValidation[ValidatedType], - usedInCallCachingValue: Boolean): RuntimeAttributesValidation[ValidatedType] = { + usedInCallCachingValue: Boolean + ): RuntimeAttributesValidation[ValidatedType] = new RuntimeAttributesValidation[ValidatedType] { override def key: String = validation.key @@ -125,10 +120,10 @@ object RuntimeAttributesValidation { override protected def staticDefaultOption = validation.staticDefaultOption } - } - def optional[ValidatedType](validation: RuntimeAttributesValidation[ValidatedType]): - OptionalRuntimeAttributesValidation[ValidatedType] = { + def optional[ValidatedType]( + validation: RuntimeAttributesValidation[ValidatedType] + ): OptionalRuntimeAttributesValidation[ValidatedType] = new OptionalRuntimeAttributesValidation[ValidatedType] { override def key: String = validation.key @@ -149,7 +144,6 @@ object RuntimeAttributesValidation { override protected def staticDefaultOption = validation.staticDefaultOption } - } /** * Returns the value from the attributes, unpacking options, and converting them to string values suitable for @@ -181,9 +175,9 @@ object RuntimeAttributesValidation { * @throws ClassCastException if the validation is called on an optional validation. */ def extract[A](runtimeAttributesValidation: RuntimeAttributesValidation[A], - validatedRuntimeAttributes: ValidatedRuntimeAttributes): A = { + validatedRuntimeAttributes: ValidatedRuntimeAttributes + ): A = extract(runtimeAttributesValidation.key, validatedRuntimeAttributes) - } /** * Returns the value from the attributes matching the key. @@ -192,14 +186,15 @@ object RuntimeAttributesValidation { * @param validatedRuntimeAttributes The values to search. * @return The value matching the key. */ - def extract[A](key: String, - validatedRuntimeAttributes: ValidatedRuntimeAttributes): A = { + def extract[A](key: String, validatedRuntimeAttributes: ValidatedRuntimeAttributes): A = { val value = extractOption(key, validatedRuntimeAttributes) value match { // NOTE: Some(innerValue) aka Some.unapply() throws a `ClassCastException` to `Nothing$` as it can't tell the type case some: Some[_] => some.get.asInstanceOf[A] - case None => throw new RuntimeException( - s"$key not found in runtime attributes ${validatedRuntimeAttributes.attributes.keys}") + case None => + throw new RuntimeException( + s"$key not found in runtime attributes ${validatedRuntimeAttributes.attributes.keys}" + ) } } @@ -211,9 +206,9 @@ object RuntimeAttributesValidation { * @return The Some(value) matching the key or None. */ def extractOption[A](runtimeAttributesValidation: RuntimeAttributesValidation[A], - validatedRuntimeAttributes: ValidatedRuntimeAttributes): Option[A] = { + validatedRuntimeAttributes: ValidatedRuntimeAttributes + ): Option[A] = extractOption(runtimeAttributesValidation.key, validatedRuntimeAttributes) - } /** * Returns Some(value) from the attributes matching the key, or None. @@ -234,13 +229,12 @@ object RuntimeAttributesValidation { * @tparam A The type to cast the unpacked value. * @return The Some(value) matching the key or None. */ - final def unpackOption[A](value: Any): Option[A] = { + final def unpackOption[A](value: Any): Option[A] = value match { case None => None case Some(innerValue) => unpackOption(innerValue) case _ => Option(value.asInstanceOf[A]) } - } } /** @@ -251,13 +245,13 @@ case class BadDefaultAttribute(badDefaultValue: WomValue) extends WomValue { val womType = WomStringType } - /** * Performs a validation on a runtime attribute and returns some value. * * @tparam ValidatedType The type of the validated value. */ trait RuntimeAttributesValidation[ValidatedType] { + /** * Returns the key of the runtime attribute. * @@ -297,8 +291,8 @@ trait RuntimeAttributesValidation[ValidatedType] { * * @return true if the value can be validated. */ - protected def validateExpression: PartialFunction[WomValue, Boolean] = { - case womValue => coercion.exists(_ == womValue.womType) + protected def validateExpression: PartialFunction[WomValue, Boolean] = { case womValue => + coercion.exists(_ == womValue.womType) } /** @@ -322,7 +316,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * * @return Wrapped invalidValueMessage. */ - protected final def invalidValueFailure(value: WomValue): ErrorOr[ValidatedType] = + final protected def invalidValueFailure(value: WomValue): ErrorOr[ValidatedType] = invalidValueMessage(value).invalidNel /** @@ -337,7 +331,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * * @return Wrapped missingValueMessage. */ - protected final lazy val missingValueFailure: ErrorOr[ValidatedType] = missingValueMessage.invalidNel + final protected lazy val missingValueFailure: ErrorOr[ValidatedType] = missingValueMessage.invalidNel /** * Runs this validation on the value matching key. @@ -347,12 +341,11 @@ trait RuntimeAttributesValidation[ValidatedType] { * @param values The full set of values. * @return The error or valid value for this key. */ - def validate(values: Map[String, WomValue]): ErrorOr[ValidatedType] = { + def validate(values: Map[String, WomValue]): ErrorOr[ValidatedType] = values.get(key) match { case Some(value) => validateValue.applyOrElse(value, (_: Any) => invalidValueFailure(value)) case None => validateNone } - } /** * Used during initialization, returning true if the expression __may be__ valid. @@ -371,7 +364,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * @param wdlExpressionMaybe The optional expression. * @return True if the expression may be evaluated. */ - def validateOptionalWomValue(wdlExpressionMaybe: Option[WomValue]): Boolean = { + def validateOptionalWomValue(wdlExpressionMaybe: Option[WomValue]): Boolean = wdlExpressionMaybe match { case None => staticDefaultOption.isDefined || validateNone.isValid case Some(wdlExpression: WdlExpression) => @@ -381,9 +374,8 @@ trait RuntimeAttributesValidation[ValidatedType] { } case Some(womValue) => validateExpression.applyOrElse(womValue, (_: Any) => false) } - } - def validateOptionalWomExpression(womExpressionMaybe: Option[WomExpression]): Boolean = { + def validateOptionalWomExpression(womExpressionMaybe: Option[WomExpression]): Boolean = womExpressionMaybe match { case None => staticDefaultOption.isDefined || validateNone.isValid case Some(womExpression) => @@ -392,7 +384,7 @@ trait RuntimeAttributesValidation[ValidatedType] { case Invalid(_) => true // If we can't evaluate it, we'll let it pass for now... } } - } + /** * Indicates whether this runtime attribute should be used in call caching calculations. * @@ -410,7 +402,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * Returns an optional version of this validation. */ final lazy val optional: OptionalRuntimeAttributesValidation[ValidatedType] = - RuntimeAttributesValidation.optional(this) + RuntimeAttributesValidation.optional(this) /** * Returns a version of this validation with the default value. @@ -432,7 +424,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * @param optionalRuntimeConfig Optional default runtime attributes config of a particular backend. * @return The new version of this validation. */ - final def configDefaultWomValue(optionalRuntimeConfig: Option[Config]): Option[WomValue] = { + final def configDefaultWomValue(optionalRuntimeConfig: Option[Config]): Option[WomValue] = optionalRuntimeConfig collect { case config if config.hasPath(key) => val value = config.getValue(key).unwrapped() @@ -442,13 +434,11 @@ trait RuntimeAttributesValidation[ValidatedType] { BadDefaultAttribute(WomString(value.toString)) } } - } - final def configDefaultValue(optionalRuntimeConfig: Option[Config]): Option[String] = { + final def configDefaultValue(optionalRuntimeConfig: Option[Config]): Option[String] = optionalRuntimeConfig collect { case config if config.hasPath(key) => config.getValue(key).unwrapped().toString } - } /* Methods below provide aliases to expose protected methods to the package. @@ -457,15 +447,15 @@ trait RuntimeAttributesValidation[ValidatedType] { access the protected values, except the `validation` package that uses these back doors. */ - private[validation] final lazy val validateValuePackagePrivate = validateValue + final private[validation] lazy val validateValuePackagePrivate = validateValue - private[validation] final lazy val validateExpressionPackagePrivate = validateExpression + final private[validation] lazy val validateExpressionPackagePrivate = validateExpression - private[validation] final def invalidValueMessagePackagePrivate(value: WomValue) = invalidValueMessage(value) + final private[validation] def invalidValueMessagePackagePrivate(value: WomValue) = invalidValueMessage(value) - private[validation] final lazy val missingValueMessagePackagePrivate = missingValueMessage + final private[validation] lazy val missingValueMessagePackagePrivate = missingValueMessage - private[validation] final lazy val usedInCallCachingPackagePrivate = usedInCallCaching + final private[validation] lazy val usedInCallCachingPackagePrivate = usedInCallCaching } /** @@ -474,6 +464,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * @tparam ValidatedType The type of the validated value. */ trait OptionalRuntimeAttributesValidation[ValidatedType] extends RuntimeAttributesValidation[Option[ValidatedType]] { + /** * Validates the wdl value. * @@ -484,13 +475,12 @@ trait OptionalRuntimeAttributesValidation[ValidatedType] extends RuntimeAttribut */ protected def validateOption: PartialFunction[WomValue, ErrorOr[ValidatedType]] - override final protected lazy val validateValue = new PartialFunction[WomValue, ErrorOr[Option[ValidatedType]]] { + final override protected lazy val validateValue = new PartialFunction[WomValue, ErrorOr[Option[ValidatedType]]] { override def isDefinedAt(womValue: WomValue): Boolean = validateOption.isDefinedAt(womValue) - override def apply(womValue: WomValue): Validated[NonEmptyList[String], Option[ValidatedType]] = { + override def apply(womValue: WomValue): Validated[NonEmptyList[String], Option[ValidatedType]] = validateOption.apply(womValue).map(Option.apply) - } } - override final protected lazy val validateNone: ErrorOr[None.type] = None.validNel[String] + final override protected lazy val validateNone: ErrorOr[None.type] = None.validNel[String] } diff --git a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala index 6e199c4c4fe..d1a6af6f58d 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala @@ -38,18 +38,14 @@ trait ValidatedRuntimeAttributesBuilder { /** * Returns validators suitable for BackendWorkflowInitializationActor.runtimeAttributeValidators. */ - final lazy val validatorMap: Map[String, Option[WomExpression] => Boolean] = { - validations.map(validation => - validation.key -> validation.validateOptionalWomExpression _ - ).toMap - } + final lazy val validatorMap: Map[String, Option[WomExpression] => Boolean] = + validations.map(validation => validation.key -> validation.validateOptionalWomExpression _).toMap /** * Returns a map of coercions suitable for RuntimeAttributesDefault.workflowOptionsDefault. */ - final lazy val coercionMap: Map[String, Iterable[WomType]] = { + final lazy val coercionMap: Map[String, Iterable[WomType]] = validations.map(validation => validation.key -> validation.coercion).toMap - } def unsupportedKeys(keys: Seq[String]): Seq[String] = keys.diff(validationKeys) @@ -61,11 +57,12 @@ trait ValidatedRuntimeAttributesBuilder { val runtimeAttributesErrorOr: ErrorOr[ValidatedRuntimeAttributes] = validate(attrs) runtimeAttributesErrorOr match { case Valid(runtimeAttributes) => runtimeAttributes - case Invalid(nel) => throw new RuntimeException with MessageAggregation with NoStackTrace { - override def exceptionContext: String = "Runtime attribute validation failed" + case Invalid(nel) => + throw new RuntimeException with MessageAggregation with NoStackTrace { + override def exceptionContext: String = "Runtime attribute validation failed" - override def errorMessages: Iterable[String] = nel.toList - } + override def errorMessages: Iterable[String] = nel.toList + } } } @@ -73,8 +70,8 @@ trait ValidatedRuntimeAttributesBuilder { val listOfKeysToErrorOrAnys: List[(String, ErrorOr[Any])] = validations.map(validation => validation.key -> validation.validate(values)).toList - val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys map { - case (key, errorOrAny) => errorOrAny map { any => (key, any) } + val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys map { case (key, errorOrAny) => + errorOrAny map { any => (key, any) } } import cats.syntax.traverse._ diff --git a/backend/src/main/scala/cromwell/backend/validation/exception/ValidationAggregatedException.scala b/backend/src/main/scala/cromwell/backend/validation/exception/ValidationAggregatedException.scala index ec3644674bb..325060aa4f6 100644 --- a/backend/src/main/scala/cromwell/backend/validation/exception/ValidationAggregatedException.scala +++ b/backend/src/main/scala/cromwell/backend/validation/exception/ValidationAggregatedException.scala @@ -3,4 +3,5 @@ package cromwell.backend.validation.exception import common.exception.MessageAggregation case class ValidationAggregatedException(override val exceptionContext: String, - override val errorMessages: Iterable[String]) extends MessageAggregation + override val errorMessages: Iterable[String] +) extends MessageAggregation diff --git a/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowPathBuilder.scala b/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowPathBuilder.scala index 43af11d8732..c112b8d5b6b 100644 --- a/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowPathBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowPathBuilder.scala @@ -2,7 +2,6 @@ package cromwell.backend.wfs import cromwell.core.path.DefaultPathBuilder - object DefaultWorkflowPathBuilder extends WorkflowPathBuilder { override def pathBuilderOption(params: WorkflowFileSystemProviderParams) = Option(DefaultPathBuilder) } diff --git a/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala b/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala index bd39ae2c911..c15ac945c12 100644 --- a/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala @@ -1,7 +1,7 @@ package cromwell.backend.wfs import com.typesafe.config.Config -import cromwell.backend.io.{WorkflowPathsWithDocker, WorkflowPaths} +import cromwell.backend.io.{WorkflowPaths, WorkflowPathsWithDocker} import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor} import cromwell.core.WorkflowOptions import cromwell.core.path.PathBuilder @@ -11,14 +11,16 @@ import scala.concurrent.ExecutionContext object WorkflowPathBuilder { def workflowPaths(configurationDescriptor: BackendConfigurationDescriptor, workflowDescriptor: BackendWorkflowDescriptor, - pathBuilders: List[PathBuilder]): WorkflowPaths = { + pathBuilders: List[PathBuilder] + ): WorkflowPaths = new WorkflowPathsWithDocker(workflowDescriptor, configurationDescriptor.backendConfig, pathBuilders) - } } -final case class WorkflowFileSystemProviderParams(fileSystemConfig: Config, globalConfig: Config, +final case class WorkflowFileSystemProviderParams(fileSystemConfig: Config, + globalConfig: Config, workflowOptions: WorkflowOptions, - fileSystemExecutionContext: ExecutionContext) + fileSystemExecutionContext: ExecutionContext +) trait WorkflowPathBuilder { def pathBuilderOption(params: WorkflowFileSystemProviderParams): Option[PathBuilder] diff --git a/backend/src/test/scala/cromwell/backend/BackendSpec.scala b/backend/src/test/scala/cromwell/backend/BackendSpec.scala index 6c4c7a654e5..835abec8eb3 100644 --- a/backend/src/test/scala/cromwell/backend/BackendSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendSpec.scala @@ -3,7 +3,12 @@ package cromwell.backend import _root_.wdl.draft2.model._ import _root_.wdl.transforms.draft2.wdlom2wom.WdlDraft2WomExecutableMakers._ import common.exception.AggregatedException -import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{ + BackendJobExecutionResponse, + JobFailedNonRetryableResponse, + JobFailedRetryableResponse, + JobSucceededResponse +} import cromwell.backend.io.TestWorkflows._ import cromwell.core.callcaching.NoDocker import cromwell.core.labels.Labels @@ -25,18 +30,17 @@ trait BackendSpec extends ScalaFutures with Matchers with ScaledTimeSpans { implicit val defaultPatience: PatienceConfig = PatienceConfig(timeout = scaled(Span(10, Seconds)), interval = Span(500, Millis)) - def testWorkflow(workflow: TestWorkflow, - backend: BackendJobExecutionActor): Unit = { + def testWorkflow(workflow: TestWorkflow, backend: BackendJobExecutionActor): Unit = executeJobAndAssertOutputs(backend, workflow.expectedResponse) - } def buildWorkflowDescriptor(workflowSource: WorkflowSource, inputFileAsJson: Option[String], options: WorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])), runtime: String = "", - labels: Labels = Labels.empty): BackendWorkflowDescriptor = { - val wdlNamespace = WdlNamespaceWithWorkflow.load(workflowSource.replaceAll("RUNTIME", runtime), - Seq.empty[Draft2ImportResolver]).get + labels: Labels = Labels.empty + ): BackendWorkflowDescriptor = { + val wdlNamespace = + WdlNamespaceWithWorkflow.load(workflowSource.replaceAll("RUNTIME", runtime), Seq.empty[Draft2ImportResolver]).get val executable = wdlNamespace.toWomExecutable(inputFileAsJson, NoIoFunctionSet, strictValidation = true) match { case Left(errors) => fail(s"Fail to build wom executable: ${errors.toList.mkString(", ")}") case Right(e) => e @@ -45,7 +49,7 @@ trait BackendSpec extends ScalaFutures with Matchers with ScaledTimeSpans { BackendWorkflowDescriptor( WorkflowId.randomId(), executable.entryPoint, - executable.resolvedExecutableInputs.flatMap({case (port, v) => v.select[WomValue] map { port -> _ }}), + executable.resolvedExecutableInputs.flatMap { case (port, v) => v.select[WomValue] map { port -> _ } }, options, labels, HogGroup("foo"), @@ -55,68 +59,84 @@ trait BackendSpec extends ScalaFutures with Matchers with ScaledTimeSpans { } def buildWdlWorkflowDescriptor(workflowSource: WorkflowSource, - inputFileAsJson: Option[String] = None, - options: WorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])), - runtime: String = "", - labels: Labels = Labels.empty): BackendWorkflowDescriptor = { - + inputFileAsJson: Option[String] = None, + options: WorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])), + runtime: String = "", + labels: Labels = Labels.empty + ): BackendWorkflowDescriptor = buildWorkflowDescriptor(workflowSource, inputFileAsJson, options, runtime, labels) - } - def fqnWdlMapToDeclarationMap(m: Map[String, WomValue]): Map[InputDefinition, WomValue] = { - m map { - case (fqn, v) => - val mockDeclaration = RequiredInputDefinition(fqn, v.womType) - mockDeclaration -> v + def fqnWdlMapToDeclarationMap(m: Map[String, WomValue]): Map[InputDefinition, WomValue] = + m map { case (fqn, v) => + val mockDeclaration = RequiredInputDefinition(fqn, v.womType) + mockDeclaration -> v } - } - def fqnMapToDeclarationMap(m: Map[OutputPort, WomValue]): Map[InputDefinition, WomValue] = { - m map { - case (outputPort, womValue) => RequiredInputDefinition(outputPort.name, womValue.womType) -> womValue + def fqnMapToDeclarationMap(m: Map[OutputPort, WomValue]): Map[InputDefinition, WomValue] = + m map { case (outputPort, womValue) => + RequiredInputDefinition(outputPort.name, womValue.womType) -> womValue } - } def jobDescriptorFromSingleCallWorkflow(workflowDescriptor: BackendWorkflowDescriptor, inputs: Map[String, WomValue], options: WorkflowOptions, - runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition]): BackendJobDescriptor = { - val call = workflowDescriptor.callable.graph.nodes.collectFirst({ case t: CommandCallNode => t}).get + runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] + ): BackendJobDescriptor = { + val call = workflowDescriptor.callable.graph.nodes.collectFirst { case t: CommandCallNode => t }.get val jobKey = BackendJobDescriptorKey(call, None, 1) val inputDeclarations: Map[InputDefinition, WomValue] = call.inputDefinitionMappings.map { - case (inputDef, resolved) => inputDef -> - resolved.select[WomValue].orElse( - resolved.select[WomExpression] - .map( - _.evaluateValue(inputs, NoIoFunctionSet).getOrElse(fail("Can't evaluate input")) + case (inputDef, resolved) => + inputDef -> + resolved + .select[WomValue] + .orElse( + resolved + .select[WomExpression] + .map( + _.evaluateValue(inputs, NoIoFunctionSet).getOrElse(fail("Can't evaluate input")) + ) ) - ).orElse( - resolved.select[OutputPort] flatMap { - case known if workflowDescriptor.knownValues.contains(known) => Option(workflowDescriptor.knownValues(known)) - case hasDefault if hasDefault.graphNode.isInstanceOf[OptionalGraphInputNodeWithDefault] => - Option(hasDefault.graphNode.asInstanceOf[OptionalGraphInputNodeWithDefault].default - .evaluateValue(inputs, NoIoFunctionSet).getOrElse(fail("Can't evaluate input"))) - case _ => None - } - ).getOrElse { - inputs(inputDef.name) - } + .orElse( + resolved.select[OutputPort] flatMap { + case known if workflowDescriptor.knownValues.contains(known) => + Option(workflowDescriptor.knownValues(known)) + case hasDefault if hasDefault.graphNode.isInstanceOf[OptionalGraphInputNodeWithDefault] => + Option( + hasDefault.graphNode + .asInstanceOf[OptionalGraphInputNodeWithDefault] + .default + .evaluateValue(inputs, NoIoFunctionSet) + .getOrElse(fail("Can't evaluate input")) + ) + case _ => None + } + ) + .getOrElse { + inputs(inputDef.name) + } }.toMap - val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.callable.runtimeAttributes, NoIoFunctionSet, Map.empty).getOrElse(fail("Failed to evaluate runtime attributes")) // .get is OK here because this is a test - val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) + val evaluatedAttributes = RuntimeAttributeDefinition + .evaluateRuntimeAttributes(call.callable.runtimeAttributes, NoIoFunctionSet, Map.empty) + .getOrElse(fail("Failed to evaluate runtime attributes")) // .get is OK here because this is a test + val runtimeAttributes = + RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, NoDocker, None, Map.empty) } def jobDescriptorFromSingleCallWorkflow(wdl: WorkflowSource, options: WorkflowOptions, - runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition]): BackendJobDescriptor = { + runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] + ): BackendJobDescriptor = { val workflowDescriptor = buildWdlWorkflowDescriptor(wdl) - val call = workflowDescriptor.callable.graph.nodes.collectFirst({ case t: CommandCallNode => t}).get + val call = workflowDescriptor.callable.graph.nodes.collectFirst { case t: CommandCallNode => t }.get val jobKey = BackendJobDescriptorKey(call, None, 1) val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.knownValues) - val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.callable.runtimeAttributes, NoIoFunctionSet, inputDeclarations).getOrElse(fail("Failed to evaluate runtime attributes")) // .get is OK here because this is a test - val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) + val evaluatedAttributes = RuntimeAttributeDefinition + .evaluateRuntimeAttributes(call.callable.runtimeAttributes, NoIoFunctionSet, inputDeclarations) + .getOrElse(fail("Failed to evaluate runtime attributes")) // .get is OK here because this is a test + val runtimeAttributes = + RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, NoDocker, None, Map.empty) } @@ -124,25 +144,33 @@ trait BackendSpec extends ScalaFutures with Matchers with ScaledTimeSpans { runtime: String, attempt: Int, options: WorkflowOptions, - runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition]): BackendJobDescriptor = { + runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] + ): BackendJobDescriptor = { val workflowDescriptor = buildWdlWorkflowDescriptor(wdl, runtime = runtime) - val call = workflowDescriptor.callable.graph.nodes.collectFirst({ case t: CommandCallNode => t}).get + val call = workflowDescriptor.callable.graph.nodes.collectFirst { case t: CommandCallNode => t }.get val jobKey = BackendJobDescriptorKey(call, None, attempt) val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.knownValues) - val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.callable.runtimeAttributes, NoIoFunctionSet, inputDeclarations).getOrElse(fail("Failed to evaluate runtime attributes")) // .get is OK here because this is a test - val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) + val evaluatedAttributes = RuntimeAttributeDefinition + .evaluateRuntimeAttributes(call.callable.runtimeAttributes, NoIoFunctionSet, inputDeclarations) + .getOrElse(fail("Failed to evaluate runtime attributes")) // .get is OK here because this is a test + val runtimeAttributes = + RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, NoDocker, None, Map.empty) } def assertResponse(executionResponse: BackendJobExecutionResponse, - expectedResponse: BackendJobExecutionResponse): Unit = { + expectedResponse: BackendJobExecutionResponse + ): Unit = { (executionResponse, expectedResponse) match { - case (JobSucceededResponse(_, _, responseOutputs, _, _, _, _), JobSucceededResponse(_, _, expectedOutputs, _, _, _, _)) => + case (JobSucceededResponse(_, _, responseOutputs, _, _, _, _), + JobSucceededResponse(_, _, expectedOutputs, _, _, _, _) + ) => responseOutputs.outputs.size shouldBe expectedOutputs.outputs.size - responseOutputs.outputs foreach { - case (fqn, out) => - val expectedOut = expectedOutputs.outputs.collectFirst({case (p, v) if p.name == fqn.name => v}) - expectedOut.getOrElse(fail(s"Output ${fqn.name} not found in ${expectedOutputs.outputs.map(_._1.name)}")).valueString shouldBe out.valueString + responseOutputs.outputs foreach { case (fqn, out) => + val expectedOut = expectedOutputs.outputs.collectFirst { case (p, v) if p.name == fqn.name => v } + expectedOut + .getOrElse(fail(s"Output ${fqn.name} not found in ${expectedOutputs.outputs.map(_._1.name)}")) + .valueString shouldBe out.valueString } case (JobFailedNonRetryableResponse(_, failure, _), JobFailedNonRetryableResponse(_, expectedFailure, _)) => failure.getClass shouldBe expectedFailure.getClass @@ -157,19 +185,20 @@ trait BackendSpec extends ScalaFutures with Matchers with ScaledTimeSpans { private def concatenateCauseMessages(t: Throwable): String = t match { case null => "" - case ae: AggregatedException => ae.getMessage + " " + ae.throwables.map(innerT => concatenateCauseMessages(innerT.getCause)).mkString("\n") + case ae: AggregatedException => + ae.getMessage + " " + ae.throwables.map(innerT => concatenateCauseMessages(innerT.getCause)).mkString("\n") case other: Throwable => other.getMessage + concatenateCauseMessages(t.getCause) } def executeJobAndAssertOutputs(backend: BackendJobExecutionActor, - expectedResponse: BackendJobExecutionResponse): Unit = { + expectedResponse: BackendJobExecutionResponse + ): Unit = whenReady(backend.execute) { executionResponse => assertResponse(executionResponse, expectedResponse) } - } def firstJobDescriptorKey(workflowDescriptor: BackendWorkflowDescriptor): BackendJobDescriptorKey = { - val call = workflowDescriptor.callable.graph.nodes.collectFirst({ case t: CommandCallNode => t}).get + val call = workflowDescriptor.callable.graph.nodes.collectFirst { case t: CommandCallNode => t }.get BackendJobDescriptorKey(call, None, 1) } } diff --git a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala index 55772d1de90..c4688d759f8 100644 --- a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala @@ -19,23 +19,23 @@ import wom.values._ import scala.concurrent.Future import scala.util.Try - -class BackendWorkflowInitializationActorSpec extends TestKitSuite - with AnyFlatSpecLike with Matchers with TableDrivenPropertyChecks { +class BackendWorkflowInitializationActorSpec + extends TestKitSuite + with AnyFlatSpecLike + with Matchers + with TableDrivenPropertyChecks { behavior of "BackendWorkflowInitializationActorSpec" - val testPredicateBackendWorkflowInitializationActorRef: - TestActorRef[TestPredicateBackendWorkflowInitializationActor] = + val testPredicateBackendWorkflowInitializationActorRef + : TestActorRef[TestPredicateBackendWorkflowInitializationActor] = TestActorRef[TestPredicateBackendWorkflowInitializationActor] - val testPredicateBackendWorkflowInitializationActor: - TestPredicateBackendWorkflowInitializationActor = + val testPredicateBackendWorkflowInitializationActor: TestPredicateBackendWorkflowInitializationActor = testPredicateBackendWorkflowInitializationActorRef.underlyingActor - val testContinueOnReturnCode: Option[WomValue] => Boolean = { + val testContinueOnReturnCode: Option[WomValue] => Boolean = testPredicateBackendWorkflowInitializationActor.continueOnReturnCodePredicate(valueRequired = false) - } val optionalConfig: Option[Config] = Option(TestConfig.optionalRuntimeConfig) @@ -62,22 +62,31 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite "read_int(\"bad file\")" ) + val starRow = Table( + "value", + "*" + ) + val invalidWdlValueRows = Table( "womValue", WomString(""), WomString("z"), - WomFloat(0.0D), + WomFloat(0.0d), WomArray(WomArrayType(WomBooleanType), Seq(WomBoolean(true))), - WomArray(WomArrayType(WomFloatType), Seq(WomFloat(0.0D))) + WomArray(WomArrayType(WomFloatType), Seq(WomFloat(0.0d))) ) forAll(booleanRows) { value => val womValue = WomBoolean(value) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(value)) } @@ -86,9 +95,13 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WomString(value.toString) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(value)) } @@ -97,7 +110,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WdlExpression.fromString(value.toString) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) // NOTE: expressions are never valid to validate } @@ -105,9 +120,13 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WomInteger(value) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -116,9 +135,13 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WomString(value.toString) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -127,7 +150,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WdlExpression.fromString(value.toString) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) // NOTE: expressions are never valid to validate } @@ -135,9 +160,13 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WomArray(WomArrayType(WomIntegerType), Seq(WomInteger(value))) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -146,9 +175,13 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WomArray(WomArrayType(WomStringType), Seq(WomString(value.toString))) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -157,7 +190,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WomArray(WomArrayType(WdlExpressionType), Seq(WdlExpression.fromString(value.toString))) val result = false testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) // NOTE: expressions are never valid to validate } @@ -165,19 +200,41 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WdlExpression.fromString(expression) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) // NOTE: expressions are never valid to validate } + forAll(starRow) { value => + val womValue = WomString(value) + val result = true + testContinueOnReturnCode(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) + val valid = + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + valid.isValid should be(result) + valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(true)) + } + forAll(invalidWdlValueRows) { womValue => val result = false testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.swap.toOption.get.toList should contain theSameElementsAs List( - "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." ) } @@ -197,17 +254,18 @@ class TestPredicateBackendWorkflowInitializationActor extends BackendWorkflowIni override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WomValue]] = throw new UnsupportedOperationException("coerceDefaultRuntimeAttributes") - override def beforeAll(): Future[Option[BackendInitializationData]] = throw new UnsupportedOperationException("beforeAll") + override def beforeAll(): Future[Option[BackendInitializationData]] = throw new UnsupportedOperationException( + "beforeAll" + ) override def validate(): Future[Unit] = throw new UnsupportedOperationException("validate") override protected def workflowDescriptor: BackendWorkflowDescriptor = throw new UnsupportedOperationException("workflowDescriptor") - override protected def configurationDescriptor: BackendConfigurationDescriptor = BackendConfigurationDescriptor(TestConfig.sampleBackendRuntimeConfig, ConfigFactory.empty()) + override protected def configurationDescriptor: BackendConfigurationDescriptor = + BackendConfigurationDescriptor(TestConfig.sampleBackendRuntimeConfig, ConfigFactory.empty()) - override def continueOnReturnCodePredicate(valueRequired: Boolean) - (wdlExpressionMaybe: Option[WomValue]): Boolean = { + override def continueOnReturnCodePredicate(valueRequired: Boolean)(wdlExpressionMaybe: Option[WomValue]): Boolean = super.continueOnReturnCodePredicate(valueRequired)(wdlExpressionMaybe) - } } diff --git a/backend/src/test/scala/cromwell/backend/MemorySizeSpec.scala b/backend/src/test/scala/cromwell/backend/MemorySizeSpec.scala index ac3c591f5df..57acada06d3 100644 --- a/backend/src/test/scala/cromwell/backend/MemorySizeSpec.scala +++ b/backend/src/test/scala/cromwell/backend/MemorySizeSpec.scala @@ -66,7 +66,7 @@ class MemorySizeSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers memorySize.to(newUnit) shouldEqual result } } - + it should "round trip" in { List( "2 GB" diff --git a/backend/src/test/scala/cromwell/backend/OutputEvaluatorSpec.scala b/backend/src/test/scala/cromwell/backend/OutputEvaluatorSpec.scala index 0959f92577b..997fd6f5df5 100644 --- a/backend/src/test/scala/cromwell/backend/OutputEvaluatorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/OutputEvaluatorSpec.scala @@ -24,7 +24,7 @@ class OutputEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc behavior of "OutputEvaluator" private val FutureTimeout = 20.seconds - final implicit val blockingEc: ExecutionContextExecutor = ExecutionContext.fromExecutor( + implicit final val blockingEc: ExecutionContextExecutor = ExecutionContext.fromExecutor( Executors.newCachedThreadPool() ) @@ -32,50 +32,56 @@ class OutputEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc private def o1Expression = new WomExpression { override def sourceString: String = "o1" override def inputs: Set[String] = Set("input") - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = { + override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = Validated.fromOption(inputValues.get("input"), NonEmptyList.one("Can't find a value for 'input'")) - } - override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = throw new UnsupportedOperationException - override def evaluateFiles(inputTypes: Map[String, WomValue], ioFunctionSet: IoFunctionSet, coerceTo: WomType): ErrorOr[Set[FileEvaluation]] = throw new UnsupportedOperationException + override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = + throw new UnsupportedOperationException + override def evaluateFiles(inputTypes: Map[String, WomValue], + ioFunctionSet: IoFunctionSet, + coerceTo: WomType + ): ErrorOr[Set[FileEvaluation]] = throw new UnsupportedOperationException } // Depends on a previous output private def o2Expression = new WomExpression { override def sourceString: String = "o2" override def inputs: Set[String] = Set("o1") - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = { + override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = Validated.fromOption(inputValues.get("o1"), NonEmptyList.one("Can't find a value for 'o1'")) - } - override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = throw new UnsupportedOperationException - override def evaluateFiles(inputTypes: Map[String, WomValue], ioFunctionSet: IoFunctionSet, coerceTo: WomType): ErrorOr[Set[FileEvaluation]] = throw new UnsupportedOperationException + override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = + throw new UnsupportedOperationException + override def evaluateFiles(inputTypes: Map[String, WomValue], + ioFunctionSet: IoFunctionSet, + coerceTo: WomType + ): ErrorOr[Set[FileEvaluation]] = throw new UnsupportedOperationException } private def invalidWomExpression1 = new WomExpression { override def sourceString: String = "invalid1" override def inputs: Set[String] = Set.empty - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = { + override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = "Invalid expression 1".invalidNel - } - override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = { + override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = "Invalid expression 1".invalidNel - } - override def evaluateFiles(inputTypes: Map[String, WomValue], ioFunctionSet: IoFunctionSet, coerceTo: WomType): ErrorOr[Set[FileEvaluation]] = { + override def evaluateFiles(inputTypes: Map[String, WomValue], + ioFunctionSet: IoFunctionSet, + coerceTo: WomType + ): ErrorOr[Set[FileEvaluation]] = "Invalid expression 1".invalidNel - } } private def invalidWomExpression2 = new WomExpression { override def sourceString: String = "invalid2" override def inputs: Set[String] = Set.empty - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = { + override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = "Invalid expression 2".invalidNel - } - override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = { + override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = "Invalid expression 2".invalidNel - } - override def evaluateFiles(inputTypes: Map[String, WomValue], ioFunctionSet: IoFunctionSet, coerceTo: WomType): ErrorOr[Set[FileEvaluation]] = { + override def evaluateFiles(inputTypes: Map[String, WomValue], + ioFunctionSet: IoFunctionSet, + coerceTo: WomType + ): ErrorOr[Set[FileEvaluation]] = "Invalid expression 2".invalidNel - } } val exception = new Exception("Expression evaluation exception") @@ -83,15 +89,15 @@ class OutputEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc private def throwingWomExpression = new WomExpression { override def sourceString: String = "throwing" override def inputs: Set[String] = Set.empty - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = { + override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = throw exception - } - override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = { + override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = throw exception - } - override def evaluateFiles(inputTypes: Map[String, WomValue], ioFunctionSet: IoFunctionSet, coerceTo: WomType): ErrorOr[Set[FileEvaluation]] = { + override def evaluateFiles(inputTypes: Map[String, WomValue], + ioFunctionSet: IoFunctionSet, + coerceTo: WomType + ): ErrorOr[Set[FileEvaluation]] = throw exception - } } val mockInputs: Map[InputDefinition, WomValue] = Map( @@ -99,7 +105,7 @@ class OutputEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc ) it should "evaluate valid jobs outputs" in { - val mockOutputs = List ( + val mockOutputs = List( OutputDefinition("o1", WomIntegerType, o1Expression), OutputDefinition("o2", WomIntegerType, o2Expression) ) @@ -109,16 +115,19 @@ class OutputEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc val jobDescriptor = BackendJobDescriptor(null, key, null, mockInputs, null, None, null) Await.result(OutputEvaluator.evaluateOutputs(jobDescriptor, NoIoFunctionSet), FutureTimeout) match { - case ValidJobOutputs(outputs) => outputs shouldBe CallOutputs(Map( - jobDescriptor.taskCall.outputPorts.find(_.name == "o1").get -> WomInteger(5), - jobDescriptor.taskCall.outputPorts.find(_.name == "o2").get -> WomInteger(5) - )) + case ValidJobOutputs(outputs) => + outputs shouldBe CallOutputs( + Map( + jobDescriptor.taskCall.outputPorts.find(_.name == "o1").get -> WomInteger(5), + jobDescriptor.taskCall.outputPorts.find(_.name == "o2").get -> WomInteger(5) + ) + ) case _ => fail("Failed to evaluate outputs") } } it should "return an InvalidJobOutputs if the evaluation returns ErrorOrs" in { - val mockOutputs = List ( + val mockOutputs = List( OutputDefinition("o1", WomIntegerType, o1Expression), OutputDefinition("invalid1", WomIntegerType, invalidWomExpression1), OutputDefinition("invalid2", WomIntegerType, invalidWomExpression2) @@ -129,15 +138,17 @@ class OutputEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc val jobDescriptor = BackendJobDescriptor(null, key, null, mockInputs, null, None, null) Await.result(OutputEvaluator.evaluateOutputs(jobDescriptor, NoIoFunctionSet), FutureTimeout) match { - case InvalidJobOutputs(errors) => errors shouldBe NonEmptyList.of( - "Bad output 'invalid1': Invalid expression 1", "Bad output 'invalid2': Invalid expression 2" - ) + case InvalidJobOutputs(errors) => + errors shouldBe NonEmptyList.of( + "Bad output 'invalid1': Invalid expression 1", + "Bad output 'invalid2': Invalid expression 2" + ) case _ => fail("Output evaluation should have failed") } } it should "return an JobOutputsEvaluationException if the evaluation throws an exception" in { - val mockOutputs = List ( + val mockOutputs = List( OutputDefinition("o1", WomIntegerType, o1Expression), OutputDefinition("invalid1", WomIntegerType, throwingWomExpression) ) diff --git a/backend/src/test/scala/cromwell/backend/RuntimeAttributeValidationSpec.scala b/backend/src/test/scala/cromwell/backend/RuntimeAttributeValidationSpec.scala index 99bec0baebd..91b6ab06617 100644 --- a/backend/src/test/scala/cromwell/backend/RuntimeAttributeValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/RuntimeAttributeValidationSpec.scala @@ -19,12 +19,14 @@ class RuntimeAttributeValidationSpec extends AnyFlatSpec with Matchers with Scal val defaultValue = womValue.asWomExpression val validator: Option[WomExpression] => Boolean = _.contains(defaultValue) assert( - BackendWorkflowInitializationActor.validateRuntimeAttributes( - taskName = taskName, - defaultRuntimeAttributes = defaultRuntimeAttributes, - runtimeAttributes = Map.empty, - runtimeAttributeValidators = Map((attributeName, validator)), - ).isValid + BackendWorkflowInitializationActor + .validateRuntimeAttributes( + taskName = taskName, + defaultRuntimeAttributes = defaultRuntimeAttributes, + runtimeAttributes = Map.empty, + runtimeAttributeValidators = Map((attributeName, validator)) + ) + .isValid ) } @@ -33,12 +35,14 @@ class RuntimeAttributeValidationSpec extends AnyFlatSpec with Matchers with Scal val defaultRuntimeAttributes = Map(attributeName -> womValue) assert( - BackendWorkflowInitializationActor.validateRuntimeAttributes( - taskName = taskName, - defaultRuntimeAttributes = defaultRuntimeAttributes, - runtimeAttributes = Map.empty, - runtimeAttributeValidators = Map((attributeName, (_: Option[WomExpression]) => false)), - ).isInvalid + BackendWorkflowInitializationActor + .validateRuntimeAttributes( + taskName = taskName, + defaultRuntimeAttributes = defaultRuntimeAttributes, + runtimeAttributes = Map.empty, + runtimeAttributeValidators = Map((attributeName, (_: Option[WomExpression]) => false)) + ) + .isInvalid ) } @@ -49,47 +53,51 @@ class RuntimeAttributeValidationSpec extends AnyFlatSpec with Matchers with Scal val validator: Option[WomExpression] => Boolean = _.contains(runtimeWomExpression) assert( - BackendWorkflowInitializationActor.validateRuntimeAttributes( - taskName = taskName, - defaultRuntimeAttributes = defaultRuntimeAttributes, - runtimeAttributes = runtimeAttributes, - runtimeAttributeValidators = Map((attributeName, validator)), - ).isValid + BackendWorkflowInitializationActor + .validateRuntimeAttributes( + taskName = taskName, + defaultRuntimeAttributes = defaultRuntimeAttributes, + runtimeAttributes = runtimeAttributes, + runtimeAttributeValidators = Map((attributeName, validator)) + ) + .isValid ) } it should "fail validation if no setting is present but it should be" in forAll { (taskName: String, attributeName: String) => - val validator: Option[WomExpression] => Boolean = { case None => false case Some(x) => throw new RuntimeException(s"expecting the runtime validator to receive a None but got $x") } assert( - BackendWorkflowInitializationActor.validateRuntimeAttributes( - taskName = taskName, - defaultRuntimeAttributes = Map.empty, - runtimeAttributes = Map.empty, - runtimeAttributeValidators = Map((attributeName, validator)), - ).isInvalid + BackendWorkflowInitializationActor + .validateRuntimeAttributes( + taskName = taskName, + defaultRuntimeAttributes = Map.empty, + runtimeAttributes = Map.empty, + runtimeAttributeValidators = Map((attributeName, validator)) + ) + .isInvalid ) } it should "use the taskName and attribute name in correct places for failures" in forAll { (taskName: String, attributeName: String) => - val validator: Option[WomExpression] => Boolean = { case None => false case Some(x) => throw new RuntimeException(s"expecting the runtime validator to receive a None but got $x") } - BackendWorkflowInitializationActor.validateRuntimeAttributes(taskName, Map.empty, Map.empty, Map((attributeName,validator))).fold( - { errors => - val error = errors.toList.head - withClue("attribute name should be set correctly")(error.runtimeAttributeName shouldBe attributeName) - withClue("task name should be set correctly")(error.jobTag shouldBe taskName) - }, - _ => fail("expecting validation to fail!") - ) + BackendWorkflowInitializationActor + .validateRuntimeAttributes(taskName, Map.empty, Map.empty, Map((attributeName, validator))) + .fold( + { errors => + val error = errors.toList.head + withClue("attribute name should be set correctly")(error.runtimeAttributeName shouldBe attributeName) + withClue("task name should be set correctly")(error.jobTag shouldBe taskName) + }, + _ => fail("expecting validation to fail!") + ) } } diff --git a/backend/src/test/scala/cromwell/backend/TestConfig.scala b/backend/src/test/scala/cromwell/backend/TestConfig.scala index 42050dbe323..82e81818d13 100644 --- a/backend/src/test/scala/cromwell/backend/TestConfig.scala +++ b/backend/src/test/scala/cromwell/backend/TestConfig.scala @@ -30,7 +30,8 @@ object TestConfig { lazy val sampleBackendRuntimeConfig = ConfigFactory.parseString(sampleBackendRuntimeConfigString) - lazy val allRuntimeAttrsConfig = ConfigFactory.parseString(allBackendRuntimeAttrsString).getConfig("default-runtime-attributes") + lazy val allRuntimeAttrsConfig = + ConfigFactory.parseString(allBackendRuntimeAttrsString).getConfig("default-runtime-attributes") lazy val optionalRuntimeConfig = sampleBackendRuntimeConfig.getConfig("default-runtime-attributes") diff --git a/backend/src/test/scala/cromwell/backend/io/DirectoryFunctionsSpec.scala b/backend/src/test/scala/cromwell/backend/io/DirectoryFunctionsSpec.scala index 1a8a101acf8..94a5b00627c 100644 --- a/backend/src/test/scala/cromwell/backend/io/DirectoryFunctionsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/DirectoryFunctionsSpec.scala @@ -18,10 +18,11 @@ class DirectoryFunctionsSpec extends AnyFlatSpec with CromwellTimeoutSpec with M override def copyFile(source: String, destination: String) = throw new UnsupportedOperationException() override def glob(pattern: String) = throw new UnsupportedOperationException() override def size(path: String) = throw new UnsupportedOperationException() - override def readFile(path: String, maxBytes: Option[Int], failOnOverflow: Boolean) = throw new UnsupportedOperationException() + override def readFile(path: String, maxBytes: Option[Int], failOnOverflow: Boolean) = + throw new UnsupportedOperationException() override def pathFunctions = throw new UnsupportedOperationException() override def writeFile(path: String, content: String) = throw new UnsupportedOperationException() - override implicit def ec = throw new UnsupportedOperationException() + implicit override def ec = throw new UnsupportedOperationException() override def createTemporaryDirectory(name: Option[String]) = throw new UnsupportedOperationException() override def asyncIo = throw new UnsupportedOperationException() } @@ -32,13 +33,12 @@ class DirectoryFunctionsSpec extends AnyFlatSpec with CromwellTimeoutSpec with M val innerDir = (rootDir / "innerDir").createDirectories() val link = innerDir / "linkToRootDirInInnerDir" link.symbolicLinkTo(rootDir) - - def listRecursively(path: String)(visited: Vector[String] = Vector.empty): Iterator[String] = { + + def listRecursively(path: String)(visited: Vector[String] = Vector.empty): Iterator[String] = Await.result(functions.listDirectory(path)(visited), Duration.Inf) flatMap { case IoFile(v) => List(v) case IoDirectory(v) => List(v) ++ listRecursively(v)(visited :+ path) } - } listRecursively(rootDir.pathAsString)().toList shouldBe List(innerDir, link).map(_.pathAsString) } diff --git a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala index 286ed796311..21709bfda58 100644 --- a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala @@ -26,7 +26,7 @@ class JobPathsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wi | } """.stripMargin - val backendConfig = ConfigFactory.parseString(configString) + val backendConfig = ConfigFactory.parseString(configString) val defaultBackendConfigDescriptor = BackendConfigurationDescriptor(backendConfig, TestConfig.globalConfig) "JobPaths" should "provide correct paths for a job" in { @@ -55,8 +55,9 @@ class JobPathsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wi fullPath(s"/cromwell-executions/wf_hello/$id/call-hello") jobPaths.callExecutionDockerRoot.pathAsString shouldBe fullPath(s"/cromwell-executions/wf_hello/$id/call-hello/execution") - jobPaths.toDockerPath(DefaultPathBuilder.get( - s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stdout")).pathAsString shouldBe + jobPaths + .toDockerPath(DefaultPathBuilder.get(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stdout")) + .pathAsString shouldBe fullPath(s"/cromwell-executions/wf_hello/$id/call-hello/execution/stdout") jobPaths.toDockerPath(DefaultPathBuilder.get("/cromwell-executions/dock/path")).pathAsString shouldBe fullPath("/cromwell-executions/dock/path") diff --git a/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala b/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala index 8e280eadf4e..f956607a573 100644 --- a/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala +++ b/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala @@ -7,25 +7,26 @@ object TestWorkflows { case class TestWorkflow(workflowDescriptor: BackendWorkflowDescriptor, config: BackendConfigurationDescriptor, - expectedResponse: BackendJobExecutionResponse) + expectedResponse: BackendJobExecutionResponse + ) val HelloWorld = s""" - |task hello { - | String addressee = "you " - | command { - | echo "Hello $${addressee}!" - | } - | output { - | String salutation = read_string(stdout()) - | } - | - | RUNTIME - |} - | - |workflow wf_hello { - | call hello - |} + |task hello { + | String addressee = "you " + | command { + | echo "Hello $${addressee}!" + | } + | output { + | String salutation = read_string(stdout()) + | } + | + | RUNTIME + |} + | + |workflow wf_hello { + | call hello + |} """.stripMargin val GoodbyeWorld = @@ -46,25 +47,25 @@ object TestWorkflows { val InputFiles = s""" - |task localize { - | File inputFileFromJson - | File inputFileFromCallInputs - | command { - | cat $${inputFileFromJson} - | echo "" - | cat $${inputFileFromCallInputs} - | } - | output { - | Array[String] out = read_lines(stdout()) - | } - | - | RUNTIME - |} - | - |workflow wf_localize { - | File workflowFile - | call localize { input: inputFileFromCallInputs = workflowFile } - |} + |task localize { + | File inputFileFromJson + | File inputFileFromCallInputs + | command { + | cat $${inputFileFromJson} + | echo "" + | cat $${inputFileFromCallInputs} + | } + | output { + | Array[String] out = read_lines(stdout()) + | } + | + | RUNTIME + |} + | + |workflow wf_localize { + | File workflowFile + | call localize { input: inputFileFromCallInputs = workflowFile } + |} """.stripMargin val Sleep20 = @@ -83,25 +84,25 @@ object TestWorkflows { val Scatter = s""" - |task scattering { - | Int intNumber - | command { - | echo $${intNumber} - | } - | output { - | Int out = read_string(stdout()) - | } - |} - | - |workflow wf_scattering { - | Array[Int] numbers = [1, 2, 3] - | scatter (i in numbers) { - | call scattering { input: intNumber = i } - | } - |} + |task scattering { + | Int intNumber + | command { + | echo $${intNumber} + | } + | output { + | Int out = read_string(stdout()) + | } + |} + | + |workflow wf_scattering { + | Array[Int] numbers = [1, 2, 3] + | scatter (i in numbers) { + | call scattering { input: intNumber = i } + | } + |} """.stripMargin - val OutputProcess = { + val OutputProcess = """ |task localize { | File inputFile @@ -121,9 +122,8 @@ object TestWorkflows { | call localize |} """.stripMargin - } - val MissingOutputProcess = { + val MissingOutputProcess = """ |task localize { | command { @@ -137,5 +137,4 @@ object TestWorkflows { | call localize |} """.stripMargin - } } diff --git a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala index cb53276f499..790d42a7887 100644 --- a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala @@ -16,16 +16,15 @@ class WorkflowPathsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche def createConfig(values: Map[String, String]): Config = { val config = mock[Config] - values.foreach { - case (key: String, value: String) => - when(config.hasPath(key)).thenReturn(true) - when(config.getString(key)).thenReturn(value) + values.foreach { case (key: String, value: String) => + when(config.hasPath(key)).thenReturn(true) + when(config.getString(key)).thenReturn(value) } config } def rootConfig(root: Option[String], dockerRoot: Option[String]): Config = { - val values: Map[String,String] = root.map("root" -> _).toMap ++ dockerRoot.map("dockerRoot" -> _).toMap + val values: Map[String, String] = root.map("root" -> _).toMap ++ dockerRoot.map("dockerRoot" -> _).toMap createConfig(values) } @@ -82,9 +81,12 @@ class WorkflowPathsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche val expectedDockerRoot = dockerRoot.getOrElse(WorkflowPathsWithDocker.DefaultDockerRoot) workflowPaths.workflowRoot.pathAsString shouldBe - DefaultPathBuilder.get( - s"$expectedRoot/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId" - ).toAbsolutePath.pathAsString + DefaultPathBuilder + .get( + s"$expectedRoot/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId" + ) + .toAbsolutePath + .pathAsString workflowPaths.dockerWorkflowRoot.pathAsString shouldBe s"$expectedDockerRoot/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId" () } diff --git a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala index 0497c8a1ba1..35c57983e6e 100644 --- a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala @@ -14,37 +14,36 @@ import spray.json.{JsArray, JsBoolean, JsNumber, JsObject, JsValue} import wom.RuntimeAttributesKeys._ import wom.values._ -class StandardValidatedRuntimeAttributesBuilderSpec extends AnyWordSpecLike with CromwellTimeoutSpec with Matchers - with MockSugar { +class StandardValidatedRuntimeAttributesBuilderSpec + extends AnyWordSpecLike + with CromwellTimeoutSpec + with Matchers + with MockSugar { val HelloWorld: String = s""" - |task hello { - | String addressee = "you" - | command { - | echo "Hello $${addressee}!" - | } - | output { - | String salutation = read_string(stdout()) - | } - | - | RUNTIME - |} - | - |workflow hello { - | call hello - |} + |task hello { + | String addressee = "you" + | command { + | echo "Hello $${addressee}!" + | } + | output { + | String salutation = read_string(stdout()) + | } + | + | RUNTIME + |} + | + |workflow hello { + | call hello + |} """.stripMargin + val defaultRuntimeAttributes: Map[String, Any] = + Map(DockerKey -> None, FailOnStderrKey -> false, ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(0))) - val defaultRuntimeAttributes: Map[String, Any] = Map( - DockerKey -> None, - FailOnStderrKey -> false, - ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(0))) - - def workflowOptionsWithDefaultRuntimeAttributes(defaults: Map[String, JsValue]): WorkflowOptions = { + def workflowOptionsWithDefaultRuntimeAttributes(defaults: Map[String, JsValue]): WorkflowOptions = WorkflowOptions(JsObject(Map("default_runtime_attributes" -> JsObject(defaults)))) - } "SharedFileSystemValidatedRuntimeAttributesBuilder" should { "validate when there are no runtime attributes defined" in { @@ -75,8 +74,11 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends AnyWordSpecLike with var warnings = List.empty[Any] val mockLogger = mock[Logger] mockLogger.warn(anyString).answers((warnings :+= _): Any => Unit) - assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, - includeDockerSupport = false, logger = mockLogger) + assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, + expectedRuntimeAttributes, + includeDockerSupport = false, + logger = mockLogger + ) warnings should contain theSameElementsAs List("Unrecognized runtime attribute keys: docker") } @@ -97,46 +99,60 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends AnyWordSpecLike with var warnings = List.empty[Any] val mockLogger = mock[Logger] mockLogger.warn(anyString).answers((warnings :+= _): Any => Unit) - assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, - includeDockerSupport = false, logger = mockLogger) + assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, + expectedRuntimeAttributes, + includeDockerSupport = false, + logger = mockLogger + ) warnings should contain theSameElementsAs List("Unrecognized runtime attribute keys: docker") } "fail to validate an invalid failOnStderr entry" in { val runtimeAttributes = Map("failOnStderr" -> WomString("yes")) - assertRuntimeAttributesFailedCreation(runtimeAttributes, - "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'") + assertRuntimeAttributesFailedCreation( + runtimeAttributes, + "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'" + ) } "use workflow options as default if failOnStdErr key is missing" in { val expectedRuntimeAttributes = defaultRuntimeAttributes + (FailOnStderrKey -> true) val workflowOptions = workflowOptionsWithDefaultRuntimeAttributes(Map(FailOnStderrKey -> JsBoolean(true))) val runtimeAttributes = Map.empty[String, WomValue] - assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, - workflowOptions = workflowOptions) + assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, + expectedRuntimeAttributes, + workflowOptions = workflowOptions + ) } "validate a valid continueOnReturnCode entry" in { val runtimeAttributes = Map("continueOnReturnCode" -> WomInteger(1)) - val expectedRuntimeAttributes = defaultRuntimeAttributes + (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1))) + val expectedRuntimeAttributes = + defaultRuntimeAttributes + (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1))) assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } "fail to validate an invalid continueOnReturnCode entry" in { val runtimeAttributes = Map("continueOnReturnCode" -> WomString("value")) - assertRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]") + assertRuntimeAttributesFailedCreation( + runtimeAttributes, + "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." + ) } "use workflow options as default if continueOnReturnCode key is missing" in { val expectedRuntimeAttributes = defaultRuntimeAttributes + (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1, 2))) val workflowOptions = workflowOptionsWithDefaultRuntimeAttributes( - Map(ContinueOnReturnCodeKey -> JsArray(Vector(JsNumber(1), JsNumber(2))))) + Map(ContinueOnReturnCodeKey -> JsArray(Vector(JsNumber(1), JsNumber(2)))) + ) val runtimeAttributes = Map.empty[String, WomValue] - assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, - workflowOptions = workflowOptions) + assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, + expectedRuntimeAttributes, + workflowOptions = workflowOptions + ) } - } val defaultLogger: Logger = LoggerFactory.getLogger(classOf[StandardValidatedRuntimeAttributesBuilderSpec]) @@ -148,44 +164,52 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends AnyWordSpecLike with expectedRuntimeAttributes: Map[String, Any], includeDockerSupport: Boolean = true, workflowOptions: WorkflowOptions = emptyWorkflowOptions, - logger: Logger = defaultLogger): Unit = { + logger: Logger = defaultLogger + ): Unit = { val builder = if (includeDockerSupport) { - StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig).withValidation(DockerValidation.optional) + StandardValidatedRuntimeAttributesBuilder + .default(mockBackendRuntimeConfig) + .withValidation(DockerValidation.optional) } else { StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig) } val runtimeAttributeDefinitions = builder.definitions.toSet - val addDefaultsToAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _ + val addDefaultsToAttributes = + RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _ val validatedRuntimeAttributes = builder.build(addDefaultsToAttributes(runtimeAttributes), logger) - val docker = RuntimeAttributesValidation.extractOption( - DockerValidation.instance, validatedRuntimeAttributes) - val failOnStderr = RuntimeAttributesValidation.extract( - FailOnStderrValidation.instance, validatedRuntimeAttributes) - val continueOnReturnCode = RuntimeAttributesValidation.extract( - ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) + val docker = RuntimeAttributesValidation.extractOption(DockerValidation.instance, validatedRuntimeAttributes) + val failOnStderr = RuntimeAttributesValidation.extract(FailOnStderrValidation.instance, validatedRuntimeAttributes) + val continueOnReturnCode = + RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) docker should be(expectedRuntimeAttributes(DockerKey).asInstanceOf[Option[String]]) failOnStderr should be(expectedRuntimeAttributes(FailOnStderrKey).asInstanceOf[Boolean]) continueOnReturnCode should be( - expectedRuntimeAttributes(ContinueOnReturnCodeKey).asInstanceOf[ContinueOnReturnCode]) + expectedRuntimeAttributes(ContinueOnReturnCodeKey) + ) () } - private def assertRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WomValue], exMsg: String, + private def assertRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WomValue], + exMsg: String, supportsDocker: Boolean = true, workflowOptions: WorkflowOptions = emptyWorkflowOptions, - logger: Logger = defaultLogger): Unit = { + logger: Logger = defaultLogger + ): Unit = { val thrown = the[RuntimeException] thrownBy { val builder = if (supportsDocker) { - StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig).withValidation(DockerValidation.optional) + StandardValidatedRuntimeAttributesBuilder + .default(mockBackendRuntimeConfig) + .withValidation(DockerValidation.optional) } else { StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig) } val runtimeAttributeDefinitions = builder.definitions.toSet - val addDefaultsToAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _ + val addDefaultsToAttributes = + RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _ builder.build(addDefaultsToAttributes(runtimeAttributes), logger) } diff --git a/backend/src/test/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManagerSpec.scala b/backend/src/test/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManagerSpec.scala index fdfa7f97e04..f50cba5834c 100644 --- a/backend/src/test/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManagerSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManagerSpec.scala @@ -8,11 +8,10 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import spray.json._ - class CallCachingBlacklistManagerSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "CallCachingBlacklistManager" - //noinspection RedundantDefaultArgument + // noinspection RedundantDefaultArgument val workflowSourcesNoGrouping = WorkflowSourceFilesWithoutImports( workflowSource = None, workflowUrl = None, diff --git a/backend/src/test/scala/cromwell/backend/standard/callcaching/RootWorkflowHashCacheActorSpec.scala b/backend/src/test/scala/cromwell/backend/standard/callcaching/RootWorkflowHashCacheActorSpec.scala index f546dd63630..0b462c74b81 100644 --- a/backend/src/test/scala/cromwell/backend/standard/callcaching/RootWorkflowHashCacheActorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/callcaching/RootWorkflowHashCacheActorSpec.scala @@ -13,8 +13,7 @@ import org.scalatest.flatspec.AnyFlatSpecLike import scala.concurrent.duration._ -class RootWorkflowHashCacheActorSpec extends TestKitSuite with ImplicitSender - with AnyFlatSpecLike { +class RootWorkflowHashCacheActorSpec extends TestKitSuite with ImplicitSender with AnyFlatSpecLike { private val fakeWorkflowId = WorkflowId.randomId() private val fakeFileName = "fakeFileName" @@ -25,17 +24,24 @@ class RootWorkflowHashCacheActorSpec extends TestKitSuite with ImplicitSender props = Props(new RootWorkflowFileHashCacheActor(ioActorProbe.ref, fakeWorkflowId) { override lazy val defaultIoTimeout: FiniteDuration = 1.second }), - name = "rootWorkflowFileHashCacheActor-without-timer", + name = "rootWorkflowFileHashCacheActor-without-timer" ) - val ioHashCommandWithContext = IoHashCommandWithContext(DefaultIoHashCommand(DefaultPathBuilder.build("").get), FileHashContext(HashKey(checkForHitOrMiss = false, List.empty), fakeFileName)) + val ioHashCommandWithContext = + IoHashCommandWithContext(DefaultIoHashCommand(DefaultPathBuilder.build("").get), + FileHashContext(HashKey(checkForHitOrMiss = false, List.empty), fakeFileName) + ) rootWorkflowFileHashCacheActor ! ioHashCommandWithContext - //wait for timeout + // wait for timeout Thread.sleep(2000) EventFilter.info(msgIoAckWithNoRequesters.format(fakeFileName), occurrences = 1).intercept { - ioActorProbe.send(rootWorkflowFileHashCacheActor, ioHashCommandWithContext.fileHashContext -> IoSuccess(ioHashCommandWithContext.ioHashCommand, "Successful result")) + ioActorProbe.send(rootWorkflowFileHashCacheActor, + ioHashCommandWithContext.fileHashContext -> IoSuccess(ioHashCommandWithContext.ioHashCommand, + "Successful result" + ) + ) } } @@ -46,17 +52,32 @@ class RootWorkflowHashCacheActorSpec extends TestKitSuite with ImplicitSender // Effectively disabling automatic timeout firing here. We'll send RequestTimeout ourselves override lazy val defaultIoTimeout: FiniteDuration = 1.hour }), - "rootWorkflowFileHashCacheActor-with-timer", + "rootWorkflowFileHashCacheActor-with-timer" ) - val ioHashCommandWithContext = IoHashCommandWithContext(DefaultIoHashCommand(DefaultPathBuilder.build("").get), FileHashContext(HashKey(checkForHitOrMiss = false, List.empty), fakeFileName)) + val ioHashCommandWithContext = + IoHashCommandWithContext(DefaultIoHashCommand(DefaultPathBuilder.build("").get), + FileHashContext(HashKey(checkForHitOrMiss = false, List.empty), fakeFileName) + ) rootWorkflowFileHashCacheActor ! ioHashCommandWithContext val hashVal = "Success" - EventFilter.info(msgTimeoutAfterIoAck.format(s"FileHashSuccess($hashVal)", ioHashCommandWithContext.fileHashContext.file), occurrences = 1).intercept { - ioActorProbe.send(rootWorkflowFileHashCacheActor, (ioHashCommandWithContext.fileHashContext, IoSuccess(ioHashCommandWithContext.ioHashCommand, hashVal))) - Thread.sleep(2000) // wait for actor to put value into cache - ioActorProbe.send(rootWorkflowFileHashCacheActor, RequestTimeout(ioHashCommandWithContext.fileHashContext -> ioHashCommandWithContext.ioHashCommand, rootWorkflowFileHashCacheActor)) - } + EventFilter + .info(msgTimeoutAfterIoAck.format(s"FileHashSuccess($hashVal)", ioHashCommandWithContext.fileHashContext.file), + occurrences = 1 + ) + .intercept { + ioActorProbe.send( + rootWorkflowFileHashCacheActor, + (ioHashCommandWithContext.fileHashContext, IoSuccess(ioHashCommandWithContext.ioHashCommand, hashVal)) + ) + Thread.sleep(2000) // wait for actor to put value into cache + ioActorProbe.send( + rootWorkflowFileHashCacheActor, + RequestTimeout(ioHashCommandWithContext.fileHashContext -> ioHashCommandWithContext.ioHashCommand, + rootWorkflowFileHashCacheActor + ) + ) + } } } diff --git a/backend/src/test/scala/cromwell/backend/standard/callcaching/StandardFileHashingActorSpec.scala b/backend/src/test/scala/cromwell/backend/standard/callcaching/StandardFileHashingActorSpec.scala index c7463cc50a0..2c7ec3ba726 100644 --- a/backend/src/test/scala/cromwell/backend/standard/callcaching/StandardFileHashingActorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/callcaching/StandardFileHashingActorSpec.scala @@ -18,8 +18,12 @@ import scala.concurrent.duration._ import scala.util.control.NoStackTrace import scala.util.{Failure, Try} -class StandardFileHashingActorSpec extends TestKitSuite with ImplicitSender - with AnyFlatSpecLike with Matchers with MockSugar { +class StandardFileHashingActorSpec + extends TestKitSuite + with ImplicitSender + with AnyFlatSpecLike + with Matchers + with MockSugar { behavior of "StandardFileHashingActor" @@ -127,20 +131,21 @@ object StandardFileHashingActorSpec { def defaultParams(): StandardFileHashingActorParams = defaultParams(testing, testing, testing, testing, testing) - def ioActorParams(ioActor: ActorRef): StandardFileHashingActorParams = { - defaultParams(withJobDescriptor = testing, + def ioActorParams(ioActor: ActorRef): StandardFileHashingActorParams = + defaultParams( + withJobDescriptor = testing, withConfigurationDescriptor = testing, withIoActor = ioActor, withServiceRegistryActor = testing, - withBackendInitializationDataOption = testing) - } + withBackendInitializationDataOption = testing + ) def defaultParams(withJobDescriptor: => BackendJobDescriptor, withConfigurationDescriptor: => BackendConfigurationDescriptor, withIoActor: => ActorRef, withServiceRegistryActor: => ActorRef, withBackendInitializationDataOption: => Option[BackendInitializationData] - ): StandardFileHashingActorParams = new StandardFileHashingActorParams { + ): StandardFileHashingActorParams = new StandardFileHashingActorParams { override def jobDescriptor: BackendJobDescriptor = withJobDescriptor @@ -150,10 +155,10 @@ object StandardFileHashingActorSpec { override def serviceRegistryActor: ActorRef = withServiceRegistryActor - override def backendInitializationDataOption: Option[BackendInitializationData] = withBackendInitializationDataOption + override def backendInitializationDataOption: Option[BackendInitializationData] = + withBackendInitializationDataOption override def fileHashCachingActor: Option[ActorRef] = None } } - diff --git a/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala b/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala index faf835c6343..bd102bc27c1 100644 --- a/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala @@ -9,12 +9,12 @@ import org.scalatest.wordspec.AnyWordSpecLike class ContinueOnReturnCodeSpec extends AnyWordSpecLike with CromwellTimeoutSpec with Matchers with BeforeAndAfterAll { "Checking for return codes" should { "continue on expected return code flags" in { - val flagTests = Table( - ("flag", "returnCode", "expectedContinue"), - (true, 0, true), - (true, 1, true), - (false, 0, true), - (false, 1, false)) + val flagTests = Table(("flag", "returnCode", "expectedContinue"), + (true, 0, true), + (true, 1, true), + (false, 0, true), + (false, 1, false) + ) forAll(flagTests) { (flag, returnCode, expectedContinue) => ContinueOnReturnCodeFlag(flag).continueFor(returnCode) should be(expectedContinue) @@ -30,11 +30,20 @@ class ContinueOnReturnCodeSpec extends AnyWordSpecLike with CromwellTimeoutSpec (Set(1), 1, true), (Set(0, 1), 0, true), (Set(0, 1), 1, true), - (Set(0, 1), 2, false)) + (Set(0, 1), 2, false) + ) forAll(setTests) { (set, returnCode, expectedContinue) => ContinueOnReturnCodeSet(set).continueFor(returnCode) should be(expectedContinue) } } + + "continue on expected return code string" in { + val flagTests = Table(("string", "returnCode", "expectedContinue"), ("*", 0, true), ("*", 1, true)) + + forAll(flagTests) { (flag, returnCode, expectedContinue) => + ContinueOnReturnCodeFlag(flag == "*").continueFor(returnCode) should be(expectedContinue) + } + } } } diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala index 766384b46fd..6123ecfe53b 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala @@ -21,9 +21,7 @@ class RuntimeAttributesDefaultSpec extends AnyFlatSpec with CromwellTimeoutSpec ) it should "coerce workflow options from Json to WdlValues" in { - val workflowOptions = WorkflowOptions(JsObject( - "default_runtime_attributes" -> JsObject(map)) - ) + val workflowOptions = WorkflowOptions(JsObject("default_runtime_attributes" -> JsObject(map))) val coercionMap: Map[String, Set[WomType]] = Map( "str" -> Set(WomStringType), @@ -43,9 +41,7 @@ class RuntimeAttributesDefaultSpec extends AnyFlatSpec with CromwellTimeoutSpec } it should "only return default values if they're in the coercionMap" in { - val workflowOptions = WorkflowOptions(JsObject( - "default_runtime_attributes" -> JsObject(map)) - ) + val workflowOptions = WorkflowOptions(JsObject("default_runtime_attributes" -> JsObject(map))) val coercionMap: Map[String, Set[WomType]] = Map( "str" -> Set(WomStringType), @@ -69,9 +65,7 @@ class RuntimeAttributesDefaultSpec extends AnyFlatSpec with CromwellTimeoutSpec } it should "throw an exception if a value can't be coerced" in { - val workflowOptions = WorkflowOptions(JsObject( - "default_runtime_attributes" -> JsObject(map)) - ) + val workflowOptions = WorkflowOptions(JsObject("default_runtime_attributes" -> JsObject(map))) val coercionMap: Map[String, Set[WomType]] = Map( "str" -> Set(WomBooleanType), diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala index 1752da9014b..dafca35e1b6 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala @@ -12,15 +12,21 @@ import wom.RuntimeAttributesKeys import wom.types._ import wom.values._ -class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeoutSpec with Matchers with BeforeAndAfterAll { +class RuntimeAttributesValidationSpec + extends AnyWordSpecLike + with CromwellTimeoutSpec + with Matchers + with BeforeAndAfterAll { val mockBackendRuntimeConfig = TestConfig.allRuntimeAttrsConfig "RuntimeAttributesValidation" should { "return success when tries to validate a valid Docker entry" in { val dockerValue = Some(WomString("someImage")) - val result = RuntimeAttributesValidation.validateDocker(dockerValue, - "Failed to get Docker mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateDocker( + dockerValue, + "Failed to get Docker mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x.get == "someImage") case Invalid(e) => fail(e.toList.mkString(" ")) @@ -38,8 +44,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure (based on defined HoF) when tries to validate a docker entry but it does not contain a value" in { val dockerValue = None - val result = RuntimeAttributesValidation.validateDocker(dockerValue, - "Failed to get Docker mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateDocker( + dockerValue, + "Failed to get Docker mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Failed to get Docker mandatory key from runtime attributes") @@ -48,8 +56,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when there is an invalid docker runtime attribute defined" in { val dockerValue = Some(WomInteger(1)) - val result = RuntimeAttributesValidation.validateDocker(dockerValue, - "Failed to get Docker mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateDocker( + dockerValue, + "Failed to get Docker mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Expecting docker runtime attribute to be a String") @@ -58,8 +68,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a failOnStderr boolean entry" in { val failOnStderrValue = Some(WomBoolean(true)) - val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateFailOnStderr( + failOnStderrValue, + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -68,8 +80,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a failOnStderr 'true' string entry" in { val failOnStderrValue = Some(WomString("true")) - val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateFailOnStderr( + failOnStderrValue, + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -78,8 +92,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a failOnStderr 'false' string entry" in { val failOnStderrValue = Some(WomString("false")) - val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateFailOnStderr( + failOnStderrValue, + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(!x) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -88,11 +104,16 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when there is an invalid failOnStderr runtime attribute defined" in { val failOnStderrValue = Some(WomInteger(1)) - val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateFailOnStderr( + failOnStderrValue, + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") - case Invalid(e) => assert(e.head == "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'") + case Invalid(e) => + assert( + e.head == "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'" + ) } } @@ -107,8 +128,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a continueOnReturnCode boolean entry" in { val continueOnReturnCodeValue = Some(WomBoolean(true)) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x == ContinueOnReturnCodeFlag(true)) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -117,8 +140,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a continueOnReturnCode 'true' string entry" in { val continueOnReturnCodeValue = Some(WomString("true")) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x == ContinueOnReturnCodeFlag(true)) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -127,8 +152,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a continueOnReturnCode 'false' string entry" in { val continueOnReturnCodeValue = Some(WomString("false")) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x == ContinueOnReturnCodeFlag(false)) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -137,8 +164,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a continueOnReturnCode int entry" in { val continueOnReturnCodeValue = Some(WomInteger(12)) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x == ContinueOnReturnCodeSet(Set(12))) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -147,19 +176,26 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when there is an invalid continueOnReturnCode runtime attribute defined" in { val continueOnReturnCodeValue = Some(WomString("yes")) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => - assert(e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]") + assert( + e.head == "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." + ) } } "return success when there is a valid integer array in continueOnReturnCode runtime attribute" in { val continueOnReturnCodeValue = Some(WomArray(WomArrayType(WomIntegerType), Seq(WomInteger(1), WomInteger(2)))) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x == ContinueOnReturnCodeSet(Set(1, 2))) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -167,29 +203,52 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo } "return failure when there is an invalid array in continueOnReturnCode runtime attribute" in { - val continueOnReturnCodeValue = Some(WomArray(WomArrayType(WomStringType), Seq(WomString("one"), WomString("two")))) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val continueOnReturnCodeValue = + Some(WomArray(WomArrayType(WomStringType), Seq(WomString("one"), WomString("two")))) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") - case Invalid(e) => assert(e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]") + case Invalid(e) => + assert( + e.head == "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." + ) } } "return success (based on defined HoF) when tries to validate a continueOnReturnCode entry but it does not contain a value" in { val continueOnReturnCodeValue = None - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, ContinueOnReturnCodeFlag(false).validNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, + ContinueOnReturnCodeFlag(false).validNel + ) result match { case Valid(x) => assert(x == ContinueOnReturnCodeFlag(false)) case Invalid(e) => fail(e.toList.mkString(" ")) } } + "return success when tries to validate a valid returnCodes string entry" in { + val returnCodesValue = Some(WomString("*")) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + returnCodesValue, + "Failed to get return code mandatory key from runtime attributes".invalidNel + ) + result match { + case Valid(x) => assert(x == ContinueOnReturnCodeFlag(true)) + case Invalid(e) => fail(e.toList.mkString(" ")) + } + } + "return success when tries to validate a valid Integer memory entry" in { val expectedGb = 1 val memoryValue = Some(WomInteger(1 << 30)) - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x.amount == expectedGb) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -198,8 +257,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when tries to validate an invalid Integer memory entry" in { val memoryValue = Some(WomInteger(-1)) - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got -1") @@ -209,8 +270,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a valid String memory entry" in { val expectedGb = 2 val memoryValue = Some(WomString("2 GB")) - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x.amount == expectedGb) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -219,8 +282,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when tries to validate an invalid size in String memory entry" in { val memoryValue = Some(WomString("0 GB")) - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got 0.0") @@ -229,28 +294,40 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when tries to validate an invalid String memory entry" in { val memoryValue = Some(WomString("value")) - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") - case Invalid(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: value should be of the form 'X Unit' where X is a number, e.g. 8 GB") + case Invalid(e) => + assert( + e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: value should be of the form 'X Unit' where X is a number, e.g. 8 GB" + ) } } "return failure when tries to validate an invalid memory entry" in { val memoryValue = Some(WomBoolean(true)) - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") - case Invalid(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: Not supported WDL type value") + case Invalid(e) => + assert( + e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: Not supported WDL type value" + ) } } "return failure when tries to validate a non-provided memory entry" in { val memoryValue = None - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Failed to get memory mandatory key from runtime attributes") @@ -259,8 +336,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a valid cpu entry" in { val cpuValue = Some(WomInteger(1)) - val result = RuntimeAttributesValidation.validateCpu(cpuValue, - "Failed to get cpu mandatory key from runtime attributes".invalidNel) + val result = + RuntimeAttributesValidation.validateCpu(cpuValue, + "Failed to get cpu mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x.value == 1) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -269,8 +348,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when tries to validate an invalid cpu entry" in { val cpuValue = Some(WomInteger(-1)) - val result = RuntimeAttributesValidation.validateCpu(cpuValue, - "Failed to get cpu mandatory key from runtime attributes".invalidNel) + val result = + RuntimeAttributesValidation.validateCpu(cpuValue, + "Failed to get cpu mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Expecting cpu runtime attribute value greater than 0") @@ -279,8 +360,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when tries to validate a non-provided cpu entry" in { val cpuValue = None - val result = RuntimeAttributesValidation.validateCpu(cpuValue, - "Failed to get cpu mandatory key from runtime attributes".invalidNel) + val result = + RuntimeAttributesValidation.validateCpu(cpuValue, + "Failed to get cpu mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Failed to get cpu mandatory key from runtime attributes") @@ -306,26 +389,25 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo } "return default values as BadDefaultAttribute when they can't be coerced to expected WdlTypes" in { - val optionalInvalidAttrsConfig = Option(ConfigFactory.parseString( - """ - |cpu = 1.4 - |failOnStderr = "notReal" - |continueOnReturnCode = 0 + val optionalInvalidAttrsConfig = Option(ConfigFactory.parseString(""" + |cpu = 1.4 + |failOnStderr = "notReal" + |continueOnReturnCode = 0 """.stripMargin)) - val defaultVals = Map( - "cpu" -> CpuValidation.configDefaultWomValue(optionalInvalidAttrsConfig).get, - "failOnStderr" -> FailOnStderrValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get, - "continueOnReturnCode" -> ContinueOnReturnCodeValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get - ) + val defaultVals = Map( + "cpu" -> CpuValidation.configDefaultWomValue(optionalInvalidAttrsConfig).get, + "failOnStderr" -> FailOnStderrValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get, + "continueOnReturnCode" -> ContinueOnReturnCodeValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get + ) - val expectedDefaultVals = Map( - "cpu" -> BadDefaultAttribute(WomString("1.4")), - "failOnStderr" -> BadDefaultAttribute(WomString("notReal")), - "continueOnReturnCode" -> WomInteger(0) - ) + val expectedDefaultVals = Map( + "cpu" -> BadDefaultAttribute(WomString("1.4")), + "failOnStderr" -> BadDefaultAttribute(WomString("notReal")), + "continueOnReturnCode" -> WomInteger(0) + ) - defaultVals shouldBe expectedDefaultVals + defaultVals shouldBe expectedDefaultVals } "should parse memory successfully" in { @@ -338,14 +420,14 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo | } |""".stripMargin - val backendConfig: Config = ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes") + val backendConfig: Config = + ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes") val memoryVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, Some(backendConfig)) - val memoryMinVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryMinKey, Some(backendConfig)) - val memoryMaxVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryMaxKey, Some(backendConfig)) - MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryKey, memoryVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some((WomLong(2147483648L))) - MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryMinKey, memoryMinVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some((WomLong(322122547L))) - MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryMaxKey, memoryMaxVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some((WomLong(429496729L))) + MemoryValidation + .withDefaultMemory(RuntimeAttributesKeys.MemoryKey, memoryVal.get) + .runtimeAttributeDefinition + .factoryDefault shouldBe Some(WomLong(2147483648L)) } "shouldn't throw up if the value for a default-runtime-attribute key cannot be coerced into an expected WomType" in { @@ -356,25 +438,33 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo | } |""".stripMargin - val backendConfig: Config = ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes") + val backendConfig: Config = + ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes") val memoryVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, Some(backendConfig)) - MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryKey, memoryVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some(BadDefaultAttribute(WomString("blahblah"))) + MemoryValidation + .withDefaultMemory(RuntimeAttributesKeys.MemoryKey, memoryVal.get) + .runtimeAttributeDefinition + .factoryDefault shouldBe Some(BadDefaultAttribute(WomString("blahblah"))) } "should be able to coerce a list of return codes into an WdlArray" in { - val optinalBackendConfig = Option(ConfigFactory.parseString( - s""" - |continueOnReturnCode = [0,1,2] - |""".stripMargin)) + val optinalBackendConfig = Option(ConfigFactory.parseString(s""" + |continueOnReturnCode = [0,1,2] + |""".stripMargin)) - ContinueOnReturnCodeValidation.configDefaultWdlValue(optinalBackendConfig).get shouldBe WomArray(WomArrayType(WomIntegerType), List(WomInteger(0), WomInteger(1), WomInteger(2))) + ContinueOnReturnCodeValidation.configDefaultWdlValue(optinalBackendConfig).get shouldBe WomArray( + WomArrayType(WomIntegerType), + List(WomInteger(0), WomInteger(1), WomInteger(2)) + ) } "return failure when tries to validate an invalid maxRetries entry" in { val maxRetries = Option(WomInteger(-1)) - val result = RuntimeAttributesValidation.validateMaxRetries(maxRetries, - "Failed to get maxRetries key from runtime attributes".invalidNel) + val result = + RuntimeAttributesValidation.validateMaxRetries(maxRetries, + "Failed to get maxRetries key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Expecting maxRetries runtime attribute value greater than or equal to 0") diff --git a/build.sbt b/build.sbt index 2c9a8068992..8e480907c5e 100644 --- a/build.sbt +++ b/build.sbt @@ -383,11 +383,7 @@ lazy val `cloud-nio-impl-ftp` = (project in cloudNio / "cloud-nio-impl-ftp") lazy val `cloud-nio-impl-drs` = (project in cloudNio / "cloud-nio-impl-drs") .withLibrarySettings(libraryName = "cloud-nio-impl-drs", dependencies = implDrsDependencies) .dependsOn(`cloud-nio-util`) - .dependsOn(common) - .dependsOn(common % "test->test") - -lazy val perf = project - .withExecutableSettings("perf", dependencies = perfDependencies, pushDocker = false) + .dependsOn(cloudSupport) .dependsOn(common) .dependsOn(common % "test->test") @@ -399,7 +395,9 @@ lazy val `cromwell-drs-localizer` = project lazy val pact4s = project.in(file("pact4s")) .settings(pact4sSettings) + .dependsOn(engine) .dependsOn(services) + .dependsOn(engine % "test->test") .disablePlugins(sbtassembly.AssemblyPlugin) lazy val server = project @@ -450,7 +448,6 @@ lazy val root = (project in file(".")) .aggregate(googleBatch) .aggregate(httpFileSystem) .aggregate(languageFactoryCore) - .aggregate(perf) .aggregate(server) .aggregate(services) .aggregate(sfsBackend) diff --git a/centaur/src/it/scala/centaur/AbstractCentaurTestCaseSpec.scala b/centaur/src/it/scala/centaur/AbstractCentaurTestCaseSpec.scala index 1ddebb6a09a..f90fb21e537 100644 --- a/centaur/src/it/scala/centaur/AbstractCentaurTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/AbstractCentaurTestCaseSpec.scala @@ -20,7 +20,10 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.Future @DoNotDiscover -abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromwellTracker: Option[CromwellTracker] = None) extends AsyncFlatSpec with Matchers { +abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], + cromwellTracker: Option[CromwellTracker] = None +) extends AsyncFlatSpec + with Matchers { /* NOTE: We need to statically initialize the object so that the exceptions appear here in the class constructor. @@ -47,10 +50,12 @@ abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromw val duplicateTestNames = allTestsCases .map(_.workflow.testName) .groupBy(identity) - .collect({ case (key, values) if values.lengthCompare(1) > 0 => key }) + .collect { case (key, values) if values.lengthCompare(1) > 0 => key } if (duplicateTestNames.nonEmpty) { - throw new RuntimeException("The following test names are duplicated in more than one test file: " + - duplicateTestNames.mkString(", ")) + throw new RuntimeException( + "The following test names are duplicated in more than one test file: " + + duplicateTestNames.mkString(", ") + ) } allTestsCases } @@ -62,75 +67,28 @@ abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromw } yield submitResponse // Make tags, but enforce lowercase: - val tags = (testCase.testOptions.tags :+ testCase.workflow.testName :+ testCase.testFormat.name) map { x => Tag(x.toLowerCase) } + val tags = (testCase.testOptions.tags :+ testCase.workflow.testName :+ testCase.testFormat.name) map { x => + Tag(x.toLowerCase) + } val isIgnored = testCase.isIgnored(cromwellBackends) val retries = if (testCase.workflow.retryTestFailures) ErrorReporters.retryAttempts else 0 runOrDont(testCase, tags, isIgnored, retries, runTestAndDeleteZippedImports()) } - def executeWdlUpgradeTest(testCase: CentaurTestCase): Unit = - executeStandardTest(wdlUpgradeTestWdl(testCase)) - - private def wdlUpgradeTestWdl(testCase: CentaurTestCase): CentaurTestCase = { - import better.files.File - import womtool.WomtoolMain - - // The suffix matters because WomGraphMaker.getBundle() uses it to choose the language factory - val rootWorkflowFile = File.newTemporaryFile(suffix = ".wdl").append(testCase.workflow.data.workflowContent.get) - val workingDir = File.newTemporaryDirectory() - val upgradedImportsDir = File.newTemporaryDirectory() - val rootWorkflowFilepath = workingDir / rootWorkflowFile.name - - // Un-upgraded imports go into the working directory - testCase.workflow.data.zippedImports match { - case Some(importsZip: File) => - importsZip.unzipTo(workingDir) - case None => () - } - - // Upgrade the imports and copy to main working dir (precludes transitive imports; no recursion yet) - workingDir.list.toList.map { file: File => - val upgradedWdl = WomtoolMain.upgrade(file.pathAsString).stdout.get - upgradedImportsDir.createChild(file.name).append(upgradedWdl) - } - - // Copy to working directory after we operate on the imports that are in it - rootWorkflowFile.copyTo(rootWorkflowFilepath) - - val upgradeResult = WomtoolMain.upgrade(rootWorkflowFilepath.pathAsString) - - upgradeResult.stderr match { - case Some(stderr) => println(stderr) - case _ => () - } - - val newCase = testCase.copy( - workflow = testCase.workflow.copy( - testName = testCase.workflow.testName + " (draft-2 to 1.0 upgrade)", - data = testCase.workflow.data.copy( - workflowContent = Option(upgradeResult.stdout.get), // this '.get' catches an error if upgrade fails - zippedImports = Option(upgradedImportsDir.zip()))))(cromwellTracker) // An empty zip appears to be completely harmless, so no special handling - - rootWorkflowFile.delete(swallowIOExceptions = true) - upgradedImportsDir.delete(swallowIOExceptions = true) - workingDir.delete(swallowIOExceptions = true) - - newCase - } - private def runOrDont(testCase: CentaurTestCase, tags: List[Tag], ignore: Boolean, retries: Int, - runTest: => IO[SubmitResponse]): Unit = { + runTest: => IO[SubmitResponse] + ): Unit = { val itShould: ItVerbString = it should testCase.name tags match { case Nil => runOrDont(itShould, ignore, testCase, retries, runTest) case head :: Nil => runOrDont(itShould taggedAs head, ignore, testCase, retries, runTest) - case head :: tail => runOrDont(itShould taggedAs(head, tail: _*), ignore, testCase, retries, runTest) + case head :: tail => runOrDont(itShould taggedAs (head, tail: _*), ignore, testCase, retries, runTest) } } @@ -138,26 +96,26 @@ abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromw ignore: Boolean, testCase: CentaurTestCase, retries: Int, - runTest: => IO[SubmitResponse]): Unit = { + runTest: => IO[SubmitResponse] + ): Unit = if (ignore) { itVerbString ignore Future.successful(succeed) } else { itVerbString in tryTryAgain(testCase, runTest, retries).unsafeToFuture().map(_ => succeed) } - } private def runOrDont(itVerbStringTaggedAs: ItVerbStringTaggedAs, ignore: Boolean, testCase: CentaurTestCase, retries: Int, - runTest: => IO[SubmitResponse]): Unit = { + runTest: => IO[SubmitResponse] + ): Unit = if (ignore) { itVerbStringTaggedAs ignore Future.successful(succeed) } else { itVerbStringTaggedAs in tryTryAgain(testCase, runTest, retries).unsafeToFuture().map(_ => succeed) } - } /** * Returns an IO effect that will recursively try to run a test. @@ -168,21 +126,27 @@ abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromw * @param attempt Current zero based attempt. * @return IO effect that will run the test, possibly retrying. */ - private def tryTryAgain(testCase: CentaurTestCase, runTest: => IO[SubmitResponse], retries: Int, attempt: Int = 0): IO[SubmitResponse] = { + private def tryTryAgain(testCase: CentaurTestCase, + runTest: => IO[SubmitResponse], + retries: Int, + attempt: Int = 0 + ): IO[SubmitResponse] = { def maybeRetry(centaurTestException: CentaurTestException): IO[SubmitResponse] = { - def clearCachedResults(workflowId: WorkflowId): IO[Unit] = CromwellDatabaseCallCaching.clearCachedResults(workflowId.toString) + def clearCachedResults(workflowId: WorkflowId): IO[Unit] = + CromwellDatabaseCallCaching.clearCachedResults(workflowId.toString) val testEnvironment = TestEnvironment(testCase, retries, attempt) for { _ <- ErrorReporters.logFailure(testEnvironment, centaurTestException) - r <- if (attempt < retries) { - testCase.submittedWorkflowTracker.cleanUpBeforeRetry(clearCachedResults) *> - tryTryAgain(testCase, runTest, retries, attempt + 1) - } else { - IO.raiseError(centaurTestException) - } + r <- + if (attempt < retries) { + testCase.submittedWorkflowTracker.cleanUpBeforeRetry(clearCachedResults) *> + tryTryAgain(testCase, runTest, retries, attempt + 1) + } else { + IO.raiseError(centaurTestException) + } } yield r } @@ -207,11 +171,10 @@ abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromw /** * Clean up temporary zip files created for Imports testing. */ - private def cleanUpImports(wfData: WorkflowData) = { + private def cleanUpImports(wfData: WorkflowData) = wfData.zippedImports match { case Some(zipFile) => zipFile.delete(swallowIOExceptions = true) case None => // } - } } diff --git a/centaur/src/it/scala/centaur/AbstractCromwellEngineOrBackendUpgradeTestCaseSpec.scala b/centaur/src/it/scala/centaur/AbstractCromwellEngineOrBackendUpgradeTestCaseSpec.scala index c55c0c5b17a..04f03d1eacf 100644 --- a/centaur/src/it/scala/centaur/AbstractCromwellEngineOrBackendUpgradeTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/AbstractCromwellEngineOrBackendUpgradeTestCaseSpec.scala @@ -13,7 +13,7 @@ import scala.concurrent.Future @DoNotDiscover abstract class AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBackends: List[String]) - extends AbstractCentaurTestCaseSpec(cromwellBackends) + extends AbstractCentaurTestCaseSpec(cromwellBackends) with CentaurTestSuiteShutdown with BeforeAndAfter { @@ -27,15 +27,20 @@ abstract class AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBacken override protected def beforeAll(): Unit = { super.beforeAll() val beforeAllIo = for { - _ <- checkIsEmpty(cromwellDatabase.engineDatabase, cromwellDatabase.engineDatabase.existsJobKeyValueEntries(), testType) - _ <- checkIsEmpty(cromwellDatabase.metadataDatabase, cromwellDatabase.metadataDatabase.existsMetadataEntries(), testType) + _ <- checkIsEmpty(cromwellDatabase.engineDatabase, + cromwellDatabase.engineDatabase.existsJobKeyValueEntries(), + testType + ) + _ <- checkIsEmpty(cromwellDatabase.metadataDatabase, + cromwellDatabase.metadataDatabase.existsMetadataEntries(), + testType + ) } yield () beforeAllIo.unsafeRunSync() } - private def failNotSlick(database: SqlDatabase): IO[Unit] = { + private def failNotSlick(database: SqlDatabase): IO[Unit] = IO.raiseError(new RuntimeException(s"Expected a slick database for ${database.connectionDescription}.")) - } after { val afterIo = for { @@ -54,26 +59,29 @@ abstract class AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBacken def isMatchingUpgradeTest(testCase: CentaurTestCase): Boolean } - object AbstractCromwellEngineOrBackendUpgradeTestCaseSpec { - private def checkIsEmpty(database: SqlDatabase, lookup: => Future[Boolean], testType: => String)(implicit cs: ContextShift[IO]): IO[Unit] = { - IO.fromFuture(IO(lookup)).flatMap(exists => - if (exists) { - IO(Assertions.fail( - s"Database ${database.connectionDescription} contains data. " + - s"$testType tests should only be run on a completely empty database. " + - "You may need to manually drop and recreate the database to continue." - )) - } else { - IO.unit - } - ) - } + private def checkIsEmpty(database: SqlDatabase, lookup: => Future[Boolean], testType: => String)(implicit + cs: ContextShift[IO] + ): IO[Unit] = + IO.fromFuture(IO(lookup)) + .flatMap(exists => + if (exists) { + IO( + Assertions.fail( + s"Database ${database.connectionDescription} contains data. " + + s"$testType tests should only be run on a completely empty database. " + + "You may need to manually drop and recreate the database to continue." + ) + ) + } else { + IO.unit + } + ) private def recreateDatabase(slickDatabase: SlickDatabase)(implicit cs: ContextShift[IO]): IO[Unit] = { import slickDatabase.dataAccess.driver.api._ val schemaName = slickDatabase.databaseConfig.getOrElse("db.cromwell-database-name", "cromwell_test") - //noinspection SqlDialectInspection + // noinspection SqlDialectInspection for { _ <- IO.fromFuture(IO(slickDatabase.database.run(sqlu"""DROP SCHEMA IF EXISTS #$schemaName"""))) _ <- IO.fromFuture(IO(slickDatabase.database.run(sqlu"""CREATE SCHEMA #$schemaName"""))) diff --git a/centaur/src/it/scala/centaur/CentaurTestSuite.scala b/centaur/src/it/scala/centaur/CentaurTestSuite.scala index 1746caee32b..c72d774c4e4 100644 --- a/centaur/src/it/scala/centaur/CentaurTestSuite.scala +++ b/centaur/src/it/scala/centaur/CentaurTestSuite.scala @@ -16,17 +16,17 @@ object CentaurTestSuite extends StrictLogging { // before we can generate the tests. startCromwell() - def startCromwell(): Unit = { + def startCromwell(): Unit = CentaurConfig.runMode match { case ManagedCromwellServer(preRestart, _, _) => CromwellManager.startCromwell(preRestart) case _ => } - } val cromwellBackends = CentaurCromwellClient.backends.unsafeRunSync().supportedBackends.map(_.toLowerCase) - - def isWdlUpgradeTest(testCase: CentaurTestCase): Boolean = testCase.containsTag("wdl_upgrade") + val defaultBackend = CentaurCromwellClient.backends.unsafeRunSync().defaultBackend.toLowerCase + logger.info(s"Cromwell under test configured with backends ${cromwellBackends.mkString(", ")}") + logger.info(s"Unless overridden by workflow options file, tests use default backend: $defaultBackend") def isEngineUpgradeTest(testCase: CentaurTestCase): Boolean = testCase.containsTag("engine_upgrade") @@ -63,9 +63,8 @@ object CentaurTestSuite extends StrictLogging { trait CentaurTestSuiteShutdown extends Suite with BeforeAndAfterAll { private var shutdownHook: Option[ShutdownHookThread] = _ - override protected def beforeAll() = { - shutdownHook = Option(sys.addShutdownHook { CromwellManager.stopCromwell("JVM Shutdown Hook") }) - } + override protected def beforeAll() = + shutdownHook = Option(sys.addShutdownHook(CromwellManager.stopCromwell("JVM Shutdown Hook"))) override protected def afterAll() = { CromwellManager.stopCromwell("ScalaTest AfterAll") @@ -78,6 +77,6 @@ trait CentaurTestSuiteShutdown extends Suite with BeforeAndAfterAll { * The main centaur test suites, runs sub suites in parallel, but allows better control over the way each nested suite runs. */ class CentaurTestSuite - extends Suites(new SequentialTestCaseSpec(), new ParallelTestCaseSpec()) + extends Suites(new SequentialTestCaseSpec(), new ParallelTestCaseSpec()) with ParallelTestExecution with CentaurTestSuiteShutdown diff --git a/centaur/src/it/scala/centaur/EngineUpgradeTestCaseSpec.scala b/centaur/src/it/scala/centaur/EngineUpgradeTestCaseSpec.scala index a16dda39b49..7a386f9d642 100644 --- a/centaur/src/it/scala/centaur/EngineUpgradeTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/EngineUpgradeTestCaseSpec.scala @@ -4,12 +4,13 @@ import centaur.test.standard.CentaurTestCase import org.scalatest.DoNotDiscover @DoNotDiscover -class EngineUpgradeTestCaseSpec(cromwellBackends: List[String]) extends - AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBackends) { +class EngineUpgradeTestCaseSpec(cromwellBackends: List[String]) + extends AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBackends) { def this() = this(CentaurTestSuite.cromwellBackends) override def testType: String = "Engine upgrade" - override def isMatchingUpgradeTest(testCase: CentaurTestCase): Boolean = CentaurTestSuite.isEngineUpgradeTest(testCase) + override def isMatchingUpgradeTest(testCase: CentaurTestCase): Boolean = + CentaurTestSuite.isEngineUpgradeTest(testCase) } diff --git a/centaur/src/it/scala/centaur/ExternalTestCaseSpec.scala b/centaur/src/it/scala/centaur/ExternalTestCaseSpec.scala index 529a507301a..84ab3e14d9d 100644 --- a/centaur/src/it/scala/centaur/ExternalTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/ExternalTestCaseSpec.scala @@ -5,7 +5,9 @@ import cats.data.Validated.{Invalid, Valid} import centaur.test.standard.CentaurTestCase import com.typesafe.scalalogging.StrictLogging -class ExternalTestCaseSpec(cromwellBackends: List[String]) extends AbstractCentaurTestCaseSpec(cromwellBackends) with StrictLogging { +class ExternalTestCaseSpec(cromwellBackends: List[String]) + extends AbstractCentaurTestCaseSpec(cromwellBackends) + with StrictLogging { def this() = this(CentaurTestSuite.cromwellBackends) @@ -15,11 +17,10 @@ class ExternalTestCaseSpec(cromwellBackends: List[String]) extends AbstractCenta logger.info("No external test to run") } - def runTestFile(testFile: String) = { + def runTestFile(testFile: String) = CentaurTestCase.fromFile(cromwellTracker = None)(File(testFile)) match { case Valid(testCase) => executeStandardTest(testCase) case Invalid(error) => fail(s"Invalid test case: ${error.toList.mkString(", ")}") } - } } diff --git a/centaur/src/it/scala/centaur/PapiUpgradeTestCaseSpec.scala b/centaur/src/it/scala/centaur/PapiUpgradeTestCaseSpec.scala index 300e50d937c..a63030ba40a 100644 --- a/centaur/src/it/scala/centaur/PapiUpgradeTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/PapiUpgradeTestCaseSpec.scala @@ -5,7 +5,7 @@ import org.scalatest.DoNotDiscover @DoNotDiscover class PapiUpgradeTestCaseSpec(cromwellBackends: List[String]) - extends AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBackends) { + extends AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBackends) { def this() = this(CentaurTestSuite.cromwellBackends) diff --git a/centaur/src/it/scala/centaur/ParallelTestCaseSpec.scala b/centaur/src/it/scala/centaur/ParallelTestCaseSpec.scala index 456c974537a..a498807b14f 100644 --- a/centaur/src/it/scala/centaur/ParallelTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/ParallelTestCaseSpec.scala @@ -3,15 +3,15 @@ package centaur import centaur.CentaurTestSuite.cromwellTracker import org.scalatest._ - /** * Runs test cases in parallel, this should be the default type for tests unless they would otherwise crosstalk in undesirable * ways with other tests and must be made sequential. */ @DoNotDiscover class ParallelTestCaseSpec(cromwellBackends: List[String]) - extends AbstractCentaurTestCaseSpec(cromwellBackends, cromwellTracker = cromwellTracker) with ParallelTestExecution { - + extends AbstractCentaurTestCaseSpec(cromwellBackends, cromwellTracker = cromwellTracker) + with ParallelTestExecution { + def this() = this(CentaurTestSuite.cromwellBackends) allTestCases.filter(_.testFormat.isParallel) foreach executeStandardTest diff --git a/centaur/src/it/scala/centaur/SequentialTestCaseSpec.scala b/centaur/src/it/scala/centaur/SequentialTestCaseSpec.scala index ab350f89fc1..12287d4686c 100644 --- a/centaur/src/it/scala/centaur/SequentialTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/SequentialTestCaseSpec.scala @@ -8,7 +8,9 @@ import org.scalatest.matchers.should.Matchers * such that the restarting tests execute sequentially to avoid a mayhem of Cromwell restarts */ @DoNotDiscover -class SequentialTestCaseSpec(cromwellBackends: List[String]) extends AbstractCentaurTestCaseSpec(cromwellBackends) with Matchers { +class SequentialTestCaseSpec(cromwellBackends: List[String]) + extends AbstractCentaurTestCaseSpec(cromwellBackends) + with Matchers { def this() = this(CentaurTestSuite.cromwellBackends) diff --git a/centaur/src/it/scala/centaur/WdlUpgradeTestCaseSpec.scala b/centaur/src/it/scala/centaur/WdlUpgradeTestCaseSpec.scala deleted file mode 100644 index 474bf0a7327..00000000000 --- a/centaur/src/it/scala/centaur/WdlUpgradeTestCaseSpec.scala +++ /dev/null @@ -1,13 +0,0 @@ -package centaur - -import org.scalatest.{DoNotDiscover, ParallelTestExecution} - -@DoNotDiscover -class WdlUpgradeTestCaseSpec(cromwellBackends: List[String]) - extends AbstractCentaurTestCaseSpec(cromwellBackends) with ParallelTestExecution with CentaurTestSuiteShutdown { - - def this() = this(CentaurTestSuite.cromwellBackends) - - // The WDL version upgrade tests are just regular draft-2 test cases tagged for re-use in testing the upgrade script - allTestCases.filter(CentaurTestSuite.isWdlUpgradeTest) foreach executeWdlUpgradeTest -} diff --git a/centaur/src/it/scala/centaur/callcaching/CromwellDatabaseCallCaching.scala b/centaur/src/it/scala/centaur/callcaching/CromwellDatabaseCallCaching.scala index b7976e64d97..6d3888767b9 100644 --- a/centaur/src/it/scala/centaur/callcaching/CromwellDatabaseCallCaching.scala +++ b/centaur/src/it/scala/centaur/callcaching/CromwellDatabaseCallCaching.scala @@ -11,7 +11,6 @@ object CromwellDatabaseCallCaching extends StrictLogging { private val cromwellDatabase = CromwellDatabase.instance - def clearCachedResults(workflowId: String)(implicit executionContext: ExecutionContext): IO[Unit] = { + def clearCachedResults(workflowId: String)(implicit executionContext: ExecutionContext): IO[Unit] = IO.fromFuture(IO(cromwellDatabase.engineDatabase.invalidateCallCacheEntryIdsForWorkflowId(workflowId))) - } } diff --git a/centaur/src/it/scala/centaur/reporting/AggregatedIo.scala b/centaur/src/it/scala/centaur/reporting/AggregatedIo.scala index e9dacd18987..5b8affee887 100644 --- a/centaur/src/it/scala/centaur/reporting/AggregatedIo.scala +++ b/centaur/src/it/scala/centaur/reporting/AggregatedIo.scala @@ -11,6 +11,7 @@ import cats.syntax.traverse._ * Validation that aggregates multiple throwable errors. */ object AggregatedIo { + /** * Similar to common.validation.ErrorOr#ErrorOr, but retains the stack traces. */ @@ -39,16 +40,16 @@ object AggregatedIo { /** * Creates an aggregated exception for multiple exceptions. */ - class AggregatedException private[reporting](exceptionContext: String, suppressed: List[Throwable]) - extends RuntimeException( - { - val suppressedZipped = suppressed.zipWithIndex - val messages = suppressedZipped map { - case (throwable, index) => s"\n ${index+1}: ${throwable.getMessage}" + class AggregatedException private[reporting] (exceptionContext: String, suppressed: List[Throwable]) + extends RuntimeException( + { + val suppressedZipped = suppressed.zipWithIndex + val messages = suppressedZipped map { case (throwable, index) => + s"\n ${index + 1}: ${throwable.getMessage}" + } + s"$exceptionContext:$messages" } - s"$exceptionContext:$messages" - } - ) { + ) { suppressed foreach addSuppressed } diff --git a/centaur/src/it/scala/centaur/reporting/BigQueryReporter.scala b/centaur/src/it/scala/centaur/reporting/BigQueryReporter.scala index 9a88a9ac109..b2335e4a794 100644 --- a/centaur/src/it/scala/centaur/reporting/BigQueryReporter.scala +++ b/centaur/src/it/scala/centaur/reporting/BigQueryReporter.scala @@ -14,7 +14,14 @@ import com.google.api.gax.retrying.RetrySettings import com.google.api.services.bigquery.BigqueryScopes import com.google.auth.Credentials import com.google.cloud.bigquery.InsertAllRequest.RowToInsert -import com.google.cloud.bigquery.{BigQuery, BigQueryError, BigQueryOptions, InsertAllRequest, InsertAllResponse, TableId} +import com.google.cloud.bigquery.{ + BigQuery, + BigQueryError, + BigQueryOptions, + InsertAllRequest, + InsertAllResponse, + TableId +} import common.util.TimeUtil._ import common.validation.Validation._ import cromwell.cloudsupport.gcp.GoogleConfiguration @@ -35,8 +42,9 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe override lazy val destination: String = bigQueryProjectOption.map(_ + "/").getOrElse("") + bigQueryDataset - private val retrySettings: RetrySettings = { - RetrySettings.newBuilder() + private val retrySettings: RetrySettings = + RetrySettings + .newBuilder() .setMaxAttempts(3) .setTotalTimeout(Duration.ofSeconds(30)) .setInitialRetryDelay(Duration.ofMillis(100)) @@ -46,7 +54,6 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe .setRpcTimeoutMultiplier(1.1) .setMaxRpcTimeout(Duration.ofSeconds(5)) .build() - } private val bigQueryCredentials: Credentials = GoogleConfiguration .apply(params.rootConfig) @@ -54,7 +61,8 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe .unsafe .credentials(Set(BigqueryScopes.BIGQUERY_INSERTDATA)) - private val bigQuery: BigQuery = BigQueryOptions.newBuilder() + private val bigQuery: BigQuery = BigQueryOptions + .newBuilder() .setRetrySettings(retrySettings) .setCredentials(bigQueryCredentials) .build() @@ -65,21 +73,19 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe private val jobKeyValueTableId = bigQueryTable("job_key_value") private val metadataTableId = bigQueryTable("metadata") - def bigQueryTable(table: String): TableId = { + def bigQueryTable(table: String): TableId = bigQueryProjectOption match { case Some(project) => TableId.of(project, bigQueryDataset, table) case None => TableId.of(bigQueryDataset, table) } - } /** * In this ErrorReporter implementation this method will send information about exceptions of type * CentaurTestException to BigQuery. Exceptions of other types will be ignored. */ - override def logFailure(testEnvironment: TestEnvironment, - ciEnvironment: CiEnvironment, - throwable: Throwable) - (implicit executionContext: ExecutionContext): IO[Unit] = { + override def logFailure(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, throwable: Throwable)(implicit + executionContext: ExecutionContext + ): IO[Unit] = throwable match { case centaurTestException: CentaurTestException => for { @@ -98,57 +104,62 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe case _ => IO.unit // this ErrorReporter only supports exceptions of CentaurTestException type } - } private def sendBigQueryFailure(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, centaurTestException: CentaurTestException, callAttemptFailures: Vector[CallAttemptFailure], jobKeyValueEntries: Seq[JobKeyValueEntry], - metadataEntries: Seq[MetadataEntry]): IO[Unit] = { + metadataEntries: Seq[MetadataEntry] + ): IO[Unit] = { val metadata: IO[List[BigQueryError]] = { - val metadataRows: List[util.List[RowToInsert]] = metadataEntries.map(toMetadataRow).grouped(10000).map(_.asJava).toList + val metadataRows: List[util.List[RowToInsert]] = + metadataEntries.map(toMetadataRow).grouped(10000).map(_.asJava).toList val metadataRequest: List[InsertAllRequest] = metadataRows.map(InsertAllRequest.of(metadataTableId, _)) if (metadataEntries.nonEmpty) - metadataRequest.traverse[IO, List[BigQueryError]](req => IO(bigQuery.insertAll(req)).map(_.getErrors)).map(_.flatten) + metadataRequest + .traverse[IO, List[BigQueryError]](req => IO(bigQuery.insertAll(req)).map(_.getErrors)) + .map(_.flatten) else - IO{Nil} + IO(Nil) } (IO { - val testFailureRow = toTestFailureRow(testEnvironment, ciEnvironment, centaurTestException) - val callAttemptFailureRows = callAttemptFailures.map(toCallAttemptFailureRow).asJava - val jobKeyValueRows = jobKeyValueEntries.map(toJobKeyValueRow).asJava - - val testFailureRequest = InsertAllRequest.of(testFailureTableId, testFailureRow) - val callAttemptFailuresRequest = InsertAllRequest.of(callAttemptFailureTableId, callAttemptFailureRows) - val jobKeyValuesRequest = InsertAllRequest.of(jobKeyValueTableId, jobKeyValueRows) - val testFailureErrors = bigQuery.insertAll(testFailureRequest).getErrors - val callAttemptFailuresErrors = - if (callAttemptFailures.nonEmpty) bigQuery.insertAll(callAttemptFailuresRequest).getErrors else Nil - val jobKeyValuesErrors = - if (jobKeyValueEntries.nonEmpty) bigQuery.insertAll(jobKeyValuesRequest).getErrors else Nil - - testFailureErrors ++ callAttemptFailuresErrors ++ jobKeyValuesErrors - }, metadata).mapN(_ ++ _).flatMap { + val testFailureRow = toTestFailureRow(testEnvironment, ciEnvironment, centaurTestException) + val callAttemptFailureRows = callAttemptFailures.map(toCallAttemptFailureRow).asJava + val jobKeyValueRows = jobKeyValueEntries.map(toJobKeyValueRow).asJava + + val testFailureRequest = InsertAllRequest.of(testFailureTableId, testFailureRow) + val callAttemptFailuresRequest = InsertAllRequest.of(callAttemptFailureTableId, callAttemptFailureRows) + val jobKeyValuesRequest = InsertAllRequest.of(jobKeyValueTableId, jobKeyValueRows) + val testFailureErrors = bigQuery.insertAll(testFailureRequest).getErrors + val callAttemptFailuresErrors = + if (callAttemptFailures.nonEmpty) bigQuery.insertAll(callAttemptFailuresRequest).getErrors else Nil + val jobKeyValuesErrors = + if (jobKeyValueEntries.nonEmpty) bigQuery.insertAll(jobKeyValuesRequest).getErrors else Nil + + testFailureErrors ++ callAttemptFailuresErrors ++ jobKeyValuesErrors + }, + metadata + ).mapN(_ ++ _).flatMap { case errors if errors.isEmpty => IO.unit - case errors => IO.raiseError { - val errorCount = errors.size - val threeErrors = errors.map(String.valueOf).distinct.sorted.take(3) - val continued = if (errorCount > 3) "\n..." else "" - val message = threeErrors.mkString( - s"$errorCount error(s) occurred uploading to BigQuery: \n", - "\n", - continued) - new RuntimeException(message) - } + case errors => + IO.raiseError { + val errorCount = errors.size + val threeErrors = errors.map(String.valueOf).distinct.sorted.take(3) + val continued = if (errorCount > 3) "\n..." else "" + val message = + threeErrors.mkString(s"$errorCount error(s) occurred uploading to BigQuery: \n", "\n", continued) + new RuntimeException(message) + } } } private def toTestFailureRow(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, - centaurTestException: CentaurTestException): RowToInsert = { + centaurTestException: CentaurTestException + ): RowToInsert = RowToInsert of Map( "ci_env_branch" -> ciEnvironment.branch, "ci_env_event" -> ciEnvironment.event, @@ -165,13 +176,12 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe "test_name" -> Option(testEnvironment.testCase.name), "test_stack_trace" -> Option(ExceptionUtils.getStackTrace(centaurTestException)), "test_timestamp" -> Option(OffsetDateTime.now.toUtcMilliString), - "test_workflow_id" -> centaurTestException.workflowIdOption, - ).collect { - case (key, Some(value)) => (key, value) + "test_workflow_id" -> centaurTestException.workflowIdOption + ).collect { case (key, Some(value)) => + (key, value) }.asJava - } - private def toCallAttemptFailureRow(callAttemptFailure: CallAttemptFailure): RowToInsert = { + private def toCallAttemptFailureRow(callAttemptFailure: CallAttemptFailure): RowToInsert = RowToInsert of Map( "call_fully_qualified_name" -> Option(callAttemptFailure.callFullyQualifiedName), "call_root" -> callAttemptFailure.callRootOption, @@ -182,24 +192,22 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe "start" -> callAttemptFailure.startOption.map(_.toUtcMilliString), "stderr" -> callAttemptFailure.stderrOption, "stdout" -> callAttemptFailure.stdoutOption, - "workflow_id" -> Option(callAttemptFailure.workflowId), - ).collect { - case (key, Some(value)) => (key, value) + "workflow_id" -> Option(callAttemptFailure.workflowId) + ).collect { case (key, Some(value)) => + (key, value) }.asJava - } - private def toJobKeyValueRow(jobKeyValueEntry: JobKeyValueEntry): RowToInsert = { + private def toJobKeyValueRow(jobKeyValueEntry: JobKeyValueEntry): RowToInsert = RowToInsert of Map[String, Any]( "call_fully_qualified_name" -> jobKeyValueEntry.callFullyQualifiedName, "job_attempt" -> jobKeyValueEntry.jobAttempt, "job_index" -> jobKeyValueEntry.jobIndex, "store_key" -> jobKeyValueEntry.storeKey, "store_value" -> jobKeyValueEntry.storeValue, - "workflow_execution_uuid" -> jobKeyValueEntry.workflowExecutionUuid, + "workflow_execution_uuid" -> jobKeyValueEntry.workflowExecutionUuid ).asJava - } - private def toMetadataRow(metadataEntry: MetadataEntry): RowToInsert = { + private def toMetadataRow(metadataEntry: MetadataEntry): RowToInsert = RowToInsert of Map( "call_fully_qualified_name" -> metadataEntry.callFullyQualifiedName, "job_attempt" -> metadataEntry.jobAttempt, @@ -208,11 +216,10 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe "metadata_timestamp" -> Option(metadataEntry.metadataTimestamp.toSystemOffsetDateTime.toUtcMilliString), "metadata_value" -> metadataEntry.metadataValue.map(_.toRawString), "metadata_value_type" -> metadataEntry.metadataValueType, - "workflow_execution_uuid" -> Option(metadataEntry.workflowExecutionUuid), - ).collect { - case (key, Some(value)) => (key, value) + "workflow_execution_uuid" -> Option(metadataEntry.workflowExecutionUuid) + ).collect { case (key, Some(value)) => + (key, value) }.asJava - } } object BigQueryReporter { diff --git a/centaur/src/it/scala/centaur/reporting/CiEnvironment.scala b/centaur/src/it/scala/centaur/reporting/CiEnvironment.scala index 6750925ca64..0845d8b8c94 100644 --- a/centaur/src/it/scala/centaur/reporting/CiEnvironment.scala +++ b/centaur/src/it/scala/centaur/reporting/CiEnvironment.scala @@ -5,8 +5,7 @@ import scala.util.Try /** * Scala representation of the CI environment values defined in `test.inc.sh`. */ -case class CiEnvironment -( +case class CiEnvironment( isCi: Option[Boolean], `type`: Option[String], branch: Option[String], @@ -20,7 +19,7 @@ case class CiEnvironment ) object CiEnvironment { - def apply(): CiEnvironment = { + def apply(): CiEnvironment = new CiEnvironment( isCi = sys.env.get("CROMWELL_BUILD_IS_CI").flatMap(tryToBoolean), `type` = sys.env.get("CROMWELL_BUILD_TYPE"), @@ -31,9 +30,8 @@ object CiEnvironment { provider = sys.env.get("CROMWELL_BUILD_PROVIDER"), os = sys.env.get("CROMWELL_BUILD_OS"), url = sys.env.get("CROMWELL_BUILD_URL"), - centaurType = sys.env.get("CROMWELL_BUILD_CENTAUR_TYPE"), + centaurType = sys.env.get("CROMWELL_BUILD_CENTAUR_TYPE") ) - } /** Try converting the value to a boolean, or return None. */ private def tryToBoolean(string: String): Option[Boolean] = Try(string.toBoolean).toOption diff --git a/centaur/src/it/scala/centaur/reporting/ErrorReporter.scala b/centaur/src/it/scala/centaur/reporting/ErrorReporter.scala index 8481bf94c51..c654ad3cfc0 100644 --- a/centaur/src/it/scala/centaur/reporting/ErrorReporter.scala +++ b/centaur/src/it/scala/centaur/reporting/ErrorReporter.scala @@ -8,6 +8,7 @@ import scala.concurrent.ExecutionContext * Reports errors during testing. */ trait ErrorReporter { + /** The various parameters for this reporter. */ def params: ErrorReporterParams @@ -15,8 +16,7 @@ trait ErrorReporter { def destination: String /** Send a report of a failure. */ - def logFailure(testEnvironment: TestEnvironment, - ciEnvironment: CiEnvironment, - throwable: Throwable) - (implicit executionContext: ExecutionContext): IO[Unit] + def logFailure(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, throwable: Throwable)(implicit + executionContext: ExecutionContext + ): IO[Unit] } diff --git a/centaur/src/it/scala/centaur/reporting/ErrorReporterCromwellDatabase.scala b/centaur/src/it/scala/centaur/reporting/ErrorReporterCromwellDatabase.scala index 89a3e5210b8..79e13c47700 100644 --- a/centaur/src/it/scala/centaur/reporting/ErrorReporterCromwellDatabase.scala +++ b/centaur/src/it/scala/centaur/reporting/ErrorReporterCromwellDatabase.scala @@ -11,25 +11,25 @@ class ErrorReporterCromwellDatabase(cromwellDatabase: CromwellDatabase) { import centaur.TestContext._ - def jobKeyValueEntriesIo(workflowExecutionUuidOption: Option[String]) - (implicit executionContext: ExecutionContext): IO[Seq[JobKeyValueEntry]] = { + def jobKeyValueEntriesIo(workflowExecutionUuidOption: Option[String])(implicit + executionContext: ExecutionContext + ): IO[Seq[JobKeyValueEntry]] = workflowExecutionUuidOption.map(jobKeyValueEntriesIo).getOrElse(IO.pure(Seq.empty)) - } - def jobKeyValueEntriesIo(workflowExecutionUuid: String) - (implicit executionContext: ExecutionContext): IO[Seq[JobKeyValueEntry]] = { + def jobKeyValueEntriesIo(workflowExecutionUuid: String)(implicit + executionContext: ExecutionContext + ): IO[Seq[JobKeyValueEntry]] = IO.fromFuture(IO(cromwellDatabase.engineDatabase.queryJobKeyValueEntries(workflowExecutionUuid))) - } - def metadataEntriesIo(workflowExecutionUuidOption: Option[String]) - (implicit executionContext: ExecutionContext): IO[Seq[MetadataEntry]] = { + def metadataEntriesIo(workflowExecutionUuidOption: Option[String])(implicit + executionContext: ExecutionContext + ): IO[Seq[MetadataEntry]] = workflowExecutionUuidOption.map(metadataEntriesIo).getOrElse(IO.pure(Seq.empty)) - } - def metadataEntriesIo(workflowExecutionUuid: String) - (implicit executionContext: ExecutionContext): IO[Seq[MetadataEntry]] = { + def metadataEntriesIo(workflowExecutionUuid: String)(implicit + executionContext: ExecutionContext + ): IO[Seq[MetadataEntry]] = // 30 seconds is less than production (60s as of 2018-08) but hopefully high enough to work on a CI machine with contended resources IO.fromFuture(IO(cromwellDatabase.metadataDatabase.queryMetadataEntries(workflowExecutionUuid, 30.seconds))) - } } diff --git a/centaur/src/it/scala/centaur/reporting/ErrorReporterParams.scala b/centaur/src/it/scala/centaur/reporting/ErrorReporterParams.scala index 6e716b82942..9a403e88138 100644 --- a/centaur/src/it/scala/centaur/reporting/ErrorReporterParams.scala +++ b/centaur/src/it/scala/centaur/reporting/ErrorReporterParams.scala @@ -5,8 +5,7 @@ import com.typesafe.config.Config /** * Collects all of the parameters to pass to a new ErrorReporter. */ -case class ErrorReporterParams -( +case class ErrorReporterParams( name: String, rootConfig: Config, reporterConfig: Config, diff --git a/centaur/src/it/scala/centaur/reporting/ErrorReporters.scala b/centaur/src/it/scala/centaur/reporting/ErrorReporters.scala index 8f989e3bd1b..2058264db29 100644 --- a/centaur/src/it/scala/centaur/reporting/ErrorReporters.scala +++ b/centaur/src/it/scala/centaur/reporting/ErrorReporters.scala @@ -25,9 +25,8 @@ class ErrorReporters(rootConfig: Config) { providersConfig.entrySet.asScala.map(_.getKey.split("\\.").toList.head).toList } - private val errorReportersIo: IO[List[ErrorReporter]] = { + private val errorReportersIo: IO[List[ErrorReporter]] = AggregatedIo.aggregateExceptions("Errors while creating ErrorReporters", errorReporterNames.map(getErrorReporter)) - } val errorReporters: List[ErrorReporter] = errorReportersIo.unsafeRunSync() @@ -42,26 +41,27 @@ class ErrorReporters(rootConfig: Config) { * @param throwable The exception that occurred while running the test. * @return An IO effect that will log the failure. */ - def logFailure(testEnvironment: TestEnvironment, - ciEnvironment: CiEnvironment, - throwable: Throwable) - (implicit executionContext: ExecutionContext): IO[Unit] = { + def logFailure(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, throwable: Throwable)(implicit + executionContext: ExecutionContext + ): IO[Unit] = if (errorReporters.isEmpty) { // If the there are no reporters, then just "throw" the exception. Do not retry to run the test. IO.raiseError(throwable) } else { val listIo = errorReporters.map(_.logFailure(testEnvironment, ciEnvironment, throwable)) - AggregatedIo.aggregateExceptions("Errors while reporting a failure", listIo).handleErrorWith(err => { - err.addSuppressed(throwable) - IO.raiseError(err) - }).void + AggregatedIo + .aggregateExceptions("Errors while reporting a failure", listIo) + .handleErrorWith { err => + err.addSuppressed(throwable) + IO.raiseError(err) + } + .void } - } /** * Constructs the IO reporter by name. */ - private def getErrorReporter(errorReporterName: String): IO[ErrorReporter] = { + private def getErrorReporter(errorReporterName: String): IO[ErrorReporter] = IO { val clazz = errorReporterConfig.getString(s"providers.$errorReporterName.class") val reporterConfig = errorReporterConfig.getOrElse(s"providers.$errorReporterName.config", ConfigFactory.empty) @@ -69,7 +69,6 @@ class ErrorReporters(rootConfig: Config) { val params = ErrorReporterParams(errorReporterName, rootConfig, reporterConfig, errorReporterCromwellDatabase) constructor.newInstance(params).asInstanceOf[ErrorReporter] } - } } object ErrorReporters extends StrictLogging { @@ -84,9 +83,8 @@ object ErrorReporters extends StrictLogging { if (retryAttempts > 0) logger.info("Error retry count: {}", retryAttempts) - def logFailure(testEnvironment: TestEnvironment, - throwable: Throwable) - (implicit executionContext: ExecutionContext): IO[Unit] = { + def logFailure(testEnvironment: TestEnvironment, throwable: Throwable)(implicit + executionContext: ExecutionContext + ): IO[Unit] = errorReporters.logFailure(testEnvironment, ciEnvironment, throwable) - } } diff --git a/centaur/src/it/scala/centaur/reporting/GcsReporter.scala b/centaur/src/it/scala/centaur/reporting/GcsReporter.scala index 41e15d80b3b..55b5be270f4 100644 --- a/centaur/src/it/scala/centaur/reporting/GcsReporter.scala +++ b/centaur/src/it/scala/centaur/reporting/GcsReporter.scala @@ -9,7 +9,10 @@ import net.ceedubs.ficus.Ficus._ import scala.concurrent.ExecutionContext -class GcsReporter(override val params: ErrorReporterParams) extends ErrorReporter with SuccessReporter with StrictLogging { +class GcsReporter(override val params: ErrorReporterParams) + extends ErrorReporter + with SuccessReporter + with StrictLogging { val storage = StorageOptions.getDefaultInstance.getService val reportBucket = params.reporterConfig.as[String]("report-bucket") val reportPath = params.reporterConfig.as[String]("report-path") @@ -21,10 +24,9 @@ class GcsReporter(override val params: ErrorReporterParams) extends ErrorReporte * In this ErrorReporter implementation this method will save information about exceptions of type * CentaurTestException to GCS. Exceptions of other types will be ignored. */ - override def logFailure(testEnvironment: TestEnvironment, - ciEnvironment: CiEnvironment, - throwable: Throwable) - (implicit executionContext: ExecutionContext): IO[Unit] = { + override def logFailure(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, throwable: Throwable)(implicit + executionContext: ExecutionContext + ): IO[Unit] = throwable match { case centaurTestException: CentaurTestException => logger.info(s"Reporting failed metadata to gs://$reportBucket/$reportPath") @@ -32,7 +34,6 @@ class GcsReporter(override val params: ErrorReporterParams) extends ErrorReporte case _ => IO.unit // this ErrorReporter only supports exceptions of CentaurTestException type } - } override def logSuccessfulRun(submitResponse: SubmitWorkflowResponse): IO[Unit] = { logger.info(s"Reporting successful metadata to gs://$reportBucket/$reportPath") @@ -44,7 +45,8 @@ class GcsReporter(override val params: ErrorReporterParams) extends ErrorReporte private def pushJsonToGcs(json: String) = IO { storage.create( - BlobInfo.newBuilder(reportBucket, reportPath) + BlobInfo + .newBuilder(reportBucket, reportPath) .setContentType("application/json") .build(), json.toArray.map(_.toByte) diff --git a/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala b/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala index 4332e01db24..ff5aeb5f7d5 100644 --- a/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala +++ b/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala @@ -13,15 +13,13 @@ import scala.concurrent.ExecutionContext * Useful as a backup in cases where another reporter is not available, for example in external PRs where secure * environment variables are not available. */ -class Slf4jReporter(override val params: ErrorReporterParams) - extends ErrorReporter with StrictLogging { +class Slf4jReporter(override val params: ErrorReporterParams) extends ErrorReporter with StrictLogging { override lazy val destination: String = "error" - override def logFailure(testEnvironment: TestEnvironment, - ciEnvironment: CiEnvironment, - throwable: Throwable) - (implicit executionContext: ExecutionContext): IO[Unit] = { + override def logFailure(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, throwable: Throwable)(implicit + executionContext: ExecutionContext + ): IO[Unit] = IO { val errorMessage = throwable match { @@ -41,9 +39,9 @@ class Slf4jReporter(override val params: ErrorReporterParams) if (testEnvironment.attempt >= testEnvironment.retries) { logger.error(message, throwable) } else { - val messageWithShortExceptionContext = message + " (" + ExceptionUtils.getMessage(throwable).replace("\n", " ").take(150) + "[...])" + val messageWithShortExceptionContext = + message + " (" + ExceptionUtils.getMessage(throwable).replace("\n", " ").take(150) + "[...])" logger.warn(messageWithShortExceptionContext) } } - } } diff --git a/centaur/src/it/scala/centaur/reporting/SuccessReporter.scala b/centaur/src/it/scala/centaur/reporting/SuccessReporter.scala index 2453ca71898..fb0c6a389e9 100644 --- a/centaur/src/it/scala/centaur/reporting/SuccessReporter.scala +++ b/centaur/src/it/scala/centaur/reporting/SuccessReporter.scala @@ -12,13 +12,13 @@ object SuccessReporters { * This is gross and piggy backs on the error reporting code but achieves what we need for now without a refactoring * of the error reporting code to handle both success and error reporting */ - private val successReporters: List[ErrorReporter with SuccessReporter] = ErrorReporters.errorReporters.errorReporters.collect({case s: SuccessReporter => s }) + private val successReporters: List[ErrorReporter with SuccessReporter] = + ErrorReporters.errorReporters.errorReporters.collect { case s: SuccessReporter => s } - def logSuccessfulRun(submitResponse: SubmitWorkflowResponse): IO[SubmitResponse] = { - if (successReporters.isEmpty) IO.pure(submitResponse) + def logSuccessfulRun(submitResponse: SubmitWorkflowResponse): IO[SubmitResponse] = + if (successReporters.isEmpty) IO.pure(submitResponse) else { val listIo = successReporters.map(_.logSuccessfulRun(submitResponse)) AggregatedIo.aggregateExceptions("Errors while reporting centaur success", listIo).map(_ => submitResponse) } - } } diff --git a/centaur/src/main/resources/standardTestCases/biscayne_new_engine_functions.test b/centaur/src/main/resources/standardTestCases/biscayne_new_engine_functions.test index 184d41b5aeb..b9f5430e57b 100644 --- a/centaur/src/main/resources/standardTestCases/biscayne_new_engine_functions.test +++ b/centaur/src/main/resources/standardTestCases/biscayne_new_engine_functions.test @@ -54,4 +54,40 @@ metadata { "outputs.biscayne_new_engine_functions.bigIntFloatComparison": 10.0 "outputs.biscayne_new_engine_functions.minMaxIntFloatComposition": 1.0 "outputs.biscayne_new_engine_functions.maxIntVsMaxFloat": 1.79769313E+308 + + "outputs.biscayne_new_engine_functions.substituted": "WATtheWAT" + + "outputs.biscayne_new_engine_functions.with_suffixes.0": "aaaS" + "outputs.biscayne_new_engine_functions.with_suffixes.1": "bbbS" + "outputs.biscayne_new_engine_functions.with_suffixes.2": "cccS" + + "outputs.biscayne_new_engine_functions.with_quotes.0": "\"1\"" + "outputs.biscayne_new_engine_functions.with_quotes.1": "\"2\"" + "outputs.biscayne_new_engine_functions.with_quotes.2": "\"3\"" + + "outputs.biscayne_new_engine_functions.string_with_quotes.0": "\"aaa\"" + "outputs.biscayne_new_engine_functions.string_with_quotes.1": "\"bbb\"" + "outputs.biscayne_new_engine_functions.string_with_quotes.2": "\"ccc\"" + + "outputs.biscayne_new_engine_functions.with_squotes.0": "'1'" + "outputs.biscayne_new_engine_functions.with_squotes.1": "'2'" + "outputs.biscayne_new_engine_functions.with_squotes.2": "'3'" + + "outputs.biscayne_new_engine_functions.string_with_squotes.0": "'aaa'" + "outputs.biscayne_new_engine_functions.string_with_squotes.1": "'bbb'" + "outputs.biscayne_new_engine_functions.string_with_squotes.2": "'ccc'" + "outputs.biscayne_new_engine_functions.unzipped_a.left.0": "A" + "outputs.biscayne_new_engine_functions.unzipped_a.right.0": "a" + + "outputs.biscayne_new_engine_functions.unzipped_b.left.0": "A" + "outputs.biscayne_new_engine_functions.unzipped_b.left.1": "B" + "outputs.biscayne_new_engine_functions.unzipped_b.right.0": "a" + "outputs.biscayne_new_engine_functions.unzipped_b.right.1": "b" + + "outputs.biscayne_new_engine_functions.unzipped_c.left.0": "one" + "outputs.biscayne_new_engine_functions.unzipped_c.left.1": "two" + "outputs.biscayne_new_engine_functions.unzipped_c.left.2": "three" + "outputs.biscayne_new_engine_functions.unzipped_c.right.0": 1.0 + "outputs.biscayne_new_engine_functions.unzipped_c.right.1": 2.0 + "outputs.biscayne_new_engine_functions.unzipped_c.right.2": 3.0 } diff --git a/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_lifesciences.test b/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_lifesciences.test new file mode 100644 index 00000000000..7fcb3fb4c28 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_lifesciences.test @@ -0,0 +1,17 @@ +name: biscayne_new_runtime_attributes_lifesciences +testFormat: workflowsuccess +tags: ["wdl_biscayne"] + +# Will run on a Cromwell that supports any one of these backends +backendsMode: any +backends: [Papi, Papiv2, GCPBatch] + +files { + workflow: wdl_biscayne/biscayne_new_runtime_attributes/biscayne_new_runtime_attributes.wdl +} + +metadata { + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.docker": "rockylinux:9", + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.cpu": 4 + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.memory": "6 GB" +} diff --git a/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_local.test b/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_local.test new file mode 100644 index 00000000000..d25d0f7e59b --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_local.test @@ -0,0 +1,16 @@ +name: biscayne_new_runtime_attributes_local +testFormat: workflowsuccess +tags: ["wdl_biscayne"] + +# This test should only run in the Local suite, on its default `Local` backend. Unfortunately the `Local` backend +# leaks into other suites, so require an irrelevant `LocalNoDocker` backend that is only found in Local suite. +backends: [Local, LocalNoDocker] + +files { + workflow: wdl_biscayne/biscayne_new_runtime_attributes/biscayne_new_runtime_attributes.wdl +} + +# CPU, memory attributes not applicable for Local backend +metadata { + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.docker": "ubuntu:latest", +} diff --git a/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_tes.test b/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_tes.test new file mode 100644 index 00000000000..8ee9cf96050 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_tes.test @@ -0,0 +1,14 @@ +name: biscayne_new_runtime_attributes_tes +testFormat: workflowsuccess +tags: ["wdl_biscayne"] +backends: [TES] + +files { + workflow: wdl_biscayne/biscayne_new_runtime_attributes/biscayne_new_runtime_attributes.wdl +} + +metadata { + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.docker": "debian:latest", + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.cpu": 4 + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.memory": "4 GB" +} diff --git a/centaur/src/main/resources/standardTestCases/composedenginefunctions.test b/centaur/src/main/resources/standardTestCases/composedenginefunctions.test index 68628483740..d51f31c9929 100644 --- a/centaur/src/main/resources/standardTestCases/composedenginefunctions.test +++ b/centaur/src/main/resources/standardTestCases/composedenginefunctions.test @@ -1,6 +1,5 @@ name: composedenginefunctions testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: composedenginefunctions/composedenginefunctions.wdl diff --git a/centaur/src/main/resources/standardTestCases/conditionals.ifs_in_scatters.test b/centaur/src/main/resources/standardTestCases/conditionals.ifs_in_scatters.test index 2bd93388199..9d20530291a 100644 --- a/centaur/src/main/resources/standardTestCases/conditionals.ifs_in_scatters.test +++ b/centaur/src/main/resources/standardTestCases/conditionals.ifs_in_scatters.test @@ -1,6 +1,6 @@ name: ifs_in_scatters testFormat: workflowsuccess -tags: [ conditionals, "wdl_upgrade" ] +tags: [ conditionals ] files { workflow: conditionals_tests/ifs_in_scatters/ifs_in_scatters.wdl diff --git a/centaur/src/main/resources/standardTestCases/conditionals.nested_lookups.test b/centaur/src/main/resources/standardTestCases/conditionals.nested_lookups.test index 6f5c0fa2c93..cf4ca22c876 100644 --- a/centaur/src/main/resources/standardTestCases/conditionals.nested_lookups.test +++ b/centaur/src/main/resources/standardTestCases/conditionals.nested_lookups.test @@ -1,6 +1,6 @@ name: nested_lookups testFormat: workflowsuccess -tags: [ conditionals, "wdl_upgrade" ] +tags: [ conditionals ] files { workflow: conditionals_tests/nested_lookups/nested_lookups.wdl diff --git a/centaur/src/main/resources/standardTestCases/conditionals.scatters_in_ifs.test b/centaur/src/main/resources/standardTestCases/conditionals.scatters_in_ifs.test index 8d9cc2193b7..f23865eae67 100644 --- a/centaur/src/main/resources/standardTestCases/conditionals.scatters_in_ifs.test +++ b/centaur/src/main/resources/standardTestCases/conditionals.scatters_in_ifs.test @@ -1,6 +1,6 @@ name: scatters_in_ifs testFormat: workflowsuccess -tags: [ conditionals, "wdl_upgrade" ] +tags: [ conditionals ] files { workflow: conditionals_tests/scatters_in_ifs/scatters_in_ifs.wdl diff --git a/centaur/src/main/resources/standardTestCases/conditionals.simple_if.test b/centaur/src/main/resources/standardTestCases/conditionals.simple_if.test index 82794b863d5..50a1c811892 100644 --- a/centaur/src/main/resources/standardTestCases/conditionals.simple_if.test +++ b/centaur/src/main/resources/standardTestCases/conditionals.simple_if.test @@ -1,6 +1,6 @@ name: simple_if testFormat: workflowsuccess -tags: [ conditionals, "wdl_upgrade" ] +tags: [ conditionals ] files { workflow: conditionals_tests/simple_if/simple_if.wdl diff --git a/centaur/src/main/resources/standardTestCases/conditionals.simple_if_workflow_outputs.test b/centaur/src/main/resources/standardTestCases/conditionals.simple_if_workflow_outputs.test index 3aed74d756b..ede32957e47 100644 --- a/centaur/src/main/resources/standardTestCases/conditionals.simple_if_workflow_outputs.test +++ b/centaur/src/main/resources/standardTestCases/conditionals.simple_if_workflow_outputs.test @@ -1,6 +1,6 @@ name: simple_if_workflow_outputs testFormat: workflowsuccess -tags: [ conditionals, "wdl_upgrade" ] +tags: [ conditionals ] files { workflow: conditionals_tests/simple_if_workflow_outputs/simple_if_workflow_outputs.wdl diff --git a/centaur/src/main/resources/standardTestCases/declarations.test b/centaur/src/main/resources/standardTestCases/declarations.test index 53766678b26..bae479145f0 100644 --- a/centaur/src/main/resources/standardTestCases/declarations.test +++ b/centaur/src/main/resources/standardTestCases/declarations.test @@ -2,7 +2,6 @@ name: declarations testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: declarations/declarations.wdl diff --git a/centaur/src/main/resources/standardTestCases/declarations_as_nodes.test b/centaur/src/main/resources/standardTestCases/declarations_as_nodes.test index d18f39677f6..ff1f4416a24 100644 --- a/centaur/src/main/resources/standardTestCases/declarations_as_nodes.test +++ b/centaur/src/main/resources/standardTestCases/declarations_as_nodes.test @@ -1,6 +1,5 @@ name: declarations_as_nodes testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: declarations_as_nodes/declarations_as_nodes.wdl diff --git a/centaur/src/main/resources/standardTestCases/failing_return_code.test b/centaur/src/main/resources/standardTestCases/failing_return_code.test new file mode 100644 index 00000000000..163715222cf --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/failing_return_code.test @@ -0,0 +1,11 @@ +name: failing_return_code +testFormat: workflowfailure + +files { + workflow: failing_return_code/failing_return_code.wdl +} + +metadata { + workflowName: FailingReturnCode + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/failing_return_code/failing_return_code.wdl b/centaur/src/main/resources/standardTestCases/failing_return_code/failing_return_code.wdl new file mode 100644 index 00000000000..f393698d2b7 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/failing_return_code/failing_return_code.wdl @@ -0,0 +1,20 @@ +version development-1.1 + +workflow FailingReturnCode { + call FailingReturnCodeSet +} + +task FailingReturnCodeSet { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + returnCodes: [0, 5, 10] + docker: "ubuntu:latest" + } +} diff --git a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_jes.test b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_jes.test index 1ac12322f4f..182a833fbdb 100644 --- a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_jes.test +++ b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_jes.test @@ -2,6 +2,7 @@ name: failures.restart_while_failing_jes testFormat: WorkflowFailureRestartWithRecover callMark: restart_while_failing.B1 backends: [Papi] +tags: [restart] files { workflow: failures/restart_while_failing/restart_while_failing.wdl diff --git a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_local.test b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_local.test index f686b16c817..58abbf78245 100644 --- a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_local.test +++ b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_local.test @@ -3,7 +3,7 @@ testFormat: WorkflowFailureRestartWithRecover callMark: restart_while_failing.B1 backendsMode: "only" backends: [Local, LocalNoDocker] -tags: [localdockertest] +tags: [localdockertest, restart] files { workflow: failures/restart_while_failing/restart_while_failing.wdl diff --git a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_tes.test b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_tes.test index 64ea5fd8906..9bb9c1a55d6 100644 --- a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_tes.test +++ b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_tes.test @@ -2,7 +2,7 @@ name: failures.restart_while_failing_tes testFormat: WorkflowFailureRestartWithoutRecover callMark: restart_while_failing.B1 backends: [TES] -tags: [localdockertest] +tags: [localdockertest, restart] files { workflow: failures/restart_while_failing/restart_while_failing.wdl diff --git a/centaur/src/main/resources/standardTestCases/failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.inputs b/centaur/src/main/resources/standardTestCases/failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.inputs new file mode 100644 index 00000000000..6f8cb64990a --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.inputs @@ -0,0 +1,4 @@ +{ + "MinimalStructExample.test": "Hello World", + "MinimalStructExample.integer": 2 +} diff --git a/centaur/src/main/resources/standardTestCases/failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.wdl b/centaur/src/main/resources/standardTestCases/failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.wdl new file mode 100644 index 00000000000..dce55da0bae --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.wdl @@ -0,0 +1,27 @@ +version 1.0 + + +struct firstLayer { + String first + Int number +} + +struct secondLayer { + String second + firstLayer lowerLayer +} + + +workflow MinimalStructExample { + input { + String test + Int integer + } + + firstLayer example1 = {"first": test, "number": integer} + secondLayer example2 = {"second": test, "lowerLayer": example1} + + output { + String example3 = example2.second + } +} diff --git a/centaur/src/main/resources/standardTestCases/failures/stderr_stdout_workflow_body/stderr_stdout_workflow_body.wdl b/centaur/src/main/resources/standardTestCases/failures/stderr_stdout_workflow_body/stderr_stdout_workflow_body.wdl new file mode 100644 index 00000000000..733d448a5ed --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/failures/stderr_stdout_workflow_body/stderr_stdout_workflow_body.wdl @@ -0,0 +1,8 @@ +version 1.0 + +workflow break_with_stderr { + + output { + File load_data_csv = select_first([stdout(), stderr()]) + } +} diff --git a/centaur/src/main/resources/standardTestCases/hello.test b/centaur/src/main/resources/standardTestCases/hello.test index 33a49bd4071..2f7fd67087f 100644 --- a/centaur/src/main/resources/standardTestCases/hello.test +++ b/centaur/src/main/resources/standardTestCases/hello.test @@ -1,6 +1,5 @@ name: hello testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: hello/hello.wdl diff --git a/centaur/src/main/resources/standardTestCases/input_localization/localize_file_larger_than_disk_space.wdl b/centaur/src/main/resources/standardTestCases/input_localization/localize_file_larger_than_disk_space.wdl deleted file mode 100644 index b90c63fe928..00000000000 --- a/centaur/src/main/resources/standardTestCases/input_localization/localize_file_larger_than_disk_space.wdl +++ /dev/null @@ -1,27 +0,0 @@ -version 1.0 - -task localize_file { - input { - File input_file - } - command { - cat "localizing file over 1 GB" - } - runtime { - docker: "ubuntu:latest" - disks: "local-disk 1 HDD" - } - output { - String out = read_string(stdout()) - } -} - -workflow localize_file_larger_than_disk_space { - File wf_input = "gs://cromwell_test_bucket/file_over_1_gb.txt" - - call localize_file { input: input_file = wf_input } - - output { - String content = localize_file.out - } -} diff --git a/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code.test b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code.test new file mode 100644 index 00000000000..8f55af82262 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code.test @@ -0,0 +1,11 @@ +name: invalid_return_codes_and_continue_on_return_code +testFormat: workflowfailure + +files { + workflow: invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl +} + +metadata { + workflowName: InvalidReturnCodeAndContinueOnReturnCode + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl new file mode 100644 index 00000000000..ea2233ab355 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl @@ -0,0 +1,21 @@ +version development-1.1 + +workflow InvalidReturnCodeAndContinueOnReturnCode { + call InvalidReturnCodeContinueOnReturnCode +} + +task InvalidReturnCodeContinueOnReturnCode { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [5, 10, 15] + continueOnReturnCode: [1] + } +} diff --git a/centaur/src/main/resources/standardTestCases/localize_file_larger_than_disk_space.test b/centaur/src/main/resources/standardTestCases/localize_file_larger_than_disk_space.test deleted file mode 100644 index bee99677b01..00000000000 --- a/centaur/src/main/resources/standardTestCases/localize_file_larger_than_disk_space.test +++ /dev/null @@ -1,17 +0,0 @@ -name: localize_file_larger_than_disk_space -testFormat: workflowfailure -backends: [Papiv2] -workflowType: WDL -workflowTypeVersion: 1.0 -tags: ["wdl_1.0"] - -files { - workflow: input_localization/localize_file_larger_than_disk_space.wdl -} - -metadata { - workflowName: localize_file_larger_than_disk_space - status: Failed - "failures.0.message": "Workflow failed" - "failures.0.causedBy.0.message": "Task localize_file_larger_than_disk_space.localize_file:NA:1 failed. The job was stopped before the command finished. PAPI error code 9. Please check the log file for more details: gs://cloud-cromwell-dev-self-cleaning/cromwell_execution/ci/localize_file_larger_than_disk_space/<>/call-localize_file/localize_file.log." -} diff --git a/centaur/src/main/resources/standardTestCases/nested_struct_bad_instantiation.test b/centaur/src/main/resources/standardTestCases/nested_struct_bad_instantiation.test new file mode 100644 index 00000000000..a2608a0262c --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/nested_struct_bad_instantiation.test @@ -0,0 +1,14 @@ +name: nested_struct_bad_instantiation +testFormat: workflowfailure + +files { + workflow: failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.wdl + inputs: failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.inputs +} + +metadata { + workflowName: MinimalStructExample + status: Failed + "failures.0.message": "Workflow failed" + "failures.0.causedBy.0.message": "Failed to evaluate 'example2' (reason 1 of 1): Evaluating { \"second\": test, \"lowerLayer\": example1 } failed: Cannot construct WomMapType(WomStringType,WomAnyType) with mixed types in map values: [WomString(Hello World), WomObject(Map(first -> WomString(Hello World), number -> WomInteger(2)),WomCompositeType(Map(first -> WomStringType, number -> WomIntegerType),Some(firstLayer)))]" +} diff --git a/centaur/src/main/resources/standardTestCases/object_access.test b/centaur/src/main/resources/standardTestCases/object_access.test index 1e824f6e78f..84d76d5a893 100644 --- a/centaur/src/main/resources/standardTestCases/object_access.test +++ b/centaur/src/main/resources/standardTestCases/object_access.test @@ -1,6 +1,5 @@ name: object_access testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: object_access/object_access.wdl diff --git a/centaur/src/main/resources/standardTestCases/papi_cpu_platform.test b/centaur/src/main/resources/standardTestCases/papi_cpu_platform.test index 5aca2634edb..7b38c3a25d7 100644 --- a/centaur/src/main/resources/standardTestCases/papi_cpu_platform.test +++ b/centaur/src/main/resources/standardTestCases/papi_cpu_platform.test @@ -8,9 +8,10 @@ files { metadata { status: Succeeded - "outputs.cpus.cascadeLake.cpuPlatform": "Intel Cascade Lake" "outputs.cpus.broadwell.cpuPlatform": "Intel Broadwell" "outputs.cpus.haswell.cpuPlatform": "Intel Haswell" + "outputs.cpus.cascadeLake.cpuPlatform": "Intel Cascade Lake" + "outputs.cpus.iceLake.cpuPlatform": "Intel Ice Lake" "outputs.cpus.rome.cpuPlatform": "AMD Rome" } diff --git a/centaur/src/main/resources/standardTestCases/papi_cpu_platform/papi_cpu_platform.wdl b/centaur/src/main/resources/standardTestCases/papi_cpu_platform/papi_cpu_platform.wdl index 39d25e6fe47..3ec1bd77cae 100644 --- a/centaur/src/main/resources/standardTestCases/papi_cpu_platform/papi_cpu_platform.wdl +++ b/centaur/src/main/resources/standardTestCases/papi_cpu_platform/papi_cpu_platform.wdl @@ -24,8 +24,9 @@ task cpu_platform { } workflow cpus { - call cpu_platform as haswell { input: cpu_platform = "Intel Haswell" } - call cpu_platform as broadwell { input: cpu_platform = "Intel Broadwell" } - call cpu_platform as cascadeLake { input: cpu_platform = "Intel Cascade Lake" } - call cpu_platform as rome {input: cpu_platform = "AMD Rome" } + call cpu_platform as haswell { input: cpu_platform = "Intel Haswell" } + call cpu_platform as broadwell { input: cpu_platform = "Intel Broadwell" } + call cpu_platform as cascadeLake { input: cpu_platform = "Intel Cascade Lake" } + call cpu_platform as iceLake { input: cpu_platform = "Intel Ice Lake" } + call cpu_platform as rome { input: cpu_platform = "AMD Rome" } } diff --git a/centaur/src/main/resources/standardTestCases/return_codes.test b/centaur/src/main/resources/standardTestCases/return_codes.test new file mode 100644 index 00000000000..7528064f4c8 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/return_codes.test @@ -0,0 +1,11 @@ +name: return_codes +testFormat: workflowsuccess + +files { + workflow: return_codes/return_codes.wdl +} + +metadata { + workflowName: ReturnCodeValidation + status: Succeeded +} diff --git a/centaur/src/main/resources/standardTestCases/return_codes/return_codes.wdl b/centaur/src/main/resources/standardTestCases/return_codes/return_codes.wdl new file mode 100644 index 00000000000..21e1da37650 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/return_codes/return_codes.wdl @@ -0,0 +1,69 @@ +version development-1.1 + +workflow ReturnCodeValidation { + call ReturnCodeSet1 + call ReturnCodeSet2 + call ReturnCodeSet3 + call ReturnCodeString +} + +task ReturnCodeSet1 { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1] + } +} + +task ReturnCodeSet2 { + meta { + volatile: true + } + + command <<< + exit 200 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1, 123, 200] + } +} + +task ReturnCodeSet3 { + meta { + volatile: true + } + + command <<< + exit 10 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: 10 + } +} + + +task ReturnCodeString { + meta { + volatile: true + } + + command <<< + exit 500 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: "*" + } +} diff --git a/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version.test b/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version.test new file mode 100644 index 00000000000..20c07a6e051 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version.test @@ -0,0 +1,11 @@ +name: return_codes_invalid_on_old_wdl_version +testFormat: workflowfailure + +files { + workflow: return_codes_invalid_on_old_wdl_version/return_codes_invalid_on_old_wdl_version.wdl +} + +metadata { + workflowName: ReturnCodesInvalidOnOldWdl + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version/return_codes_invalid_on_old_wdl_version.wdl b/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version/return_codes_invalid_on_old_wdl_version.wdl new file mode 100644 index 00000000000..296dc833f02 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version/return_codes_invalid_on_old_wdl_version.wdl @@ -0,0 +1,18 @@ +workflow ReturnCodesInvalidOnOldWdl { + call ReturnCodesInvalidOnOldWdlTask +} + +task ReturnCodesInvalidOnOldWdlTask { + meta { + volatile: "true" + } + + command <<< + exit 5 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [5, 10, 15] + } +} diff --git a/centaur/src/main/resources/standardTestCases/scatterchain.test b/centaur/src/main/resources/standardTestCases/scatterchain.test index 26fdb95b27b..c1b02be7abe 100644 --- a/centaur/src/main/resources/standardTestCases/scatterchain.test +++ b/centaur/src/main/resources/standardTestCases/scatterchain.test @@ -1,6 +1,5 @@ name: scatterchain testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: scatter_chain/scatter_chain.wdl diff --git a/centaur/src/main/resources/standardTestCases/scattergather.test b/centaur/src/main/resources/standardTestCases/scattergather.test index fad78c0a08c..425be6fd3c4 100644 --- a/centaur/src/main/resources/standardTestCases/scattergather.test +++ b/centaur/src/main/resources/standardTestCases/scattergather.test @@ -1,6 +1,5 @@ name: scattergather testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: scattergather/scattergather.wdl diff --git a/centaur/src/main/resources/standardTestCases/short_circuit.test b/centaur/src/main/resources/standardTestCases/short_circuit.test index a0727dc949f..a961f39071b 100644 --- a/centaur/src/main/resources/standardTestCases/short_circuit.test +++ b/centaur/src/main/resources/standardTestCases/short_circuit.test @@ -2,7 +2,7 @@ name: short_circuit testFormat: workflowsuccess backends: [Local] -tags: [localdockertest, "wdl_upgrade"] +tags: [localdockertest] files { diff --git a/centaur/src/main/resources/standardTestCases/square.test b/centaur/src/main/resources/standardTestCases/square.test index ef0ac9a9ef8..db4e5851ead 100644 --- a/centaur/src/main/resources/standardTestCases/square.test +++ b/centaur/src/main/resources/standardTestCases/square.test @@ -1,6 +1,5 @@ name: square testFormat: workflowsuccess -tags: [ "wdl_upgrade" ] files { workflow: square/square.wdl diff --git a/centaur/src/main/resources/standardTestCases/stderr_stdout_workflow_body.test b/centaur/src/main/resources/standardTestCases/stderr_stdout_workflow_body.test new file mode 100644 index 00000000000..71cd116b44a --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/stderr_stdout_workflow_body.test @@ -0,0 +1,13 @@ +name: stderr_stdout_workflow_body +testFormat: workflowfailure + +files { + workflow: failures/stderr_stdout_workflow_body/stderr_stdout_workflow_body.wdl +} + +metadata { + workflowName: break_with_stderr + status: Failed + "failures.0.message": "Workflow failed" + "failures.0.causedBy.0.message": "Failed to evaluate 'break_with_stderr.load_data_csv' (reason 1 of 2): Evaluating select_first([stdout(), stderr()]) failed: stdout is not implemented at the workflow level, Failed to evaluate 'break_with_stderr.load_data_csv' (reason 2 of 2): Evaluating select_first([stdout(), stderr()]) failed: stderr is not implemented at the workflow level" +} diff --git a/centaur/src/main/resources/standardTestCases/string_interpolation.test b/centaur/src/main/resources/standardTestCases/string_interpolation.test index 7e7e894f9fa..04736e59ea4 100644 --- a/centaur/src/main/resources/standardTestCases/string_interpolation.test +++ b/centaur/src/main/resources/standardTestCases/string_interpolation.test @@ -1,6 +1,5 @@ name: string_interpolation testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: string_interpolation/string_interpolation.wdl diff --git a/centaur/src/main/resources/standardTestCases/struct_literal.test b/centaur/src/main/resources/standardTestCases/struct_literal.test new file mode 100644 index 00000000000..72755145a02 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/struct_literal.test @@ -0,0 +1,12 @@ +name: struct_literal +testFormat: workflowsuccess + +files { + workflow: struct_literal/struct_literal.wdl +} + +metadata { + workflowName: struct_literal + status: Succeeded + "outputs.struct_literal.out": 44 +} diff --git a/centaur/src/main/resources/standardTestCases/struct_literal/struct_literal.wdl b/centaur/src/main/resources/standardTestCases/struct_literal/struct_literal.wdl new file mode 100644 index 00000000000..215fd40f8d3 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/struct_literal/struct_literal.wdl @@ -0,0 +1,68 @@ +version development-1.1 + +struct Plant { + String color + Int id +} + +struct Fungi { + File fungiFile +} + + +struct Animal { + Plant jacket + Fungi hat +} + +task a { + input { + Plant in_plant_literal = Plant{color: "red", id: 44} + } + + command { + echo "${in_plant_literal.id}" + } + + output { + Animal out_animal = Animal{jacket: Plant{color: "green", id: 10}, hat: Fungi{fungiFile: stdout()}} + } + + runtime { + docker: "ubuntu:latest" + } + + meta { + volatile: true + } +} + +task b { + input { + Animal in_animal + } + + command { + cat ${in_animal.hat.fungiFile} + } + + output { + Int out = read_int(stdout()) + } + + runtime { + docker: "ubuntu:latest" + } + + meta { + volatile: true + } +} + +workflow struct_literal { + call a + call b {input: in_animal=a.out_animal} + output { + Int out = b.out + } +} diff --git a/centaur/src/main/resources/standardTestCases/sub_workflow_hello_world.test b/centaur/src/main/resources/standardTestCases/sub_workflow_hello_world.test index 4b216940cb2..cc18f753e42 100644 --- a/centaur/src/main/resources/standardTestCases/sub_workflow_hello_world.test +++ b/centaur/src/main/resources/standardTestCases/sub_workflow_hello_world.test @@ -1,6 +1,6 @@ name: sub_workflow_hello_world testFormat: workflowsuccess -tags: [subworkflow, "wdl_upgrade"] +tags: [subworkflow] files { workflow: sub_workflow_hello_world/sub_workflow_hello_world.wdl diff --git a/centaur/src/main/resources/standardTestCases/sub_workflow_var_refs.test b/centaur/src/main/resources/standardTestCases/sub_workflow_var_refs.test index 390c089359b..deca93aa195 100644 --- a/centaur/src/main/resources/standardTestCases/sub_workflow_var_refs.test +++ b/centaur/src/main/resources/standardTestCases/sub_workflow_var_refs.test @@ -1,6 +1,6 @@ name: sub_workflow_var_refs testFormat: workflowsuccess -tags: [subworkflow, "wdl_upgrade"] +tags: [subworkflow] files { workflow: sub_workflow_var_refs/sub_workflow_var_refs.wdl diff --git a/centaur/src/main/resources/standardTestCases/valid_labels.test b/centaur/src/main/resources/standardTestCases/valid_labels.test index 1ab748f8a86..6f7f4e7ceec 100644 --- a/centaur/src/main/resources/standardTestCases/valid_labels.test +++ b/centaur/src/main/resources/standardTestCases/valid_labels.test @@ -1,6 +1,6 @@ name: valid_labels testFormat: workflowsuccess -tags: [ labels, "wdl_upgrade" ] +tags: [ labels ] files { workflow: hello/hello.wdl diff --git a/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl new file mode 100644 index 00000000000..41acb028221 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl @@ -0,0 +1,55 @@ +version development-1.1 + +workflow ValidReturnCodeAndContinueOnReturnCode { + call ReturnCodeContinueOnReturnCode1 + call ReturnCodeContinueOnReturnCode2 + call ReturnCodeContinueOnReturnCode3 +} + +task ReturnCodeContinueOnReturnCode1 { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1] + continueOnReturnCode: [0] + } +} + +task ReturnCodeContinueOnReturnCode2 { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1] + continueOnReturnCode: false + } +} + +task ReturnCodeContinueOnReturnCode3 { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1, 4, 7] + continueOnReturnCode: [1, 3, 5] + } +} diff --git a/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_codes.test b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_codes.test new file mode 100644 index 00000000000..1a8354b9482 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_codes.test @@ -0,0 +1,11 @@ +name: valid_return_codes_and_continue_on_return_code +testFormat: workflowsuccess + +files { + workflow: valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl +} + +metadata { + workflowName: ValidReturnCodeAndContinueOnReturnCode + status: Succeeded +} diff --git a/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl index 220fcb535e5..979af2dc778 100644 --- a/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl +++ b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl @@ -4,7 +4,7 @@ workflow biscayne_new_engine_functions { meta { description: "This test makes sure that these functions work in a real workflow" - functions_under_test: [ "keys", "as_map", "as_pairs", "collect_by_key" ] + functions_under_test: [ "keys", "as_map", "as_pairs", "collect_by_key", "quote", "squote", "sub", "suffix", "unzip" ] } Map[String, Int] x_map_in = {"a": 1, "b": 2, "c": 3} @@ -15,6 +15,10 @@ workflow biscayne_new_engine_functions { Array[Pair[String,Int]] z_pairs_in = [("a", 1), ("b", 2), ("a", 3)] + Array[String] some_strings = ["aaa", "bbb", "ccc"] + + Array[Int] some_ints = [1, 2, 3] + Int smallestInt = 1 Float smallFloat = 2.718 Float bigFloat = 3.141 @@ -24,6 +28,10 @@ workflow biscayne_new_engine_functions { # max float... near enough: Float maxFloat = 179769313000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0 + Array[Pair[String,String]] zipped_a = [("A", "a")] + Array[Pair[String,String]] zipped_b = [("A", "a"),("B", "b")] + Array[Pair[String,Float]] zipped_c = [("one", 1.0),("two", 2.0),("three", 3.0)] + output { # keys(), as_map(), as_pairs(), collect_by_key(): @@ -48,6 +56,31 @@ workflow biscayne_new_engine_functions { Float bigIntFloatComparison = max(bigFloat, biggestInt) # 10.0 Float minMaxIntFloatComposition = min(max(biggestInt, smallFloat), smallestInt) # 1.0 Float maxIntVsMaxFloat = max(maxInt, maxFloat) + + # sub(): + # (Exists before Biscayne, but uses different regex flavor here) + # ================================================= + String substituted = sub("AtheZ", "[[:upper:]]", "WAT") + + # suffix(): + # ================================================= + Array[String] with_suffixes = suffix("S", some_strings) + + # quote(): + # ================================================= + Array[String] with_quotes = quote(some_ints) + Array[String] string_with_quotes = quote(some_strings) + + # squote(): + # ================================================= + Array[String] with_squotes = squote(some_ints) + Array[String] string_with_squotes = squote(some_strings) + + # unzip(): + # ================================================= + Pair[Array[String], Array[String]] unzipped_a = unzip(zipped_a) + Pair[Array[String], Array[String]] unzipped_b = unzip(zipped_b) + Pair[Array[String], Array[String]] unzipped_c = unzip(zipped_c) } } diff --git a/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_runtime_attributes/biscayne_new_runtime_attributes.wdl b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_runtime_attributes/biscayne_new_runtime_attributes.wdl new file mode 100644 index 00000000000..1174d5c02d6 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_runtime_attributes/biscayne_new_runtime_attributes.wdl @@ -0,0 +1,48 @@ +version development-1.1 + +workflow runtime_attributes_wf { + call runtime_attributes_task + output { + String out = runtime_attributes_task.out + } +} + +task runtime_attributes_task { + + command <<< + echo "Zardoz" + >>> + + meta { + volatile: true + } + + runtime { + # Meaningless keys are ignored + banana: object { + cpuPlatform: "Banana Lake" + } + + gcp: object { + # Platform-specific keys take precedence + docker: "rockylinux:9", + memory: "6 GB" + } + + azure: object { + memory: "4 GB", + docker: "debian:latest" + } + + # Generic keys are ignored in favor of platform ones + docker: "ubuntu:latest" + memory: "8 GB" + + # We still read generic keys that are not overridden + cpu: 4 + } + + output { + String out = read_string(stdout()) + } +} diff --git a/centaur/src/main/resources/standardTestCases/write_lines.test b/centaur/src/main/resources/standardTestCases/write_lines.test index 977459a50c7..fa0c6edd973 100644 --- a/centaur/src/main/resources/standardTestCases/write_lines.test +++ b/centaur/src/main/resources/standardTestCases/write_lines.test @@ -1,6 +1,5 @@ name: write_lines testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: write_lines/write_lines.wdl @@ -13,4 +12,3 @@ metadata { "calls.write_lines.f2a.executionStatus": Done "outputs.write_lines.a2f_second.x": "a\nb\nc\nd" } - diff --git a/centaur/src/main/scala/centaur/CentaurConfig.scala b/centaur/src/main/scala/centaur/CentaurConfig.scala index f17c288283a..c15ae31f453 100644 --- a/centaur/src/main/scala/centaur/CentaurConfig.scala +++ b/centaur/src/main/scala/centaur/CentaurConfig.scala @@ -41,16 +41,19 @@ sealed trait CentaurRunMode { def cromwellUrl: URL } -case class UnmanagedCromwellServer(cromwellUrl : URL) extends CentaurRunMode -case class ManagedCromwellServer(preRestart: CromwellConfiguration, postRestart: CromwellConfiguration, withRestart: Boolean) extends CentaurRunMode { +case class UnmanagedCromwellServer(cromwellUrl: URL) extends CentaurRunMode +case class ManagedCromwellServer(preRestart: CromwellConfiguration, + postRestart: CromwellConfiguration, + withRestart: Boolean +) extends CentaurRunMode { override val cromwellUrl = new URL(s"http://localhost:${CromwellManager.ManagedCromwellPort}") } object CentaurConfig { lazy val conf: Config = ConfigFactory.load().getConfig("centaur") - + lazy val runMode: CentaurRunMode = CentaurRunMode(conf) - + lazy val cromwellUrl: URL = runMode.cromwellUrl lazy val workflowProgressTimeout: FiniteDuration = conf.getDuration("workflow-progress-timeout").toScala lazy val sendReceiveTimeout: FiniteDuration = conf.getDuration("sendReceiveTimeout").toScala diff --git a/centaur/src/main/scala/centaur/CromwellManager.scala b/centaur/src/main/scala/centaur/CromwellManager.scala index a53416a0ded..55c317b41b3 100644 --- a/centaur/src/main/scala/centaur/CromwellManager.scala +++ b/centaur/src/main/scala/centaur/CromwellManager.scala @@ -18,15 +18,15 @@ object CromwellManager extends StrictLogging { private var cromwellProcess: Option[CromwellProcess] = None private var _ready: Boolean = false private var _isManaged: Boolean = false - + /** * Returns true if Cromwell is ready to be queried, false otherwise * In Unmanaged mode, this is irrelevant so always return true. * In managed mode return the value of _ready */ def isReady: Boolean = !_isManaged || _ready - - // Check that we have a cromwellProcess, that this process is alive, and that cromwell is ready to accept requests + + // Check that we have a cromwellProcess, that this process is alive, and that cromwell is ready to accept requests private def isAlive(checkType: String): Boolean = { val processAlive = cromwellProcess.exists(_.isAlive) logger.info(s"Cromwell process alive $checkType = $processAlive") @@ -76,14 +76,14 @@ object CromwellManager extends StrictLogging { def stopCromwell(reason: String) = { _ready = false logger.info(s"Stopping Cromwell... ($reason)") - try { + try cromwellProcess foreach { _.stop() } - } catch { - case e: Exception => + catch { + case e: Exception => logger.error("Caught exception while stopping Cromwell") e.printStackTrace() } - + cromwellProcess = None } } diff --git a/centaur/src/main/scala/centaur/CromwellTracker.scala b/centaur/src/main/scala/centaur/CromwellTracker.scala index 4a89dea302d..1ec6fefe54c 100644 --- a/centaur/src/main/scala/centaur/CromwellTracker.scala +++ b/centaur/src/main/scala/centaur/CromwellTracker.scala @@ -14,7 +14,6 @@ import org.apache.commons.math3.stat.inference.ChiSquareTest import scala.language.postfixOps - case class CromwellTracker(backendCount: Int, configuredSignificance: Double) extends StrictLogging { var counts: Map[String, Int] = Map() def track(metadata: WorkflowMetadata): Unit = { @@ -45,10 +44,15 @@ case class CromwellTracker(backendCount: Int, configuredSignificance: Double) ex val actual: Array[Long] = counts.values map { _.toLong } toArray val observedSignificance = new ChiSquareTest().chiSquareTest(expected, actual) - logger.info(f"configured/observed horicromtal significance levels: $configuredSignificance%.4f/$observedSignificance%.4f", configuredSignificance, observedSignificance) + logger.info( + f"configured/observed horicromtal significance levels: $configuredSignificance%.4f/$observedSignificance%.4f", + configuredSignificance, + observedSignificance + ) if (observedSignificance < configuredSignificance) { - val message = f"Failed horicromtal check: observed significance level $observedSignificance%.4f, minimum of $configuredSignificance%.4f was required" + val message = + f"Failed horicromtal check: observed significance level $observedSignificance%.4f, minimum of $configuredSignificance%.4f was required" throw new RuntimeException(message) } } diff --git a/centaur/src/main/scala/centaur/DockerComposeCromwellConfiguration.scala b/centaur/src/main/scala/centaur/DockerComposeCromwellConfiguration.scala index 6fab385537b..eeb1df1557f 100644 --- a/centaur/src/main/scala/centaur/DockerComposeCromwellConfiguration.scala +++ b/centaur/src/main/scala/centaur/DockerComposeCromwellConfiguration.scala @@ -14,13 +14,17 @@ object DockerComposeCromwellConfiguration { } } -case class DockerComposeCromwellConfiguration(dockerTag: String, dockerComposeFile: String, conf: String, logFile: String) extends CromwellConfiguration { +case class DockerComposeCromwellConfiguration(dockerTag: String, + dockerComposeFile: String, + conf: String, + logFile: String +) extends CromwellConfiguration { override def createProcess: CromwellProcess = { - case class DockerComposeCromwellProcess(override val cromwellConfiguration: DockerComposeCromwellConfiguration) extends CromwellProcess { + case class DockerComposeCromwellProcess(override val cromwellConfiguration: DockerComposeCromwellConfiguration) + extends CromwellProcess { - private def composeCommand(command: String*): Array[String] = { + private def composeCommand(command: String*): Array[String] = Array("docker-compose", "-f", dockerComposeFile) ++ command - } private val startCommand = composeCommand("up", "--abort-on-container-exit") private val logsCommand = composeCommand("logs") @@ -29,14 +33,13 @@ case class DockerComposeCromwellConfiguration(dockerTag: String, dockerComposeFi private val envVariables = Map[String, String]( "CROMWELL_BUILD_CENTAUR_MANAGED_PORT" -> ManagedCromwellPort.toString, "CROMWELL_BUILD_CENTAUR_MANAGED_TAG" -> dockerTag, - "CROMWELL_BUILD_CENTAUR_MANAGED_CONFIG" -> conf, + "CROMWELL_BUILD_CENTAUR_MANAGED_CONFIG" -> conf ) private var process: Option[Process] = None - override def start(): Unit = { + override def start(): Unit = process = Option(runProcess(startCommand, envVariables)) - } override def stop(): Unit = { if (!isAlive) { diff --git a/centaur/src/main/scala/centaur/JarCromwellConfiguration.scala b/centaur/src/main/scala/centaur/JarCromwellConfiguration.scala index c95316d1a32..1bf83b16ee1 100644 --- a/centaur/src/main/scala/centaur/JarCromwellConfiguration.scala +++ b/centaur/src/main/scala/centaur/JarCromwellConfiguration.scala @@ -15,20 +15,15 @@ object JarCromwellConfiguration { case class JarCromwellConfiguration(jar: String, conf: String, logFile: String) extends CromwellConfiguration { override def createProcess: CromwellProcess = { - case class JarCromwellProcess(override val cromwellConfiguration: JarCromwellConfiguration) extends CromwellProcess { - private val command = Array( - "java", - s"-Dconfig.file=$conf", - s"-Dwebservice.port=$ManagedCromwellPort", - "-jar", - jar, - "server") + case class JarCromwellProcess(override val cromwellConfiguration: JarCromwellConfiguration) + extends CromwellProcess { + private val command = + Array("java", s"-Dconfig.file=$conf", s"-Dwebservice.port=$ManagedCromwellPort", "-jar", jar, "server") private var process: Option[Process] = None - override def start(): Unit = { + override def start(): Unit = process = Option(runProcess(command, Map.empty)) - } override def stop(): Unit = { process foreach { @@ -37,7 +32,7 @@ case class JarCromwellConfiguration(jar: String, conf: String, logFile: String) process = None } - override def isAlive: Boolean = process.exists { _.isAlive } + override def isAlive: Boolean = process.exists(_.isAlive) override def logFile: String = cromwellConfiguration.logFile } diff --git a/centaur/src/main/scala/centaur/api/CentaurCromwellClient.scala b/centaur/src/main/scala/centaur/api/CentaurCromwellClient.scala index 33f96974b42..defc30bc108 100644 --- a/centaur/src/main/scala/centaur/api/CentaurCromwellClient.scala +++ b/centaur/src/main/scala/centaur/api/CentaurCromwellClient.scala @@ -30,20 +30,20 @@ object CentaurCromwellClient extends StrictLogging { // Do not use scala.concurrent.ExecutionContext.Implicits.global as long as this is using Await.result // See https://github.com/akka/akka-http/issues/602 // And https://github.com/viktorklang/blog/blob/master/Futures-in-Scala-2.12-part-7.md - final implicit val blockingEc: ExecutionContextExecutor = ExecutionContext.fromExecutor( - Executors.newFixedThreadPool(100, DaemonizedDefaultThreadFactory)) + implicit final val blockingEc: ExecutionContextExecutor = + ExecutionContext.fromExecutor(Executors.newFixedThreadPool(100, DaemonizedDefaultThreadFactory)) // Akka HTTP needs both the actor system and a materializer - final implicit val system: ActorSystem = ActorSystem("centaur-acting-like-a-system") - final implicit val materializer: ActorMaterializer = ActorMaterializer(ActorMaterializerSettings(system)) + implicit final val system: ActorSystem = ActorSystem("centaur-acting-like-a-system") + implicit final val materializer: ActorMaterializer = ActorMaterializer(ActorMaterializerSettings(system)) final val apiVersion = "v1" val cromwellClient = new CromwellClient(CentaurConfig.cromwellUrl, apiVersion) val defaultMetadataArgs: Option[Map[String, List[String]]] = config.getAs[Map[String, List[String]]]("centaur.metadata-args") - def submit(workflow: Workflow): IO[SubmittedWorkflow] = { - sendReceiveFutureCompletion(() => { + def submit(workflow: Workflow): IO[SubmittedWorkflow] = + sendReceiveFutureCompletion { () => val submitted = cromwellClient.submit(workflow.toWorkflowSubmission) submitted.biSemiflatMap( httpResponse => @@ -59,44 +59,40 @@ object CentaurCromwellClient extends StrictLogging { _ = workflow.submittedWorkflowTracker.add(submittedWorkflow) } yield submittedWorkflow ) - }) - } + } - def describe(workflow: Workflow): IO[WaasDescription] = { + def describe(workflow: Workflow): IO[WaasDescription] = sendReceiveFutureCompletion(() => cromwellClient.describe(workflow.toWorkflowDescribeRequest)) - } - def status(workflow: SubmittedWorkflow): IO[WorkflowStatus] = { + def status(workflow: SubmittedWorkflow): IO[WorkflowStatus] = sendReceiveFutureCompletion(() => cromwellClient.status(workflow.id)) - } - def abort(workflow: SubmittedWorkflow): IO[WorkflowStatus] = { + def abort(workflow: SubmittedWorkflow): IO[WorkflowStatus] = sendReceiveFutureCompletion(() => cromwellClient.abort(workflow.id)) - } - def outputs(workflow: SubmittedWorkflow): IO[WorkflowOutputs] = { + def outputs(workflow: SubmittedWorkflow): IO[WorkflowOutputs] = sendReceiveFutureCompletion(() => cromwellClient.outputs(workflow.id)) - } - def callCacheDiff(workflowA: SubmittedWorkflow, callA: String, workflowB: SubmittedWorkflow, callB: String): IO[CallCacheDiff] = { - sendReceiveFutureCompletion(() => cromwellClient.callCacheDiff(workflowA.id, callA, ShardIndex(None), workflowB.id, callB, ShardIndex(None))) - } + def callCacheDiff(workflowA: SubmittedWorkflow, + callA: String, + workflowB: SubmittedWorkflow, + callB: String + ): IO[CallCacheDiff] = + sendReceiveFutureCompletion(() => + cromwellClient.callCacheDiff(workflowA.id, callA, ShardIndex(None), workflowB.id, callB, ShardIndex(None)) + ) - def logs(workflow: SubmittedWorkflow): IO[WorkflowMetadata] = { + def logs(workflow: SubmittedWorkflow): IO[WorkflowMetadata] = sendReceiveFutureCompletion(() => cromwellClient.logs(workflow.id)) - } - def labels(workflow: SubmittedWorkflow): IO[WorkflowLabels] = { + def labels(workflow: SubmittedWorkflow): IO[WorkflowLabels] = sendReceiveFutureCompletion(() => cromwellClient.labels(workflow.id)) - } - def addLabels(workflow: SubmittedWorkflow, newLabels: List[Label]): IO[WorkflowLabels] = { + def addLabels(workflow: SubmittedWorkflow, newLabels: List[Label]): IO[WorkflowLabels] = sendReceiveFutureCompletion(() => cromwellClient.addLabels(workflow.id, newLabels)) - } - def version: IO[CromwellVersion] = { + def version: IO[CromwellVersion] = sendReceiveFutureCompletion(() => cromwellClient.version) - } /* Sends a quick ping to the Cromwell query endpoint. The query endpoint is the only one which both hits the @@ -104,7 +100,9 @@ object CentaurCromwellClient extends StrictLogging { currently does not support query. */ def isAlive: Boolean = { - val response = Http().singleRequest(HttpRequest(uri=s"${CentaurConfig.cromwellUrl}/api/workflows/$apiVersion/query?status=Succeeded")) + val response = Http().singleRequest( + HttpRequest(uri = s"${CentaurConfig.cromwellUrl}/api/workflows/$apiVersion/query?status=Succeeded") + ) // Silence the following warning by discarding the result of a successful query: // Response entity was not subscribed after 1 second. Make sure to read the response entity body or call `discardBytes()` on it. val successOrFailure = response map { _.entity.discardBytes() } @@ -113,18 +111,19 @@ object CentaurCromwellClient extends StrictLogging { def metadata(workflow: SubmittedWorkflow, args: Option[Map[String, List[String]]] = defaultMetadataArgs, - expandSubworkflows: Boolean = false): IO[WorkflowMetadata] = { + expandSubworkflows: Boolean = false + ): IO[WorkflowMetadata] = { val mandatoryArgs = Map("expandSubWorkflows" -> List(expandSubworkflows.toString)) metadataWithId(workflow.id, Option(args.getOrElse(Map.empty) ++ mandatoryArgs)) } - def metadataWithId(id: WorkflowId, args: Option[Map[String, List[String]]] = defaultMetadataArgs): IO[WorkflowMetadata] = { + def metadataWithId(id: WorkflowId, + args: Option[Map[String, List[String]]] = defaultMetadataArgs + ): IO[WorkflowMetadata] = sendReceiveFutureCompletion(() => cromwellClient.metadata(id, args)) - } - def archiveStatus(id: WorkflowId): IO[String] = { + def archiveStatus(id: WorkflowId): IO[String] = sendReceiveFutureCompletion(() => cromwellClient.query(id)).map(_.results.head.metadataArchiveStatus) - } implicit private val timer: Timer[IO] = IO.timer(blockingEc) implicit private val contextShift: ContextShift[IO] = IO.contextShift(blockingEc) @@ -137,42 +136,43 @@ object CentaurCromwellClient extends StrictLogging { val stackTraceString = ExceptionUtils.getStackTrace(new Exception) - ioDelay.flatMap( _ => + ioDelay.flatMap(_ => // Could probably use IO to do the retrying too. For now use a copyport of Retry from cromwell core. Retry 5 times, // wait 5 seconds between retries. Timeout the whole thing using the IO timeout. // https://github.com/cb372/cats-retry // https://typelevel.org/cats-effect/datatypes/io.html#example-retrying-with-exponential-backoff - IO.fromFuture(IO(Retry.withRetry( - () => func().asIo.unsafeToFuture(), - Option(5), - 5.seconds, - isTransient = isTransient, - isFatal = isFatal - )).timeoutTo(timeout, - { + IO.fromFuture( + IO( + Retry.withRetry( + () => func().asIo.unsafeToFuture(), + Option(5), + 5.seconds, + isTransient = isTransient, + isFatal = isFatal + ) + ).timeoutTo( + timeout, IO.raiseError(new TimeoutException("Timeout from retryRequest " + timeout.toString + ": " + stackTraceString)) - } - ))) + ) + ) + ) } - def sendReceiveFutureCompletion[T](x: () => FailureResponseOrT[T]): IO[T] = { + def sendReceiveFutureCompletion[T](x: () => FailureResponseOrT[T]): IO[T] = retryRequest(x, CentaurConfig.sendReceiveTimeout) - } private def isFatal(f: Throwable) = f match { case _: DeserializationException => true case _ => false } - private def isTransient(f: Throwable) = { + private def isTransient(f: Throwable) = f match { - case _: StreamTcpException | - _: IOException | - _: UnsupportedContentTypeException => true + case _: StreamTcpException | _: IOException | _: UnsupportedContentTypeException => true case BufferOverflowException(message) => message.contains("Please retry the request later.") case unsuccessful: UnsuccessfulRequestException => unsuccessful.httpResponse.status == StatusCodes.NotFound - case unexpected: RuntimeException => unexpected.getMessage.contains("The http server closed the connection unexpectedly") + case unexpected: RuntimeException => + unexpected.getMessage.contains("The http server closed the connection unexpectedly") case _ => false } - } } diff --git a/centaur/src/main/scala/centaur/api/Retry.scala b/centaur/src/main/scala/centaur/api/Retry.scala index f384bc37684..1cbee9a1901 100644 --- a/centaur/src/main/scala/centaur/api/Retry.scala +++ b/centaur/src/main/scala/centaur/api/Retry.scala @@ -7,6 +7,7 @@ import scala.concurrent.duration._ import scala.concurrent.{ExecutionContext, Future} object Retry { + /** * Copied from cromwell.core * Replaced the backoff with a fixed retry delay @@ -16,8 +17,8 @@ object Retry { delay: FiniteDuration, isTransient: Throwable => Boolean = throwableToFalse, isFatal: Throwable => Boolean = throwableToFalse, - onRetry: Throwable => Unit = noopOnRetry) - (implicit actorSystem: ActorSystem): Future[A] = { + onRetry: Throwable => Unit = noopOnRetry + )(implicit actorSystem: ActorSystem): Future[A] = { // In the future we might want EC passed in separately but at the moment it caused more issues than it solved to do so implicit val ec: ExecutionContext = actorSystem.dispatcher @@ -25,10 +26,18 @@ object Retry { case throwable if isFatal(throwable) => Future.failed(throwable) case throwable if !isFatal(throwable) => val retriesLeft = if (isTransient(throwable)) maxRetries else maxRetries map { _ - 1 } - + if (retriesLeft.forall(_ > 0)) { onRetry(throwable) - after(delay, actorSystem.scheduler)(withRetry(f, delay = delay, maxRetries = retriesLeft, isTransient = isTransient, isFatal = isFatal, onRetry = onRetry)) + after(delay, actorSystem.scheduler)( + withRetry(f, + delay = delay, + maxRetries = retriesLeft, + isTransient = isTransient, + isFatal = isFatal, + onRetry = onRetry + ) + ) } else { Future.failed(throwable) } @@ -38,4 +47,3 @@ object Retry { def throwableToFalse(t: Throwable) = false def noopOnRetry(t: Throwable) = {} } - diff --git a/centaur/src/main/scala/centaur/json/JsonUtils.scala b/centaur/src/main/scala/centaur/json/JsonUtils.scala index 1af725d595b..90e3bd91a98 100644 --- a/centaur/src/main/scala/centaur/json/JsonUtils.scala +++ b/centaur/src/main/scala/centaur/json/JsonUtils.scala @@ -8,6 +8,7 @@ object JsonUtils { val attemptNumber = "attempt" implicit class EnhancedJsValue(val jsValue: JsValue) extends AnyVal { + /** * Modified from http://stackoverflow.com/a/31592156 - changes were made both to port from using * Play-JSON to Spray-JSON as well as to handle some cases specific to Cromwell's metadata response @@ -32,9 +33,8 @@ object JsonUtils { */ def flatten(prefix: String = ""): JsObject = { - def flattenShardAndAttempt(k:String, v: JsArray, f: JsObject => String): JsObject = { - v.elements.map(_.asJsObject).fold(JsObject.empty) { (x, y) => x ++ y.flatten(s"$k.${f(y)}") } - } + def flattenShardAndAttempt(k: String, v: JsArray, f: JsObject => String): JsObject = + v.elements.map(_.asJsObject).fold(JsObject.empty)((x, y) => x ++ y.flatten(s"$k.${f(y)}")) jsValue.asJsObject.fields.foldLeft(JsObject.empty) { case (acc, (k, v: JsArray)) if v.isSingleCallArray => acc ++ JsObject(k -> v.elements.head).flatten(prefix) @@ -44,10 +44,13 @@ object JsonUtils { to avoid lossy conversion for multiple attempts of the same shard. The older way of flattening shards with only shard index in the flattened structure is also kept so that the new structure doesn't fail tests that rely on the older flattened structure. This should be cleaned up in https://broadworkbench.atlassian.net/browse/BW-483 - */ + */ acc ++ flattenShardAndAttempt(k, v, (y: JsObject) => y.getField(shardIndex).get) ++ - flattenShardAndAttempt(k, v, (y: JsObject) => s"${y.getField(shardIndex).get}.${y.getField(attemptNumber).get}") + flattenShardAndAttempt(k, + v, + (y: JsObject) => s"${y.getField(shardIndex).get}.${y.getField(attemptNumber).get}" + ) case (acc, (k, v: JsArray)) => v.elements.zipWithIndex.foldLeft(acc) { case (accumulator, (element, idx)) => val maybePrefix = if (prefix.isEmpty) "" else s"$prefix." @@ -76,7 +79,7 @@ object JsonUtils { // A couple of helper functions to assist with flattening Cromwell metadata responses def hasField(fieldName: String): Boolean = jsObject.fields.keySet contains fieldName def getField(fieldName: String): Option[String] = jsObject.fields.get(fieldName) map { _.toString() } - def flattenToMap: Map [String, JsValue] = jsObject.flatten().fields map { case (k, v: JsValue) => k -> v} + def flattenToMap: Map[String, JsValue] = jsObject.flatten().fields map { case (k, v: JsValue) => k -> v } } /** @@ -89,9 +92,8 @@ object JsonUtils { def nonEmptyObjectArray = jsArray.isObjectArray && jsArray.nonEmpty def isSingleCallArray = jsArray.hasField(shardIndex) && jsArray.size == 1 - def hasField(fieldName: String): Boolean = { + def hasField(fieldName: String): Boolean = if (jsArray.nonEmptyObjectArray) jsArray.elements.map(_.asJsObject) forall { _.hasField(fieldName) } else false - } } } diff --git a/centaur/src/main/scala/centaur/test/CentaurTestException.scala b/centaur/src/main/scala/centaur/test/CentaurTestException.scala index fa83a3afb1a..ffc50a8dfdb 100644 --- a/centaur/src/main/scala/centaur/test/CentaurTestException.scala +++ b/centaur/src/main/scala/centaur/test/CentaurTestException.scala @@ -12,12 +12,12 @@ import cromwell.api.model.{SubmittedWorkflow, WorkflowMetadata} * @param metadataJsonOption The optional metadata. * @param causeOption The optional underlying cause. */ -case class CentaurTestException private(message: String, - testName: String, - workflowIdOption: Option[String], - metadataJsonOption: Option[String], - causeOption: Option[Exception]) - extends RuntimeException(message, causeOption.orNull) +case class CentaurTestException private (message: String, + testName: String, + workflowIdOption: Option[String], + metadataJsonOption: Option[String], + causeOption: Option[Exception] +) extends RuntimeException(message, causeOption.orNull) object CentaurTestException { @@ -25,7 +25,8 @@ object CentaurTestException { def apply(message: String, workflowDefinition: Workflow, submittedWorkflow: SubmittedWorkflow, - actualMetadata: WorkflowMetadata): CentaurTestException = { + actualMetadata: WorkflowMetadata + ): CentaurTestException = new CentaurTestException( message, workflowDefinition.testName, @@ -33,12 +34,9 @@ object CentaurTestException { Option(actualMetadata.value), None ) - } /** Create a new CentaurTestException for a submitted workflow. */ - def apply(message: String, - workflowDefinition: Workflow, - submittedWorkflow: SubmittedWorkflow): CentaurTestException = { + def apply(message: String, workflowDefinition: Workflow, submittedWorkflow: SubmittedWorkflow): CentaurTestException = new CentaurTestException( message, workflowDefinition.testName, @@ -46,10 +44,9 @@ object CentaurTestException { None, None ) - } /** Create a new CentaurTestException for only a workflow definition. */ - def apply(message: String, workflowDefinition: Workflow): CentaurTestException = { + def apply(message: String, workflowDefinition: Workflow): CentaurTestException = new CentaurTestException( message, workflowDefinition.testName, @@ -57,10 +54,9 @@ object CentaurTestException { None, None ) - } /** Create a new CentaurTestException for only a workflow definition, including a root cause. */ - def apply(message: String, workflowDefinition: Workflow, cause: Exception): CentaurTestException = { + def apply(message: String, workflowDefinition: Workflow, cause: Exception): CentaurTestException = new CentaurTestException( message, workflowDefinition.testName, @@ -68,5 +64,4 @@ object CentaurTestException { None, Option(cause) ) - } } diff --git a/centaur/src/main/scala/centaur/test/ObjectCounter.scala b/centaur/src/main/scala/centaur/test/ObjectCounter.scala index 124f78e1dc8..a5cb51ea190 100644 --- a/centaur/src/main/scala/centaur/test/ObjectCounter.scala +++ b/centaur/src/main/scala/centaur/test/ObjectCounter.scala @@ -40,20 +40,19 @@ object ObjectCounterInstances { listObjectsAtPath(_).size } - implicit val blobObjectCounter: ObjectCounter[BlobContainerClient] = (containerClient : BlobContainerClient) => { + implicit val blobObjectCounter: ObjectCounter[BlobContainerClient] = (containerClient: BlobContainerClient) => { val pathToInt: Path => Int = providedPath => { - //Our path parsing is somewhat GCP centric. Convert to a blob path starting from the container root. - def pathToBlobPath(parsedPath : Path) : String = { + // Our path parsing is somewhat GCP centric. Convert to a blob path starting from the container root. + def pathToBlobPath(parsedPath: Path): String = (Option(parsedPath.bucket), Option(parsedPath.directory)) match { case (None, _) => "" case (Some(_), None) => parsedPath.bucket case (Some(_), Some(_)) => parsedPath.bucket + "/" + parsedPath.directory } - } val fullPath = pathToBlobPath(providedPath) - val blobsInFolder = containerClient.listBlobsByHierarchy(fullPath) - //if something "isPrefix", it's a directory. Otherwise, its a file. We just want to count files. + val blobsInFolder = containerClient.listBlobsByHierarchy(fullPath) + // if something "isPrefix", it's a directory. Otherwise, its a file. We just want to count files. blobsInFolder.asScala.count(!_.isPrefix) } pathToInt(_) @@ -63,7 +62,8 @@ object ObjectCounterInstances { object ObjectCounterSyntax { implicit class ObjectCounterSyntax[A](client: A) { - def countObjects(regex: String)(implicit c: ObjectCounter[A]): String => Int = c.parsePath(regex) andThen c.countObjectsAtPath(client) + def countObjects(regex: String)(implicit c: ObjectCounter[A]): String => Int = + c.parsePath(regex) andThen c.countObjectsAtPath(client) } } diff --git a/centaur/src/main/scala/centaur/test/Test.scala b/centaur/src/main/scala/centaur/test/Test.scala index d0a56a2cbf5..66a4655a107 100644 --- a/centaur/src/main/scala/centaur/test/Test.scala +++ b/centaur/src/main/scala/centaur/test/Test.scala @@ -22,9 +22,20 @@ import com.typesafe.scalalogging.StrictLogging import common.validation.Validation._ import configs.syntax._ import cromwell.api.CromwellClient.UnsuccessfulRequestException -import cromwell.api.model.{CallCacheDiff, Failed, HashDifference, SubmittedWorkflow, Succeeded, TerminalStatus, WaasDescription, WorkflowId, WorkflowMetadata, WorkflowStatus} +import cromwell.api.model.{ + CallCacheDiff, + Failed, + HashDifference, + SubmittedWorkflow, + Succeeded, + TerminalStatus, + WaasDescription, + WorkflowId, + WorkflowMetadata, + WorkflowStatus +} import cromwell.cloudsupport.aws.AwsConfiguration -import cromwell.cloudsupport.azure.{AzureUtils} +import cromwell.cloudsupport.azure.AzureUtils import cromwell.cloudsupport.gcp.GoogleConfiguration import cromwell.cloudsupport.gcp.auth.GoogleAuthMode import io.circe.parser._ @@ -53,32 +64,28 @@ sealed abstract class Test[A] { object Test { def successful[A](value: A): Test[A] = testMonad.pure(value) - def invalidTestDefinition[A](message: String, workflowDefinition: Workflow): Test[A] = { + def invalidTestDefinition[A](message: String, workflowDefinition: Workflow): Test[A] = new Test[A] { override def run: IO[Nothing] = IO.raiseError(CentaurTestException(message, workflowDefinition)) } - } implicit val testMonad: Monad[Test] = new Monad[Test] { - override def flatMap[A, B](fa: Test[A])(f: A => Test[B]): Test[B] = { + override def flatMap[A, B](fa: Test[A])(f: A => Test[B]): Test[B] = new Test[B] { override def run: IO[B] = fa.run flatMap { f(_).run } } - } - override def pure[A](x: A): Test[A] = { + override def pure[A](x: A): Test[A] = new Test[A] { override def run: IO[A] = IO.pure(x) } - } /** Call the default non-stack-safe but correct version of this method. */ - override def tailRecM[A, B](a: A)(f: A => Test[Either[A, B]]): Test[B] = { + override def tailRecM[A, B](a: A)(f: A => Test[Either[A, B]]): Test[B] = flatMap(f(a)) { case Right(b) => pure(b) case Left(nextA) => tailRecM(nextA)(f) } - } } implicit class TestableIO[A](a: IO[A]) { @@ -103,14 +110,14 @@ object Operations extends StrictLogging { lazy val authName: String = googleConf.getString("auth") lazy val genomicsEndpointUrl: String = googleConf.getString("genomics.endpoint-url") lazy val genomicsAndStorageScopes = List(StorageScopes.CLOUD_PLATFORM_READ_ONLY, GenomicsScopes.GENOMICS) - lazy val credentials: Credentials = configuration.auth(authName) + lazy val credentials: Credentials = configuration + .auth(authName) .unsafe .credentials(genomicsAndStorageScopes) - lazy val credentialsProjectOption: Option[String] = { - Option(credentials) collect { - case serviceAccountCredentials: ServiceAccountCredentials => serviceAccountCredentials.getProjectId + lazy val credentialsProjectOption: Option[String] = + Option(credentials) collect { case serviceAccountCredentials: ServiceAccountCredentials => + serviceAccountCredentials.getProjectId } - } lazy val confProjectOption: Option[String] = googleConf.get[Option[String]]("project") valueOrElse None // The project from the config or from the credentials. By default the project is read from the system environment. lazy val projectOption: Option[String] = confProjectOption orElse credentialsProjectOption @@ -140,13 +147,14 @@ object Operations extends StrictLogging { lazy val awsConfiguration: AwsConfiguration = AwsConfiguration(CentaurConfig.conf) lazy val awsConf: Config = CentaurConfig.conf.getConfig("aws") lazy val awsAuthName: String = awsConf.getString("auths") - lazy val region: String = awsConf.getString("region") - lazy val accessKeyId: String = awsConf.getString("access-key") + lazy val region: String = awsConf.getString("region") + lazy val accessKeyId: String = awsConf.getString("access-key") lazy val secretAccessKey: String = awsConf.getString("secret-key") def buildAmazonS3Client: S3Client = { val basicAWSCredentials = AwsBasicCredentials.create(accessKeyId, secretAccessKey) - S3Client.builder() + S3Client + .builder() .region(Region.of(region)) .credentialsProvider(StaticCredentialsProvider.create(basicAWSCredentials)) .build() @@ -156,16 +164,16 @@ object Operations extends StrictLogging { val azureSubscription = azureConfig.getString("subscription") val blobContainer = azureConfig.getString("container") val azureEndpoint = azureConfig.getString("endpoint") - //NB: Centaur will throw an exception if it isn't able to authenticate with Azure blob storage via the local environment. - lazy val blobContainerClient: BlobContainerClient = AzureUtils.buildContainerClientFromLocalEnvironment(blobContainer, azureEndpoint, Option(azureSubscription)).get + // NB: Centaur will throw an exception if it isn't able to authenticate with Azure blob storage via the local environment. + lazy val blobContainerClient: BlobContainerClient = + AzureUtils.buildContainerClientFromLocalEnvironment(blobContainer, azureEndpoint, Option(azureSubscription)).get - def submitWorkflow(workflow: Workflow): Test[SubmittedWorkflow] = { + def submitWorkflow(workflow: Workflow): Test[SubmittedWorkflow] = new Test[SubmittedWorkflow] { override def run: IO[SubmittedWorkflow] = for { id <- CentaurCromwellClient.submit(workflow) } yield id } - } /** * A smoke test of the version endpoint, this confirms that a) nothing explodes and b) the result must be a JSON object @@ -180,15 +188,15 @@ object Operations extends StrictLogging { def checkTimingRequirement(timeRequirement: Option[FiniteDuration]): Test[FiniteDuration] = new Test[FiniteDuration] { override def run: IO[FiniteDuration] = timeRequirement match { case Some(duration) => IO.pure(duration) - case None => IO.raiseError(new Exception("Duration value for 'maximumTime' required but not supplied in test config")) + case None => + IO.raiseError(new Exception("Duration value for 'maximumTime' required but not supplied in test config")) } } def checkFastEnough(before: Long, after: Long, allowance: FiniteDuration): Test[Unit] = new Test[Unit] { - override def run: IO[Unit] = { + override def run: IO[Unit] = if (after - before < allowance.toSeconds) IO.pure(()) else IO.raiseError(new Exception(s"Test took too long. Allowance was $allowance. Actual time: ${after - before}")) - } } def timingVerificationNotSupported(timingRequirement: Option[FiniteDuration]): Test[Unit] = new Test[Unit] { @@ -198,7 +206,7 @@ object Operations extends StrictLogging { } - def checkDescription(workflow: Workflow, validityExpectation: Option[Boolean], retries: Int = 3): Test[Unit] = { + def checkDescription(workflow: Workflow, validityExpectation: Option[Boolean], retries: Int = 3): Test[Unit] = new Test[Unit] { private val timeout = 60.seconds @@ -210,77 +218,79 @@ object Operations extends StrictLogging { case None => IO.pure(()) case Some(d.valid) => IO.pure(()) case Some(otherExpectation) => - logger.error(s"Unexpected 'valid=${d.valid}' response when expecting $otherExpectation. Full unexpected description:${System.lineSeparator()}$d") - IO.raiseError(new Exception(s"Expected this workflow's /describe validity to be '$otherExpectation' but got: '${d.valid}' (errors: ${d.errors.mkString(", ")})")) + logger.error( + s"Unexpected 'valid=${d.valid}' response when expecting $otherExpectation. Full unexpected description:${System + .lineSeparator()}$d" + ) + IO.raiseError( + new Exception( + s"Expected this workflow's /describe validity to be '$otherExpectation' but got: '${d.valid}' (errors: ${d.errors + .mkString(", ")})" + ) + ) } - }).timeoutTo(timeout, IO { - if (alreadyTried + 1 >= retries) { - throw new TimeoutException("Timeout from checkDescription 60 seconds: " + timeoutStackTraceString) - } else { - logger.warn(s"checkDescription timeout on attempt ${alreadyTried + 1}. ") - checkDescriptionInner(alreadyTried + 1) - () + }).timeoutTo( + timeout, + IO { + if (alreadyTried + 1 >= retries) { + throw new TimeoutException("Timeout from checkDescription 60 seconds: " + timeoutStackTraceString) + } else { + logger.warn(s"checkDescription timeout on attempt ${alreadyTried + 1}. ") + checkDescriptionInner(alreadyTried + 1) + () + } } - }) + ) } - - override def run: IO[Unit] = { - - + override def run: IO[Unit] = // We can't describe workflows based on zipped imports, so don't try: if (workflow.skipDescribeEndpointValidation || workflow.data.zippedImports.nonEmpty) { IO.pure(()) } else { checkDescriptionInner(0) } - } } - } - def submitInvalidWorkflow(workflow: Workflow): Test[SubmitHttpResponse] = { + def submitInvalidWorkflow(workflow: Workflow): Test[SubmitHttpResponse] = new Test[SubmitHttpResponse] { - override def run: IO[SubmitHttpResponse] = { - CentaurCromwellClient.submit(workflow).redeemWith( - { - case unsuccessfulRequestException: UnsuccessfulRequestException => - val httpResponse = unsuccessfulRequestException.httpResponse - val statusCode = httpResponse.status.intValue() - httpResponse.entity match { - case akka.http.scaladsl.model.HttpEntity.Strict(_, data) => - IO.pure(SubmitHttpResponse(statusCode, data.utf8String)) - case _ => - val message = s"Expected a strict http response entity but got ${httpResponse.entity}" - IO.raiseError(CentaurTestException(message, workflow, unsuccessfulRequestException)) - } - case unexpected: Exception => - val message = s"Unexpected error: ${unexpected.getMessage}" - IO.raiseError(CentaurTestException(message, workflow, unexpected)) - case throwable: Throwable => throw throwable - }, - { - submittedWorkflow => { + override def run: IO[SubmitHttpResponse] = + CentaurCromwellClient + .submit(workflow) + .redeemWith( + { + case unsuccessfulRequestException: UnsuccessfulRequestException => + val httpResponse = unsuccessfulRequestException.httpResponse + val statusCode = httpResponse.status.intValue() + httpResponse.entity match { + case akka.http.scaladsl.model.HttpEntity.Strict(_, data) => + IO.pure(SubmitHttpResponse(statusCode, data.utf8String)) + case _ => + val message = s"Expected a strict http response entity but got ${httpResponse.entity}" + IO.raiseError(CentaurTestException(message, workflow, unsuccessfulRequestException)) + } + case unexpected: Exception => + val message = s"Unexpected error: ${unexpected.getMessage}" + IO.raiseError(CentaurTestException(message, workflow, unexpected)) + case throwable: Throwable => throw throwable + }, + { submittedWorkflow => val message = s"Expected a failure but got a successfully submitted workflow with id ${submittedWorkflow.id}" IO.raiseError(CentaurTestException(message, workflow)) } - } - ) - } + ) } - } - def abortWorkflow(workflow: SubmittedWorkflow): Test[WorkflowStatus] = { + def abortWorkflow(workflow: SubmittedWorkflow): Test[WorkflowStatus] = new Test[WorkflowStatus] { override def run: IO[WorkflowStatus] = CentaurCromwellClient.abort(workflow) } - } - def waitFor(duration: FiniteDuration): Test[Unit] = { + def waitFor(duration: FiniteDuration): Test[Unit] = new Test[Unit] { override def run: IO[Unit] = IO.sleep(duration) } - } /** * Polls until a valid status is reached. @@ -290,16 +300,18 @@ object Operations extends StrictLogging { def expectSomeProgress(workflow: SubmittedWorkflow, testDefinition: Workflow, expectedStatuses: Set[WorkflowStatus], - timeout: FiniteDuration): Test[SubmittedWorkflow] = { + timeout: FiniteDuration + ): Test[SubmittedWorkflow] = new Test[SubmittedWorkflow] { - def status(remainingTimeout: FiniteDuration): IO[SubmittedWorkflow] = { + def status(remainingTimeout: FiniteDuration): IO[SubmittedWorkflow] = for { workflowStatus <- CentaurCromwellClient.status(workflow) mappedStatus <- workflowStatus match { case s if expectedStatuses.contains(s) => IO.pure(workflow) case s: TerminalStatus => CentaurCromwellClient.metadata(workflow) flatMap { metadata => - val message = s"Unexpected terminal status $s while waiting for one of [${expectedStatuses.mkString(", ")}] (workflow ID: ${workflow.id})" + val message = + s"Unexpected terminal status $s while waiting for one of [${expectedStatuses.mkString(", ")}] (workflow ID: ${workflow.id})" IO.raiseError(CentaurTestException(message, testDefinition, workflow, metadata)) } case _ if remainingTimeout > 0.seconds => @@ -308,16 +320,15 @@ object Operations extends StrictLogging { s <- status(remainingTimeout - 10.seconds) } yield s case other => - val message = s"Cromwell failed to progress into any of the statuses [${expectedStatuses.mkString(", ")}]. Was still '$other' after $timeout (workflow ID: ${workflow.id})" + val message = + s"Cromwell failed to progress into any of the statuses [${expectedStatuses.mkString(", ")}]. Was still '$other' after $timeout (workflow ID: ${workflow.id})" IO.raiseError(CentaurTestException(message, testDefinition, workflow)) } } yield mappedStatus - } override def run: IO[SubmittedWorkflow] = status(timeout).timeout(CentaurConfig.maxWorkflowLength) } - } /** * Polls until a specific status is reached. @@ -326,9 +337,10 @@ object Operations extends StrictLogging { */ def pollUntilStatus(workflow: SubmittedWorkflow, testDefinition: Workflow, - expectedStatus: WorkflowStatus): Test[SubmittedWorkflow] = { + expectedStatus: WorkflowStatus + ): Test[SubmittedWorkflow] = new Test[SubmittedWorkflow] { - def status: IO[SubmittedWorkflow] = { + def status: IO[SubmittedWorkflow] = for { workflowStatus <- CentaurCromwellClient.status(workflow) mappedStatus <- workflowStatus match { @@ -336,35 +348,38 @@ object Operations extends StrictLogging { case s: TerminalStatus => val reducedMetadataOptions: Map[String, List[String]] = CentaurCromwellClient.defaultMetadataArgs.getOrElse(Map.empty) ++ Map( - "includeKey" -> (List("status") ++ (if (expectedStatus == Succeeded) List("failures") else List.empty)), + "includeKey" -> (List("status") ++ (if (expectedStatus == Succeeded) List("failures") + else List.empty)), "expandSubWorkflows" -> List("false") ) - CentaurCromwellClient.metadata(workflow = workflow, args = Option(reducedMetadataOptions)) flatMap { metadata => - val failuresString = if (expectedStatus == Succeeded) { - (for { - metadataJson <- parse(metadata.value).toOption - asObject <- metadataJson.asObject - failures <- asObject.toMap.get("failures") - } yield s" Metadata 'failures' content: ${failures.spaces2}").getOrElse("No additional failure information found in metadata.") - } else { - "" - } - - val message = s"Unexpected terminal status $s but was waiting for $expectedStatus (workflow ID: ${workflow.id}).$failuresString" - IO.raiseError(CentaurTestException(message, testDefinition, workflow, metadata)) + CentaurCromwellClient.metadata(workflow = workflow, args = Option(reducedMetadataOptions)) flatMap { + metadata => + val failuresString = if (expectedStatus == Succeeded) { + (for { + metadataJson <- parse(metadata.value).toOption + asObject <- metadataJson.asObject + failures <- asObject.toMap.get("failures") + } yield s" Metadata 'failures' content: ${failures.spaces2}") + .getOrElse("No additional failure information found in metadata.") + } else { + "" + } + + val message = + s"Unexpected terminal status $s but was waiting for $expectedStatus (workflow ID: ${workflow.id}).$failuresString" + IO.raiseError(CentaurTestException(message, testDefinition, workflow, metadata)) } - case _ => for { - _ <- IO.sleep(10.seconds) - s <- status - } yield s + case _ => + for { + _ <- IO.sleep(10.seconds) + s <- status + } yield s } } yield mappedStatus - } override def run: IO[SubmittedWorkflow] = status.timeout(CentaurConfig.maxWorkflowLength) } - } /** * Validate that the given jobId matches the one in the metadata @@ -373,7 +388,8 @@ object Operations extends StrictLogging { workflow: SubmittedWorkflow, metadata: WorkflowMetadata, callFqn: String, - formerJobId: String): Test[Unit] = { + formerJobId: String + ): Test[Unit] = new Test[Unit] { override def run: IO[Unit] = CentaurCromwellClient.metadata(workflow) flatMap { s => s.asFlat.value.get(s"calls.$callFqn.jobId") match { @@ -387,56 +403,58 @@ object Operations extends StrictLogging { } } } - } - def validatePAPIAborted(workflowDefinition: Workflow, workflow: SubmittedWorkflow, jobId: String): Test[Unit] = { + def validatePAPIAborted(workflowDefinition: Workflow, workflow: SubmittedWorkflow, jobId: String): Test[Unit] = new Test[Unit] { - def checkPAPIAborted(): IO[Unit] = { + def checkPAPIAborted(): IO[Unit] = for { - operation <- IO { genomics.projects().operations().get(jobId).execute() } + operation <- IO(genomics.projects().operations().get(jobId).execute()) done = operation.getDone operationError = Option(operation.getError) - aborted = operationError.exists(_.getCode == 1) && operationError.exists(_.getMessage.startsWith("Operation canceled")) - result <- if (!(done && aborted)) { - CentaurCromwellClient.metadata(workflow) flatMap { metadata => - val message = s"Underlying JES job was not aborted properly. " + - s"Done = $done. Error = ${operationError.map(_.getMessage).getOrElse("N/A")} (workflow ID: ${workflow.id})" - IO.raiseError(CentaurTestException(message, workflowDefinition, workflow, metadata)) - } - } else IO.unit + aborted = operationError.exists(_.getCode == 1) && operationError.exists( + _.getMessage.startsWith("Operation canceled") + ) + result <- + if (!(done && aborted)) { + CentaurCromwellClient.metadata(workflow) flatMap { metadata => + val message = s"Underlying JES job was not aborted properly. " + + s"Done = $done. Error = ${operationError.map(_.getMessage).getOrElse("N/A")} (workflow ID: ${workflow.id})" + IO.raiseError(CentaurTestException(message, workflowDefinition, workflow, metadata)) + } + } else IO.unit } yield result - } override def run: IO[Unit] = if (jobId.startsWith("operations/")) { checkPAPIAborted() } else IO.unit } - } /** * Polls until a specific call is in Running state. Returns the job id. */ - def pollUntilCallIsRunning(workflowDefinition: Workflow, workflow: SubmittedWorkflow, callFqn: String): Test[String] = { + def pollUntilCallIsRunning(workflowDefinition: Workflow, + workflow: SubmittedWorkflow, + callFqn: String + ): Test[String] = { // Special case for sub workflow testing - def findJobIdInSubWorkflow(subWorkflowId: String): IO[Option[String]] = { + def findJobIdInSubWorkflow(subWorkflowId: String): IO[Option[String]] = for { metadata <- CentaurCromwellClient .metadataWithId(WorkflowId.fromString(subWorkflowId)) .redeem(_ => None, Option.apply) jobId <- IO.pure(metadata.flatMap(_.asFlat.value.get("calls.inner_abort.aborted.jobId"))) } yield jobId.map(_.asInstanceOf[JsString].value) - } - def valueAsString(key: String, metadata: WorkflowMetadata) = { + def valueAsString(key: String, metadata: WorkflowMetadata) = metadata.asFlat.value.get(key).map(_.asInstanceOf[JsString].value) - } def findCallStatus(metadata: WorkflowMetadata): IO[Option[(String, String)]] = { val status = metadata.asFlat.value.get(s"calls.$callFqn.executionStatus") val statusString = status.map(_.asInstanceOf[JsString].value) for { - jobId <- valueAsString(s"calls.$callFqn.jobId", metadata).map(jobId => IO.pure(Option(jobId))) + jobId <- valueAsString(s"calls.$callFqn.jobId", metadata) + .map(jobId => IO.pure(Option(jobId))) .orElse(valueAsString(s"calls.$callFqn.subWorkflowId", metadata).map(findJobIdInSubWorkflow)) .getOrElse(IO.pure(None)) pair = (statusString, jobId) match { @@ -447,7 +465,7 @@ object Operations extends StrictLogging { } new Test[String] { - def doPerform(): IO[String] = { + def doPerform(): IO[String] = for { // We don't want to keep going forever if the workflow failed status <- CentaurCromwellClient.status(workflow) @@ -464,13 +482,13 @@ object Operations extends StrictLogging { case Some(("Failed", _)) => val message = s"$callFqn failed" IO.raiseError(CentaurTestException(message, workflowDefinition, workflow, metadata)) - case _ => for { - _ <- IO.sleep(5.seconds) - recurse <- doPerform() - } yield recurse + case _ => + for { + _ <- IO.sleep(5.seconds) + recurse <- doPerform() + } yield recurse } } yield result - } override def run: IO[String] = doPerform().timeout(CentaurConfig.maxWorkflowLength) } @@ -483,34 +501,40 @@ object Operations extends StrictLogging { for { md <- CentaurCromwellClient.metadata(workflowB) - calls = md.asFlat.value.keySet.flatMap({ + calls = md.asFlat.value.keySet.flatMap { case callNameRegexp(name) => Option(name) case _ => None - }) - diffs <- calls.toList.traverse[IO, CallCacheDiff]({ callName => + } + diffs <- calls.toList.traverse[IO, CallCacheDiff] { callName => CentaurCromwellClient.callCacheDiff(workflowA, callName, workflowB, callName) - }) + } } yield diffs.flatMap(_.hashDifferential) } - override def run: IO[Unit] = { + override def run: IO[Unit] = hashDiffOfAllCalls map { case diffs if diffs.nonEmpty && CentaurCromwellClient.LogFailures => Console.err.println(s"Hash differential for ${workflowA.id} and ${workflowB.id}") - diffs.map({ diff => - s"For key ${diff.hashKey}:\nCall A: ${diff.callA.getOrElse("N/A")}\nCall B: ${diff.callB.getOrElse("N/A")}" - }).foreach(Console.err.println) + diffs + .map { diff => + s"For key ${diff.hashKey}:\nCall A: ${diff.callA.getOrElse("N/A")}\nCall B: ${diff.callB.getOrElse("N/A")}" + } + .foreach(Console.err.println) case _ => } - } } /* Select only those flat metadata items whose keys begin with the specified prefix, removing the prefix from the keys. Also * perform variable substitutions for UUID and WORKFLOW_ROOT and remove any ~> Centaur metadata expectation metacharacters. */ - private def selectMetadataExpectationSubsetByPrefix(workflow: Workflow, prefix: String, workflowId: WorkflowId, workflowRoot: String): List[(String, JsValue)] = { + private def selectMetadataExpectationSubsetByPrefix(workflow: Workflow, + prefix: String, + workflowId: WorkflowId, + workflowRoot: String + ): List[(String, JsValue)] = { import WorkflowFlatMetadata._ def replaceVariables(value: JsValue): JsValue = value match { - case s: JsString => JsString(s.value.replaceExpectationVariables(workflowId, workflowRoot).replaceFirst("^~>", "")) + case s: JsString => + JsString(s.value.replaceExpectationVariables(workflowId, workflowRoot).replaceFirst("^~>", "")) case o => o } val filterLabels: PartialFunction[(String, JsValue), (String, JsValue)] = { @@ -525,7 +549,8 @@ object Operations extends StrictLogging { def fetchAndValidateOutputs(submittedWorkflow: SubmittedWorkflow, workflow: Workflow, - workflowRoot: String): Test[JsObject] = new Test[JsObject] { + workflowRoot: String + ): Test[JsObject] = new Test[JsObject] { def checkOutputs(expectedOutputs: List[(String, JsValue)])(actualOutputs: Map[String, JsValue]): IO[Unit] = { val expected = expectedOutputs.toSet @@ -535,11 +560,13 @@ object Operations extends StrictLogging { lazy val inExpectedButNotInActual = expected.diff(actual) if (!workflow.allowOtherOutputs && inActualButNotInExpected.nonEmpty) { - val message = s"In actual outputs but not in expected and other outputs not allowed: ${inActualButNotInExpected.mkString(", ")}" + val message = + s"In actual outputs but not in expected and other outputs not allowed: ${inActualButNotInExpected.mkString(", ")}" IO.raiseError(CentaurTestException(message, workflow, submittedWorkflow)) } else if (inExpectedButNotInActual.nonEmpty) { - val message = s"In actual outputs but not in expected: ${inExpectedButNotInActual.mkString(", ")}" + System.lineSeparator + - s"In expected outputs but not in actual: ${inExpectedButNotInActual.mkString(", ")}" + val message = + s"In actual outputs but not in expected: ${inExpectedButNotInActual.mkString(", ")}" + System.lineSeparator + + s"In expected outputs but not in actual: ${inExpectedButNotInActual.mkString(", ")}" IO.raiseError(CentaurTestException(message, workflow, submittedWorkflow)) } else { IO.unit @@ -549,7 +576,8 @@ object Operations extends StrictLogging { override def run: IO[JsObject] = { import centaur.test.metadata.WorkflowFlatOutputs._ - val expectedOutputs: List[(String, JsValue)] = selectMetadataExpectationSubsetByPrefix(workflow, "outputs.", submittedWorkflow.id, workflowRoot) + val expectedOutputs: List[(String, JsValue)] = + selectMetadataExpectationSubsetByPrefix(workflow, "outputs.", submittedWorkflow.id, workflowRoot) for { outputs <- CentaurCromwellClient.outputs(submittedWorkflow) @@ -562,7 +590,8 @@ object Operations extends StrictLogging { def fetchAndValidateLabels(submittedWorkflow: SubmittedWorkflow, workflow: Workflow, - workflowRoot: String): Test[Unit] = new Test[Unit] { + workflowRoot: String + ): Test[Unit] = new Test[Unit] { override def run: IO[Unit] = { import centaur.test.metadata.WorkflowFlatLabels._ @@ -570,17 +599,18 @@ object Operations extends StrictLogging { val expectedLabels: List[(String, JsValue)] = workflowIdLabel :: selectMetadataExpectationSubsetByPrefix(workflow, "labels.", submittedWorkflow.id, workflowRoot) - def validateLabels(actualLabels: Map[String, JsValue]) = { val diff = expectedLabels.toSet.diff(actualLabels.toSet) if (diff.isEmpty) { IO.unit } else { - IO.raiseError(CentaurTestException( - s"In expected labels but not in actual: ${diff.mkString(", ")}", - workflow, - submittedWorkflow - )) + IO.raiseError( + CentaurTestException( + s"In expected labels but not in actual: ${diff.mkString(", ")}", + workflow, + submittedWorkflow + ) + ) } } @@ -593,75 +623,77 @@ object Operations extends StrictLogging { } /** Compares logs filtered from the raw `metadata` endpoint with the `logs` endpoint. */ - def validateLogs(metadata: WorkflowMetadata, - submittedWorkflow: SubmittedWorkflow, - workflow: Workflow): Test[Unit] = new Test[Unit] { - val suffixes = Set("stdout", "shardIndex", "stderr", "attempt", "backendLogs.log") - - def removeSubworkflowKeys(flattened: Map[String, JsValue]): Map[String, JsValue] = { - val subWorkflowIdPrefixes = flattened.keys.filter(_.endsWith(".subWorkflowId")).map(s => s.substring(0, s.lastIndexOf('.'))) - flattened filter { case (k, _) => !subWorkflowIdPrefixes.exists(k.startsWith) } - } - - // Filter to only include the fields in the flattened metadata that should appear in the logs endpoint. - def filterForLogsFields(flattened: Map[String, JsValue]): Map[String, JsValue] = removeSubworkflowKeys(flattened).filter { - case (k, _) => k == "id" || suffixes.exists(s => k.endsWith("." + s) && !k.contains(".outputs.") && !k.startsWith("outputs.")) - } + def validateLogs(metadata: WorkflowMetadata, submittedWorkflow: SubmittedWorkflow, workflow: Workflow): Test[Unit] = + new Test[Unit] { + val suffixes = Set("stdout", "shardIndex", "stderr", "attempt", "backendLogs.log") - override def run: IO[Unit] = { + def removeSubworkflowKeys(flattened: Map[String, JsValue]): Map[String, JsValue] = { + val subWorkflowIdPrefixes = + flattened.keys.filter(_.endsWith(".subWorkflowId")).map(s => s.substring(0, s.lastIndexOf('.'))) + flattened filter { case (k, _) => !subWorkflowIdPrefixes.exists(k.startsWith) } + } - def validateLogsMetadata(flatLogs: Map[String, JsValue], flatFilteredMetadata: Map[String, JsValue]): IO[Unit] = - if (flatLogs.equals(flatFilteredMetadata)) { - IO.unit - } else { - val message = (List("actual logs endpoint output did not equal filtered metadata", "flat logs: ") ++ - flatLogs.toList ++ List("flat filtered metadata: ") ++ flatFilteredMetadata.toList).mkString("\n") - IO.raiseError(CentaurTestException(message, workflow, submittedWorkflow)) + // Filter to only include the fields in the flattened metadata that should appear in the logs endpoint. + def filterForLogsFields(flattened: Map[String, JsValue]): Map[String, JsValue] = + removeSubworkflowKeys(flattened).filter { case (k, _) => + k == "id" || suffixes.exists(s => + k.endsWith("." + s) && !k.contains(".outputs.") && !k.startsWith("outputs.") + ) } - for { - logs <- CentaurCromwellClient.logs(submittedWorkflow) - flatLogs = logs.asFlat.value - flatFilteredMetadata = metadata.asFlat.value |> filterForLogsFields - _ <- validateLogsMetadata(flatLogs, flatFilteredMetadata) - } yield () + override def run: IO[Unit] = { + + def validateLogsMetadata(flatLogs: Map[String, JsValue], flatFilteredMetadata: Map[String, JsValue]): IO[Unit] = + if (flatLogs.equals(flatFilteredMetadata)) { + IO.unit + } else { + val message = (List("actual logs endpoint output did not equal filtered metadata", "flat logs: ") ++ + flatLogs.toList ++ List("flat filtered metadata: ") ++ flatFilteredMetadata.toList).mkString("\n") + IO.raiseError(CentaurTestException(message, workflow, submittedWorkflow)) + } + + for { + logs <- CentaurCromwellClient.logs(submittedWorkflow) + flatLogs = logs.asFlat.value + flatFilteredMetadata = metadata.asFlat.value |> filterForLogsFields + _ <- validateLogsMetadata(flatLogs, flatFilteredMetadata) + } yield () + } } - } - def fetchMetadata(submittedWorkflow: SubmittedWorkflow, - expandSubworkflows: Boolean): IO[WorkflowMetadata] = { + def fetchMetadata(submittedWorkflow: SubmittedWorkflow, expandSubworkflows: Boolean): IO[WorkflowMetadata] = CentaurCromwellClient.metadata(submittedWorkflow, expandSubworkflows = expandSubworkflows) - } def fetchAndValidateNonSubworkflowMetadata(submittedWorkflow: SubmittedWorkflow, workflowSpec: Workflow, - cacheHitUUID: Option[UUID] = None): Test[WorkflowMetadata] = { + cacheHitUUID: Option[UUID] = None + ): Test[WorkflowMetadata] = new Test[WorkflowMetadata] { def fetchOnce(): IO[WorkflowMetadata] = fetchMetadata(submittedWorkflow, expandSubworkflows = false) def eventuallyMetadata(workflow: SubmittedWorkflow, - expectedMetadata: WorkflowFlatMetadata): IO[WorkflowMetadata] = { - validateMetadata(workflow, expectedMetadata).handleErrorWith({ _ => + expectedMetadata: WorkflowFlatMetadata + ): IO[WorkflowMetadata] = + validateMetadata(workflow, expectedMetadata).handleErrorWith { _ => for { _ <- IO.sleep(2.seconds) recurse <- eventuallyMetadata(workflow, expectedMetadata) } yield recurse - }) - } + } def validateMetadata(workflow: SubmittedWorkflow, - expectedMetadata: WorkflowFlatMetadata): IO[WorkflowMetadata] = { - def checkDiff(diffs: Iterable[String], actualMetadata: WorkflowMetadata): IO[Unit] = { + expectedMetadata: WorkflowFlatMetadata + ): IO[WorkflowMetadata] = { + def checkDiff(diffs: Iterable[String], actualMetadata: WorkflowMetadata): IO[Unit] = if (diffs.nonEmpty) { val message = s"Invalid metadata response:\n -${diffs.mkString("\n -")}\n" IO.raiseError(CentaurTestException(message, workflowSpec, workflow, actualMetadata)) } else { IO.unit } - } - def validateUnwantedMetadata(actualMetadata: WorkflowMetadata): IO[Unit] = { + def validateUnwantedMetadata(actualMetadata: WorkflowMetadata): IO[Unit] = if (workflowSpec.notInMetadata.nonEmpty) { // Check that none of the "notInMetadata" keys are in the actual metadata val absentMdIntersect = workflowSpec.notInMetadata.toSet.intersect(actualMetadata.asFlat.value.keySet) @@ -674,23 +706,23 @@ object Operations extends StrictLogging { } else { IO.unit } - } - def validateAllowOtherOutputs(actualMetadata: WorkflowMetadata): IO[Unit] = { + def validateAllowOtherOutputs(actualMetadata: WorkflowMetadata): IO[Unit] = if (workflowSpec.allowOtherOutputs) IO.unit else { val flat = actualMetadata.asFlat.value val actualOutputs: Iterable[String] = flat.keys.filter(_.startsWith("outputs.")) - val expectedOutputs: Iterable[String] = workflowSpec.metadata.map(w => w.value.keys.filter(_.startsWith("outputs."))).getOrElse(List.empty) + val expectedOutputs: Iterable[String] = + workflowSpec.metadata.map(w => w.value.keys.filter(_.startsWith("outputs."))).getOrElse(List.empty) val diff = actualOutputs.toSet.diff(expectedOutputs.toSet) if (diff.nonEmpty) { - val message = s"Found unwanted keys in metadata with `allow-other-outputs` = false: ${diff.mkString(", ")}" + val message = + s"Found unwanted keys in metadata with `allow-other-outputs` = false: ${diff.mkString(", ")}" IO.raiseError(CentaurTestException(message, workflowSpec, workflow, actualMetadata)) } else { IO.unit } } - } for { actualMetadata <- fetchOnce() @@ -709,14 +741,14 @@ object Operations extends StrictLogging { case None => fetchOnce() } } - } def validateMetadataJson(testType: String, expected: JsObject, actual: JsObject, submittedWorkflow: SubmittedWorkflow, workflow: Workflow, - allowableAddedOneWordFields: List[String]): IO[Unit] = { + allowableAddedOneWordFields: List[String] + ): IO[Unit] = if (actual.equals(expected)) { IO.unit } else { @@ -748,46 +780,66 @@ object Operations extends StrictLogging { } else { val writer: JsonWriter[Vector[Operation[JsValue]]] = new JsonWriter[Vector[Operation[JsValue]]] { def processOperation(op: Operation[JsValue]): JsValue = op match { - case Add(path, value) => JsObject(Map[String, JsValue]( - "description" -> JsString("Unexpected value found"), - "path" -> JsString(path.toString), - "value" -> value)) - case Copy(from, path) => JsObject(Map[String, JsValue]( - "description" -> JsString("Value(s) unexpectedly copied"), - "expected_at" -> JsString(from.toString), - "also_at" -> JsString(path.toString))) - case Move(from, path) => JsObject(Map[String, JsValue]( - "description" -> JsString("Value(s) unexpectedly moved"), - "expected_location" -> JsString(from.toString), - "actual_location" -> JsString(path.toString))) - case Remove(path, old) => JsObject(Map[String, JsValue]( - "description" -> JsString("Value missing"), - "expected_location" -> JsString(path.toString)) ++ - old.map(o => "expected_value" -> o)) - case Replace(path, value, old) => JsObject(Map[String, JsValue]( - "description" -> JsString("Incorrect value found"), - "path" -> JsString(path.toString), - "found_value" -> value) ++ old.map(o => "expected_value" -> o)) - case diffson.jsonpatch.Test(path, value) => JsObject(Map[String, JsValue]( - "op" -> JsString("test"), - "path" -> JsString(path.toString), - "value" -> value)) + case Add(path, value) => + JsObject( + Map[String, JsValue]("description" -> JsString("Unexpected value found"), + "path" -> JsString(path.toString), + "value" -> value + ) + ) + case Copy(from, path) => + JsObject( + Map[String, JsValue]("description" -> JsString("Value(s) unexpectedly copied"), + "expected_at" -> JsString(from.toString), + "also_at" -> JsString(path.toString) + ) + ) + case Move(from, path) => + JsObject( + Map[String, JsValue]("description" -> JsString("Value(s) unexpectedly moved"), + "expected_location" -> JsString(from.toString), + "actual_location" -> JsString(path.toString) + ) + ) + case Remove(path, old) => + JsObject( + Map[String, JsValue]("description" -> JsString("Value missing"), + "expected_location" -> JsString(path.toString) + ) ++ + old.map(o => "expected_value" -> o) + ) + case Replace(path, value, old) => + JsObject( + Map[String, JsValue]("description" -> JsString("Incorrect value found"), + "path" -> JsString(path.toString), + "found_value" -> value + ) ++ old.map(o => "expected_value" -> o) + ) + case diffson.jsonpatch.Test(path, value) => + JsObject( + Map[String, JsValue]("op" -> JsString("test"), "path" -> JsString(path.toString), "value" -> value) + ) } - override def write(vector: Vector[Operation[JsValue]]): JsValue = { + override def write(vector: Vector[Operation[JsValue]]): JsValue = JsArray(vector.map(processOperation)) - } } val jsonDiff = filteredDifferences.toJson(writer).prettyPrint - IO.raiseError(CentaurTestException(s"Error during $testType metadata comparison. Diff: $jsonDiff Expected: $expected Actual: $actual", workflow, submittedWorkflow)) + IO.raiseError( + CentaurTestException( + s"Error during $testType metadata comparison. Diff: $jsonDiff Expected: $expected Actual: $actual", + workflow, + submittedWorkflow + ) + ) } } - } def fetchAndValidateJobManagerStyleMetadata(submittedWorkflow: SubmittedWorkflow, workflow: Workflow, - prefetchedOriginalNonSubWorkflowMetadata: Option[String]): Test[WorkflowMetadata] = new Test[WorkflowMetadata] { + prefetchedOriginalNonSubWorkflowMetadata: Option[String] + ): Test[WorkflowMetadata] = new Test[WorkflowMetadata] { // If the non-subworkflow metadata was already fetched, there's no need to fetch it again. def originalMetadataStringIO: IO[String] = prefetchedOriginalNonSubWorkflowMetadata match { @@ -799,19 +851,43 @@ object Operations extends StrictLogging { originalMetadata <- originalMetadataStringIO jmMetadata <- CentaurCromwellClient.metadata( workflow = submittedWorkflow, - Option(CentaurCromwellClient.defaultMetadataArgs.getOrElse(Map.empty) ++ jmArgs)) + Option(CentaurCromwellClient.defaultMetadataArgs.getOrElse(Map.empty) ++ jmArgs) + ) jmMetadataObject <- IO.fromTry(Try(jmMetadata.value.parseJson.asJsObject)) expectation <- IO.fromTry(Try(extractJmStyleMetadataFields(originalMetadata.parseJson.asJsObject))) - _ <- validateMetadataJson(testType = s"fetchAndValidateJobManagerStyleMetadata", expectation, jmMetadataObject, submittedWorkflow, workflow, allowableOneWordAdditionsInJmMetadata) + _ <- validateMetadataJson(testType = s"fetchAndValidateJobManagerStyleMetadata", + expectation, + jmMetadataObject, + submittedWorkflow, + workflow, + allowableOneWordAdditionsInJmMetadata + ) } yield jmMetadata } val oneWordJmIncludeKeys = List( - "attempt", "callRoot", "end", - "executionStatus", "failures", "inputs", "jobId", - "calls", "outputs", "shardIndex", "start", "stderr", "stdout", - "description", "executionEvents", "labels", "parentWorkflowId", - "returnCode", "status", "submission", "subWorkflowId", "workflowName" + "attempt", + "callRoot", + "end", + "executionStatus", + "failures", + "inputs", + "jobId", + "calls", + "outputs", + "shardIndex", + "start", + "stderr", + "stdout", + "description", + "executionEvents", + "labels", + "parentWorkflowId", + "returnCode", + "status", + "submission", + "subWorkflowId", + "workflowName" ) // Our Job Manager metadata validation works by comparing the first pull of metadata which satisfies all test requirements @@ -839,12 +915,13 @@ object Operations extends StrictLogging { // NB: this filter to remove "calls" is because - although it is a single word in the JM request, // it gets treated specially by the API (so has to be treated specially here too) - def processOneWordIncludes(json: JsObject) = (oneWordJmIncludeKeys.filterNot(_ == "calls") :+ "id").foldRight(JsObject.empty) { (toInclude, current) => - json.fields.get(toInclude) match { - case Some(jsonToInclude) => JsObject(current.fields + (toInclude -> jsonToInclude)) - case None => current + def processOneWordIncludes(json: JsObject) = + (oneWordJmIncludeKeys.filterNot(_ == "calls") :+ "id").foldRight(JsObject.empty) { (toInclude, current) => + json.fields.get(toInclude) match { + case Some(jsonToInclude) => JsObject(current.fields + (toInclude -> jsonToInclude)) + case None => current + } } - } def processCallCacheField(callJson: JsObject) = for { originalCallCachingField <- callJson.fields.get("callCaching") @@ -864,7 +941,9 @@ object Operations extends StrictLogging { } val workflowLevelWithOneWordIncludes = processOneWordIncludes(originalWorkflowMetadataJson) - val callsField = originalCallMetadataJson map { calls => Map("calls" -> processCallsSection(calls)) } getOrElse Map.empty + val callsField = originalCallMetadataJson map { calls => + Map("calls" -> processCallsSection(calls)) + } getOrElse Map.empty JsObject(workflowLevelWithOneWordIncludes.fields ++ callsField) } @@ -875,7 +954,8 @@ object Operations extends StrictLogging { def validateCacheResultField(workflowDefinition: Workflow, submittedWorkflow: SubmittedWorkflow, metadata: WorkflowMetadata, - blacklistedValue: String): Test[Unit] = { + blacklistedValue: String + ): Test[Unit] = new Test[Unit] { override def run: IO[Unit] = { val badCacheResults = metadata.asFlat.value collect { @@ -891,24 +971,24 @@ object Operations extends StrictLogging { } } } - } def validateDirectoryContentsCounts(workflowDefinition: Workflow, submittedWorkflow: SubmittedWorkflow, - metadata: WorkflowMetadata): Test[Unit] = new Test[Unit] { + metadata: WorkflowMetadata + ): Test[Unit] = new Test[Unit] { private val workflowId = submittedWorkflow.id.id.toString override def run: IO[Unit] = workflowDefinition.directoryContentCounts match { case None => IO.unit case Some(directoryContentCountCheck) => - val counts = directoryContentCountCheck.expectedDirectoryContentsCounts map { - case (directory, count) => - val substitutedDir = directory.replaceAll("<>", workflowId) - (substitutedDir, count, directoryContentCountCheck.checkFiles.countObjectsAtPath(substitutedDir)) + val counts = directoryContentCountCheck.expectedDirectoryContentsCounts map { case (directory, count) => + val substitutedDir = directory.replaceAll("<>", workflowId) + (substitutedDir, count, directoryContentCountCheck.checkFiles.countObjectsAtPath(substitutedDir)) } val badCounts = counts collect { - case (directory, expectedCount, actualCount) if expectedCount != actualCount => s"Expected to find $expectedCount item(s) at $directory but got $actualCount" + case (directory, expectedCount, actualCount) if expectedCount != actualCount => + s"Expected to find $expectedCount item(s) at $directory but got $actualCount" } if (badCounts.isEmpty) { IO.unit @@ -921,21 +1001,22 @@ object Operations extends StrictLogging { def validateNoCacheHits(submittedWorkflow: SubmittedWorkflow, metadata: WorkflowMetadata, - workflowDefinition: Workflow): Test[Unit] = { + workflowDefinition: Workflow + ): Test[Unit] = validateCacheResultField(workflowDefinition, submittedWorkflow, metadata, "Cache Hit") - } def validateNoCacheMisses(submittedWorkflow: SubmittedWorkflow, metadata: WorkflowMetadata, - workflowDefinition: Workflow): Test[Unit] = { + workflowDefinition: Workflow + ): Test[Unit] = validateCacheResultField(workflowDefinition, submittedWorkflow, metadata, "Cache Miss") - } def validateSubmitFailure(workflow: Workflow, expectedSubmitResponse: SubmitHttpResponse, - actualSubmitResponse: SubmitHttpResponse): Test[Unit] = { + actualSubmitResponse: SubmitHttpResponse + ): Test[Unit] = new Test[Unit] { - override def run: IO[Unit] = { + override def run: IO[Unit] = if (expectedSubmitResponse == actualSubmitResponse) { IO.unit } else { @@ -949,7 +1030,5 @@ object Operations extends StrictLogging { |""".stripMargin IO.raiseError(CentaurTestException(message, workflow)) } - } } - } } diff --git a/centaur/src/main/scala/centaur/test/TestOptions.scala b/centaur/src/main/scala/centaur/test/TestOptions.scala index 10354c756bb..ceba238a4da 100644 --- a/centaur/src/main/scala/centaur/test/TestOptions.scala +++ b/centaur/src/main/scala/centaur/test/TestOptions.scala @@ -18,15 +18,13 @@ object TestOptions { Apply[ErrorOr].map2(tags, ignore)((t, i) => TestOptions(t, i)) } - def tagsFromConfig(conf: Config): ErrorOr[List[String]] = { + def tagsFromConfig(conf: Config): ErrorOr[List[String]] = conf.get[List[String]]("tags") match { case Success(tagStrings) => Valid(tagStrings.map(_.toLowerCase).distinct) case Failure(_) => Valid(List.empty[String]) } - } - - def ignoreFromConfig(conf: Config): ErrorOr[Boolean] = { + def ignoreFromConfig(conf: Config): ErrorOr[Boolean] = if (conf.hasPath("ignore")) { conf.get[Boolean]("ignore") match { case Success(ignore) => Valid(ignore) @@ -35,6 +33,4 @@ object TestOptions { } else { Valid(false) } - } } - diff --git a/centaur/src/main/scala/centaur/test/formulas/TestFormulas.scala b/centaur/src/main/scala/centaur/test/formulas/TestFormulas.scala index 774f852237d..6c006283894 100644 --- a/centaur/src/main/scala/centaur/test/formulas/TestFormulas.scala +++ b/centaur/src/main/scala/centaur/test/formulas/TestFormulas.scala @@ -12,11 +12,21 @@ import centaur.test.workflow.Workflow import centaur.test.{Operations, Test} import centaur.{CentaurConfig, CromwellManager, CromwellTracker, ManagedCromwellServer} import com.typesafe.scalalogging.StrictLogging -import cromwell.api.model.{Aborted, Aborting, Failed, Running, SubmittedWorkflow, Succeeded, TerminalStatus, WorkflowMetadata} +import cromwell.api.model.{ + Aborted, + Aborting, + Failed, + Running, + SubmittedWorkflow, + Succeeded, + TerminalStatus, + WorkflowMetadata +} import scala.concurrent.duration._ import centaur.test.metadata.WorkflowFlatMetadata._ import spray.json.JsString + /** * A collection of test formulas which can be used, building upon operations by chaining them together via a * for comprehension. These assembled formulas can then be run by a client @@ -33,7 +43,7 @@ object TestFormulas extends StrictLogging { |""".stripMargin.trim ) - private def runWorkflowUntilTerminalStatus(workflow: Workflow, status: TerminalStatus): Test[SubmittedWorkflow] = { + private def runWorkflowUntilTerminalStatus(workflow: Workflow, status: TerminalStatus): Test[SubmittedWorkflow] = for { _ <- checkVersion() s <- submitWorkflow(workflow) @@ -41,14 +51,15 @@ object TestFormulas extends StrictLogging { workflow = s, testDefinition = workflow, expectedStatuses = Set(Running, status), - timeout = CentaurConfig.workflowProgressTimeout, + timeout = CentaurConfig.workflowProgressTimeout ) _ <- pollUntilStatus(s, workflow, status) } yield s - } - private def runSuccessfulWorkflow(workflow: Workflow): Test[SubmittedWorkflow] = runWorkflowUntilTerminalStatus(workflow, Succeeded) - private def runFailingWorkflow(workflow: Workflow): Test[SubmittedWorkflow] = runWorkflowUntilTerminalStatus(workflow, Failed) + private def runSuccessfulWorkflow(workflow: Workflow): Test[SubmittedWorkflow] = + runWorkflowUntilTerminalStatus(workflow, Succeeded) + private def runFailingWorkflow(workflow: Workflow): Test[SubmittedWorkflow] = + runWorkflowUntilTerminalStatus(workflow, Failed) def runSuccessfulWorkflowAndVerifyTimeAndOutputs(workflowDefinition: Workflow): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = Option(true)) @@ -56,18 +67,28 @@ object TestFormulas extends StrictLogging { beforeTimestamp = OffsetDateTime.now().toInstant.getEpochSecond submittedWorkflow <- runSuccessfulWorkflow(workflowDefinition) afterTimestamp = OffsetDateTime.now().toInstant.getEpochSecond - _ <- fetchAndValidateOutputs(submittedWorkflow, workflowDefinition, "ROOT NOT SUPPORTED IN TIMING/OUTPUT ONLY TESTS") + _ <- fetchAndValidateOutputs(submittedWorkflow, + workflowDefinition, + "ROOT NOT SUPPORTED IN TIMING/OUTPUT ONLY TESTS" + ) _ <- checkFastEnough(beforeTimestamp, afterTimestamp, timeAllowance) } yield SubmitResponse(submittedWorkflow) - def runSuccessfulWorkflowAndVerifyMetadata(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { + def runSuccessfulWorkflowAndVerifyMetadata( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = Option(true)) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) submittedWorkflow <- runSuccessfulWorkflow(workflowDefinition) metadata <- fetchAndValidateNonSubworkflowMetadata(submittedWorkflow, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = Option(metadata.value)) + _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = Option(metadata.value) + ) notArchivedFlatMetadata = metadata.asFlat - workflowRoot = notArchivedFlatMetadata.value.get("workflowRoot").collectFirst { case JsString(r) => r } getOrElse "No Workflow Root" + workflowRoot = notArchivedFlatMetadata.value.get("workflowRoot").collectFirst { case JsString(r) => + r + } getOrElse "No Workflow Root" _ <- fetchAndValidateOutputs(submittedWorkflow, workflowDefinition, workflowRoot) _ <- fetchAndValidateLabels(submittedWorkflow, workflowDefinition, workflowRoot) _ <- validateLogs(metadata, submittedWorkflow, workflowDefinition) @@ -75,17 +96,24 @@ object TestFormulas extends StrictLogging { _ <- validateDirectoryContentsCounts(workflowDefinition, submittedWorkflow, metadata) } yield SubmitResponse(submittedWorkflow) - def runFailingWorkflowAndVerifyMetadata(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { + def runFailingWorkflowAndVerifyMetadata( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = None) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) submittedWorkflow <- runFailingWorkflow(workflowDefinition) metadata <- fetchAndValidateNonSubworkflowMetadata(submittedWorkflow, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = Option(metadata.value)) + _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = Option(metadata.value) + ) _ = cromwellTracker.track(metadata) _ <- validateDirectoryContentsCounts(workflowDefinition, submittedWorkflow, metadata) } yield SubmitResponse(submittedWorkflow) - def runWorkflowTwiceExpectingCaching(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + def runWorkflowTwiceExpectingCaching( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = Option(true)) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) @@ -93,14 +121,18 @@ object TestFormulas extends StrictLogging { secondWf <- runSuccessfulWorkflow(workflowDefinition.secondRun) _ <- printHashDifferential(firstWF, secondWf) metadata <- fetchAndValidateNonSubworkflowMetadata(secondWf, workflowDefinition, Option(firstWF.id.id)) - _ <- fetchAndValidateJobManagerStyleMetadata(secondWf, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(secondWf, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) _ <- validateNoCacheMisses(secondWf, metadata, workflowDefinition) _ <- validateDirectoryContentsCounts(workflowDefinition, secondWf, metadata) } yield SubmitResponse(secondWf) - } - def runWorkflowThriceExpectingCaching(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + def runWorkflowThriceExpectingCaching( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = Option(true)) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) @@ -115,41 +147,48 @@ object TestFormulas extends StrictLogging { _ <- validateNoCacheMisses(thirdWf, metadataThree, workflowDefinition) _ <- validateDirectoryContentsCounts(workflowDefinition, thirdWf, metadataThree) } yield SubmitResponse(thirdWf) - } - def runWorkflowTwiceExpectingNoCaching(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + def runWorkflowTwiceExpectingNoCaching( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = Option(true)) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) _ <- runSuccessfulWorkflow(workflowDefinition) // Build caches testWf <- runSuccessfulWorkflow(workflowDefinition.secondRun) metadata <- fetchAndValidateNonSubworkflowMetadata(testWf, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(testWf, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(testWf, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) _ <- validateNoCacheHits(testWf, metadata, workflowDefinition) _ <- validateDirectoryContentsCounts(workflowDefinition, testWf, metadata) } yield SubmitResponse(testWf) - } - def runFailingWorkflowTwiceExpectingNoCaching(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + def runFailingWorkflowTwiceExpectingNoCaching( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = None) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) _ <- runFailingWorkflow(workflowDefinition) // Build caches testWf <- runFailingWorkflow(workflowDefinition) metadata <- fetchAndValidateNonSubworkflowMetadata(testWf, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(testWf, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(testWf, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) _ <- validateNoCacheHits(testWf, metadata, workflowDefinition) _ <- validateDirectoryContentsCounts(workflowDefinition, testWf, metadata) } yield SubmitResponse(testWf) - } private def cromwellRestart(workflowDefinition: Workflow, callMarker: CallMarker, testRecover: Boolean, - finalStatus: TerminalStatus)( - implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + finalStatus: TerminalStatus + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = CentaurConfig.runMode match { case ManagedCromwellServer(_, postRestart, withRestart) if withRestart => for { @@ -163,27 +202,31 @@ object TestFormulas extends StrictLogging { workflow = submittedWorkflow, testDefinition = workflowDefinition, expectedStatuses = Set(Running, finalStatus), - timeout = CentaurConfig.workflowProgressTimeout, + timeout = CentaurConfig.workflowProgressTimeout ) _ <- pollUntilStatus(submittedWorkflow, workflowDefinition, finalStatus) metadata <- fetchAndValidateNonSubworkflowMetadata(submittedWorkflow, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) - _ <- if (testRecover) { - validateRecovered(workflowDefinition, submittedWorkflow, metadata, callMarker.callKey, jobId) - } - else { - Test.successful(()) - } + _ <- + if (testRecover) { + validateRecovered(workflowDefinition, submittedWorkflow, metadata, callMarker.callKey, jobId) + } else { + Test.successful(()) + } _ <- validateDirectoryContentsCounts(workflowDefinition, submittedWorkflow, metadata) } yield SubmitResponse(submittedWorkflow) case _ if finalStatus == Succeeded => runSuccessfulWorkflowAndVerifyMetadata(workflowDefinition) case _ if finalStatus == Failed => runFailingWorkflowAndVerifyMetadata(workflowDefinition) case _ => Test.invalidTestDefinition("This test can only run successful or failed workflow", workflowDefinition) } - } - def instantAbort(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { + def instantAbort( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = Option(true)) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) submittedWorkflow <- submitWorkflow(workflowDefinition) @@ -192,16 +235,21 @@ object TestFormulas extends StrictLogging { workflow = submittedWorkflow, testDefinition = workflowDefinition, expectedStatuses = Set(Running, Aborting, Aborted), - timeout = CentaurConfig.workflowProgressTimeout, + timeout = CentaurConfig.workflowProgressTimeout ) _ <- pollUntilStatus(submittedWorkflow, workflowDefinition, Aborted) metadata <- fetchAndValidateNonSubworkflowMetadata(submittedWorkflow, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) _ <- validateDirectoryContentsCounts(workflowDefinition, submittedWorkflow, metadata) } yield SubmitResponse(submittedWorkflow) - def scheduledAbort(workflowDefinition: Workflow, callMarker: CallMarker, restart: Boolean)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + def scheduledAbort(workflowDefinition: Workflow, callMarker: CallMarker, restart: Boolean)(implicit + cromwellTracker: Option[CromwellTracker] + ): Test[SubmitResponse] = { def withRestart(): Unit = CentaurConfig.runMode match { case ManagedCromwellServer(_, postRestart, withRestart) if withRestart => CromwellManager.stopCromwell(s"Scheduled restart from ${workflowDefinition.testName}") @@ -217,42 +265,45 @@ object TestFormulas extends StrictLogging { // The Cromwell call status could be running but the backend job might not have started yet, give it some time _ <- waitFor(30.seconds) _ <- abortWorkflow(submittedWorkflow) - _ = if(restart) withRestart() + _ = if (restart) withRestart() _ <- expectSomeProgress( workflow = submittedWorkflow, testDefinition = workflowDefinition, expectedStatuses = Set(Running, Aborting, Aborted), - timeout = CentaurConfig.workflowProgressTimeout, + timeout = CentaurConfig.workflowProgressTimeout ) _ <- pollUntilStatus(submittedWorkflow, workflowDefinition, Aborted) _ <- validatePAPIAborted(workflowDefinition, submittedWorkflow, jobId) // Wait a little to make sure that if the abort didn't work and calls start running we see them in the metadata _ <- waitFor(30.seconds) metadata <- fetchAndValidateNonSubworkflowMetadata(submittedWorkflow, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) _ <- validateDirectoryContentsCounts(workflowDefinition, submittedWorkflow, metadata) } yield SubmitResponse(submittedWorkflow) } def workflowRestart(workflowDefinition: Workflow, - callMarker: CallMarker, - recover: Boolean, - finalStatus: TerminalStatus)( - implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + callMarker: CallMarker, + recover: Boolean, + finalStatus: TerminalStatus + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = cromwellRestart(workflowDefinition, callMarker, testRecover = recover, finalStatus = finalStatus) - } - def submitInvalidWorkflow(workflow: Workflow, expectedSubmitResponse: SubmitHttpResponse): Test[SubmitResponse] = { + def submitInvalidWorkflow(workflow: Workflow, expectedSubmitResponse: SubmitHttpResponse): Test[SubmitResponse] = for { _ <- checkDescription(workflow, validityExpectation = None) _ <- timingVerificationNotSupported(workflow.maximumAllowedTime) actualSubmitResponse <- Operations.submitInvalidWorkflow(workflow) _ <- validateSubmitFailure(workflow, expectedSubmitResponse, actualSubmitResponse) } yield actualSubmitResponse - } - def papiUpgrade(workflowDefinition: Workflow, callMarker: CallMarker)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + def papiUpgrade(workflowDefinition: Workflow, callMarker: CallMarker)(implicit + cromwellTracker: Option[CromwellTracker] + ): Test[SubmitResponse] = CentaurConfig.runMode match { case ManagedCromwellServer(_, postRestart, withRestart) if withRestart => for { @@ -266,21 +317,25 @@ object TestFormulas extends StrictLogging { workflow = first, testDefinition = workflowDefinition, expectedStatuses = Set(Running, Succeeded), - timeout = CentaurConfig.workflowProgressTimeout, + timeout = CentaurConfig.workflowProgressTimeout ) _ <- pollUntilStatus(first, workflowDefinition, Succeeded) _ <- checkDescription(workflowDefinition.secondRun, validityExpectation = Option(true)) - second <- runSuccessfulWorkflow(workflowDefinition.secondRun) // Same WDL and config but a "backend" runtime option targeting PAPI v2. + second <- runSuccessfulWorkflow( + workflowDefinition.secondRun + ) // Same WDL and config but a "backend" runtime option targeting PAPI v2. _ <- printHashDifferential(first, second) metadata <- fetchAndValidateNonSubworkflowMetadata(second, workflowDefinition, Option(first.id.id)) - _ <- fetchAndValidateJobManagerStyleMetadata(second, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(second, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) _ <- validateNoCacheMisses(second, metadata, workflowDefinition) _ <- validateDirectoryContentsCounts(workflowDefinition, second, metadata) } yield SubmitResponse(second) case _ => Test.invalidTestDefinition("Configuration not supported by PapiUpgradeTest", workflowDefinition) } - } implicit class EnhancedCromwellTracker(val tracker: Option[CromwellTracker]) extends AnyVal { def track(metadata: WorkflowMetadata): Unit = tracker foreach { _.track(metadata) } diff --git a/centaur/src/main/scala/centaur/test/markers/CallMarker.scala b/centaur/src/main/scala/centaur/test/markers/CallMarker.scala index 08eafc09213..56274a07ed4 100644 --- a/centaur/src/main/scala/centaur/test/markers/CallMarker.scala +++ b/centaur/src/main/scala/centaur/test/markers/CallMarker.scala @@ -7,12 +7,11 @@ import configs.Result.{Failure, Success} import configs.syntax._ object CallMarker { - def fromConfig(config: Config): ErrorOr[Option[CallMarker]] = { + def fromConfig(config: Config): ErrorOr[Option[CallMarker]] = config.get[Option[String]]("callMark") match { case Success(marker) => (marker map CallMarker.apply).validNel case Failure(f) => s"Invalid restart marker $f".invalidNel } - } } /** diff --git a/centaur/src/main/scala/centaur/test/metadata/CallAttemptFailure.scala b/centaur/src/main/scala/centaur/test/metadata/CallAttemptFailure.scala index 098328d0d41..f79aaca6826 100644 --- a/centaur/src/main/scala/centaur/test/metadata/CallAttemptFailure.scala +++ b/centaur/src/main/scala/centaur/test/metadata/CallAttemptFailure.scala @@ -12,8 +12,7 @@ import io.circe.parser._ * * https://github.com/DataBiosphere/job-manager/blob/f83e4284e2419389b7e515720c9d960d2eb81a29/servers/cromwell/jobs/controllers/jobs_controller.py#L155-L162 */ -case class CallAttemptFailure -( +case class CallAttemptFailure( workflowId: String, callFullyQualifiedName: String, jobIndex: Int, @@ -27,29 +26,25 @@ case class CallAttemptFailure ) object CallAttemptFailure { - def buildFailures(jsonOption: Option[String]): IO[Vector[CallAttemptFailure]] = { + def buildFailures(jsonOption: Option[String]): IO[Vector[CallAttemptFailure]] = jsonOption.map(buildFailures).getOrElse(IO.pure(Vector.empty)) - } - def buildFailures(json: String): IO[Vector[CallAttemptFailure]] = { + def buildFailures(json: String): IO[Vector[CallAttemptFailure]] = IO.fromEither(decode[Vector[CallAttemptFailure]](json)) - } - private implicit val decodeFailures: Decoder[Vector[CallAttemptFailure]] = { + implicit private val decodeFailures: Decoder[Vector[CallAttemptFailure]] = Decoder.instance { c => for { workflowId <- c.get[String]("id") calls <- c.get[Map[String, Json]]("calls").map(_.toVector) - callAttemptFailures <- calls.flatTraverse[Decoder.Result, CallAttemptFailure] { - case (callName, callJson) => - val decoderCallAttempt = decodeFromCallAttempt(workflowId, callName) - callJson.as[Vector[Option[CallAttemptFailure]]](Decoder.decodeVector(decoderCallAttempt)).map(_.flatten) + callAttemptFailures <- calls.flatTraverse[Decoder.Result, CallAttemptFailure] { case (callName, callJson) => + val decoderCallAttempt = decodeFromCallAttempt(workflowId, callName) + callJson.as[Vector[Option[CallAttemptFailure]]](Decoder.decodeVector(decoderCallAttempt)).map(_.flatten) } } yield callAttemptFailures } or Decoder.const(Vector.empty) - } - private def decodeFromCallAttempt(workflowId: String, callName: String): Decoder[Option[CallAttemptFailure]] = { + private def decodeFromCallAttempt(workflowId: String, callName: String): Decoder[Option[CallAttemptFailure]] = Decoder.instance { c => for { shardIndexOption <- c.get[Option[Int]]("shardIndex") @@ -77,5 +72,4 @@ object CallAttemptFailure { } } yield callAttemptFailureOption } or Decoder.const(None) - } } diff --git a/centaur/src/main/scala/centaur/test/metadata/WorkflowFlatMetadata.scala b/centaur/src/main/scala/centaur/test/metadata/WorkflowFlatMetadata.scala index 182d1fad58f..5054105c154 100644 --- a/centaur/src/main/scala/centaur/test/metadata/WorkflowFlatMetadata.scala +++ b/centaur/src/main/scala/centaur/test/metadata/WorkflowFlatMetadata.scala @@ -17,7 +17,6 @@ import spray.json._ import scala.language.postfixOps import scala.util.{Failure, Success, Try} - /** * Workflow metadata that has been flattened for Centaur test purposes. The keys are similar to the simpleton-syntax * stored in the Cromwell database, and values are primitive types, not nested JSON objects or arrays. @@ -27,15 +26,23 @@ case class WorkflowFlatMetadata(value: Map[String, JsValue]) extends AnyVal { def diff(actual: WorkflowFlatMetadata, workflowID: UUID, cacheHitUUID: Option[UUID] = None): Iterable[String] = { // If the test fails in initialization there wouldn't be workflow root metadata, and if that's the expectation // then that's ok. - val workflowRoot = actual.value.get("workflowRoot").collectFirst { case JsString(r) => r } getOrElse "No Workflow Root" + val workflowRoot = + actual.value.get("workflowRoot").collectFirst { case JsString(r) => r } getOrElse "No Workflow Root" val missingErrors = value.keySet.diff(actual.value.keySet) map { k => s"Missing key: $k" } - val mismatchErrors = value.keySet.intersect(actual.value.keySet) flatMap { k => diffValues(k, value(k), actual.value(k), - workflowID, workflowRoot, cacheHitUUID)} + val mismatchErrors = value.keySet.intersect(actual.value.keySet) flatMap { k => + diffValues(k, value(k), actual.value(k), workflowID, workflowRoot, cacheHitUUID) + } mismatchErrors ++ missingErrors } - private def diffValues(key: String, expected: JsValue, actual: JsValue, workflowID: UUID, workflowRoot: String, cacheHitUUID: Option[UUID]): Option[String] = { + private def diffValues(key: String, + expected: JsValue, + actual: JsValue, + workflowID: UUID, + workflowRoot: String, + cacheHitUUID: Option[UUID] + ): Option[String] = { /* FIXME/TODO: @@ -59,8 +66,10 @@ case class WorkflowFlatMetadata(value: Map[String, JsValue]) extends AnyVal { val stripped = stripQuotes(cacheSubstitutions).stripPrefix("~~") (!stripQuotes(o.toString).contains(stripped)).option(s"Actual value ${o.toString()} does not contain $stripped") case o: JsString => (cacheSubstitutions != o.toString).option(s"expected: $cacheSubstitutions but got: $actual") - case o: JsNumber => (expected != JsString(o.value.toString)).option(s"expected: $cacheSubstitutions but got: $actual") - case o: JsBoolean => (expected != JsString(o.value.toString)).option(s"expected: $cacheSubstitutions but got: $actual") + case o: JsNumber => + (expected != JsString(o.value.toString)).option(s"expected: $cacheSubstitutions but got: $actual") + case o: JsBoolean => + (expected != JsString(o.value.toString)).option(s"expected: $cacheSubstitutions but got: $actual") case o: JsArray if stripQuotes(cacheSubstitutions).startsWith("~>") => val stripped = stripQuotes(cacheSubstitutions).stripPrefix("~>") val replaced = stripped.replaceAll("\\\\\"", "\"") @@ -76,12 +85,11 @@ case class WorkflowFlatMetadata(value: Map[String, JsValue]) extends AnyVal { object WorkflowFlatMetadata { - def fromConfig(config: Config): ErrorOr[WorkflowFlatMetadata] = { + def fromConfig(config: Config): ErrorOr[WorkflowFlatMetadata] = config.extract[Map[String, Option[String]]] match { case Result.Success(m) => Valid(WorkflowFlatMetadata(m safeMapValues { _.map(JsString.apply).getOrElse(JsNull) })) case Result.Failure(_) => invalidNel(s"Metadata block can not be converted to a Map: $config") } - } def fromWorkflowMetadata(workflowMetadata: WorkflowMetadata): ErrorOr[WorkflowFlatMetadata] = { val jsValue: ErrorOr[JsValue] = Try(workflowMetadata.value.parseJson) match { @@ -96,9 +104,8 @@ object WorkflowFlatMetadata { } implicit class EnhancedWorkflowMetadata(val workflowMetadata: WorkflowMetadata) { - def asFlat: WorkflowFlatMetadata = { + def asFlat: WorkflowFlatMetadata = WorkflowFlatMetadata.fromWorkflowMetadata(workflowMetadata).unsafe - } } implicit class EnhancedWorkflowFlatMetadata(val workflowFlatMetadata: WorkflowFlatMetadata) { @@ -113,25 +120,22 @@ object WorkflowFlatMetadata { } implicit class EnhancedExpectation(val expectation: String) extends AnyVal { - def replaceExpectationVariables(workflowId: WorkflowId, workflowRoot: String): String = { + def replaceExpectationVariables(workflowId: WorkflowId, workflowRoot: String): String = expectation.replaceAll("<>", workflowId.toString).replaceAll("<>", workflowRoot) - } } } object WorkflowFlatOutputs { implicit class EnhancedWorkflowOutputs(val workflowOutputs: WorkflowOutputs) extends AnyVal { - def asFlat: WorkflowFlatMetadata = { + def asFlat: WorkflowFlatMetadata = workflowOutputs.outputs.asMap map WorkflowFlatMetadata.apply unsafe - } } } object WorkflowFlatLabels { implicit class EnhancedWorkflowLabels(val workflowLabels: WorkflowLabels) extends AnyVal { - def asFlat: WorkflowFlatMetadata = { + def asFlat: WorkflowFlatMetadata = workflowLabels.labels.asMap map WorkflowFlatMetadata.apply unsafe - } } } @@ -140,11 +144,10 @@ object JsValueEnhancer { import DefaultJsonProtocol._ import centaur.json.JsonUtils._ - def asMap: ErrorOr[Map[String, JsValue]] = { + def asMap: ErrorOr[Map[String, JsValue]] = Try(jsValue.asJsObject.flatten().convertTo[Map[String, JsValue]]) match { case Success(m) => Valid(m) case Failure(e) => invalidNel(s"Unable to convert JsValue to JsObject: ${e.getMessage}") } - } } } diff --git a/centaur/src/main/scala/centaur/test/standard/CentaurTestCase.scala b/centaur/src/main/scala/centaur/test/standard/CentaurTestCase.scala index 258c211c900..e462ea83c90 100644 --- a/centaur/src/main/scala/centaur/test/standard/CentaurTestCase.scala +++ b/centaur/src/main/scala/centaur/test/standard/CentaurTestCase.scala @@ -19,8 +19,8 @@ case class CentaurTestCase(workflow: Workflow, testFormat: CentaurTestFormat, testOptions: TestOptions, submittedWorkflowTracker: SubmittedWorkflowTracker, - submitResponseOption: Option[SubmitHttpResponse])( - implicit cromwellTracker: Option[CromwellTracker]) { + submitResponseOption: Option[SubmitHttpResponse] +)(implicit cromwellTracker: Option[CromwellTracker]) { def testFunction: Test[SubmitResponse] = this.testFormat match { case WorkflowSuccessTest => TestFormulas.runSuccessfulWorkflowAndVerifyMetadata(workflow) @@ -32,10 +32,14 @@ case class CentaurTestCase(workflow: Workflow, case RunFailingTwiceExpectingNoCallCachingTest => TestFormulas.runFailingWorkflowTwiceExpectingNoCaching(workflow) case SubmitFailureTest => TestFormulas.submitInvalidWorkflow(workflow, submitResponseOption.get) case InstantAbort => TestFormulas.instantAbort(workflow) - case CromwellRestartWithRecover(callMarker)=> TestFormulas.workflowRestart(workflow, callMarker, recover = true, finalStatus = Succeeded) - case WorkflowFailureRestartWithRecover(callMarker)=> TestFormulas.workflowRestart(workflow, callMarker, recover = true, finalStatus = Failed) - case WorkflowFailureRestartWithoutRecover(callMarker)=> TestFormulas.workflowRestart(workflow, callMarker, recover = false, finalStatus = Failed) - case CromwellRestartWithoutRecover(callMarker) => TestFormulas.workflowRestart(workflow, callMarker, recover = false, finalStatus = Succeeded) + case CromwellRestartWithRecover(callMarker) => + TestFormulas.workflowRestart(workflow, callMarker, recover = true, finalStatus = Succeeded) + case WorkflowFailureRestartWithRecover(callMarker) => + TestFormulas.workflowRestart(workflow, callMarker, recover = true, finalStatus = Failed) + case WorkflowFailureRestartWithoutRecover(callMarker) => + TestFormulas.workflowRestart(workflow, callMarker, recover = false, finalStatus = Failed) + case CromwellRestartWithoutRecover(callMarker) => + TestFormulas.workflowRestart(workflow, callMarker, recover = false, finalStatus = Succeeded) case ScheduledAbort(callMarker) => TestFormulas.scheduledAbort(workflow, callMarker, restart = false) case ScheduledAbortWithRestart(callMarker) => TestFormulas.scheduledAbort(workflow, callMarker, restart = true) case PapiUpgradeTest(callMarker) => TestFormulas.papiUpgrade(workflow, callMarker) @@ -44,9 +48,15 @@ case class CentaurTestCase(workflow: Workflow, def isIgnored(supportedBackends: List[String]): Boolean = { val backendSupported = workflow.backends match { - case AllBackendsRequired(allBackends) => allBackends forall supportedBackends.contains - case AnyBackendRequired(anyBackend) => anyBackend exists supportedBackends.contains - case OnlyBackendsAllowed(onlyBackends) => supportedBackends forall onlyBackends.contains + case AllBackendsRequired(testBackends) => + // Test will run on servers that support all of the test's backends (or more) (default) + testBackends forall supportedBackends.contains + case AnyBackendRequired(testBackends) => + // Test will run on servers that support at least one of the test's backends (or more) + testBackends exists supportedBackends.contains + case OnlyBackendsAllowed(testBackends) => + // Test will run on servers that only support backends the test specifies (or fewer) + supportedBackends forall testBackends.contains } testOptions.ignore || !backendSupported @@ -58,13 +68,14 @@ case class CentaurTestCase(workflow: Workflow, } object CentaurTestCase { - def fromFile(cromwellTracker: Option[CromwellTracker])(file: File): ErrorOr[CentaurTestCase] = { + def fromFile(cromwellTracker: Option[CromwellTracker])(file: File): ErrorOr[CentaurTestCase] = Try(ConfigFactory.parseFile(file.toJava).resolve()) match { case Success(c) => - CentaurTestCase.fromConfig(c, file.parent, cromwellTracker) flatMap validateTestCase leftMap { s"Error in test file '$file'." :: _ } + CentaurTestCase.fromConfig(c, file.parent, cromwellTracker) flatMap validateTestCase leftMap { + s"Error in test file '$file'." :: _ + } case Failure(f) => invalidNel(s"Invalid test config: $file (${f.getMessage})") } - } def fromConfig(conf: Config, configFile: File, cromwellTracker: Option[CromwellTracker]): ErrorOr[CentaurTestCase] = { val submittedWorkflowTracker = new SubmittedWorkflowTracker() @@ -77,18 +88,18 @@ object CentaurTestCase { } } - private def validateTestCase(testCase: CentaurTestCase): ErrorOr[CentaurTestCase] = { + private def validateTestCase(testCase: CentaurTestCase): ErrorOr[CentaurTestCase] = testCase.testFormat match { - case SubmitFailureTest => validateSubmitFailure(testCase.workflow, testCase.submitResponseOption).map(_ => testCase) + case SubmitFailureTest => + validateSubmitFailure(testCase.workflow, testCase.submitResponseOption).map(_ => testCase) case _ => Valid(testCase) } - } private def validateSubmitFailure(workflow: Workflow, - submitResponseOption: Option[SubmitHttpResponse]): ErrorOr[SubmitResponse] = { + submitResponseOption: Option[SubmitHttpResponse] + ): ErrorOr[SubmitResponse] = submitResponseOption match { case None => invalidNel("No submit stanza included in test config") case Some(response) => Valid(response) } - } } diff --git a/centaur/src/main/scala/centaur/test/standard/CentaurTestFormat.scala b/centaur/src/main/scala/centaur/test/standard/CentaurTestFormat.scala index 1cb2345f25c..eae961d6e5f 100644 --- a/centaur/src/main/scala/centaur/test/standard/CentaurTestFormat.scala +++ b/centaur/src/main/scala/centaur/test/standard/CentaurTestFormat.scala @@ -9,7 +9,7 @@ import configs.syntax._ sealed abstract class CentaurTestFormat(val name: String) { val lowerCaseName = name.toLowerCase - + def testSpecString: String = this match { case WorkflowSuccessTest => "successfully run" case WorkflowSuccessAndTimedOutputsTest => "successfully run" @@ -20,12 +20,14 @@ sealed abstract class CentaurTestFormat(val name: String) { case RunFailingTwiceExpectingNoCallCachingTest => "Fail the first run and NOT call cache the second run of" case SubmitFailureTest => "fail to submit" case InstantAbort => "abort a workflow immediately after submission" - case _: PapiUpgradeTest => "make sure a PAPI upgrade preserves call caching when the `name-for-call-caching-purposes` attribute is used" + case _: PapiUpgradeTest => + "make sure a PAPI upgrade preserves call caching when the `name-for-call-caching-purposes` attribute is used" case _: CromwellRestartWithRecover => "survive a Cromwell restart and recover jobs" case _: CromwellRestartWithoutRecover => "survive a Cromwell restart" case _: ScheduledAbort => "abort a workflow mid run" case _: ScheduledAbortWithRestart => "abort a workflow mid run and restart immediately" - case _: WorkflowFailureRestartWithRecover => "survive a Cromwell restart when a workflow was failing and recover jobs" + case _: WorkflowFailureRestartWithRecover => + "survive a Cromwell restart when a workflow was failing and recover jobs" case _: WorkflowFailureRestartWithoutRecover => "survive a Cromwell restart when a workflow was failing" case other => s"unrecognized format $other" } @@ -39,71 +41,89 @@ object CentaurTestFormat { sealed trait SequentialTestFormat extends CentaurTestFormat { override def isParallel: Boolean = false } - + sealed trait RestartFormat extends SequentialTestFormat - sealed trait WithCallMarker { this: CentaurTestFormat => val build: CallMarker => CentaurTestFormat } - + sealed trait WithCallMarker { this: CentaurTestFormat => + val build: CallMarker => CentaurTestFormat + } + case object WorkflowSuccessTest extends CentaurTestFormat("WorkflowSuccess") case object WorkflowSuccessAndTimedOutputsTest extends CentaurTestFormat("WorkflowSuccessAndTimedOutputs") case object WorkflowFailureTest extends CentaurTestFormat("WorkflowFailure") case object RunTwiceExpectingCallCachingTest extends CentaurTestFormat("RunTwiceExpectingCallCaching") case object RunThriceExpectingCallCachingTest extends CentaurTestFormat(name = "RunThriceExpectingCallCaching") case object RunTwiceExpectingNoCallCachingTest extends CentaurTestFormat("RunTwiceExpectingNoCallCaching") - case object RunFailingTwiceExpectingNoCallCachingTest extends CentaurTestFormat("RunFailingTwiceExpectingNoCallCaching") + case object RunFailingTwiceExpectingNoCallCachingTest + extends CentaurTestFormat("RunFailingTwiceExpectingNoCallCaching") case object SubmitFailureTest extends CentaurTestFormat("SubmitFailure") case object InstantAbort extends CentaurTestFormat("InstantAbort") with SequentialTestFormat object CromwellRestartWithRecover extends CentaurTestFormat("CromwellRestartWithRecover") with WithCallMarker { val build = CromwellRestartWithRecover.apply _ } - case class CromwellRestartWithRecover(callMarker: CallMarker) extends CentaurTestFormat(CromwellRestartWithRecover.name) with RestartFormat - + case class CromwellRestartWithRecover(callMarker: CallMarker) + extends CentaurTestFormat(CromwellRestartWithRecover.name) + with RestartFormat + object CromwellRestartWithoutRecover extends CentaurTestFormat("CromwellRestartWithoutRecover") with WithCallMarker { val build = CromwellRestartWithoutRecover.apply _ } - case class CromwellRestartWithoutRecover(callMarker: CallMarker) extends CentaurTestFormat(CromwellRestartWithoutRecover.name) with RestartFormat + case class CromwellRestartWithoutRecover(callMarker: CallMarker) + extends CentaurTestFormat(CromwellRestartWithoutRecover.name) + with RestartFormat object ScheduledAbort extends CentaurTestFormat("ScheduledAbort") with WithCallMarker { val build = ScheduledAbort.apply _ } - case class ScheduledAbort(callMarker: CallMarker) extends CentaurTestFormat(ScheduledAbort.name) with SequentialTestFormat + case class ScheduledAbort(callMarker: CallMarker) + extends CentaurTestFormat(ScheduledAbort.name) + with SequentialTestFormat object ScheduledAbortWithRestart extends CentaurTestFormat("ScheduledAbortWithRestart") with WithCallMarker { val build = ScheduledAbortWithRestart.apply _ } - case class ScheduledAbortWithRestart(callMarker: CallMarker) extends CentaurTestFormat(ScheduledAbortWithRestart.name) with RestartFormat + case class ScheduledAbortWithRestart(callMarker: CallMarker) + extends CentaurTestFormat(ScheduledAbortWithRestart.name) + with RestartFormat - object WorkflowFailureRestartWithRecover extends CentaurTestFormat("WorkflowFailureRestartWithRecover") with WithCallMarker { + object WorkflowFailureRestartWithRecover + extends CentaurTestFormat("WorkflowFailureRestartWithRecover") + with WithCallMarker { val build = WorkflowFailureRestartWithRecover.apply _ } - case class WorkflowFailureRestartWithRecover(callMarker: CallMarker) extends CentaurTestFormat(WorkflowFailureRestartWithRecover.name) with RestartFormat + case class WorkflowFailureRestartWithRecover(callMarker: CallMarker) + extends CentaurTestFormat(WorkflowFailureRestartWithRecover.name) + with RestartFormat - object WorkflowFailureRestartWithoutRecover extends CentaurTestFormat("WorkflowFailureRestartWithoutRecover") with WithCallMarker { + object WorkflowFailureRestartWithoutRecover + extends CentaurTestFormat("WorkflowFailureRestartWithoutRecover") + with WithCallMarker { val build = WorkflowFailureRestartWithoutRecover.apply _ } - case class WorkflowFailureRestartWithoutRecover(callMarker: CallMarker) extends CentaurTestFormat(WorkflowFailureRestartWithoutRecover.name) with RestartFormat + case class WorkflowFailureRestartWithoutRecover(callMarker: CallMarker) + extends CentaurTestFormat(WorkflowFailureRestartWithoutRecover.name) + with RestartFormat object PapiUpgradeTest extends CentaurTestFormat("PapiUpgrade") with WithCallMarker { val build = PapiUpgradeTest.apply _ } case class PapiUpgradeTest(callMarker: CallMarker) extends CentaurTestFormat(PapiUpgradeTest.name) with RestartFormat - def fromConfig(conf: Config): Checked[CentaurTestFormat] = { - + def fromConfig(conf: Config): Checked[CentaurTestFormat] = CallMarker.fromConfig(conf).toEither flatMap { callMarker => conf.get[String]("testFormat") match { case Success(f) => CentaurTestFormat.fromString(f, callMarker) case Failure(_) => "No testFormat string provided".invalidNelCheck[CentaurTestFormat] } } - } private def fromString(testFormat: String, callMarker: Option[CallMarker]): Checked[CentaurTestFormat] = { def withCallMarker(name: String, constructor: CallMarker => CentaurTestFormat) = callMarker match { case Some(marker) => constructor(marker).validNelCheck - case None => s"$name needs a callMarker to know on which call to trigger the restart".invalidNelCheck[CentaurTestFormat] + case None => + s"$name needs a callMarker to know on which call to trigger the restart".invalidNelCheck[CentaurTestFormat] } - + List( WorkflowSuccessTest, WorkflowSuccessAndTimedOutputsTest, @@ -121,9 +141,10 @@ object CentaurTestFormat { WorkflowFailureRestartWithRecover, WorkflowFailureRestartWithoutRecover, PapiUpgradeTest - ).collectFirst({ - case format: WithCallMarker if format.name.equalsIgnoreCase(testFormat) => withCallMarker(format.name, format.build) + ).collectFirst { + case format: WithCallMarker if format.name.equalsIgnoreCase(testFormat) => + withCallMarker(format.name, format.build) case format if format.name.equalsIgnoreCase(testFormat) => format.validNelCheck - }).getOrElse(s"No such test format: $testFormat".invalidNelCheck[CentaurTestFormat]) + }.getOrElse(s"No such test format: $testFormat".invalidNelCheck[CentaurTestFormat]) } } diff --git a/centaur/src/main/scala/centaur/test/submit/SubmitResponse.scala b/centaur/src/main/scala/centaur/test/submit/SubmitResponse.scala index ed160482bef..faed88cc8cd 100644 --- a/centaur/src/main/scala/centaur/test/submit/SubmitResponse.scala +++ b/centaur/src/main/scala/centaur/test/submit/SubmitResponse.scala @@ -14,13 +14,11 @@ import cromwell.api.model.SubmittedWorkflow sealed trait SubmitResponse object SubmitResponse { - def apply(submittedWorkflow: SubmittedWorkflow): SubmitResponse = { + def apply(submittedWorkflow: SubmittedWorkflow): SubmitResponse = SubmitWorkflowResponse(submittedWorkflow) - } - def apply(statusCode: Int, message: String): SubmitResponse = { + def apply(statusCode: Int, message: String): SubmitResponse = SubmitHttpResponse(statusCode, message) - } } case class SubmitWorkflowResponse(submittedWorkflow: SubmittedWorkflow) extends SubmitResponse @@ -29,7 +27,7 @@ case class SubmitHttpResponse(statusCode: Int, message: String) extends SubmitRe object SubmitHttpResponse { - def fromConfig(conf: Config): ErrorOr[Option[SubmitHttpResponse]] = { + def fromConfig(conf: Config): ErrorOr[Option[SubmitHttpResponse]] = conf.get[Config]("submit") match { case Result.Failure(_) => Valid(None) case Result.Success(submitConf) => @@ -41,17 +39,13 @@ object SubmitHttpResponse { Option(_) } } - } - private def toErrorOr[A](result: Result[A]): ErrorOr[A] = { + private def toErrorOr[A](result: Result[A]): ErrorOr[A] = result match { case Result.Success(value) => Valid(value) case Result.Failure(error) => - error.messages - .toList - .toNel + error.messages.toList.toNel .getOrElse(throw new RuntimeException("Paranoia... error.messages is a Nel exposed as a Seq.")) .invalid } - } } diff --git a/centaur/src/main/scala/centaur/test/workflow/DirectoryContentCountCheck.scala b/centaur/src/main/scala/centaur/test/workflow/DirectoryContentCountCheck.scala index 2bf90619dab..4d064455f8c 100644 --- a/centaur/src/main/scala/centaur/test/workflow/DirectoryContentCountCheck.scala +++ b/centaur/src/main/scala/centaur/test/workflow/DirectoryContentCountCheck.scala @@ -2,7 +2,7 @@ package centaur.test.workflow import cats.data.Validated._ import cats.syntax.all._ -import centaur.test.{AWSFilesChecker, FilesChecker, LocalFilesChecker, PipelinesFilesChecker, BlobFilesChecker} +import centaur.test.{AWSFilesChecker, BlobFilesChecker, FilesChecker, LocalFilesChecker, PipelinesFilesChecker} import com.typesafe.config.Config import common.validation.ErrorOr.ErrorOr import configs.Result @@ -16,18 +16,23 @@ object DirectoryContentCountCheck { if (!keepGoing) { valid(None) } else { - val directoryContentCountsValidation: ErrorOr[Map[String, Int]] = conf.get[Map[String, Int]]("outputExpectations") match { - case Result.Success(a) => valid(a) - case Result.Failure(_) => invalidNel(s"Test '$name': Unable to read outputExpectations as a Map[String, Int]") - } + val directoryContentCountsValidation: ErrorOr[Map[String, Int]] = + conf.get[Map[String, Int]]("outputExpectations") match { + case Result.Success(a) => valid(a) + case Result.Failure(_) => invalidNel(s"Test '$name': Unable to read outputExpectations as a Map[String, Int]") + } val fileSystemChecker: ErrorOr[FilesChecker] = conf.get[String]("fileSystemCheck") match { case Result.Success("gcs") => valid(PipelinesFilesChecker) case Result.Success("local") => valid(LocalFilesChecker) case Result.Success("aws") => valid(AWSFilesChecker) case Result.Success("blob") => valid(BlobFilesChecker) - case Result.Success(_) => invalidNel(s"Test '$name': Invalid 'fileSystemCheck' value (must be either 'local', 'gcs', 'blob', or 'aws'") - case Result.Failure(_) => invalidNel(s"Test '$name': Must specify a 'fileSystemCheck' value (must be either 'local', 'gcs', 'blob', or 'aws'") + case Result.Success(_) => + invalidNel(s"Test '$name': Invalid 'fileSystemCheck' value (must be either 'local', 'gcs', 'blob', or 'aws'") + case Result.Failure(_) => + invalidNel( + s"Test '$name': Must specify a 'fileSystemCheck' value (must be either 'local', 'gcs', 'blob', or 'aws'" + ) } (directoryContentCountsValidation, fileSystemChecker) mapN { (d, f) => Option(DirectoryContentCountCheck(d, f)) } diff --git a/centaur/src/main/scala/centaur/test/workflow/SubmittedWorkflowTracker.scala b/centaur/src/main/scala/centaur/test/workflow/SubmittedWorkflowTracker.scala index dc3f0355f80..718db740dab 100644 --- a/centaur/src/main/scala/centaur/test/workflow/SubmittedWorkflowTracker.scala +++ b/centaur/src/main/scala/centaur/test/workflow/SubmittedWorkflowTracker.scala @@ -25,7 +25,6 @@ class SubmittedWorkflowTracker { * object require a retry. Prevents unwanted cache hits from partially successful attempts when retrying a call * caching test case. */ - def add(submittedWorkflow: SubmittedWorkflow): Unit = { + def add(submittedWorkflow: SubmittedWorkflow): Unit = submittedWorkflowIds = submittedWorkflow.id :: submittedWorkflowIds - } } diff --git a/centaur/src/main/scala/centaur/test/workflow/Workflow.scala b/centaur/src/main/scala/centaur/test/workflow/Workflow.scala index 743924bb9c6..212064acfe1 100644 --- a/centaur/src/main/scala/centaur/test/workflow/Workflow.scala +++ b/centaur/src/main/scala/centaur/test/workflow/Workflow.scala @@ -15,17 +15,18 @@ import cromwell.api.model.{WorkflowDescribeRequest, WorkflowSingleSubmission} import java.nio.file.Path import scala.concurrent.duration.FiniteDuration -final case class Workflow private(testName: String, - data: WorkflowData, - metadata: Option[WorkflowFlatMetadata], - notInMetadata: List[String], - directoryContentCounts: Option[DirectoryContentCountCheck], - backends: BackendsRequirement, - retryTestFailures: Boolean, - allowOtherOutputs: Boolean, - skipDescribeEndpointValidation: Boolean, - submittedWorkflowTracker: SubmittedWorkflowTracker, - maximumAllowedTime: Option[FiniteDuration]) { +final case class Workflow private (testName: String, + data: WorkflowData, + metadata: Option[WorkflowFlatMetadata], + notInMetadata: List[String], + directoryContentCounts: Option[DirectoryContentCountCheck], + backends: BackendsRequirement, + retryTestFailures: Boolean, + allowOtherOutputs: Boolean, + skipDescribeEndpointValidation: Boolean, + submittedWorkflowTracker: SubmittedWorkflowTracker, + maximumAllowedTime: Option[FiniteDuration] +) { def toWorkflowSubmission: WorkflowSingleSubmission = WorkflowSingleSubmission( workflowSource = data.workflowContent, @@ -36,7 +37,8 @@ final case class Workflow private(testName: String, inputsJson = data.inputs.map(_.unsafeRunSync()), options = data.options.map(_.unsafeRunSync()), labels = Option(data.labels), - zippedImports = data.zippedImports) + zippedImports = data.zippedImports + ) def toWorkflowDescribeRequest: WorkflowDescribeRequest = WorkflowDescribeRequest( workflowSource = data.workflowContent, @@ -46,22 +48,26 @@ final case class Workflow private(testName: String, inputsJson = data.inputs.map(_.unsafeRunSync()) ) - def secondRun: Workflow = { + def secondRun: Workflow = copy(data = data.copy(options = data.secondOptions)) - } - def thirdRun: Workflow = { + def thirdRun: Workflow = copy(data = data.copy(options = data.thirdOptions)) - } } object Workflow { - def fromConfig(conf: Config, configFile: File, submittedWorkflowTracker: SubmittedWorkflowTracker): ErrorOr[Workflow] = { + def fromConfig(conf: Config, + configFile: File, + submittedWorkflowTracker: SubmittedWorkflowTracker + ): ErrorOr[Workflow] = conf.get[String]("name") match { case Result.Success(n) => // If backend is provided, Centaur will only run this test if that backend is available on Cromwell - val backendsRequirement = BackendsRequirement.fromConfig(conf.get[String]("backendsMode").map(_.toLowerCase).valueOrElse("all"), conf.get[List[String]]("backends").valueOrElse(List.empty[String]).map(_.toLowerCase)) + val backendsRequirement = BackendsRequirement.fromConfig( + conf.get[String]("backendsMode").map(_.toLowerCase).valueOrElse("all"), + conf.get[List[String]]("backends").valueOrElse(List.empty[String]).map(_.toLowerCase) + ) // If basePath is provided it'll be used as basis for finding other files, otherwise use the dir the config was in val basePath = conf.get[Option[Path]]("basePath") valueOrElse None map (File(_)) getOrElse configFile val metadata: ErrorOr[Option[WorkflowFlatMetadata]] = conf.get[Config]("metadata") match { @@ -73,7 +79,8 @@ object Workflow { case Result.Failure(_) => List.empty } - val directoryContentCheckValidation: ErrorOr[Option[DirectoryContentCountCheck]] = DirectoryContentCountCheck.forConfig(n, conf) + val directoryContentCheckValidation: ErrorOr[Option[DirectoryContentCountCheck]] = + DirectoryContentCountCheck.forConfig(n, conf) val files = conf.get[Config]("files") match { case Result.Success(f) => WorkflowData.fromConfig(filesConfig = f, fullConfig = conf, basePath = basePath) case Result.Failure(_) => invalidNel(s"No 'files' block in $configFile") @@ -89,10 +96,21 @@ object Workflow { val maximumTime: Option[FiniteDuration] = conf.get[Option[FiniteDuration]]("maximumTime").value (files, directoryContentCheckValidation, metadata, retryTestFailuresErrorOr) mapN { - (f, d, m, retryTestFailures) => Workflow(n, f, m, absentMetadata, d, backendsRequirement, retryTestFailures, allowOtherOutputs, validateDescription, submittedWorkflowTracker, maximumTime) + (f, d, m, retryTestFailures) => + Workflow(n, + f, + m, + absentMetadata, + d, + backendsRequirement, + retryTestFailures, + allowOtherOutputs, + validateDescription, + submittedWorkflowTracker, + maximumTime + ) } case Result.Failure(_) => invalidNel(s"No test 'name' for: $configFile") } - } } diff --git a/centaur/src/main/scala/centaur/test/workflow/WorkflowData.scala b/centaur/src/main/scala/centaur/test/workflow/WorkflowData.scala index 72e66b31e18..2096a5f88d7 100644 --- a/centaur/src/main/scala/centaur/test/workflow/WorkflowData.scala +++ b/centaur/src/main/scala/centaur/test/workflow/WorkflowData.scala @@ -34,7 +34,8 @@ case class WorkflowData(workflowContent: Option[String], labels: List[Label], zippedImports: Option[File], secondOptions: Option[IO[String]] = None, - thirdOptions: Option[IO[String]] = None) + thirdOptions: Option[IO[String]] = None +) object WorkflowData { val blockingEC = ExecutionContext.fromExecutorService(Executors.newFixedThreadPool(5)) @@ -47,55 +48,64 @@ object WorkflowData { val workflowSourcePath = filesConfig.as[Option[String]]("workflow") (workflowSourcePath, workflowUrl) match { - case (Some(workflowPath), None) => Valid(WorkflowData( - workflowPath = Option(workflowPath), - workflowUrl = None, - filesConfig = filesConfig, - fullConfig = fullConfig, - basePath = basePath)) - case (None, Some(_)) => Valid(WorkflowData( - workflowPath = None, - workflowUrl = workflowUrl, - filesConfig = filesConfig, - fullConfig = fullConfig, - basePath = basePath)) + case (Some(workflowPath), None) => + Valid( + WorkflowData(workflowPath = Option(workflowPath), + workflowUrl = None, + filesConfig = filesConfig, + fullConfig = fullConfig, + basePath = basePath + ) + ) + case (None, Some(_)) => + Valid( + WorkflowData(workflowPath = None, + workflowUrl = workflowUrl, + filesConfig = filesConfig, + fullConfig = fullConfig, + basePath = basePath + ) + ) case (Some(_), Some(_)) => invalidNel(s"Both 'workflow' path or 'workflowUrl' can't be provided.") case (None, None) => invalidNel(s"No 'workflow' path or 'workflowUrl' provided.") } } - def apply(workflowPath: Option[String], workflowUrl: Option[String], filesConfig: Config, fullConfig: Config, basePath: File): WorkflowData = { + def apply(workflowPath: Option[String], + workflowUrl: Option[String], + filesConfig: Config, + fullConfig: Config, + basePath: File + ): WorkflowData = { def slurp(file: String): IO[String] = file match { - case http if http.startsWith("http://") || http.startsWith("https://") => + case http if http.startsWith("http://") || http.startsWith("https://") => httpClient.expect[String](http) case gcs if gcs.startsWith("gs://") => val noScheme = gcs.stripPrefix("gs://") val firstSlashPosition = noScheme.indexOf("/") val blob = BlobId.of(noScheme.substring(0, firstSlashPosition), noScheme.substring(firstSlashPosition + 1)) - IO { gcsStorage.readAllBytes(blob).map(_.toChar).mkString } + IO(gcsStorage.readAllBytes(blob).map(_.toChar).mkString) case local => - IO { basePath./(local).contentAsString } + IO(basePath./(local).contentAsString) } - - def getOptionalFileContent(name: String): Option[IO[String]] = { + + def getOptionalFileContent(name: String): Option[IO[String]] = filesConfig.getAs[String](name).map(slurp) - } def getImports = filesConfig.get[List[String]]("imports") match { case Success(paths) => zipImports(paths map basePath./) case Failure(_) => None } - def getImportsDirName(workflowPath: Option[File], workflowUrl: Option[String]): String = { + def getImportsDirName(workflowPath: Option[File], workflowUrl: Option[String]): String = workflowPath match { case Some(file) => file.name.replaceAll("\\.[^.]*$", "") case None => // workflow url is defined val fileName = workflowUrl.get.split("/").last fileName.replaceAll("\\.[^.]*$", "") } - } - def zipImports(imports: List[File]): Option[File] = { + def zipImports(imports: List[File]): Option[File] = imports match { case Nil => None case _ => @@ -109,7 +119,6 @@ object WorkflowData { Option(importsDir.zip()) } - } def getLabels: List[Label] = { import cromwell.api.model.LabelsJsonFormatter._ diff --git a/centaur/src/test/scala/centaur/api/DaemonizedDefaultThreadFactorySpec.scala b/centaur/src/test/scala/centaur/api/DaemonizedDefaultThreadFactorySpec.scala index afd3918d738..7069cee9c40 100644 --- a/centaur/src/test/scala/centaur/api/DaemonizedDefaultThreadFactorySpec.scala +++ b/centaur/src/test/scala/centaur/api/DaemonizedDefaultThreadFactorySpec.scala @@ -9,7 +9,7 @@ class DaemonizedDefaultThreadFactorySpec extends AnyFlatSpec with CromwellTimeou behavior of "DaemonizedDefaultThreadFactory" it should "create a non-blocking execution context" in { - val thread = DaemonizedDefaultThreadFactory.newThread(() => {}) + val thread = DaemonizedDefaultThreadFactory.newThread { () => } thread.getName should startWith("daemonpool-thread-") thread.isDaemon should be(true) } diff --git a/centaur/src/test/scala/centaur/json/JsonUtilsSpec.scala b/centaur/src/test/scala/centaur/json/JsonUtilsSpec.scala index 1f89477d388..91fe7123ed8 100644 --- a/centaur/src/test/scala/centaur/json/JsonUtilsSpec.scala +++ b/centaur/src/test/scala/centaur/json/JsonUtilsSpec.scala @@ -111,7 +111,7 @@ class JsonUtilsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "calls.wf_hello.hello.outputs.salutation" -> "Hello Mr. Bean!", "calls.wf_hello.hello.runtimeAttributes.bootDiskSizeGb" -> "10", "calls.wf_hello.hello.runtimeAttributes.continueOnReturnCode" -> "0", - "calls.wf_hello.hello.runtimeAttributes.maxRetries" -> "0", + "calls.wf_hello.hello.runtimeAttributes.maxRetries" -> "0" ).map(x => (x._1, JsString(x._2))) val actualFlattenedMetadata: Map[String, JsValue] = metadata.parseJson.asJsObject.flatten().fields @@ -169,7 +169,7 @@ class JsonUtilsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "calls.wf_hello.task1.executionEvents.0.description" -> "task 1 step 1", "calls.wf_hello.task2.executionEvents.0.description" -> "task 2 step 1", "calls.wf_hello.task1.runtimeAttributes.bootDiskSizeGb" -> "10", - "calls.wf_hello.task2.runtimeAttributes.bootDiskSizeGb" -> "10", + "calls.wf_hello.task2.runtimeAttributes.bootDiskSizeGb" -> "10" ).map(x => (x._1, JsString(x._2))) val actualFlattenedMetadata: Map[String, JsValue] = metadata.parseJson.asJsObject.flatten().fields @@ -249,7 +249,7 @@ class JsonUtilsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "wf_hello.hello.0.2.shardIndex" -> 0, "wf_hello.hello.1.shardIndex" -> 1, "wf_hello.hello.1.1.shardIndex" -> 1, - "wf_hello.hello.1.2.shardIndex" -> 1, + "wf_hello.hello.1.2.shardIndex" -> 1 ).map(x => (x._1, JsNumber(x._2))) ++ Map( "id" -> "5abfaa90-570f-48d4-a35b-81d5ad4ea0f7", "status" -> "Succeeded", @@ -265,7 +265,7 @@ class JsonUtilsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "wf_hello.hello.0.2.runtimeAttributes.memory" -> "1.1 GB", "wf_hello.hello.1.runtimeAttributes.memory" -> "1.1 GB", "wf_hello.hello.1.1.runtimeAttributes.memory" -> "1 GB", - "wf_hello.hello.1.2.runtimeAttributes.memory" -> "1.1 GB", + "wf_hello.hello.1.2.runtimeAttributes.memory" -> "1.1 GB" ).map(x => (x._1, JsString(x._2))) val actualFlattenedMetadata: Map[String, JsValue] = metadata.parseJson.asJsObject.flatten().fields diff --git a/centaur/src/test/scala/centaur/test/CentaurOperationsSpec.scala b/centaur/src/test/scala/centaur/test/CentaurOperationsSpec.scala index 9d09827027a..28a74274192 100644 --- a/centaur/src/test/scala/centaur/test/CentaurOperationsSpec.scala +++ b/centaur/src/test/scala/centaur/test/CentaurOperationsSpec.scala @@ -15,23 +15,29 @@ import scala.concurrent.duration._ class CentaurOperationsSpec extends AnyFlatSpec with Matchers { behavior of "validateMetadataJson" - val placeholderSubmittedWorkflow: SubmittedWorkflow = SubmittedWorkflow(id = WorkflowId(UUID.randomUUID()), null, null) - val placeholderWorkflow: Workflow = Workflow(testName = "", null, null, null, null, null, false, false, false, null, null) + val placeholderSubmittedWorkflow: SubmittedWorkflow = + SubmittedWorkflow(id = WorkflowId(UUID.randomUUID()), null, null) + val placeholderWorkflow: Workflow = + Workflow(testName = "", null, null, null, null, null, false, false, false, null, null) val allowableOneWordAdditions = List("farmer") def runTest(json1: String, json2: String, expectMatching: Boolean): Unit = { - val validation = Operations.validateMetadataJson("", - json1.parseJson.asJsObject, - json2.parseJson.asJsObject, - placeholderSubmittedWorkflow, - placeholderWorkflow, - allowableAddedOneWordFields = allowableOneWordAdditions).unsafeToFuture() + val validation = Operations + .validateMetadataJson( + "", + json1.parseJson.asJsObject, + json2.parseJson.asJsObject, + placeholderSubmittedWorkflow, + placeholderWorkflow, + allowableAddedOneWordFields = allowableOneWordAdditions + ) + .unsafeToFuture() Await.ready(validation, atMost = 10.seconds) validation.value.get match { case Success(()) if expectMatching => // great case Success(_) if !expectMatching => fail("Metadata unexpectedly matches") - case Failure(e) if expectMatching => fail("Metadata unexpectedly mismatches", e) + case Failure(e) if expectMatching => fail("Metadata unexpectedly mismatches", e) case Failure(_) if !expectMatching => // great case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") } diff --git a/centaur/src/test/scala/centaur/test/metadata/CallAttemptFailureSpec.scala b/centaur/src/test/scala/centaur/test/metadata/CallAttemptFailureSpec.scala index 2476bb4b231..0be16d9a57a 100644 --- a/centaur/src/test/scala/centaur/test/metadata/CallAttemptFailureSpec.scala +++ b/centaur/src/test/scala/centaur/test/metadata/CallAttemptFailureSpec.scala @@ -6,7 +6,6 @@ import io.circe.ParsingFailure import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class CallAttemptFailureSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "CallAttemptFailure" diff --git a/centaur/src/test/scala/centaur/test/metadata/ExtractJobManagerStyleMetadataFieldsSpec.scala b/centaur/src/test/scala/centaur/test/metadata/ExtractJobManagerStyleMetadataFieldsSpec.scala index 2b9cb409352..f4ce7300385 100644 --- a/centaur/src/test/scala/centaur/test/metadata/ExtractJobManagerStyleMetadataFieldsSpec.scala +++ b/centaur/src/test/scala/centaur/test/metadata/ExtractJobManagerStyleMetadataFieldsSpec.scala @@ -6,7 +6,6 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import spray.json._ - class ExtractJobManagerStyleMetadataFieldsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "extracting Job Manager style metadata fields" @@ -45,6 +44,8 @@ class ExtractJobManagerStyleMetadataFieldsSpec extends AnyFlatSpec with Cromwell | } |}""".stripMargin - Operations.extractJmStyleMetadataFields(originalMetadata.parseJson.asJsObject) should be(expectedExpectedMetadata.parseJson.asJsObject) + Operations.extractJmStyleMetadataFields(originalMetadata.parseJson.asJsObject) should be( + expectedExpectedMetadata.parseJson.asJsObject + ) } } diff --git a/centaur/src/test/scala/centaur/testfilecheck/FileCheckerSpec.scala b/centaur/src/test/scala/centaur/testfilecheck/FileCheckerSpec.scala index f389192aa7f..0f9984d882e 100644 --- a/centaur/src/test/scala/centaur/testfilecheck/FileCheckerSpec.scala +++ b/centaur/src/test/scala/centaur/testfilecheck/FileCheckerSpec.scala @@ -11,7 +11,6 @@ import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.model.{ListObjectsRequest, ListObjectsResponse, S3Object} import org.scalatest.flatspec.AnyFlatSpec - class FileCheckerSpec extends AnyFlatSpec with CromwellTimeoutSpec with MockSugar { import centaur.test.ObjectCounterInstances._ @@ -26,24 +25,29 @@ class FileCheckerSpec extends AnyFlatSpec with CromwellTimeoutSpec with MockSuga private val wrongBucketPrefix = "s3Bucket://my-not-so-cool-bucket/somelogs/empty" private val EmptyTestPath = "" private val testGsPath = "gs://my-cool-bucket/path/to/file" - private val objResponse = ListObjectsResponse.builder() - .contents(util.Arrays.asList(S3Object.builder() - .build())) + private val objResponse = ListObjectsResponse + .builder() + .contents( + util.Arrays.asList( + S3Object + .builder() + .build() + ) + ) .build() private val objRequest = ListObjectsRequest.builder().bucket(bucketName).prefix(dirName).build() private val awsS3Path = awsS3ObjectCounter.parsePath(s3PrefixRegex)(testPath) private val gsPath = gcsObjectCounter.parsePath(gsPrefixRegex)(testGsPath) - "parsePath" should "return a bucket and directories" in { assert(awsS3Path.bucket == bucketName) assert(awsS3Path.directory == dirName) } "parsePath" should "throw Exception for wrong path" in { - assertThrows[centaur.test.IllegalPathException] {awsS3ObjectCounter.parsePath(s3PrefixRegex)(wrongBucketPrefix)} - assertThrows[centaur.test.IllegalPathException] {awsS3ObjectCounter.parsePath(s3PrefixRegex)(testGsPath)} - assertThrows[centaur.test.IllegalPathException] {awsS3ObjectCounter.parsePath(s3PrefixRegex)(EmptyTestPath)} + assertThrows[centaur.test.IllegalPathException](awsS3ObjectCounter.parsePath(s3PrefixRegex)(wrongBucketPrefix)) + assertThrows[centaur.test.IllegalPathException](awsS3ObjectCounter.parsePath(s3PrefixRegex)(testGsPath)) + assertThrows[centaur.test.IllegalPathException](awsS3ObjectCounter.parsePath(s3PrefixRegex)(EmptyTestPath)) } "countObjectAtPath" should "should return 1 if the file exist" in { diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileProvider.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileProvider.scala index 93f4cf77d34..d4a50238176 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileProvider.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileProvider.scala @@ -11,11 +11,10 @@ import common.exception._ import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.http.HttpStatus +class DrsCloudNioFileProvider(drsPathResolver: DrsPathResolver, drsReadInterpreter: DrsReadInterpreter) + extends CloudNioFileProvider { -class DrsCloudNioFileProvider(drsPathResolver: EngineDrsPathResolver, - drsReadInterpreter: DrsReadInterpreter) extends CloudNioFileProvider { - - private def checkIfPathExistsThroughDrsResolver(drsPath: String): IO[Boolean] = { + private def checkIfPathExistsThroughDrsResolver(drsPath: String): IO[Boolean] = /* * Unlike other cloud providers where directories are identified with a trailing slash at the end like `gs://bucket/dir/`, * DRS has a concept of bundles for directories (not supported yet). Hence for method `checkDirectoryExists` which appends a trailing '/' @@ -23,12 +22,14 @@ class DrsCloudNioFileProvider(drsPathResolver: EngineDrsPathResolver, */ if (drsPath.endsWith("/")) IO(false) else { - drsPathResolver.rawDrsResolverResponse(drsPath, NonEmptyList.one(DrsResolverField.GsUri)).use { drsResolverResponse => - val errorMsg = s"Status line was null for DRS Resolver response $drsResolverResponse." - toIO(Option(drsResolverResponse.getStatusLine), errorMsg) - }.map(_.getStatusCode == HttpStatus.SC_OK) + drsPathResolver + .rawDrsResolverResponse(drsPath, NonEmptyList.one(DrsResolverField.GsUri)) + .use { drsResolverResponse => + val errorMsg = s"Status line was null for DRS Resolver response $drsResolverResponse." + toIO(Option(drsResolverResponse.getStatusLine), errorMsg) + } + .map(_.getStatusCode == HttpStatus.SC_OK) } - } override def existsPath(drsPath: String, unused: String): Boolean = checkIfPathExistsThroughDrsResolver(drsPath).unsafeRunSync() @@ -36,34 +37,46 @@ class DrsCloudNioFileProvider(drsPathResolver: EngineDrsPathResolver, override def existsPaths(cloudHost: String, cloudPathPrefix: String): Boolean = existsPath(cloudHost, cloudPathPrefix) - override def listObjects(drsPath: String, unused: String, markerOption: Option[String]): CloudNioFileList = { + override def listObjects(drsPath: String, unused: String, markerOption: Option[String]): CloudNioFileList = throw new UnsupportedOperationException("DRS currently doesn't support list.") - } - override def copy(sourceCloudHost: String, sourceCloudPath: String, targetCloudHost: String, targetCloudPath: String): Unit = + override def copy(sourceCloudHost: String, + sourceCloudPath: String, + targetCloudHost: String, + targetCloudPath: String + ): Unit = throw new UnsupportedOperationException("DRS currently doesn't support copy.") override def deleteIfExists(cloudHost: String, cloudPath: String): Boolean = throw new UnsupportedOperationException("DRS currently doesn't support delete.") override def read(drsPath: String, unused: String, offset: Long): ReadableByteChannel = { - val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl) + val fields = + NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl) val byteChannelIO = for { drsResolverResponse <- drsPathResolver.resolveDrs(drsPath, fields) byteChannel <- drsReadInterpreter(drsPathResolver, drsResolverResponse) } yield byteChannel - byteChannelIO.handleErrorWith { - e => IO.raiseError(new RuntimeException(s"Error while reading from DRS path: $drsPath. Error: ${ExceptionUtils.getMessage(e)}")) - }.unsafeRunSync() + byteChannelIO + .handleErrorWith { e => + IO.raiseError( + new RuntimeException(s"Error while reading from DRS path: $drsPath. Error: ${ExceptionUtils.getMessage(e)}") + ) + } + .unsafeRunSync() } override def write(cloudHost: String, cloudPath: String): WritableByteChannel = throw new UnsupportedOperationException("DRS currently doesn't support write.") override def fileAttributes(drsPath: String, unused: String): Option[CloudNioRegularFileAttributes] = { - val fields = NonEmptyList.of(DrsResolverField.Size, DrsResolverField.TimeCreated, DrsResolverField.TimeUpdated, DrsResolverField.Hashes) + val fields = NonEmptyList.of(DrsResolverField.Size, + DrsResolverField.TimeCreated, + DrsResolverField.TimeUpdated, + DrsResolverField.Hashes + ) val fileAttributesIO = for { drsResolverResponse <- drsPathResolver.resolveDrs(drsPath, fields) diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProvider.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProvider.scala index 884072e4a31..216c20524f6 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProvider.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProvider.scala @@ -8,14 +8,13 @@ import com.typesafe.config.Config class DrsCloudNioFileSystemProvider(rootConfig: Config, val drsCredentials: DrsCredentials, - drsReadInterpreter: DrsReadInterpreter, - ) extends CloudNioFileSystemProvider { + drsReadInterpreter: DrsReadInterpreter +) extends CloudNioFileSystemProvider { - lazy val drsResolverConfig = if (rootConfig.hasPath("resolver")) rootConfig.getConfig("resolver") else rootConfig.getConfig("martha") + lazy val drsResolverConfig = rootConfig.getConfig("resolver") lazy val drsConfig: DrsConfig = DrsConfig.fromConfig(drsResolverConfig) - lazy val drsPathResolver: EngineDrsPathResolver = - EngineDrsPathResolver(drsConfig, drsCredentials) + lazy val drsPathResolver: DrsPathResolver = new DrsPathResolver(drsConfig, drsCredentials) override def config: Config = rootConfig diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala index 778d9b8380e..b09606294c6 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala @@ -11,8 +11,8 @@ class DrsCloudNioRegularFileAttributes(drsPath: String, sizeOption: Option[Long], hashOption: Option[FileHash], timeCreatedOption: Option[FileTime], - timeUpdatedOption: Option[FileTime], - ) extends CloudNioRegularFileAttributes{ + timeUpdatedOption: Option[FileTime] +) extends CloudNioRegularFileAttributes { override def fileKey(): String = drsPath @@ -33,50 +33,47 @@ object DrsCloudNioRegularFileAttributes { ("etag", HashType.S3Etag) ) - def getPreferredHash(hashesOption: Option[Map[String, String]]): Option[FileHash] = { + def getPreferredHash(hashesOption: Option[Map[String, String]]): Option[FileHash] = hashesOption match { case Some(hashes: Map[String, String]) if hashes.nonEmpty => priorityHashList collectFirst { case (key, hashType) if hashes.contains(key) => FileHash(hashType, hashes(key)) } - // if no preferred hash was found, go ahead and return none because we don't support anything that the DRS object is offering + // if no preferred hash was found, go ahead and return none because we don't support anything that the DRS object is offering case _ => None } - } - private def convertToOffsetDateTime(timeInString: String): IO[OffsetDateTime] = { + private def convertToOffsetDateTime(timeInString: String): IO[OffsetDateTime] = // Here timeInString is assumed to be a ISO-8601 DateTime with timezone IO(OffsetDateTime.parse(timeInString)) - .handleErrorWith( - offsetDateTimeException => - // As a fallback timeInString is assumed to be a ISO-8601 DateTime without timezone - IO(LocalDateTime.parse(timeInString).atOffset(ZoneOffset.UTC)) - .handleErrorWith(_ => IO.raiseError(offsetDateTimeException)) + .handleErrorWith(offsetDateTimeException => + // As a fallback timeInString is assumed to be a ISO-8601 DateTime without timezone + IO(LocalDateTime.parse(timeInString).atOffset(ZoneOffset.UTC)) + .handleErrorWith(_ => IO.raiseError(offsetDateTimeException)) ) - } - private def convertToFileTime(timeInString: String): IO[FileTime] = { + private def convertToFileTime(timeInString: String): IO[FileTime] = convertToOffsetDateTime(timeInString) .map(_.toInstant) .map(FileTime.from) - } - def convertToFileTime(drsPath: String, key: DrsResolverField.Value, timeInStringOption: Option[String]): IO[Option[FileTime]] = { + def convertToFileTime(drsPath: String, + key: DrsResolverField.Value, + timeInStringOption: Option[String] + ): IO[Option[FileTime]] = timeInStringOption match { case None => IO.pure(None) case Some(timeInString) => convertToFileTime(timeInString) .map(Option(_)) - .handleErrorWith( - throwable => - IO.raiseError( - new RuntimeException( - s"Error while parsing '$key' value from DRS Resolver to FileTime for DRS path $drsPath. " + - s"Reason: ${ExceptionUtils.getMessage(throwable)}.", - throwable, - ) + .handleErrorWith(throwable => + IO.raiseError( + new RuntimeException( + s"Error while parsing '$key' value from DRS Resolver to FileTime for DRS path $drsPath. " + + s"Reason: ${ExceptionUtils.getMessage(throwable)}.", + throwable ) + ) ) } - } } diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsConfig.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsConfig.scala index a2b0a385680..28b1ee57b7a 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsConfig.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsConfig.scala @@ -11,8 +11,8 @@ final case class DrsConfig(drsResolverUrl: String, waitInitial: FiniteDuration, waitMaximum: FiniteDuration, waitMultiplier: Double, - waitRandomizationFactor: Double, - ) + waitRandomizationFactor: Double +) object DrsConfig { // If you update these values also update Filesystems.md! @@ -29,20 +29,17 @@ object DrsConfig { private val EnvDrsResolverWaitMultiplier = "DRS_RESOLVER_WAIT_MULTIPLIER" private val EnvDrsResolverWaitRandomizationFactor = "DRS_RESOLVER_WAIT_RANDOMIZATION_FACTOR" - - def fromConfig(drsResolverConfig: Config): DrsConfig = { + def fromConfig(drsResolverConfig: Config): DrsConfig = DrsConfig( drsResolverUrl = drsResolverConfig.getString("url"), numRetries = drsResolverConfig.getOrElse("num-retries", DefaultNumRetries), waitInitial = drsResolverConfig.getOrElse("wait-initial", DefaultWaitInitial), waitMaximum = drsResolverConfig.getOrElse("wait-maximum", DefaultWaitMaximum), waitMultiplier = drsResolverConfig.getOrElse("wait-multiplier", DefaultWaitMultiplier), - waitRandomizationFactor = - drsResolverConfig.getOrElse("wait-randomization-factor", DefaultWaitRandomizationFactor), + waitRandomizationFactor = drsResolverConfig.getOrElse("wait-randomization-factor", DefaultWaitRandomizationFactor) ) - } - def fromEnv(env: Map[String, String]): DrsConfig = { + def fromEnv(env: Map[String, String]): DrsConfig = DrsConfig( drsResolverUrl = env(EnvDrsResolverUrl), numRetries = env.get(EnvDrsResolverNumRetries).map(_.toInt).getOrElse(DefaultNumRetries), @@ -50,18 +47,16 @@ object DrsConfig { waitMaximum = env.get(EnvDrsResolverWaitMaximumSeconds).map(_.toLong.seconds).getOrElse(DefaultWaitMaximum), waitMultiplier = env.get(EnvDrsResolverWaitMultiplier).map(_.toDouble).getOrElse(DefaultWaitMultiplier), waitRandomizationFactor = - env.get(EnvDrsResolverWaitRandomizationFactor).map(_.toDouble).getOrElse(DefaultWaitRandomizationFactor), + env.get(EnvDrsResolverWaitRandomizationFactor).map(_.toDouble).getOrElse(DefaultWaitRandomizationFactor) ) - } - def toEnv(drsConfig: DrsConfig): Map[String, String] = { + def toEnv(drsConfig: DrsConfig): Map[String, String] = Map( EnvDrsResolverUrl -> drsConfig.drsResolverUrl, EnvDrsResolverNumRetries -> s"${drsConfig.numRetries}", EnvDrsResolverWaitInitialSeconds -> s"${drsConfig.waitInitial.toSeconds}", EnvDrsResolverWaitMaximumSeconds -> s"${drsConfig.waitMaximum.toSeconds}", EnvDrsResolverWaitMultiplier -> s"${drsConfig.waitMultiplier}", - EnvDrsResolverWaitRandomizationFactor -> s"${drsConfig.waitRandomizationFactor}", + EnvDrsResolverWaitRandomizationFactor -> s"${drsConfig.waitRandomizationFactor}" ) - } } diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCredentials.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCredentials.scala index 2d3e972508a..c8a4fb93743 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCredentials.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCredentials.scala @@ -1,17 +1,13 @@ package cloud.nio.impl.drs import cats.syntax.validated._ -import com.azure.core.credential.TokenRequestContext -import com.azure.core.management.AzureEnvironment -import com.azure.core.management.profile.AzureProfile -import com.azure.identity.DefaultAzureCredentialBuilder import com.google.auth.oauth2.{AccessToken, GoogleCredentials, OAuth2Credentials} import com.typesafe.config.Config import common.validation.ErrorOr.ErrorOr import net.ceedubs.ficus.Ficus._ +import cromwell.cloudsupport.azure.AzureCredentials import scala.concurrent.duration._ -import scala.jdk.DurationConverters._ import scala.util.{Failure, Success, Try} /** @@ -27,11 +23,10 @@ trait DrsCredentials { * is designed for use within the Cromwell engine. */ case class GoogleOauthDrsCredentials(credentials: OAuth2Credentials, acceptableTTL: Duration) extends DrsCredentials { - //Based on method from GoogleRegistry + // Based on method from GoogleRegistry def getAccessToken: ErrorOr[String] = { - def accessTokenTTLIsAcceptable(accessToken: AccessToken): Boolean = { + def accessTokenTTLIsAcceptable(accessToken: AccessToken): Boolean = (accessToken.getExpirationTime.getTime - System.currentTimeMillis()).millis.gteq(acceptableTTL) - } Option(credentials.getAccessToken) match { case Some(accessToken) if accessTokenTTLIsAcceptable(accessToken) => @@ -51,26 +46,25 @@ object GoogleOauthDrsCredentials { GoogleOauthDrsCredentials(credentials, config.as[FiniteDuration]("access-token-acceptable-ttl")) } - /** * Strategy for obtaining an access token from Google Application Default credentials that are assumed to already exist * in the environment. This class is designed for use by standalone executables running in environments * that have direct access to a Google identity (ex. CromwellDrsLocalizer). */ case object GoogleAppDefaultTokenStrategy extends DrsCredentials { - private final val UserInfoEmailScope = "https://www.googleapis.com/auth/userinfo.email" - private final val UserInfoProfileScope = "https://www.googleapis.com/auth/userinfo.profile" + final private val UserInfoEmailScope = "https://www.googleapis.com/auth/userinfo.email" + final private val UserInfoProfileScope = "https://www.googleapis.com/auth/userinfo.profile" - def getAccessToken: ErrorOr[String] = { + def getAccessToken: ErrorOr[String] = Try { - val scopedCredentials = GoogleCredentials.getApplicationDefault().createScoped(UserInfoEmailScope, UserInfoProfileScope) + val scopedCredentials = + GoogleCredentials.getApplicationDefault().createScoped(UserInfoEmailScope, UserInfoProfileScope) scopedCredentials.refreshAccessToken().getTokenValue } match { case Success(null) => "null token value attempting to refresh access token".invalidNel case Success(value) => value.validNel case Failure(e) => s"Failed to refresh access token: ${e.getMessage}".invalidNel } - } } /** @@ -78,36 +72,6 @@ case object GoogleAppDefaultTokenStrategy extends DrsCredentials { * If you need to disambiguate among multiple active user-assigned managed identities, pass * in the client id of the identity that should be used. */ -case class AzureDrsCredentials(identityClientId: Option[String]) extends DrsCredentials { - - final val tokenAcquisitionTimeout = 5.seconds - - val azureProfile = new AzureProfile(AzureEnvironment.AZURE) - val tokenScope = "https://management.azure.com/.default" - - def tokenRequestContext: TokenRequestContext = { - val trc = new TokenRequestContext() - trc.addScopes(tokenScope) - trc - } - - def defaultCredentialBuilder: DefaultAzureCredentialBuilder = - new DefaultAzureCredentialBuilder() - .authorityHost(azureProfile.getEnvironment.getActiveDirectoryEndpoint) - - def getAccessToken: ErrorOr[String] = { - val credentials = identityClientId.foldLeft(defaultCredentialBuilder) { - (builder, clientId) => builder.managedIdentityClientId(clientId) - }.build() - - Try( - credentials - .getToken(tokenRequestContext) - .block(tokenAcquisitionTimeout.toJava) - ) match { - case Success(null) => "null token value attempting to obtain access token".invalidNel - case Success(token) => token.getToken.validNel - case Failure(error) => s"Failed to refresh access token: ${error.getMessage}".invalidNel - } - } +case class AzureDrsCredentials(identityClientId: Option[String] = None) extends DrsCredentials { + def getAccessToken: ErrorOr[String] = AzureCredentials.getAccessToken(identityClientId) } diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala index 22d86c31726..028d3ee4849 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala @@ -6,7 +6,7 @@ import cats.effect.{IO, Resource} import cats.implicits._ import cloud.nio.impl.drs.DrsPathResolver.{FatalRetryDisposition, RegularRetryDisposition} import cloud.nio.impl.drs.DrsResolverResponseSupport._ -import common.exception.{AggregatedMessageException, toIO} +import common.exception.{toIO, AggregatedMessageException} import common.validation.ErrorOr.ErrorOr import io.circe._ import io.circe.generic.semiauto._ @@ -25,7 +25,7 @@ import java.nio.ByteBuffer import java.nio.channels.{Channels, ReadableByteChannel} import scala.util.Try -abstract class DrsPathResolver(drsConfig: DrsConfig) { +class DrsPathResolver(drsConfig: DrsConfig, drsCredentials: DrsCredentials) { protected lazy val httpClientBuilder: HttpClientBuilder = { val clientBuilder = HttpClientBuilder.create() @@ -38,24 +38,39 @@ abstract class DrsPathResolver(drsConfig: DrsConfig) { clientBuilder } - def getAccessToken: ErrorOr[String] + def getAccessToken: ErrorOr[String] = drsCredentials.getAccessToken - private def makeHttpRequestToDrsResolver(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): Resource[IO, HttpPost] = { + private lazy val currentCloudPlatform: Option[DrsCloudPlatform.Value] = drsCredentials match { + case _: GoogleOauthDrsCredentials => Option(DrsCloudPlatform.GoogleStorage) + case GoogleAppDefaultTokenStrategy => Option(DrsCloudPlatform.GoogleStorage) + case _: AzureDrsCredentials => Option(DrsCloudPlatform.Azure) + case _ => None + } + + def makeDrsResolverRequest(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): DrsResolverRequest = + DrsResolverRequest(drsPath, currentCloudPlatform, fields) + + private def makeHttpRequestToDrsResolver(drsPath: String, + fields: NonEmptyList[DrsResolverField.Value] + ): Resource[IO, HttpPost] = { val io = getAccessToken match { - case Valid(token) => IO { - val postRequest = new HttpPost(drsConfig.drsResolverUrl) - val requestJson = DrsResolverRequest(drsPath, fields).asJson.noSpaces - postRequest.setEntity(new StringEntity(requestJson, ContentType.APPLICATION_JSON)) - postRequest.setHeader("Authorization", s"Bearer $token") - postRequest - } + case Valid(token) => + IO { + val postRequest = new HttpPost(drsConfig.drsResolverUrl) + val requestJson = makeDrsResolverRequest(drsPath, fields).asJson.noSpaces + postRequest.setEntity(new StringEntity(requestJson, ContentType.APPLICATION_JSON)) + postRequest.setHeader("Authorization", s"Bearer $token") + postRequest + } case Invalid(errors) => IO.raiseError(AggregatedMessageException("Error getting access token", errors.toList)) } Resource.eval(io) } - private def httpResponseToDrsResolverResponse(drsPathForDebugging: String)(httpResponse: HttpResponse): IO[DrsResolverResponse] = { + private def httpResponseToDrsResolverResponse( + drsPathForDebugging: String + )(httpResponse: HttpResponse): IO[DrsResolverResponse] = { val responseStatusLine = httpResponse.getStatusLine val status = responseStatusLine.getStatusCode @@ -73,45 +88,55 @@ abstract class DrsPathResolver(drsConfig: DrsConfig) { IO.raiseError(new RuntimeException(retryMessage) with RegularRetryDisposition) case _ => val drsResolverResponseEntityOption = Option(httpResponse.getEntity).map(EntityUtils.toString) - val exceptionMsg = errorMessageFromResponse(drsPathForDebugging, drsResolverResponseEntityOption, responseStatusLine, drsConfig.drsResolverUrl) - val responseEntityOption = (responseStatusLine.getStatusCode == HttpStatus.SC_OK).valueOrZero(drsResolverResponseEntityOption) + val exceptionMsg = errorMessageFromResponse(drsPathForDebugging, + drsResolverResponseEntityOption, + responseStatusLine, + drsConfig.drsResolverUrl + ) + val responseEntityOption = + (responseStatusLine.getStatusCode == HttpStatus.SC_OK).valueOrZero(drsResolverResponseEntityOption) val responseContentIO = toIO(responseEntityOption, exceptionMsg) - responseContentIO.flatMap { responseContent => - IO.fromEither(decode[DrsResolverResponse](responseContent)) - }.handleErrorWith { - e => IO.raiseError(new RuntimeException(s"Unexpected response during DRS resolution: ${ExceptionUtils.getMessage(e)}")) - } + responseContentIO + .flatMap { responseContent => + IO.fromEither(decode[DrsResolverResponse](responseContent)) + } + .handleErrorWith { e => + IO.raiseError( + new RuntimeException(s"Unexpected response during DRS resolution: ${ExceptionUtils.getMessage(e)}") + ) + } } } - private def executeDrsResolverRequest(httpPost: HttpPost): Resource[IO, HttpResponse]= { + private def executeDrsResolverRequest(httpPost: HttpPost): Resource[IO, HttpResponse] = for { httpClient <- Resource.fromAutoCloseable(IO(httpClientBuilder.build())) httpResponse <- Resource.fromAutoCloseable(IO(httpClient.execute(httpPost))) } yield httpResponse - } - def rawDrsResolverResponse(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): Resource[IO, HttpResponse] = { + def rawDrsResolverResponse(drsPath: String, + fields: NonEmptyList[DrsResolverField.Value] + ): Resource[IO, HttpResponse] = for { httpPost <- makeHttpRequestToDrsResolver(drsPath, fields) response <- executeDrsResolverRequest(httpPost) } yield response - } /** * * Resolves the DRS path through DRS Resolver url provided in the config. * Please note, this method returns an IO that would make a synchronous HTTP request to DRS Resolver when run. */ - def resolveDrs(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): IO[DrsResolverResponse] = { - rawDrsResolverResponse(drsPath, fields).use(httpResponseToDrsResolverResponse(drsPathForDebugging = drsPath)) - } + def resolveDrs(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): IO[DrsResolverResponse] = + rawDrsResolverResponse(drsPath, fields).use( + httpResponseToDrsResolverResponse(drsPathForDebugging = drsPath) + ) - def openChannel(accessUrl: AccessUrl): IO[ReadableByteChannel] = { + def openChannel(accessUrl: AccessUrl): IO[ReadableByteChannel] = IO { val httpGet = new HttpGet(accessUrl.url) - accessUrl.headers.getOrElse(Map.empty).toList foreach { - case (name, value) => httpGet.addHeader(name, value) + accessUrl.headers.getOrElse(Map.empty).toList foreach { case (name, value) => + httpGet.addHeader(name, value) } val client = httpClientBuilder.build() val response = client.execute(httpGet) @@ -130,7 +155,7 @@ abstract class DrsPathResolver(drsConfig: DrsConfig) { override def isOpen: Boolean = inner.isOpen - //noinspection ScalaUnusedExpression + // noinspection ScalaUnusedExpression override def close(): Unit = { val innerTry = Try(inner.close()) val responseTry = Try(response.close()) @@ -141,7 +166,6 @@ abstract class DrsPathResolver(drsConfig: DrsConfig) { } } } - } } object DrsPathResolver { @@ -166,7 +190,20 @@ object DrsResolverField extends Enumeration { val LocalizationPath: DrsResolverField.Value = Value("localizationPath") } -final case class DrsResolverRequest(url: String, fields: NonEmptyList[DrsResolverField.Value]) +// We supply a cloud platform value to the DRS service. In cases where the DRS repository +// has multiple cloud files associated with a DRS link, it will prefer sending a file on the same +// platform as this Cromwell instance. That is, if a DRS file has copies on both GCP and Azure, +// we'll get the GCP one when running on GCP and the Azure one when running on Azure. +object DrsCloudPlatform extends Enumeration { + val GoogleStorage: DrsCloudPlatform.Value = Value("gs") + val Azure: DrsCloudPlatform.Value = Value("azure") + val AmazonS3: DrsCloudPlatform.Value = Value("s3") // supported by DRSHub but not currently used by us +} + +final case class DrsResolverRequest(url: String, + cloudPlatform: Option[DrsCloudPlatform.Value], + fields: NonEmptyList[DrsResolverField.Value] +) final case class SADataObject(data: Json) @@ -198,14 +235,17 @@ final case class DrsResolverResponse(size: Option[Long] = None, hashes: Option[Map[String, String]] = None, accessUrl: Option[AccessUrl] = None, localizationPath: Option[String] = None - ) +) final case class DrsResolverFailureResponse(response: DrsResolverFailureResponsePayload) final case class DrsResolverFailureResponsePayload(text: String) object DrsResolverResponseSupport { - implicit lazy val drsResolverFieldEncoder: Encoder[DrsResolverField.Value] = Encoder.encodeEnumeration(DrsResolverField) + implicit lazy val drsResolverFieldEncoder: Encoder[DrsResolverField.Value] = + Encoder.encodeEnumeration(DrsResolverField) + implicit lazy val drsResolverCloudPlatformEncoder: Encoder[DrsCloudPlatform.Value] = + Encoder.encodeEnumeration(DrsCloudPlatform) implicit lazy val drsResolverRequestEncoder: Encoder[DrsResolverRequest] = deriveEncoder implicit lazy val saDataObjectDecoder: Decoder[SADataObject] = deriveDecoder @@ -219,12 +259,17 @@ object DrsResolverResponseSupport { private val GcsScheme = "gs://" def getGcsBucketAndName(gcsUrl: String): (String, String) = { - val array = gcsUrl.substring(GcsScheme.length).split("/", 2) - (array(0), array(1)) + val array = gcsUrl.substring(GcsScheme.length).split("/", 2) + (array(0), array(1)) } - def errorMessageFromResponse(drsPathForDebugging: String, drsResolverResponseEntityOption: Option[String], responseStatusLine: StatusLine, drsResolverUri: String): String = { - val baseMessage = s"Could not access object \'$drsPathForDebugging\'. Status: ${responseStatusLine.getStatusCode}, reason: \'${responseStatusLine.getReasonPhrase}\', DRS Resolver location: \'$drsResolverUri\', message: " + def errorMessageFromResponse(drsPathForDebugging: String, + drsResolverResponseEntityOption: Option[String], + responseStatusLine: StatusLine, + drsResolverUri: String + ): String = { + val baseMessage = + s"Could not access object \'$drsPathForDebugging\'. Status: ${responseStatusLine.getStatusCode}, reason: \'${responseStatusLine.getReasonPhrase}\', DRS Resolver location: \'$drsResolverUri\', message: " drsResolverResponseEntityOption match { case Some(entity) => diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategy.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategy.scala index 39020f5a684..e39e7b49c81 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategy.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategy.scala @@ -8,7 +8,8 @@ import org.apache.http.client.{HttpRequestRetryHandler, ServiceUnavailableRetryS import org.apache.http.protocol.HttpContext class DrsResolverHttpRequestRetryStrategy(drsConfig: DrsConfig) - extends ServiceUnavailableRetryStrategy with HttpRequestRetryHandler { + extends ServiceUnavailableRetryStrategy + with HttpRequestRetryHandler { // We can execute a total of one time, plus the number of retries private val executionMax: Int = drsConfig.numRetries + 1 @@ -17,23 +18,21 @@ class DrsResolverHttpRequestRetryStrategy(drsConfig: DrsConfig) initialInterval = drsConfig.waitInitial, maxInterval = drsConfig.waitMaximum, multiplier = drsConfig.waitMultiplier, - randomizationFactor = drsConfig.waitRandomizationFactor, + randomizationFactor = drsConfig.waitRandomizationFactor ) private var transientFailures: Int = 0 /** Returns true if an IOException should be immediately retried. */ - override def retryRequest(exception: IOException, executionCount: Int, context: HttpContext): Boolean = { + override def retryRequest(exception: IOException, executionCount: Int, context: HttpContext): Boolean = retryRequest(executionCount) - } /** Returns true if HttpResponse should be retried after getRetryInterval. */ - override def retryRequest(response: HttpResponse, executionCount: Int, context: HttpContext): Boolean = { + override def retryRequest(response: HttpResponse, executionCount: Int, context: HttpContext): Boolean = response.getStatusLine.getStatusCode match { case code if code == 408 || code == 429 => retryRequestTransient(executionCount) case code if 500 <= code && code <= 599 => retryRequest(executionCount) case _ => false } - } /** Returns the number of milliseconds to wait before retrying an HttpResponse. */ override def getRetryInterval: Long = { @@ -47,8 +46,7 @@ class DrsResolverHttpRequestRetryStrategy(drsConfig: DrsConfig) retryRequest(executionCount) } - private def retryRequest(executionCount: Int): Boolean = { + private def retryRequest(executionCount: Int): Boolean = // The first execution is executionCount == 1 executionCount - transientFailures <= executionMax - } } diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/EngineDrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/EngineDrsPathResolver.scala deleted file mode 100644 index 01f7a488eb3..00000000000 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/EngineDrsPathResolver.scala +++ /dev/null @@ -1,11 +0,0 @@ -package cloud.nio.impl.drs - -import common.validation.ErrorOr.ErrorOr - -case class EngineDrsPathResolver(drsConfig: DrsConfig, - drsCredentials: DrsCredentials, - ) - extends DrsPathResolver(drsConfig) { - - override def getAccessToken: ErrorOr[String] = drsCredentials.getAccessToken -} diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileProviderSpec.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileProviderSpec.scala index f61c608bd71..c3b7cb70349 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileProviderSpec.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileProviderSpec.scala @@ -52,11 +52,10 @@ class DrsCloudNioFileProviderSpec extends AnyFlatSpecLike with CromwellTimeoutSp "drs://dg.4DFC:0027045b-9ed6-45af-a68e-f55037b5184c", "drs://dg.4503:dg.4503/fc046e84-6cf9-43a3-99cc-ffa2964b88cb", "drs://dg.ANV0:dg.ANV0/0db6577e-57bd-48a1-93c6-327c292bcb6b", - "drs://dg.F82A1A:ed6be7ab-068e-46c8-824a-f39cfbb885cc", + "drs://dg.F82A1A:ed6be7ab-068e-46c8-824a-f39cfbb885cc" ) - for (exampleUri <- exampleUris) { + for (exampleUri <- exampleUris) fileSystemProvider.getHost(exampleUri) should be(exampleUri) - } } it should "check existing drs objects" in { @@ -78,62 +77,54 @@ class DrsCloudNioFileProviderSpec extends AnyFlatSpecLike with CromwellTimeoutSp } it should "return a file provider that can read bytes from gcs" in { - val drsPathResolver = new MockEngineDrsPathResolver() { - override def resolveDrs(drsPath: String, - fields: NonEmptyList[DrsResolverField.Value], - ): IO[DrsResolverResponse] = { + val drsPathResolver = new MockDrsPathResolver() { + override def resolveDrs(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): IO[DrsResolverResponse] = IO(DrsResolverResponse(gsUri = Option("gs://bucket/object/path"))) - } } val readChannel = mock[ReadableByteChannel] - val drsReadInterpreter: DrsReadInterpreter = (_, drsResolverResponse) => { + val drsReadInterpreter: DrsReadInterpreter = (_, drsResolverResponse) => IO( (drsResolverResponse.gsUri, drsResolverResponse.googleServiceAccount) match { case (Some("gs://bucket/object/path"), None) => readChannel case _ => fail(s"Unexpected parameters passed: $drsResolverResponse") } ) - } val fileSystemProvider = new MockDrsCloudNioFileSystemProvider( mockResolver = Option(drsPathResolver), - drsReadInterpreter = drsReadInterpreter, + drsReadInterpreter = drsReadInterpreter ) fileSystemProvider.fileProvider.read("dg.123", "abc", 0) should be(readChannel) } it should "return a file provider that can read bytes from an access url" in { - val drsPathResolver = new MockEngineDrsPathResolver() { - override def resolveDrs(drsPath: String, - fields: NonEmptyList[DrsResolverField.Value], - ): IO[DrsResolverResponse] = { + val drsPathResolver = new MockDrsPathResolver() { + override def resolveDrs(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): IO[DrsResolverResponse] = IO(DrsResolverResponse(accessUrl = Option(AccessUrl("https://host/object/path", None)))) - } } val readChannel = mock[ReadableByteChannel] - val drsReadInterpreter: DrsReadInterpreter = (_, drsResolverResponse) => { + val drsReadInterpreter: DrsReadInterpreter = (_, drsResolverResponse) => IO( drsResolverResponse.accessUrl match { case Some(AccessUrl("https://host/object/path", None)) => readChannel case _ => fail(s"Unexpected parameters passed: $drsResolverResponse") } ) - } val fileSystemProvider = new MockDrsCloudNioFileSystemProvider( mockResolver = Option(drsPathResolver), - drsReadInterpreter = drsReadInterpreter, + drsReadInterpreter = drsReadInterpreter ) fileSystemProvider.fileProvider.read("dg.123", "abc", 0) should be(readChannel) } it should "return a file provider that can return file attributes" in { - val drsPathResolver = new MockEngineDrsPathResolver() { + val drsPathResolver = new MockDrsPathResolver() { override def resolveDrs(drsPath: String, - fields: NonEmptyList[DrsResolverField.Value], - ): IO[DrsResolverResponse] = { + fields: NonEmptyList[DrsResolverField.Value] + ): IO[DrsResolverResponse] = { val instantCreated = Instant.ofEpochMilli(123L) val instantUpdated = Instant.ofEpochMilli(456L) IO( @@ -141,7 +132,7 @@ class DrsCloudNioFileProviderSpec extends AnyFlatSpecLike with CromwellTimeoutSp size = Option(789L), timeCreated = Option(OffsetDateTime.ofInstant(instantCreated, ZoneOffset.UTC).toString), timeUpdated = Option(OffsetDateTime.ofInstant(instantUpdated, ZoneOffset.UTC).toString), - hashes = Option(Map("md5" -> "gg0217869")), + hashes = Option(Map("md5" -> "gg0217869")) ) ) } diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProviderSpec.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProviderSpec.scala index 88b95dc76e8..13df061d7a6 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProviderSpec.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProviderSpec.scala @@ -17,6 +17,6 @@ class DrsCloudNioFileSystemProviderSpec extends org.scalatest.flatspec.AnyFlatSp val path = fileSystemProvider.getCloudNioPath("drs://foo/bar/") the[UnsupportedOperationException] thrownBy { fileSystemProvider.deleteIfExists(path) - } should have message("DRS currently doesn't support delete.") + } should have message "DRS currently doesn't support delete." } } diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsPathResolverSpec.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsPathResolverSpec.scala index b4e5968e048..adbb2adf43b 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsPathResolverSpec.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsPathResolverSpec.scala @@ -1,8 +1,9 @@ package cloud.nio.impl.drs +import cats.data.NonEmptyList + import java.nio.file.attribute.FileTime import java.time.OffsetDateTime - import cloud.nio.impl.drs.DrsCloudNioRegularFileAttributes._ import cloud.nio.spi.{FileHash, HashType} import common.assertion.CromwellTimeoutSpec @@ -12,7 +13,7 @@ import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with Matchers { - private val mockGSA = SADataObject(data = Json.fromJsonObject(JsonObject("key"-> Json.fromString("value")))) + private val mockGSA = SADataObject(data = Json.fromJsonObject(JsonObject("key" -> Json.fromString("value")))) private val crcHashValue = "8a366443" private val md5HashValue = "336ea55913bc261b72875bd259753046" private val shaHashValue = "f76877f8e86ec3932fd2ae04239fbabb8c90199dab0019ae55fa42b31c314c44" @@ -40,54 +41,70 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with .copy(timeUpdated = fullDrsResolverResponse.timeUpdated.map(_.stripSuffix("Z") + "BADTZ")) private val etagHashValue = "something" - private val completeHashesMap = Option(Map( - "betty" -> "abc123", - "charles" -> "456", - "alfred" -> "xrd", - "sha256" -> shaHashValue, - "crc32c" -> crcHashValue, - "md5" -> md5HashValue, - "etag" -> etagHashValue, - )) - - private val missingCRCHashesMap = Option(Map( - "alfred" -> "xrd", - "sha256" -> shaHashValue, - "betty" -> "abc123", - "md5" -> md5HashValue, - "charles" -> "456", - )) - - private val onlySHAHashesMap = Option(Map( - "betty" -> "abc123", - "charles" -> "456", - "alfred" -> "xrd", - "sha256" -> shaHashValue, - )) - - private val onlyEtagHashesMap = Option(Map( - "alfred" -> "xrd", - "betty" -> "abc123", - "charles" -> "456", - "etag" -> etagHashValue, - )) + private val completeHashesMap = Option( + Map( + "betty" -> "abc123", + "charles" -> "456", + "alfred" -> "xrd", + "sha256" -> shaHashValue, + "crc32c" -> crcHashValue, + "md5" -> md5HashValue, + "etag" -> etagHashValue + ) + ) + + private val missingCRCHashesMap = Option( + Map( + "alfred" -> "xrd", + "sha256" -> shaHashValue, + "betty" -> "abc123", + "md5" -> md5HashValue, + "charles" -> "456" + ) + ) + + private val onlySHAHashesMap = Option( + Map( + "betty" -> "abc123", + "charles" -> "456", + "alfred" -> "xrd", + "sha256" -> shaHashValue + ) + ) + + private val onlyEtagHashesMap = Option( + Map( + "alfred" -> "xrd", + "betty" -> "abc123", + "charles" -> "456", + "etag" -> etagHashValue + ) + ) behavior of "fileHash()" it should "return crc32c hash from `hashes` in DRS Resolver response when there is a crc32c" in { - DrsCloudNioRegularFileAttributes.getPreferredHash(completeHashesMap) shouldBe Option(FileHash(HashType.Crc32c, crcHashValue)) + DrsCloudNioRegularFileAttributes.getPreferredHash(completeHashesMap) shouldBe Option( + FileHash(HashType.Crc32c, crcHashValue) + ) } it should "return md5 hash from `hashes` in DRS Resolver response when there is no crc32c" in { - DrsCloudNioRegularFileAttributes.getPreferredHash(missingCRCHashesMap) shouldBe Option(FileHash(HashType.Md5, md5HashValue)) + DrsCloudNioRegularFileAttributes.getPreferredHash(missingCRCHashesMap) shouldBe Option( + FileHash(HashType.Md5, md5HashValue) + ) } it should "return sha256 hash from `hashes` in DRS Resolver response when there is only a sha256" in { - DrsCloudNioRegularFileAttributes.getPreferredHash(onlySHAHashesMap) shouldBe Option(FileHash(HashType.Sha256, shaHashValue)) + DrsCloudNioRegularFileAttributes.getPreferredHash(onlySHAHashesMap) shouldBe Option( + FileHash(HashType.Sha256, shaHashValue) + ) } it should "return etag hash from `hashes` in DRS Resolver response when there is only an etag" in { - DrsCloudNioRegularFileAttributes.getPreferredHash(onlyEtagHashesMap) shouldBe Option(FileHash(HashType.S3Etag, etagHashValue)) + DrsCloudNioRegularFileAttributes.getPreferredHash(onlyEtagHashesMap) shouldBe Option( + FileHash(HashType.S3Etag, etagHashValue) + ) } it should "return None when no hashes object is returned" in { @@ -138,19 +155,39 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with import org.apache.http.message.BasicStatusLine val drsPathForDebugging = "drs://my_awesome_drs" - val responseStatusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 2) , 345, "test-reason") + val responseStatusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 2), 345, "test-reason") val testDrsResolverUri = "www.drshub_v4.com" + it should "construct the right request when using Azure creds" in { + val resolver = new MockDrsPathResolver(drsCredentials = AzureDrsCredentials()) + val drsRequest = resolver.makeDrsResolverRequest(drsPathForDebugging, NonEmptyList.of(DrsResolverField.AccessUrl)) + drsRequest.cloudPlatform shouldBe Option(DrsCloudPlatform.Azure) + } + + it should "construct the right request when using Google creds" in { + val resolver = new MockDrsPathResolver() + val drsRequest = resolver.makeDrsResolverRequest(drsPathForDebugging, NonEmptyList.of(DrsResolverField.AccessUrl)) + drsRequest.cloudPlatform shouldBe Option(DrsCloudPlatform.GoogleStorage) + } + it should "construct an error message from a populated, well-formed failure response" in { val failureResponse = Option(failureResponseJson) - DrsResolverResponseSupport.errorMessageFromResponse(drsPathForDebugging, failureResponse, responseStatusLine, testDrsResolverUri) shouldBe { + DrsResolverResponseSupport.errorMessageFromResponse(drsPathForDebugging, + failureResponse, + responseStatusLine, + testDrsResolverUri + ) shouldBe { "Could not access object 'drs://my_awesome_drs'. Status: 345, reason: 'test-reason', DRS Resolver location: 'www.drshub_v4.com', message: '{\"msg\":\"User 'null' does not have required action: read_data\",\"status_code\":500}'" } } it should "construct an error message from an empty failure response" in { - DrsResolverResponseSupport.errorMessageFromResponse(drsPathForDebugging, None, responseStatusLine, testDrsResolverUri) shouldBe { + DrsResolverResponseSupport.errorMessageFromResponse(drsPathForDebugging, + None, + responseStatusLine, + testDrsResolverUri + ) shouldBe { "Could not access object 'drs://my_awesome_drs'. Status: 345, reason: 'test-reason', DRS Resolver location: 'www.drshub_v4.com', message: (empty response)" } } @@ -160,7 +197,11 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with it should "construct an error message from a malformed failure response" in { val unparsableFailureResponse = Option("something went horribly wrong") - DrsResolverResponseSupport.errorMessageFromResponse(drsPathForDebugging, unparsableFailureResponse, responseStatusLine, testDrsResolverUri) shouldBe { + DrsResolverResponseSupport.errorMessageFromResponse(drsPathForDebugging, + unparsableFailureResponse, + responseStatusLine, + testDrsResolverUri + ) shouldBe { "Could not access object 'drs://my_awesome_drs'. Status: 345, reason: 'test-reason', DRS Resolver location: 'www.drshub_v4.com', message: 'something went horribly wrong'" } } @@ -169,7 +210,7 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with val lastModifiedTimeIO = convertToFileTime( "drs://my_awesome_drs", DrsResolverField.TimeUpdated, - fullDrsResolverResponse.timeUpdated, + fullDrsResolverResponse.timeUpdated ) lastModifiedTimeIO.unsafeRunSync() should be(Option(FileTime.from(OffsetDateTime.parse("2020-04-27T15:56:09.696Z").toInstant))) @@ -179,7 +220,7 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with val lastModifiedTimeIO = convertToFileTime( "drs://my_awesome_drs", DrsResolverField.TimeUpdated, - fullDrsResolverResponseNoTz.timeUpdated, + fullDrsResolverResponseNoTz.timeUpdated ) lastModifiedTimeIO.unsafeRunSync() should be(Option(FileTime.from(OffsetDateTime.parse("2020-04-27T15:56:09.696Z").toInstant))) @@ -189,7 +230,7 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with val lastModifiedTimeIO = convertToFileTime( "drs://my_awesome_drs", DrsResolverField.TimeUpdated, - fullDrsResolverResponseNoTime.timeUpdated, + fullDrsResolverResponseNoTime.timeUpdated ) lastModifiedTimeIO.unsafeRunSync() should be(None) } @@ -198,10 +239,10 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with val lastModifiedTimeIO = convertToFileTime( "drs://my_awesome_drs", DrsResolverField.TimeUpdated, - fullDrsResolverResponseBadTz.timeUpdated, + fullDrsResolverResponseBadTz.timeUpdated ) the[RuntimeException] thrownBy lastModifiedTimeIO.unsafeRunSync() should have message "Error while parsing 'timeUpdated' value from DRS Resolver to FileTime for DRS path drs://my_awesome_drs. " + - "Reason: DateTimeParseException: Text '2020-04-27T15:56:09.696BADTZ' could not be parsed at index 23." + "Reason: DateTimeParseException: Text '2020-04-27T15:56:09.696BADTZ' could not be parsed at index 23." } } diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategySpec.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategySpec.scala index b221038bedb..5be2b155391 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategySpec.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategySpec.scala @@ -93,7 +93,7 @@ class DrsResolverHttpRequestRetryStrategySpec extends AnyFlatSpec with Matchers waitInitial = 10.seconds, waitMultiplier = 2.0d, waitMaximum = 1.minute, - waitRandomizationFactor = 0d, + waitRandomizationFactor = 0d ) val retryStrategy = new DrsResolverHttpRequestRetryStrategy(drsConfig) diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsCloudNioFileSystemProvider.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsCloudNioFileSystemProvider.scala index 07a02e096dc..5a2ba6172f1 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsCloudNioFileSystemProvider.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsCloudNioFileSystemProvider.scala @@ -7,26 +7,24 @@ import com.google.cloud.NoCredentials import com.typesafe.config.{Config, ConfigFactory} import org.apache.http.impl.client.HttpClientBuilder -import scala.concurrent.duration.Duration - class MockDrsCloudNioFileSystemProvider(config: Config = mockConfig, httpClientBuilder: Option[HttpClientBuilder] = None, drsReadInterpreter: DrsReadInterpreter = (_, _) => IO.raiseError( new UnsupportedOperationException("mock did not specify a read interpreter") ), - mockResolver: Option[EngineDrsPathResolver] = None, - ) - extends DrsCloudNioFileSystemProvider(config, GoogleOauthDrsCredentials(NoCredentials.getInstance, config), drsReadInterpreter) { + mockResolver: Option[DrsPathResolver] = None +) extends DrsCloudNioFileSystemProvider(config, + GoogleOauthDrsCredentials(NoCredentials.getInstance, config), + drsReadInterpreter + ) { - override lazy val drsPathResolver: EngineDrsPathResolver = { + override lazy val drsPathResolver: DrsPathResolver = mockResolver getOrElse - new MockEngineDrsPathResolver( + new MockDrsPathResolver( drsConfig = drsConfig, - httpClientBuilderOverride = httpClientBuilder, - accessTokenAcceptableTTL = Duration.Inf, + httpClientBuilderOverride = httpClientBuilder ) - } } object MockDrsCloudNioFileSystemProvider { diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockEngineDrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPathResolver.scala similarity index 75% rename from cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockEngineDrsPathResolver.scala rename to cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPathResolver.scala index 9e22544c38b..9439c7d94ab 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockEngineDrsPathResolver.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPathResolver.scala @@ -10,11 +10,11 @@ import common.mock.MockSugar import scala.concurrent.duration.Duration -class MockEngineDrsPathResolver(drsConfig: DrsConfig = MockDrsPaths.mockDrsConfig, - httpClientBuilderOverride: Option[HttpClientBuilder] = None, - accessTokenAcceptableTTL: Duration = Duration.Inf, - ) - extends EngineDrsPathResolver(drsConfig, GoogleOauthDrsCredentials(NoCredentials.getInstance, accessTokenAcceptableTTL)) { +class MockDrsPathResolver(drsConfig: DrsConfig = MockDrsPaths.mockDrsConfig, + httpClientBuilderOverride: Option[HttpClientBuilder] = None, + drsCredentials: DrsCredentials = + GoogleOauthDrsCredentials(NoCredentials.getInstance, Duration.Inf) +) extends DrsPathResolver(drsConfig, drsCredentials) { override protected lazy val httpClientBuilder: HttpClientBuilder = httpClientBuilderOverride getOrElse MockSugar.mock[HttpClientBuilder] @@ -38,13 +38,15 @@ class MockEngineDrsPathResolver(drsConfig: DrsConfig = MockDrsPaths.mockDrsConfi private val drsResolverObjWithFileName = drsResolverObjWithGcsPath.copy(fileName = Option("file.txt")) - private val drsResolverObjWithLocalizationPath = drsResolverObjWithGcsPath.copy(localizationPath = Option("/dir/subdir/file.txt")) + private val drsResolverObjWithLocalizationPath = + drsResolverObjWithGcsPath.copy(localizationPath = Option("/dir/subdir/file.txt")) - private val drsResolverObjWithAllThePaths = drsResolverObjWithLocalizationPath.copy(fileName = drsResolverObjWithFileName.fileName) + private val drsResolverObjWithAllThePaths = + drsResolverObjWithLocalizationPath.copy(fileName = drsResolverObjWithFileName.fileName) private val drsResolverObjWithNoGcsPath = drsResolverObjWithGcsPath.copy(gsUri = None) - override def resolveDrs(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): IO[DrsResolverResponse] = { + override def resolveDrs(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): IO[DrsResolverResponse] = drsPath match { case MockDrsPaths.drsPathResolvingGcsPath => IO(drsResolverObjWithGcsPath) case MockDrsPaths.drsPathWithNonPathChars => IO(drsResolverObjWithGcsPath) @@ -60,7 +62,6 @@ class MockEngineDrsPathResolver(drsConfig: DrsConfig = MockDrsPaths.mockDrsConfi ) ) } - } override lazy val getAccessToken: ErrorOr[String] = MockDrsPaths.mockToken.validNel } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpClientPool.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpClientPool.scala index d93b0aa6f82..b13eb844a24 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpClientPool.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpClientPool.scala @@ -7,25 +7,26 @@ import org.apache.commons.net.ftp.FTPClient import scala.concurrent.duration._ object FtpClientPool extends StrictLogging { - def dispose(ftpClient: FTPClient) = try { + def dispose(ftpClient: FTPClient) = try if (ftpClient.isConnected) { ftpClient.logout() ftpClient.disconnect() } - } catch { + catch { case e: Exception => logger.debug("Failed to disconnect ftp client", e) } } -class FtpClientPool(capacity: Int, maxIdleTime: FiniteDuration, factory: () => FTPClient) extends ExpiringPool[FTPClient]( - capacity = capacity, - maxIdleTime = maxIdleTime, - referenceType = ReferenceType.Strong, - _factory = factory, - // Reset is called every time a client is added or released back to the pool. We don't want to actually reset the connection here - // otherwise we'd need to login again and reconfigure the connection every time - _reset = Function.const(()), - _dispose = FtpClientPool.dispose, - // Could not find a good health check at the moment (isAvailable and isConnected on the socket seem to both return false sometimes even if the client is fine) - _healthCheck = Function.const(true) -) +class FtpClientPool(capacity: Int, maxIdleTime: FiniteDuration, factory: () => FTPClient) + extends ExpiringPool[FTPClient]( + capacity = capacity, + maxIdleTime = maxIdleTime, + referenceType = ReferenceType.Strong, + _factory = factory, + // Reset is called every time a client is added or released back to the pool. We don't want to actually reset the connection here + // otherwise we'd need to login again and reconfigure the connection every time + _reset = Function.const(()), + _dispose = FtpClientPool.dispose, + // Could not find a good health check at the moment (isAvailable and isConnected on the socket seem to both return false sometimes even if the client is fine) + _healthCheck = Function.const(true) + ) diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileProvider.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileProvider.scala index 96e05702034..37ace6c250b 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileProvider.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileProvider.scala @@ -37,17 +37,25 @@ class FtpCloudNioFileProvider(fsProvider: FtpCloudNioFileSystemProvider) extends * Returns a listing of keys within a bucket starting with prefix. The returned keys should include the prefix. The * paths must be absolute, but the key should not begin with a slash. */ - override def listObjects(cloudHost: String, cloudPathPrefix: String, markerOption: Option[String]): CloudNioFileList = withAutoRelease(cloudHost) { client => - FtpListFiles(cloudHost, cloudPathPrefix, "list objects") - .run(client) - .map({ files => - val cleanFiles = files.map(_.getName).map(cloudPathPrefix.stripPrefix("/").ensureSlashed + _) - CloudNioFileList(cleanFiles, markerOption) - }) - }.unsafeRunSync() - - override def copy(sourceCloudHost: String, sourceCloudPath: String, targetCloudHost: String, targetCloudPath: String): Unit = { - if (sourceCloudHost != targetCloudHost) throw new UnsupportedOperationException(s"Cannot copy files across different ftp servers: Source host: $sourceCloudHost, Target host: $targetCloudHost") + override def listObjects(cloudHost: String, cloudPathPrefix: String, markerOption: Option[String]): CloudNioFileList = + withAutoRelease(cloudHost) { client => + FtpListFiles(cloudHost, cloudPathPrefix, "list objects") + .run(client) + .map { files => + val cleanFiles = files.map(_.getName).map(cloudPathPrefix.stripPrefix("/").ensureSlashed + _) + CloudNioFileList(cleanFiles, markerOption) + } + }.unsafeRunSync() + + override def copy(sourceCloudHost: String, + sourceCloudPath: String, + targetCloudHost: String, + targetCloudPath: String + ): Unit = { + if (sourceCloudHost != targetCloudHost) + throw new UnsupportedOperationException( + s"Cannot copy files across different ftp servers: Source host: $sourceCloudHost, Target host: $targetCloudHost" + ) val fileSystem = findFileSystem(sourceCloudHost) @@ -63,7 +71,11 @@ class FtpCloudNioFileProvider(fsProvider: FtpCloudNioFileSystemProvider) extends Util.copyStream(ios.inputStream, ios.outputStream) } match { case Success(_) => - case Failure(failure) => throw new IOException(s"Failed to copy ftp://$sourceCloudHost/$sourceCloudPath to ftp://$targetCloudHost/$targetCloudPath", failure) + case Failure(failure) => + throw new IOException( + s"Failed to copy ftp://$sourceCloudHost/$sourceCloudPath to ftp://$targetCloudHost/$targetCloudPath", + failure + ) } } @@ -71,12 +83,15 @@ class FtpCloudNioFileProvider(fsProvider: FtpCloudNioFileSystemProvider) extends FtpDeleteFile(cloudHost, cloudPath, "delete").run(client) }.unsafeRunSync() - private def inputStream(cloudHost: String, cloudPath: String, offset: Long, lease: Lease[FTPClient]): IO[LeasedInputStream] = { + private def inputStream(cloudHost: String, + cloudPath: String, + offset: Long, + lease: Lease[FTPClient] + ): IO[LeasedInputStream] = FtpInputStream(cloudHost, cloudPath, offset) .run(lease.get()) // Wrap the input stream in a LeasedInputStream so that the lease can be released when the stream is closed .map(new LeasedInputStream(cloudHost, cloudPath, _, lease)) - } override def read(cloudHost: String, cloudPath: String, offset: Long): ReadableByteChannel = { for { @@ -85,11 +100,10 @@ class FtpCloudNioFileProvider(fsProvider: FtpCloudNioFileSystemProvider) extends } yield Channels.newChannel(is) }.unsafeRunSync() - private def outputStream(cloudHost: String, cloudPath: String, lease: Lease[FTPClient]): IO[LeasedOutputStream] = { + private def outputStream(cloudHost: String, cloudPath: String, lease: Lease[FTPClient]): IO[LeasedOutputStream] = FtpOutputStream(cloudHost, cloudPath) .run(lease.get()) .map(new LeasedOutputStream(cloudHost, cloudPath, _, lease)) - } override def write(cloudHost: String, cloudPath: String): WritableByteChannel = { for { @@ -98,15 +112,16 @@ class FtpCloudNioFileProvider(fsProvider: FtpCloudNioFileSystemProvider) extends } yield Channels.newChannel(os) }.unsafeRunSync() - override def fileAttributes(cloudHost: String, cloudPath: String): Option[CloudNioRegularFileAttributes] = withAutoRelease(cloudHost) { client => - FtpListFiles(cloudHost, cloudPath, "get file attributes") - .run(client) - .map( - _.headOption map { file => - new FtpCloudNioRegularFileAttributes(file, cloudHost + cloudPath) - } - ) - }.unsafeRunSync() + override def fileAttributes(cloudHost: String, cloudPath: String): Option[CloudNioRegularFileAttributes] = + withAutoRelease(cloudHost) { client => + FtpListFiles(cloudHost, cloudPath, "get file attributes") + .run(client) + .map( + _.headOption map { file => + new FtpCloudNioRegularFileAttributes(file, cloudHost + cloudPath) + } + ) + }.unsafeRunSync() override def createDirectory(cloudHost: String, cloudPath: String) = withAutoRelease(cloudHost) { client => val operation = FtpCreateDirectory(cloudHost, cloudPath) @@ -129,7 +144,8 @@ class FtpCloudNioFileProvider(fsProvider: FtpCloudNioFileSystemProvider) extends private def findFileSystem(host: String): FtpCloudNioFileSystem = fsProvider.newCloudNioFileSystemFromHost(host) - private def acquireLease[A](host: String): IO[Lease[FTPClient]] = IO { findFileSystem(host).leaseClient } + private def acquireLease[A](host: String): IO[Lease[FTPClient]] = IO(findFileSystem(host).leaseClient) - private def withAutoRelease[A](cloudHost: String): (FTPClient => IO[A]) => IO[A] = autoRelease[A](acquireLease(cloudHost)) + private def withAutoRelease[A](cloudHost: String): (FTPClient => IO[A]) => IO[A] = + autoRelease[A](acquireLease(cloudHost)) } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystem.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystem.scala index 91a26bc8f8e..dd0bb79cfa7 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystem.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystem.scala @@ -12,7 +12,9 @@ object FtpCloudNioFileSystem { val logger = LoggerFactory.getLogger("FtpFileSystem") } -class FtpCloudNioFileSystem(provider: FtpCloudNioFileSystemProvider, host: String) extends CloudNioFileSystem(provider, host) with StrictLogging { +class FtpCloudNioFileSystem(provider: FtpCloudNioFileSystemProvider, host: String) + extends CloudNioFileSystem(provider, host) + with StrictLogging { private val credentials = provider.credentials private val ftpConfig = provider.ftpConfig private val connectionModeFunction: FTPClient => Unit = ftpConfig.connectionMode match { @@ -20,7 +22,7 @@ class FtpCloudNioFileSystem(provider: FtpCloudNioFileSystemProvider, host: Strin case Active => client: FTPClient => client.enterLocalActiveMode() } - private [ftp] lazy val clientFactory = () => { + private[ftp] lazy val clientFactory = () => { val client = new FTPClient() client.setDefaultPort(ftpConfig.connectionPort) client.connect(host) @@ -33,7 +35,10 @@ class FtpCloudNioFileSystem(provider: FtpCloudNioFileSystemProvider, host: Strin private val clientPool = new FtpClientPool(ftpConfig.capacity, ftpConfig.idleConnectionTimeout, clientFactory) def leaseClient = ftpConfig.leaseTimeout match { - case Some(timeout) => clientPool.tryAcquire(timeout).getOrElse(throw new TimeoutException("Timed out waiting for an available connection, try again later.")) + case Some(timeout) => + clientPool + .tryAcquire(timeout) + .getOrElse(throw new TimeoutException("Timed out waiting for an available connection, try again later.")) case _ => clientPool.acquire() } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProvider.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProvider.scala index e20edfc76ff..bb2faaa5ccd 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProvider.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProvider.scala @@ -11,7 +11,11 @@ import com.typesafe.scalalogging.StrictLogging import scala.util.{Failure, Success, Try} -class FtpCloudNioFileSystemProvider(override val config: Config, val credentials: FtpCredentials, ftpFileSystems: FtpFileSystems) extends CloudNioFileSystemProvider with StrictLogging { +class FtpCloudNioFileSystemProvider(override val config: Config, + val credentials: FtpCredentials, + ftpFileSystems: FtpFileSystems +) extends CloudNioFileSystemProvider + with StrictLogging { val ftpConfig = ftpFileSystems.config override def fileProvider = new FtpCloudNioFileProvider(this) @@ -29,18 +33,19 @@ class FtpCloudNioFileSystemProvider(override val config: Config, val credentials * will try to get it using the fileProvider which will require a new client lease and can result in a deadlock of the client pool, since * the read channel holds on to its lease until its closed. */ - val preComputedFileSize = retry.from(() => fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath).map(_.size())) + val preComputedFileSize = + retry.from(() => fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath).map(_.size())) new CloudNioReadChannel(fileProvider, retry, cloudNioPath) { override def fileSize = preComputedFileSize } } - override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = { + override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = Try { - retry.from(() => { + retry.from { () => val cloudNioPath = CloudNioPath.checkPath(dir) fileProvider.createDirectory(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - }) + } } match { case Success(_) => case Failure(f: FileAlreadyExistsException) => throw f @@ -51,15 +56,12 @@ class FtpCloudNioFileSystemProvider(override val config: Config, val credentials throw f case Failure(f) => throw f } - } override def usePseudoDirectories = false - override def newCloudNioFileSystem(uriAsString: String, config: Config) = { + override def newCloudNioFileSystem(uriAsString: String, config: Config) = newCloudNioFileSystemFromHost(getHost(uriAsString)) - } - - def newCloudNioFileSystemFromHost(host: String) = { + + def newCloudNioFileSystemFromHost(host: String) = ftpFileSystems.getFileSystem(host, this) - } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCredentials.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCredentials.scala index 2e553a5f66b..c63efad1de7 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCredentials.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCredentials.scala @@ -11,7 +11,8 @@ sealed trait FtpCredentials { } // Yes, FTP uses plain text username / password -case class FtpAuthenticatedCredentials(username: String, password: String, account: Option[String]) extends FtpCredentials { +case class FtpAuthenticatedCredentials(username: String, password: String, account: Option[String]) + extends FtpCredentials { override def login(ftpClient: FTPClient) = { lazy val replyString = Option(ftpClient.getReplyString).getOrElse("N/A") diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystems.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystems.scala index 006e19982ff..63f0c10f655 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystems.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystems.scala @@ -9,7 +9,7 @@ import scala.concurrent.duration._ object FtpFileSystems { val DefaultConfig = FtpFileSystemsConfiguration(1.day, Option(1.hour), 5, 1.hour, 21, Passive) val Default = new FtpFileSystems(DefaultConfig) - private [ftp] case class FtpCacheKey(host: String, ftpProvider: FtpCloudNioFileSystemProvider) + private[ftp] case class FtpCacheKey(host: String, ftpProvider: FtpCloudNioFileSystemProvider) } /** @@ -21,18 +21,18 @@ class FtpFileSystems(val config: FtpFileSystemsConfiguration) { private val fileSystemTTL = config.cacheTTL - private val fileSystemsCache: LoadingCache[FtpCacheKey, FtpCloudNioFileSystem] = CacheBuilder.newBuilder() + private val fileSystemsCache: LoadingCache[FtpCacheKey, FtpCloudNioFileSystem] = CacheBuilder + .newBuilder() .expireAfterAccess(fileSystemTTL.length, fileSystemTTL.unit) - .removalListener((notification: RemovalNotification[FtpCacheKey, FtpCloudNioFileSystem]) => { + .removalListener { (notification: RemovalNotification[FtpCacheKey, FtpCloudNioFileSystem]) => notification.getValue.close() - }) + } .build[FtpCacheKey, FtpCloudNioFileSystem](new CacheLoader[FtpCacheKey, FtpCloudNioFileSystem] { override def load(key: FtpCacheKey) = createFileSystem(key) }) - - private [ftp] def createFileSystem(key: FtpCacheKey) = new FtpCloudNioFileSystem(key.ftpProvider, key.host) - def getFileSystem(host: String, ftpCloudNioFileProvider: FtpCloudNioFileSystemProvider) = { + private[ftp] def createFileSystem(key: FtpCacheKey) = new FtpCloudNioFileSystem(key.ftpProvider, key.host) + + def getFileSystem(host: String, ftpCloudNioFileProvider: FtpCloudNioFileSystemProvider) = fileSystemsCache.get(FtpCacheKey(host, ftpCloudNioFileProvider)) - } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystemsConfiguration.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystemsConfiguration.scala index 2c305f82edf..8e5071f604a 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystemsConfiguration.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystemsConfiguration.scala @@ -9,7 +9,8 @@ case class FtpFileSystemsConfiguration(cacheTTL: FiniteDuration, capacity: Int, idleConnectionTimeout: FiniteDuration, connectionPort: Int, - connectionMode: ConnectionMode) + connectionMode: ConnectionMode +) object FtpFileSystemsConfiguration { sealed trait ConnectionMode diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpUtil.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpUtil.scala index 9138401f803..049c54d9d89 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpUtil.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpUtil.scala @@ -12,16 +12,16 @@ object FtpUtil { def ensureSlashed = if (cloudPath.endsWith("/")) cloudPath else s"$cloudPath/" } - case class FtpIoException(message: String, code: Int, replyString: String, cause: Option[Throwable] = None) extends IOException(s"$message: $replyString", cause.orNull) { + case class FtpIoException(message: String, code: Int, replyString: String, cause: Option[Throwable] = None) + extends IOException(s"$message: $replyString", cause.orNull) { def isTransient = FTPReply.isNegativeTransient(code) def isFatal = FTPReply.isNegativePermanent(code) } - def autoRelease[A](acquire: IO[Lease[FTPClient]])(action: FTPClient => IO[A]): IO[A] = { - acquire.bracketCase(lease => action(lease.get()))({ - // If there's a cause, the call to the FTP client threw an exception, assume the connection is compromised and invalidate the lease - case (lease, ExitCase.Error(FtpIoException(_, _, _, Some(_)))) => IO { lease.invalidate() } - case (lease, _) => IO { lease.release() } - }) - } + def autoRelease[A](acquire: IO[Lease[FTPClient]])(action: FTPClient => IO[A]): IO[A] = + acquire.bracketCase(lease => action(lease.get())) { + // If there's a cause, the call to the FTP client threw an exception, assume the connection is compromised and invalidate the lease + case (lease, ExitCase.Error(FtpIoException(_, _, _, Some(_)))) => IO(lease.invalidate()) + case (lease, _) => IO(lease.release()) + } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/InputOutputStreams.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/InputOutputStreams.scala index 2455fa2b86a..a1a8b158ec7 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/InputOutputStreams.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/InputOutputStreams.scala @@ -3,11 +3,9 @@ package cloud.nio.impl.ftp import java.io.{InputStream, OutputStream} class InputOutputStreams(val inputStream: InputStream, val outputStream: OutputStream) extends AutoCloseable { - override def close() = { - try { + override def close() = + try inputStream.close() - } finally { + finally outputStream.close() - } - } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedInputStream.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedInputStream.scala index 29d0b7c6bf7..2e794a96791 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedInputStream.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedInputStream.scala @@ -8,7 +8,8 @@ import cloud.nio.impl.ftp.operations.FtpCompletePendingCommand import io.github.andrebeat.pool.Lease import org.apache.commons.net.ftp.FTPClient -class LeasedInputStream(cloudHost: String, cloudPath: String, inputStream: InputStream, lease: Lease[FTPClient]) extends InputStream { +class LeasedInputStream(cloudHost: String, cloudPath: String, inputStream: InputStream, lease: Lease[FTPClient]) + extends InputStream { override def read() = inputStream.read() override def read(b: Array[Byte]) = inputStream.read(b) override def read(b: Array[Byte], off: Int, len: Int): Int = inputStream.read(b, off, len) @@ -16,7 +17,8 @@ class LeasedInputStream(cloudHost: String, cloudPath: String, inputStream: Input override def available = inputStream.available() override def close() = { inputStream.close() - autoRelease(IO.pure(lease))(FtpCompletePendingCommand(cloudHost, cloudPath, "close input steam").run).void.unsafeRunSync() + autoRelease(IO.pure(lease))(FtpCompletePendingCommand(cloudHost, cloudPath, "close input steam").run).void + .unsafeRunSync() } override def mark(readlimit: Int) = inputStream.mark(readlimit) override def reset() = inputStream.reset() diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedOutputStream.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedOutputStream.scala index 5a4f43acabb..7fe09f3fbf1 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedOutputStream.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedOutputStream.scala @@ -8,13 +8,15 @@ import cloud.nio.impl.ftp.operations.FtpCompletePendingCommand import io.github.andrebeat.pool.Lease import org.apache.commons.net.ftp.FTPClient -class LeasedOutputStream(cloudHost: String, cloudPath: String, outputStream: OutputStream, lease: Lease[FTPClient]) extends OutputStream { +class LeasedOutputStream(cloudHost: String, cloudPath: String, outputStream: OutputStream, lease: Lease[FTPClient]) + extends OutputStream { override def write(b: Int) = outputStream.write(b) override def write(b: Array[Byte]) = outputStream.write(b) override def write(b: Array[Byte], off: Int, len: Int): Unit = outputStream.write(b, off, len) override def flush() = outputStream.flush() override def close() = { outputStream.close() - autoRelease(IO.pure(lease))(FtpCompletePendingCommand(cloudHost, cloudPath, "close input steam").run).void.unsafeRunSync() + autoRelease(IO.pure(lease))(FtpCompletePendingCommand(cloudHost, cloudPath, "close input steam").run).void + .unsafeRunSync() } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/operations/FtpOperation.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/operations/FtpOperation.scala index 400b0b26740..dee465cff4c 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/operations/FtpOperation.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/operations/FtpOperation.scala @@ -14,93 +14,101 @@ sealed trait FtpOperation[A] { def description: String def action: FTPClient => A def run(client: FTPClient): IO[A] - + def fullPath = s"ftp://$cloudHost/$cloudPath" protected def errorMessage = s"Failed to $description at $fullPath" - protected def fail(client: FTPClient, cause: Option[Throwable] = None) = { + protected def fail(client: FTPClient, cause: Option[Throwable] = None) = IO.raiseError[A](generateException(client, cause)) - } - private [operations] def generateException(client: FTPClient, cause: Option[Throwable]) = cause match { - case None if client.getReplyCode == FTPReply.FILE_UNAVAILABLE && client.getReplyString.toLowerCase.contains("exists") => + private[operations] def generateException(client: FTPClient, cause: Option[Throwable]) = cause match { + case None + if client.getReplyCode == FTPReply.FILE_UNAVAILABLE && client.getReplyString.toLowerCase.contains("exists") => new FileAlreadyExistsException(fullPath) - case None if client.getReplyCode == FTPReply.FILE_UNAVAILABLE && client.getReplyString.toLowerCase.contains("no such file") => + case None + if client.getReplyCode == FTPReply.FILE_UNAVAILABLE && client.getReplyString.toLowerCase.contains( + "no such file" + ) => new NoSuchFileException(fullPath) case None => FtpIoException(errorMessage, client.getReplyCode, Option(client.getReplyString).getOrElse("N/A")) case Some(c) => FtpIoException(errorMessage, client.getReplyCode, Option(client.getReplyString).getOrElse("N/A"), Option(c)) } - + protected def handleError(client: FTPClient)(failure: Throwable) = fail(client, Option(failure)) - protected def commonRun(client: FTPClient, bind: A => IO[A]): IO[A] = { - IO { action(client) } redeemWith (handleError(client), bind) - } - + protected def commonRun(client: FTPClient, bind: A => IO[A]): IO[A] = + IO(action(client)) redeemWith (handleError(client), bind) + override def toString = s"$description at $fullPath" } sealed trait FtpBooleanOperation extends FtpOperation[Boolean] { protected def failOnFalse: Boolean = true - - def run(client: FTPClient): IO[Boolean] = { + + def run(client: FTPClient): IO[Boolean] = commonRun(client, - { - // Operation didn't throw but the result is false which means it failed - case false if failOnFalse => fail(client) - case result => IO.pure(result) - } + { + // Operation didn't throw but the result is false which means it failed + case false if failOnFalse => fail(client) + case result => IO.pure(result) + } ) - } } sealed trait FtpValueOperation[A <: AnyRef] extends FtpOperation[A] { - def run(client: FTPClient): IO[A] = { - commonRun(client, - { - // Operation didn't throw but the result is null which means it failed - case null => fail(client) - case result => IO.pure(result) - } + def run(client: FTPClient): IO[A] = + commonRun(client, + { + // Operation didn't throw but the result is null which means it failed + case null => fail(client) + case result => IO.pure(result) + } ) - } } -case class FtpListFiles(cloudHost: String, cloudPath: String, description: String = "List files") extends FtpValueOperation[Array[FTPFile]] { +case class FtpListFiles(cloudHost: String, cloudPath: String, description: String = "List files") + extends FtpValueOperation[Array[FTPFile]] { override val action = _.listFiles(cloudPath.ensureSlashedPrefix) } -case class FtpListDirectories(cloudHost: String, cloudPath: String, description: String = "List files") extends FtpValueOperation[Array[FTPFile]] { +case class FtpListDirectories(cloudHost: String, cloudPath: String, description: String = "List files") + extends FtpValueOperation[Array[FTPFile]] { // We need to list the directories in the parent and see if any matches the name, hence the string manipulations - lazy val parts = cloudPath.ensureSlashedPrefix.stripSuffix(CloudNioFileSystem.Separator).split(CloudNioFileSystem.Separator) + lazy val parts = + cloudPath.ensureSlashedPrefix.stripSuffix(CloudNioFileSystem.Separator).split(CloudNioFileSystem.Separator) lazy val parent = parts.init.mkString(CloudNioFileSystem.Separator) lazy val directoryName = parts.last override val action = _.listDirectories(parent) } -case class FtpDeleteFile(cloudHost: String, cloudPath: String, description: String = "delete file") extends FtpBooleanOperation { +case class FtpDeleteFile(cloudHost: String, cloudPath: String, description: String = "delete file") + extends FtpBooleanOperation { override val action = _.deleteFile(cloudPath.ensureSlashedPrefix) override val failOnFalse = false } -case class FtpInputStream(cloudHost: String, cloudPath: String, offset: Long, description: String = "read") extends FtpValueOperation[InputStream] { +case class FtpInputStream(cloudHost: String, cloudPath: String, offset: Long, description: String = "read") + extends FtpValueOperation[InputStream] { override val action = { client => client.setRestartOffset(offset) client.retrieveFileStream(cloudPath.ensureSlashedPrefix) } } -case class FtpOutputStream(cloudHost: String, cloudPath: String, description: String = "write") extends FtpValueOperation[OutputStream] { +case class FtpOutputStream(cloudHost: String, cloudPath: String, description: String = "write") + extends FtpValueOperation[OutputStream] { override val action = _.storeFileStream(cloudPath.ensureSlashedPrefix) } -case class FtpCreateDirectory(cloudHost: String, cloudPath: String, description: String = "create directory") extends FtpBooleanOperation { +case class FtpCreateDirectory(cloudHost: String, cloudPath: String, description: String = "create directory") + extends FtpBooleanOperation { override val action = _.makeDirectory(cloudPath.ensureSlashedPrefix) } -case class FtpCompletePendingCommand(cloudHost: String, cloudPath: String, description: String = "close stream") extends FtpBooleanOperation { +case class FtpCompletePendingCommand(cloudHost: String, cloudPath: String, description: String = "close stream") + extends FtpBooleanOperation { override val action = _.completePendingCommand() } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpClientPoolSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpClientPoolSpec.scala index 91b69bf7e94..1837ee27139 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpClientPoolSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpClientPoolSpec.scala @@ -17,15 +17,13 @@ class FtpClientPoolSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche var loggedOut: Boolean = false var disconnected: Boolean = false client.isConnected.returns(true) - client.logout().responds(_ => { + client.logout().responds { _ => loggedOut = true true - }) - client.disconnect().responds(_ => { - disconnected = true - }) + } + client.disconnect().responds(_ => disconnected = true) - val clientPool = new FtpClientPool(1, 10.minutes, () => { client }) + val clientPool = new FtpClientPool(1, 10.minutes, () => client) clientPool.acquire().invalidate() loggedOut shouldBe true diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileProviderSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileProviderSpec.scala index f68f67cbf67..f1ca24c847a 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileProviderSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileProviderSpec.scala @@ -46,7 +46,8 @@ class FtpCloudNioFileProviderSpec extends AnyFlatSpec with CromwellTimeoutSpec w fakeUnixFileSystem.add(new DirectoryEntry(directory)) fileProvider.listObjects("localhost", root, None).paths should contain theSameElementsAs List( - file.stripPrefix("/"), directory.stripPrefix("/") + file.stripPrefix("/"), + directory.stripPrefix("/") ) } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProviderSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProviderSpec.scala index e23e8ba450e..952e171f9ec 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProviderSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProviderSpec.scala @@ -13,8 +13,12 @@ import org.mockito.Mockito._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -class FtpCloudNioFileSystemProviderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with MockSugar - with MockFtpFileSystem { +class FtpCloudNioFileSystemProviderSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with MockSugar + with MockFtpFileSystem { behavior of "FtpCloudNioFileSystemProviderSpec" @@ -45,7 +49,9 @@ class FtpCloudNioFileSystemProviderSpec extends AnyFlatSpec with CromwellTimeout it should "pre compute the size before opening a read channel to avoid deadlocks" in { val mockSizeFunction = mock[() => Long] val provider: FtpCloudNioFileSystemProvider = new FtpCloudNioFileSystemProvider( - ConfigFactory.empty, FtpAnonymousCredentials, ftpFileSystems + ConfigFactory.empty, + FtpAnonymousCredentials, + ftpFileSystems ) { override def fileProvider: FtpCloudNioFileProvider = new FtpCloudNioFileProvider(this) { @@ -59,9 +65,8 @@ class FtpCloudNioFileSystemProviderSpec extends AnyFlatSpec with CromwellTimeout } ) - override def read(cloudHost: String, cloudPath: String, offset: Long): ReadableByteChannel = { + override def read(cloudHost: String, cloudPath: String, offset: Long): ReadableByteChannel = mock[ReadableByteChannel] - } } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemSpec.scala index d12db7ec612..f2b86fbd621 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemSpec.scala @@ -11,7 +11,6 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.TimeoutException import scala.concurrent.duration._ - class FtpCloudNioFileSystemSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Eventually { behavior of "FtpCloudNioFileSystemSpec" @@ -20,11 +19,12 @@ class FtpCloudNioFileSystemSpec extends AnyFlatSpec with CromwellTimeoutSpec wit implicit val patience = patienceConfig it should "lease the number of clients configured, not more, not less" in { - val fileSystems = new FtpFileSystems(FtpFileSystems.DefaultConfig.copy(leaseTimeout = Option(1.second), capacity = 3)) + val fileSystems = + new FtpFileSystems(FtpFileSystems.DefaultConfig.copy(leaseTimeout = Option(1.second), capacity = 3)) val provider = new FtpCloudNioFileSystemProvider(ConfigFactory.empty, FtpAnonymousCredentials, fileSystems) val fileSystem = new FtpCloudNioFileSystem(provider, "ftp.example.com") { // Override so we don't try to connect to anything - override private[ftp] lazy val clientFactory = () => { new FTPClient() } + override private[ftp] lazy val clientFactory = () => new FTPClient() } val client1 = fileSystem.leaseClient @@ -46,7 +46,7 @@ class FtpCloudNioFileSystemSpec extends AnyFlatSpec with CromwellTimeoutSpec wit val provider = new FtpCloudNioFileSystemProvider(ConfigFactory.empty, FtpAnonymousCredentials, fileSystems) val fileSystem = new FtpCloudNioFileSystem(provider, "ftp.example.com") { // Override so we don't try to connect to anything - override private[ftp] lazy val clientFactory = () => { new FTPClient() } + override private[ftp] lazy val clientFactory = () => new FTPClient() override def leaseClient = { val lease = super.leaseClient diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCredentialsSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCredentialsSpec.scala index 0ba19060ab2..b4262452e35 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCredentialsSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCredentialsSpec.scala @@ -17,21 +17,21 @@ class FtpCredentialsSpec extends AnyFlatSpec with Matchers with MockSugar with C var loggedInWithAccount: Boolean = false var loggedInWithoutAccount: Boolean = false val client = mock[FTPClient] - client.login(anyString, anyString).responds(_ => { + client.login(anyString, anyString).responds { _ => loggedInWithoutAccount = true true - }) - client.login(anyString, anyString, anyString).responds(_ => { + } + client.login(anyString, anyString, anyString).responds { _ => loggedInWithAccount = true true - }) + } FtpAuthenticatedCredentials("user", "password", None).login(client) loggedInWithoutAccount shouldBe true loggedInWithAccount shouldBe false // reset - loggedInWithoutAccount= false + loggedInWithoutAccount = false FtpAuthenticatedCredentials("user", "password", Option("account")).login(client) loggedInWithAccount shouldBe true diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpFileSystemsSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpFileSystemsSpec.scala index 86f86914103..6a186a0d5f2 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpFileSystemsSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpFileSystemsSpec.scala @@ -35,8 +35,8 @@ class FtpFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match verify(mockCreateFunction).apply(ftpCacheKey) } - class MockFtpFileSystems(conf: FtpFileSystemsConfiguration, - mockCreateFunction: FtpCacheKey => FtpCloudNioFileSystem) extends FtpFileSystems(conf) { + class MockFtpFileSystems(conf: FtpFileSystemsConfiguration, mockCreateFunction: FtpCacheKey => FtpCloudNioFileSystem) + extends FtpFileSystems(conf) { override private[ftp] def createFileSystem(key: FtpCacheKey) = mockCreateFunction(key) } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpUtilSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpUtilSpec.scala index 0e86194ecbd..1796cef16b9 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpUtilSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpUtilSpec.scala @@ -16,7 +16,7 @@ class FtpUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "autoRelease" it should "release the lease when the client fails the operation without throwing" in { - val clientPool = new FtpClientPool(1, 10.minutes, () => { new FTPClient }) + val clientPool = new FtpClientPool(1, 10.minutes, () => new FTPClient) val lease = clientPool.acquire() val action = autoRelease(IO.pure(lease)) { _ => @@ -28,7 +28,7 @@ class FtpUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "invalidate the lease when the client fails the operation by throwing" in { - val clientPool = new FtpClientPool(1, 10.minutes, () => { new FTPClient }) + val clientPool = new FtpClientPool(1, 10.minutes, () => new FTPClient) val lease = clientPool.acquire() val action = autoRelease(IO.pure(lease)) { _ => @@ -40,7 +40,7 @@ class FtpUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "release the lease when the operation succeeds" in { - val clientPool = new FtpClientPool(1, 10.minutes, () => { new FTPClient }) + val clientPool = new FtpClientPool(1, 10.minutes, () => new FTPClient) val lease = clientPool.acquire() val action = autoRelease(IO.pure(lease)) { _ => diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseInputStreamSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseInputStreamSpec.scala index 6b70679a239..7042571b993 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseInputStreamSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseInputStreamSpec.scala @@ -22,11 +22,11 @@ class LeaseInputStreamSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat } val mockClient = mock[FTPClient] var completed: Boolean = false - mockClient.completePendingCommand().returns({ + mockClient.completePendingCommand().returns { completed = true true - }) - val clientPool = new FtpClientPool(1, 10.minutes, () => { mockClient }) + } + val clientPool = new FtpClientPool(1, 10.minutes, () => mockClient) val lease = clientPool.acquire() val leasedInputStream = new LeasedInputStream("host", "path", is, lease) diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseOutputStreamSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseOutputStreamSpec.scala index ee82de6ffea..5c5bef9b9f4 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseOutputStreamSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseOutputStreamSpec.scala @@ -17,11 +17,11 @@ class LeaseOutputStreamSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma val os = new TestOutputStream val mockClient = mock[FTPClient] var completed: Boolean = false - mockClient.completePendingCommand().returns({ + mockClient.completePendingCommand().returns { completed = true true - }) - val clientPool = new FtpClientPool(1, 10.minutes, () => { mockClient }) + } + val clientPool = new FtpClientPool(1, 10.minutes, () => mockClient) val lease = clientPool.acquire() val leasedOutputStream = new LeasedOutputStream("host", "path", os, lease) diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/MockFtpFileSystem.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/MockFtpFileSystem.scala index e3c954c8d24..42716da92cd 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/MockFtpFileSystem.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/MockFtpFileSystem.scala @@ -22,15 +22,18 @@ trait MockFtpFileSystem extends BeforeAndAfterAll { this: Suite => connectionPort = Option(fakeFtpServer.getServerControlPort) } - override def afterAll() = { + override def afterAll() = fakeFtpServer.stop() - } - lazy val ftpFileSystemsConfiguration = FtpFileSystems.DefaultConfig.copy(connectionPort = connectionPort.getOrElse(throw new RuntimeException("Fake FTP server has not been started"))) + lazy val ftpFileSystemsConfiguration = FtpFileSystems.DefaultConfig.copy(connectionPort = + connectionPort.getOrElse(throw new RuntimeException("Fake FTP server has not been started")) + ) lazy val ftpFileSystems = new FtpFileSystems(ftpFileSystemsConfiguration) // Do not call this before starting the server - lazy val mockProvider = { - new FtpCloudNioFileSystemProvider(ConfigFactory.empty, FtpAuthenticatedCredentials("test_user", "test_password", None), ftpFileSystems) - } + lazy val mockProvider = + new FtpCloudNioFileSystemProvider(ConfigFactory.empty, + FtpAuthenticatedCredentials("test_user", "test_password", None), + ftpFileSystems + ) } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioBackoff.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioBackoff.scala index e55031f5caa..eb0ae65df29 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioBackoff.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioBackoff.scala @@ -6,8 +6,10 @@ import com.google.api.client.util.ExponentialBackOff import scala.concurrent.duration.{Duration, FiniteDuration} trait CloudNioBackoff { + /** Next interval in millis */ def backoffMillis: Long + /** Get the next instance of backoff. This should be called after every call to backoffMillis */ def next: CloudNioBackoff } @@ -16,8 +18,8 @@ object CloudNioBackoff { private[spi] def newExponentialBackOff(initialInterval: FiniteDuration, maxInterval: FiniteDuration, multiplier: Double, - randomizationFactor: Double, - ): ExponentialBackOff = { + randomizationFactor: Double + ): ExponentialBackOff = new ExponentialBackOff.Builder() .setInitialIntervalMillis(initialInterval.toMillis.toInt) .setMaxIntervalMillis(maxInterval.toMillis.toInt) @@ -25,7 +27,6 @@ object CloudNioBackoff { .setRandomizationFactor(randomizationFactor) .setMaxElapsedTimeMillis(Int.MaxValue) .build() - } } object CloudNioInitialGapBackoff { @@ -33,24 +34,25 @@ object CloudNioInitialGapBackoff { initialInterval: FiniteDuration, maxInterval: FiniteDuration, multiplier: Double, - randomizationFactor: Double = ExponentialBackOff.DEFAULT_RANDOMIZATION_FACTOR, - ): CloudNioInitialGapBackoff = { + randomizationFactor: Double = ExponentialBackOff.DEFAULT_RANDOMIZATION_FACTOR + ): CloudNioInitialGapBackoff = new CloudNioInitialGapBackoff( initialGap, newExponentialBackOff( initialInterval = initialInterval, maxInterval = maxInterval, multiplier = multiplier, - randomizationFactor = randomizationFactor, + randomizationFactor = randomizationFactor ) ) - } } -case class CloudNioInitialGapBackoff(initialGapMillis: FiniteDuration, googleBackoff: ExponentialBackOff) extends CloudNioBackoff { +case class CloudNioInitialGapBackoff(initialGapMillis: FiniteDuration, googleBackoff: ExponentialBackOff) + extends CloudNioBackoff { assert(initialGapMillis.compareTo(Duration.Zero) != 0, "Initial gap cannot be null, use SimpleBackoff instead.") override val backoffMillis: Long = initialGapMillis.toMillis + /** Switch to a SimpleExponentialBackoff after the initial gap has been used */ override def next = new CloudNioSimpleExponentialBackoff(googleBackoff) } @@ -59,21 +61,21 @@ object CloudNioSimpleExponentialBackoff { def apply(initialInterval: FiniteDuration, maxInterval: FiniteDuration, multiplier: Double, - randomizationFactor: Double = ExponentialBackOff.DEFAULT_RANDOMIZATION_FACTOR, - ): CloudNioSimpleExponentialBackoff = { + randomizationFactor: Double = ExponentialBackOff.DEFAULT_RANDOMIZATION_FACTOR + ): CloudNioSimpleExponentialBackoff = new CloudNioSimpleExponentialBackoff( newExponentialBackOff( initialInterval = initialInterval, maxInterval = maxInterval, multiplier = multiplier, - randomizationFactor = randomizationFactor, + randomizationFactor = randomizationFactor ) ) - } } case class CloudNioSimpleExponentialBackoff(googleBackoff: ExponentialBackOff) extends CloudNioBackoff { override def backoffMillis: Long = googleBackoff.nextBackOffMillis() + /** google ExponentialBackOff is mutable so we can keep returning the same instance */ override def next: CloudNioBackoff = this } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioDirectoryStream.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioDirectoryStream.scala index 909c1fc54db..25d0f34e7dc 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioDirectoryStream.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioDirectoryStream.scala @@ -13,24 +13,19 @@ class CloudNioDirectoryStream( override def iterator(): java.util.Iterator[Path] = pathStream().filterNot(_ == prefix).iterator.asJava - private[this] def pathStream(markerOption: Option[String] = None): LazyList[Path] = { + private[this] def pathStream(markerOption: Option[String] = None): LazyList[Path] = listNext(markerOption) match { case CloudNioFileList(keys, Some(marker)) => keys.to(LazyList).map(toPath) ++ pathStream(Option(marker)) case CloudNioFileList(keys, None) => keys.to(LazyList).map(toPath) } - } - private[this] def toPath(key: String): Path = { + private[this] def toPath(key: String): Path = prefix.getFileSystem.getPath("/" + key) - } - private[this] def listNext(markerOption: Option[String]): CloudNioFileList = { - retry.from( - () => fileProvider.listObjects(prefix.cloudHost, prefix.cloudPath, markerOption) - ) - } + private[this] def listNext(markerOption: Option[String]): CloudNioFileList = + retry.from(() => fileProvider.listObjects(prefix.cloudHost, prefix.cloudPath, markerOption)) override def close(): Unit = {} diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileAttributeView.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileAttributeView.scala index 7cc34400676..852951784ac 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileAttributeView.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileAttributeView.scala @@ -11,21 +11,17 @@ final case class CloudNioFileAttributeView( ) extends BasicFileAttributeView { override def name(): String = CloudNioFileAttributeView.Name - override def readAttributes(): CloudNioFileAttributes = { + override def readAttributes(): CloudNioFileAttributes = if (isDirectory) { CloudNioDirectoryAttributes(cloudNioPath) } else { retry - .from( - () => fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - ) + .from(() => fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath)) .getOrElse(throw new FileNotFoundException(cloudNioPath.uriAsString)) } - } - override def setTimes(lastModifiedTime: FileTime, lastAccessTime: FileTime, createTime: FileTime): Unit = { + override def setTimes(lastModifiedTime: FileTime, lastAccessTime: FileTime, createTime: FileTime): Unit = throw new UnsupportedOperationException - } } object CloudNioFileAttributeView { diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystem.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystem.scala index 8b93419f350..43384330388 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystem.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystem.scala @@ -20,49 +20,40 @@ class CloudNioFileSystem(override val provider: CloudNioFileSystemProvider, val // do nothing currently. } - override def isOpen: Boolean = { + override def isOpen: Boolean = true - } - override def isReadOnly: Boolean = { + override def isReadOnly: Boolean = false - } - override def getSeparator: String = { + override def getSeparator: String = CloudNioFileSystem.Separator - } - override def getRootDirectories: java.lang.Iterable[Path] = { + override def getRootDirectories: java.lang.Iterable[Path] = Set[Path](getPath(UnixPath.Root)).asJava - } - override def getFileStores: java.lang.Iterable[FileStore] = { + override def getFileStores: java.lang.Iterable[FileStore] = Set.empty[FileStore].asJava - } - override def getPathMatcher(syntaxAndPattern: String): PathMatcher = { + override def getPathMatcher(syntaxAndPattern: String): PathMatcher = FileSystems.getDefault.getPathMatcher(syntaxAndPattern) - } - override def getUserPrincipalLookupService: UserPrincipalLookupService = { + override def getUserPrincipalLookupService: UserPrincipalLookupService = throw new UnsupportedOperationException - } - override def newWatchService(): WatchService = { + override def newWatchService(): WatchService = throw new UnsupportedOperationException - } - override def supportedFileAttributeViews(): java.util.Set[String] = { + override def supportedFileAttributeViews(): java.util.Set[String] = Set("basic", CloudNioFileAttributeView.Name).asJava - } def canEqual(other: Any): Boolean = other.isInstanceOf[CloudNioFileSystem] override def equals(other: Any): Boolean = other match { case that: CloudNioFileSystem => (that canEqual this) && - provider == that.provider && - host == that.host + provider == that.provider && + host == that.host case _ => false } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystemProvider.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystemProvider.scala index 9b79e308afd..c78d30d2dce 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystemProvider.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystemProvider.scala @@ -3,7 +3,7 @@ package cloud.nio.spi import java.net.URI import java.nio.channels.SeekableByteChannel import java.nio.file._ -import java.nio.file.attribute.{BasicFileAttributeView, BasicFileAttributes, FileAttribute, FileAttributeView} +import java.nio.file.attribute.{BasicFileAttributes, BasicFileAttributeView, FileAttribute, FileAttributeView} import java.nio.file.spi.FileSystemProvider import com.typesafe.config.{Config, ConfigFactory} @@ -64,9 +64,8 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { newCloudNioFileSystem(uri.toString, config) } - override def getPath(uri: URI): CloudNioPath = { + override def getPath(uri: URI): CloudNioPath = getFileSystem(uri).getPath(uri.getPath) - } override def newByteChannel( path: Path, @@ -75,7 +74,7 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { ): SeekableByteChannel = { val cloudNioPath = CloudNioPath.checkPath(path) - for (opt <- options.asScala) { + for (opt <- options.asScala) opt match { case StandardOpenOption.READ | StandardOpenOption.WRITE | StandardOpenOption.SPARSE | StandardOpenOption.TRUNCATE_EXISTING | StandardOpenOption.CREATE | StandardOpenOption.CREATE_NEW => @@ -84,7 +83,6 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { StandardOpenOption.SYNC => throw new UnsupportedOperationException(opt.toString) } - } if (options.contains(StandardOpenOption.READ) && options.contains(StandardOpenOption.WRITE)) { throw new UnsupportedOperationException("Cannot open a READ+WRITE channel") @@ -95,70 +93,64 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { } } - protected def cloudNioReadChannel(retry: CloudNioRetry, cloudNioPath: CloudNioPath): CloudNioReadChannel = new CloudNioReadChannel(fileProvider, retry, cloudNioPath) - protected def cloudNioWriteChannel(retry: CloudNioRetry, cloudNioPath: CloudNioPath): CloudNioWriteChannel = new CloudNioWriteChannel(fileProvider, retry, cloudNioPath) + protected def cloudNioReadChannel(retry: CloudNioRetry, cloudNioPath: CloudNioPath): CloudNioReadChannel = + new CloudNioReadChannel(fileProvider, retry, cloudNioPath) + protected def cloudNioWriteChannel(retry: CloudNioRetry, cloudNioPath: CloudNioPath): CloudNioWriteChannel = + new CloudNioWriteChannel(fileProvider, retry, cloudNioPath) - override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = retry.from(() => { + override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = retry.from { () => val cloudNioPath = CloudNioPath.checkPath(dir) fileProvider.createDirectory(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - }) + } override def deleteIfExists(path: Path): Boolean = { val cloudNioPath = CloudNioPath.checkPath(path) if (checkDirectoryExists(cloudNioPath)) { - val hasObjects = retry.from( - () => fileProvider.existsPaths(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - ) + val hasObjects = retry.from(() => fileProvider.existsPaths(cloudNioPath.cloudHost, cloudNioPath.cloudPath)) if (hasObjects) { throw new UnsupportedOperationException("Can not delete a non-empty directory") } else { true } } else { - retry.from( - () => fileProvider.deleteIfExists(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - ) + retry.from(() => fileProvider.deleteIfExists(cloudNioPath.cloudHost, cloudNioPath.cloudPath)) } } - override def delete(path: Path): Unit = { + override def delete(path: Path): Unit = if (!deleteIfExists(path)) { val cloudNioPath = CloudNioPath.checkPath(path) throw new NoSuchFileException(cloudNioPath.uriAsString) } - } override def copy(source: Path, target: Path, options: CopyOption*): Unit = { val sourceCloudNioPath = CloudNioPath.checkPath(source) val targetCloudNioPath = CloudNioPath.checkPath(target) if (sourceCloudNioPath != targetCloudNioPath) { - retry.from( - () => - fileProvider.copy( - sourceCloudNioPath.cloudHost, - sourceCloudNioPath.cloudPath, - targetCloudNioPath.cloudHost, - targetCloudNioPath.cloudPath - ) + retry.from(() => + fileProvider.copy( + sourceCloudNioPath.cloudHost, + sourceCloudNioPath.cloudPath, + targetCloudNioPath.cloudHost, + targetCloudNioPath.cloudPath + ) ) } } override def move(source: Path, target: Path, options: CopyOption*): Unit = { - for (option <- options) { + for (option <- options) if (option == StandardCopyOption.ATOMIC_MOVE) throw new AtomicMoveNotSupportedException(null, null, "Atomic move unsupported") - } copy(source, target, options: _*) delete(source) () } - override def isSameFile(path: Path, path2: Path): Boolean = { + override def isSameFile(path: Path, path2: Path): Boolean = CloudNioPath.checkPath(path).equals(CloudNioPath.checkPath(path2)) - } override def isHidden(path: Path): Boolean = { CloudNioPath.checkPath(path) @@ -174,8 +166,8 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { val cloudNioPath = CloudNioPath.checkPath(path) - val exists = checkDirectoryExists(cloudNioPath) || retry.from( - () => fileProvider.existsPath(cloudNioPath.cloudHost, cloudNioPath.cloudPath) + val exists = checkDirectoryExists(cloudNioPath) || retry.from(() => + fileProvider.existsPath(cloudNioPath.cloudHost, cloudNioPath.cloudPath) ) if (!exists) { @@ -183,12 +175,11 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { } } - def checkDirectoryExists(cloudNioPath: CloudNioPath): Boolean = { + def checkDirectoryExists(cloudNioPath: CloudNioPath): Boolean = // Anything that "seems" like a directory exists. Otherwise see if the path with a "/" contains files on the cloud. - (usePseudoDirectories && cloudNioPath.seemsLikeDirectory) || retry.from( - () => fileProvider.existsPaths(cloudNioPath.cloudHost, cloudNioPath.cloudPath + "/") + (usePseudoDirectories && cloudNioPath.seemsLikeDirectory) || retry.from(() => + fileProvider.existsPaths(cloudNioPath.cloudHost, cloudNioPath.cloudPath + "/") ) - } override def getFileAttributeView[V <: FileAttributeView]( path: Path, @@ -205,9 +196,8 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { CloudNioFileAttributeView(fileProvider, retry, cloudNioPath, isDirectory).asInstanceOf[V] } - override def readAttributes(path: Path, attributes: String, options: LinkOption*): java.util.Map[String, AnyRef] = { + override def readAttributes(path: Path, attributes: String, options: LinkOption*): java.util.Map[String, AnyRef] = throw new UnsupportedOperationException - } override def readAttributes[A <: BasicFileAttributes]( path: Path, @@ -224,9 +214,7 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { CloudNioDirectoryAttributes(cloudNioPath).asInstanceOf[A] } else { retry - .from( - () => fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - ) + .from(() => fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath)) .map(_.asInstanceOf[A]) .getOrElse(throw new NoSuchFileException(cloudNioPath.uriAsString)) } @@ -237,16 +225,15 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { new CloudNioDirectoryStream(fileProvider, retry, cloudNioPath, filter) } - override def setAttribute(path: Path, attribute: String, value: scala.Any, options: LinkOption*): Unit = { + override def setAttribute(path: Path, attribute: String, value: scala.Any, options: LinkOption*): Unit = throw new UnsupportedOperationException - } def canEqual(other: Any): Boolean = other.isInstanceOf[CloudNioFileSystemProvider] override def equals(other: Any): Boolean = other match { case that: CloudNioFileSystemProvider => (that canEqual this) && - config == that.config + config == that.config case _ => false } @@ -258,7 +245,6 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { object CloudNioFileSystemProvider { - def defaultConfig(scheme: String): Config = { + def defaultConfig(scheme: String): Config = ConfigFactory.load.getOrElse(s"cloud.nio.default.$scheme", ConfigFactory.empty) - } } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala index ec0c701a9b9..530ba5526ef 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala @@ -9,12 +9,11 @@ import scala.jdk.CollectionConverters._ object CloudNioPath { - def checkPath(path: Path): CloudNioPath = { + def checkPath(path: Path): CloudNioPath = path match { case cloudNioPath: CloudNioPath => cloudNioPath - case _ => throw new ProviderMismatchException(s"Not a CloudNioPath: $path") + case _ => throw new ProviderMismatchException(s"Not a CloudNioPath: $path") } - } } class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: UnixPath) extends Path { @@ -65,13 +64,12 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un /** * If is relative, returns just the normalized path. If is absolute, return the host + the absolute path. */ - def relativeDependentPath: String = { + def relativeDependentPath: String = if (unixPath.isAbsolute) { cloudHost + "/" + unixPath.toString.stripPrefix("/") } else { unixPath.normalize().toString } - } /** * Returns true if the path probably represents a directory, but won't be known until contacting the host. @@ -84,30 +82,25 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un override def isAbsolute: Boolean = unixPath.isAbsolute - override def getRoot: CloudNioPath = { + override def getRoot: CloudNioPath = unixPath.getRoot.map(newPath).orNull - } - override def getFileName: CloudNioPath = { + override def getFileName: CloudNioPath = unixPath.getFileName.map(newPath).orNull - } - override def getParent: CloudNioPath = { + override def getParent: CloudNioPath = unixPath.getParent.map(newPath).orNull - } override def getNameCount: Int = unixPath.getNameCount - override def getName(index: Int): CloudNioPath = { + override def getName(index: Int): CloudNioPath = unixPath.getName(index).map(newPath).getOrElse(throw new IllegalArgumentException(s"Bad index $index")) - } - override def subpath(beginIndex: Int, endIndex: Int): CloudNioPath = { + override def subpath(beginIndex: Int, endIndex: Int): CloudNioPath = unixPath .subPath(beginIndex, endIndex) .map(newPath) .getOrElse(throw new IllegalArgumentException(s"Bad range $beginIndex-$endIndex")) - } override def startsWith(other: Path): Boolean = { if (!other.isInstanceOf[CloudNioPath]) { @@ -122,9 +115,8 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un unixPath.startsWith(that.unixPath) } - override def startsWith(other: String): Boolean = { + override def startsWith(other: String): Boolean = unixPath.startsWith(UnixPath.getPath(other)) - } override def endsWith(other: Path): Boolean = { if (!other.isInstanceOf[CloudNioPath]) { @@ -138,9 +130,8 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un unixPath.endsWith(that.unixPath) } - override def endsWith(other: String): Boolean = { + override def endsWith(other: String): Boolean = unixPath.endsWith(UnixPath.getPath(other)) - } override def normalize(): CloudNioPath = newPath(unixPath.normalize()) @@ -150,9 +141,8 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un newPath(unixPath.resolve(that.unixPath)) } - override def resolve(other: String): CloudNioPath = { + override def resolve(other: String): CloudNioPath = newPath(unixPath.resolve(UnixPath.getPath(other))) - } override def resolveSibling(other: Path): CloudNioPath = { val that = CloudNioPath.checkPath(other) @@ -160,9 +150,8 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un newPath(unixPath.resolveSibling(that.unixPath)) } - override def resolveSibling(other: String): CloudNioPath = { + override def resolveSibling(other: String): CloudNioPath = newPath(unixPath.resolveSibling(UnixPath.getPath(other))) - } override def relativize(other: Path): CloudNioPath = { val that = CloudNioPath.checkPath(other) @@ -170,9 +159,8 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un newPath(unixPath.relativize(that.unixPath)) } - override def toAbsolutePath: CloudNioPath = { + override def toAbsolutePath: CloudNioPath = newPath(unixPath.toAbsolutePath) - } override def toRealPath(options: LinkOption*): CloudNioPath = toAbsolutePath @@ -187,13 +175,12 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un modifiers: WatchEvent.Modifier* ): WatchKey = throw new UnsupportedOperationException - override def iterator(): java.util.Iterator[Path] = { - if (unixPath.isEmpty || unixPath.isRoot) { + override def iterator(): java.util.Iterator[Path] = + if (unixPath.izEmpty || unixPath.isRoot) { java.util.Collections.emptyIterator() } else { unixPath.split().to(LazyList).map(part => newPath(UnixPath.getPath(part)).asInstanceOf[Path]).iterator.asJava } - } override def compareTo(other: Path): Int = { if (other.isInstanceOf[CloudNioPath]) { @@ -209,22 +196,19 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un unixPath.compareTo(that.unixPath) } - override def equals(obj: scala.Any): Boolean = { + override def equals(obj: scala.Any): Boolean = (this eq obj.asInstanceOf[AnyRef]) || - obj.isInstanceOf[CloudNioPath] && - obj.asInstanceOf[CloudNioPath].cloudHost.equals(cloudHost) && - obj.asInstanceOf[CloudNioPath].unixPath.equals(unixPath) - } + obj.isInstanceOf[CloudNioPath] && + obj.asInstanceOf[CloudNioPath].cloudHost.equals(cloudHost) && + obj.asInstanceOf[CloudNioPath].unixPath.equals(unixPath) - override def hashCode(): Int = { + override def hashCode(): Int = Objects.hash(cloudHost, unixPath) - } - protected def newPath(unixPath: UnixPath): CloudNioPath = { + protected def newPath(unixPath: UnixPath): CloudNioPath = if (this.unixPath == unixPath) { this } else { new CloudNioPath(filesystem, unixPath) } - } } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioReadChannel.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioReadChannel.scala index 1e6020307fa..b8260b9a599 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioReadChannel.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioReadChannel.scala @@ -2,35 +2,28 @@ package cloud.nio.spi import java.io.FileNotFoundException import java.nio.ByteBuffer -import java.nio.channels.{ - ClosedChannelException, - NonWritableChannelException, - ReadableByteChannel, - SeekableByteChannel -} +import java.nio.channels.{ClosedChannelException, NonWritableChannelException, ReadableByteChannel, SeekableByteChannel} class CloudNioReadChannel(fileProvider: CloudNioFileProvider, retry: CloudNioRetry, cloudNioPath: CloudNioPath) - extends SeekableByteChannel { + extends SeekableByteChannel { private var internalPosition: Long = 0 private var channel: ReadableByteChannel = channelPosition(0) override def read(dst: ByteBuffer): Int = { var resetConnection = false - val count = retry.from( - () => { - try { - if (resetConnection) { - if (channel.isOpen) channel.close() - channel = fileProvider.read(cloudNioPath.cloudHost, cloudNioPath.cloudPath, internalPosition) - } - channel.read(dst) - } catch { - case exception: Exception => - resetConnection = true - throw exception + val count = retry.from { () => + try { + if (resetConnection) { + if (channel.isOpen) channel.close() + channel = fileProvider.read(cloudNioPath.cloudHost, cloudNioPath.cloudPath, internalPosition) } + channel.read(dst) + } catch { + case exception: Exception => + resetConnection = true + throw exception } - ) + } if (count > 0) internalPosition += count count @@ -50,25 +43,19 @@ class CloudNioReadChannel(fileProvider: CloudNioFileProvider, retry: CloudNioRet this } - private def channelPosition(newPosition: Long): ReadableByteChannel = { - retry.from( - () => fileProvider.read(cloudNioPath.cloudHost, cloudNioPath.cloudPath, newPosition) - ) - } + private def channelPosition(newPosition: Long): ReadableByteChannel = + retry.from(() => fileProvider.read(cloudNioPath.cloudHost, cloudNioPath.cloudPath, newPosition)) - override def size(): Long = { + override def size(): Long = retry - .from( - () => fileSize - ) + .from(() => fileSize) .getOrElse(throw new FileNotFoundException(cloudNioPath.uriAsString)) - } override def truncate(size: Long): SeekableByteChannel = throw new NonWritableChannelException override def isOpen: Boolean = channel.isOpen override def close(): Unit = channel.close() - + protected def fileSize = fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath).map(_.size()) } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioRetry.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioRetry.scala index 9a4028ed859..40c8cba7377 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioRetry.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioRetry.scala @@ -24,7 +24,7 @@ class CloudNioRetry(config: Config) { val delay = backoff.backoffMillis f() match { - case Success(ret) => ret + case Success(ret) => ret case Failure(exception: Exception) if isFatal(exception) => throw exception case Failure(exception: Exception) if !isFatal(exception) => val retriesLeft = if (isTransient(exception)) maxRetries else maxRetries map { _ - 1 } @@ -38,11 +38,13 @@ class CloudNioRetry(config: Config) { } } - def from[A](f: () => A, maxRetries: Option[Int] = Option(defaultMaxRetries), backoff: CloudNioBackoff = defaultBackOff): A = { + def from[A](f: () => A, + maxRetries: Option[Int] = Option(defaultMaxRetries), + backoff: CloudNioBackoff = defaultBackOff + ): A = fromTry[A]( () => Try(f()), maxRetries, backoff ) - } } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioWriteChannel.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioWriteChannel.scala index 15a9d97213b..232cc25297b 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioWriteChannel.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioWriteChannel.scala @@ -6,11 +6,8 @@ import java.nio.channels._ class CloudNioWriteChannel(fileProvider: CloudNioFileProvider, retry: CloudNioRetry, cloudNioPath: CloudNioPath) extends SeekableByteChannel { private var internalPosition: Long = 0 - private val channel: WritableByteChannel = { - retry.from( - () => fileProvider.write(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - ) - } + private val channel: WritableByteChannel = + retry.from(() => fileProvider.write(cloudNioPath.cloudHost, cloudNioPath.cloudPath)) override def read(dst: ByteBuffer): Int = throw new NonReadableChannelException diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala index fb3a2bf8f50..41d742a4e66 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala @@ -24,7 +24,7 @@ private[spi] object UnixPath { private def hasTrailingSeparator(path: String): Boolean = !path.isEmpty && path.charAt(path.length - 1) == Separator - def getPath(path: String): UnixPath = { + def getPath(path: String): UnixPath = if (path.isEmpty) { EmptyPath } else if (isRoot(path)) { @@ -32,7 +32,6 @@ private[spi] object UnixPath { } else { UnixPath(path) } - } def getPath(first: String, more: String*): UnixPath = { if (more.isEmpty) { @@ -40,7 +39,7 @@ private[spi] object UnixPath { } val builder = new StringBuilder(first) - for ((part, index) <- more.view.zipWithIndex) { + for ((part, index) <- more.view.zipWithIndex) if (part.isEmpty) { // do nothing } else if (isAbsolute(part)) { @@ -55,7 +54,6 @@ private[spi] object UnixPath { builder.append(Separator) builder.append(part) } - } UnixPath(builder.toString) } @@ -69,18 +67,21 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { def isAbsolute: Boolean = UnixPath.isAbsolute(path) - def isEmpty: Boolean = path.isEmpty + // Named this way because isEmpty is a name collision new in 17. + // The initial compile error is that it needs an override. + // Adding the override results in a second error saying it overrides nothing! + // So, we just renamed it. + def izEmpty: Boolean = path.isEmpty def hasTrailingSeparator: Boolean = UnixPath.hasTrailingSeparator(path) - def seemsLikeDirectory(): Boolean = { + def seemsLikeDirectory(): Boolean = path.isEmpty || - hasTrailingSeparator || - path.endsWith(".") && (length == 1 || path.charAt(length - 2) == UnixPath.Separator) || - path.endsWith("..") && (length == 2 || path.charAt(length - 3) == UnixPath.Separator) - } + hasTrailingSeparator || + path.endsWith(".") && (length == 1 || path.charAt(length - 2) == UnixPath.Separator) || + path.endsWith("..") && (length == 2 || path.charAt(length - 3) == UnixPath.Separator) - def getFileName: Option[UnixPath] = { + def getFileName: Option[UnixPath] = if (path.isEmpty || isRoot) { None } else { @@ -90,7 +91,6 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { Some(UnixPath(parts.last)) } } - } def getParent: Option[UnixPath] = { if (path.isEmpty || isRoot) { @@ -103,7 +103,7 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { else path.lastIndexOf(UnixPath.Separator.toInt) index match { - case -1 => if (isAbsolute) Some(UnixPath.RootPath) else None + case -1 => if (isAbsolute) Some(UnixPath.RootPath) else None case pos => Some(UnixPath(path.substring(0, pos + 1))) } } @@ -122,7 +122,7 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { Try(UnixPath(parts.slice(beginIndex, endIndex).mkString(UnixPath.Separator.toString))) } - def getNameCount: Int = { + def getNameCount: Int = if (path.isEmpty) { 1 } else if (isRoot) { @@ -130,7 +130,6 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { } else { parts.length } - } def getName(index: Int): Try[UnixPath] = { if (path.isEmpty) { @@ -144,7 +143,7 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { Success(UnixPath(parts(2))) } - def resolve(other: UnixPath): UnixPath = { + def resolve(other: UnixPath): UnixPath = if (other.path.isEmpty) { this } else if (other.isAbsolute) { @@ -154,15 +153,13 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { } else { new UnixPath(path + UnixPath.Separator.toString + other.path) } - } - def resolveSibling(other: UnixPath): UnixPath = { + def resolveSibling(other: UnixPath): UnixPath = getParent match { case Some(parent: UnixPath) => parent.resolve(other) case None => other } - } def relativize(other: UnixPath): UnixPath = { if (path.isEmpty) { @@ -247,21 +244,18 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { def splitReverse(): Iterator[String] = parts.reverseIterator - def removeBeginningSeparator(): UnixPath = { + def removeBeginningSeparator(): UnixPath = if (isAbsolute) new UnixPath(path.substring(1)) else this - } - def addTrailingSeparator(): UnixPath = { + def addTrailingSeparator(): UnixPath = if (hasTrailingSeparator) this else new UnixPath(path + UnixPath.Separator) - } - def removeTrailingSeparator(): UnixPath = { + def removeTrailingSeparator(): UnixPath = if (!isRoot && hasTrailingSeparator) { new UnixPath(path.substring(0, length - 1)) } else { this } - } def startsWith(other: UnixPath): Boolean = { val me = removeTrailingSeparator() @@ -279,11 +273,10 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { } def startsWith(left: Iterator[String], right: Iterator[String]): Boolean = { - while (right.hasNext) { + while (right.hasNext) if (!left.hasNext || right.next() != left.next()) { return false } - } true } @@ -310,9 +303,8 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { if (isAbsolute) Success(this) else Success(currentWorkingDirectory.resolve(this)) } - def toAbsolutePath: UnixPath = { + def toAbsolutePath: UnixPath = if (isAbsolute) this else UnixPath.RootPath.resolve(this) - } def compareTo(other: UnixPath): Int = { val me = parts.toList @@ -327,29 +319,24 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { } } - override def equals(obj: scala.Any): Boolean = { + override def equals(obj: scala.Any): Boolean = (this eq obj.asInstanceOf[AnyRef]) || { obj.isInstanceOf[UnixPath] && obj.asInstanceOf[UnixPath].path.equals(path) } - } - override def length(): Int = { + override def length(): Int = path.length - } - override def charAt(index: Int): Char = { + override def charAt(index: Int): Char = path.charAt(index) - } - override def subSequence(start: Int, end: Int): CharSequence = { + override def subSequence(start: Int, end: Int): CharSequence = path.subSequence(start, end) - } - override def toString: String = { + override def toString: String = path - } - def initParts(): Array[String] = { + def initParts(): Array[String] = if (path.isEmpty) { Array.empty[String] } else { @@ -359,5 +346,4 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { path.split(UnixPath.Separator) } } - } } diff --git a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/ChannelUtil.scala b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/ChannelUtil.scala index 04c9d4cdd3f..bbd1cbaf4db 100644 --- a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/ChannelUtil.scala +++ b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/ChannelUtil.scala @@ -26,13 +26,12 @@ object ChannelUtil { def pipedStreamWriter(threadName: String)(consumer: InputStream => Unit): WritableByteChannel = { val pipe = Pipe.open() var threadResult: Option[Try[Unit]] = None - val runnable: Runnable = () => { + val runnable: Runnable = () => threadResult = Option( Try( consumer(Channels.newInputStream(pipe.source)) ) ) - } val thread = new Thread(runnable, threadName) thread.setDaemon(true) thread.start() diff --git a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioFiles.scala b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioFiles.scala index 162d47d08db..e15cb574031 100644 --- a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioFiles.scala +++ b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioFiles.scala @@ -12,13 +12,12 @@ object CloudNioFiles { /** * Lists all files under a path. */ - def listRegularFiles(path: Path): Iterator[Path] = { + def listRegularFiles(path: Path): Iterator[Path] = Files .walk(path, Int.MaxValue) .iterator .asScala .filter(Files.isRegularFile(_)) - } /** * Returns an iterator of all regular files under sourcePath mapped relatively to targetPath. diff --git a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioPaths.scala b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioPaths.scala index 7448b272426..66ab48100c6 100644 --- a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioPaths.scala +++ b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioPaths.scala @@ -21,15 +21,14 @@ object CloudNioPaths { * @see [[cloud.nio.util.CloudNioPaths#showAbsolute(java.nio.file.Path)]] * @see [[cloud.nio.spi.CloudNioPath#uriAsString()]] */ - def get(filePath: String): Path = { - try { + def get(filePath: String): Path = + try // TODO: softer parsing using Guava UrlEscapers. May also be better to list the providers ourselves if possible. Paths.get(new URI(filePath)) - } catch { - case _: URISyntaxException => Paths.get(filePath) + catch { + case _: URISyntaxException => Paths.get(filePath) case iae: IllegalArgumentException if iae.getMessage == "Missing scheme" => Paths.get(filePath) } - } /** * Return a path in a way reciprocal with [[cloud.nio.util.CloudNioPaths#get]]. @@ -38,12 +37,11 @@ object CloudNioPaths { * @see [[cloud.nio.util.CloudNioPaths#showRelative(java.nio.file.Path)]] * @see [[cloud.nio.spi.CloudNioPath#uriAsString()]] */ - def showAbsolute(path: Path): String = { + def showAbsolute(path: Path): String = path match { case cloudNioPath: CloudNioPath => cloudNioPath.uriAsString - case _ => path.toAbsolutePath.toString + case _ => path.toAbsolutePath.toString } - } /** * When the path is relative returns a relative path in a way reciprocal with resolve. @@ -53,11 +51,10 @@ object CloudNioPaths { * @see [[java.nio.file.Path#resolve(java.nio.file.Path)]] * @see [[cloud.nio.spi.CloudNioPath#uriAsString()]] */ - def showRelative(path: Path): String = { + def showRelative(path: Path): String = path match { case cloudNioPath: CloudNioPath => cloudNioPath.relativeDependentPath - case _ if !path.isAbsolute => path.normalize().toString - case _ => path.getRoot.relativize(path).normalize().toString + case _ if !path.isAbsolute => path.normalize().toString + case _ => path.getRoot.relativize(path).normalize().toString } - } } diff --git a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/IoUtil.scala b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/IoUtil.scala index 0ff44ce2dd0..f8c7921d0d8 100644 --- a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/IoUtil.scala +++ b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/IoUtil.scala @@ -11,24 +11,21 @@ object IoUtil { type ACC = IO[Either[NonEmptyList[Exception], A]] - def attemptHead(headIo: IO[A]): ACC = { + def attemptHead(headIo: IO[A]): ACC = attemptIo(NonEmptyList.one)(headIo) - } - def attemptAcc(accIo: ACC, nextIo: IO[A]): ACC = { + def attemptAcc(accIo: ACC, nextIo: IO[A]): ACC = accIo flatMap { - case Right(previousSuccess) => IO.pure(Right(previousSuccess)) + case Right(previousSuccess) => IO.pure(Right(previousSuccess)) case Left(previousExceptions) => attemptIo(_ :: previousExceptions)(nextIo) } - } - def attemptIo(f: Exception => NonEmptyList[Exception])(io: IO[A]): ACC = { + def attemptIo(f: Exception => NonEmptyList[Exception])(io: IO[A]): ACC = io.attempt flatMap { - case Right(success) => IO.pure(Right(success)) + case Right(success) => IO.pure(Right(success)) case Left(exception: Exception) => IO.pure(Left(f(exception))) - case Left(throwable) => throw throwable + case Left(throwable) => throw throwable } - } val res: ACC = tries.tail.foldLeft(attemptHead(tries.head))(attemptAcc) diff --git a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/TryWithResource.scala b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/TryWithResource.scala index 326a1821d84..f43ad24cb1f 100644 --- a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/TryWithResource.scala +++ b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/TryWithResource.scala @@ -20,17 +20,17 @@ object TryWithResource { case x: Throwable => t = Option(x) throw x - } finally { + } finally resource foreach { r => - try { + try r.close() - } catch { - case y: Throwable => t match { - case Some(_t) => _t.addSuppressed(y) - case None => throw y - } + catch { + case y: Throwable => + t match { + case Some(_t) => _t.addSuppressed(y) + case None => throw y + } } } - } } } diff --git a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/VersionUtil.scala b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/VersionUtil.scala index 019dbe81caf..c11651879fa 100644 --- a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/VersionUtil.scala +++ b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/VersionUtil.scala @@ -35,19 +35,17 @@ object VersionUtil { * @param default What to return when the version cannot be found. The parameter passed is the `projectName`. * @return The version from the conf or the default */ - def getVersion(projectName: String, default: String => String = defaultMessage): String = { + def getVersion(projectName: String, default: String => String = defaultMessage): String = ConfigFactory .load(versionConf(projectName)) .as[Option[String]](versionProperty(projectName)) .getOrElse(default(projectName)) - } /** * Instead of returning a version, states that the version conf will be generated by sbt. */ - def defaultMessage(projectName: String): String = { + def defaultMessage(projectName: String): String = s"${versionConf(projectName)}-to-be-generated-by-sbt" - } /** * A regex compatible with the dependency constants in project/Dependencies.scala. @@ -62,7 +60,7 @@ object VersionUtil { * @return The dependency version from project/Dependencies.scala * @throws RuntimeException If the dependency cannot be found */ - def sbtDependencyVersion(dependencyName: String)(projectName: String): String = { + def sbtDependencyVersion(dependencyName: String)(projectName: String): String = try { val dependencies = Paths.get("project/Dependencies.scala").toAbsolutePath val lines = Files.readAllLines(dependencies).asScala @@ -79,6 +77,5 @@ object VersionUtil { e ) } - } } diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala index 1aca98c9efd..df2cef13bde 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala @@ -46,16 +46,16 @@ import software.amazon.awssdk.regions.Region final case class AwsConfiguration private (applicationName: String, authsByName: Map[String, AwsAuthMode], - strRegion: Option[String]) { + strRegion: Option[String] +) { - def auth(name: String): ErrorOr[AwsAuthMode] = { + def auth(name: String): ErrorOr[AwsAuthMode] = authsByName.get(name) match { case None => val knownAuthNames = authsByName.keys.mkString(", ") s"`aws` configuration stanza does not contain an auth named '$name'. Known auth names: $knownAuthNames".invalidNel case Some(a) => a.validNel } - } def region: Option[Region] = strRegion.map(Region.of) } @@ -77,7 +77,7 @@ object AwsConfiguration { val awsConfig = config.getConfig("aws") - val appName = validate { awsConfig.as[String]("application-name") } + val appName = validate(awsConfig.as[String]("application-name")) val region: Option[String] = awsConfig.getAs[String]("region") @@ -88,13 +88,16 @@ object AwsConfiguration { (authConfig.getAs[String]("access-key"), authConfig.getAs[String]("secret-key")) match { case (Some(accessKey), Some(secretKey)) => CustomKeyMode(name, accessKey, secretKey, region) - case _ => throw new ConfigException.Generic(s"""Access key and/or secret """ + - s"""key missing for service account "$name". See reference.conf under the aws.auth, """ + - s"""custom key section for details of required configuration.""") + case _ => + throw new ConfigException.Generic( + s"""Access key and/or secret """ + + s"""key missing for service account "$name". See reference.conf under the aws.auth, """ + + s"""custom key section for details of required configuration.""" + ) } } - def defaultAuth(authConfig: Config, name: String, region: Option[String]): ErrorOr[AwsAuthMode] = validate { + def defaultAuth(authConfig: Config, name: String, region: Option[String]): ErrorOr[AwsAuthMode] = validate { DefaultMode(name, region) } diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala index b8c351c4185..7335441ea80 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala @@ -52,9 +52,8 @@ sealed trait AwsAuthMode { /** * Validate the auth mode against provided options */ - def validate(options: OptionLookup): Unit = { + def validate(options: OptionLookup): Unit = () - } /** * The name of the auth mode @@ -72,27 +71,30 @@ sealed trait AwsAuthMode { * All traits in this file are sealed, all classes final, meaning things * like Mockito or other java/scala overrides cannot work. */ - private[auth] var credentialValidation: (AwsCredentialsProvider, Option[String]) => Unit = - (provider: AwsCredentialsProvider, region: Option[String]) => { - val builder = StsClient.builder - - //If the region argument exists in config, set it in the builder. - //Otherwise it is left unset and the AwsCredentialsProvider will be responsible for sourcing a region - region.map(Region.of).foreach(builder.region) - - // make an essentially no-op call just to assure ourselves the credentials from our provider are valid - builder.credentialsProvider(provider) - .build - .getCallerIdentity(GetCallerIdentityRequest.builder.build) - () - } - - protected def validateCredential(provider: AwsCredentialsProvider, region: Option[String]) = { + private[auth] var credentialValidation: (AwsCredentialsProvider, Option[String]) => Unit = + (provider: AwsCredentialsProvider, region: Option[String]) => { + val builder = StsClient.builder + + // If the region argument exists in config, set it in the builder. + // Otherwise it is left unset and the AwsCredentialsProvider will be responsible for sourcing a region + region.map(Region.of).foreach(builder.region) + + // make an essentially no-op call just to assure ourselves the credentials from our provider are valid + builder + .credentialsProvider(provider) + .build + .getCallerIdentity(GetCallerIdentityRequest.builder.build) + () + } + + protected def validateCredential(provider: AwsCredentialsProvider, region: Option[String]) = Try(credentialValidation(provider, region)) match { - case Failure(ex) => throw new RuntimeException(s"Credentials produced by the AWS provider ${name} are invalid: ${ex.getMessage}", ex) + case Failure(ex) => + throw new RuntimeException(s"Credentials produced by the AWS provider ${name} are invalid: ${ex.getMessage}", + ex + ) case Success(_) => provider } - } } /** @@ -114,11 +116,8 @@ object CustomKeyMode * @param secretKey static AWS secret key * @param region an optional AWS region */ -final case class CustomKeyMode(override val name: String, - accessKey: String, - secretKey: String, - region: Option[String] - ) extends AwsAuthMode { +final case class CustomKeyMode(override val name: String, accessKey: String, secretKey: String, region: Option[String]) + extends AwsAuthMode { private lazy val _provider: AwsCredentialsProvider = { // make a provider locked to the given access and secret val p = StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey)) @@ -159,17 +158,17 @@ final case class DefaultMode(override val name: String, region: Option[String]) * @param region an optional AWS region */ final case class AssumeRoleMode(override val name: String, - baseAuthName: String, - roleArn: String, - externalId: String, - region: Option[String] - ) extends AwsAuthMode { + baseAuthName: String, + roleArn: String, + externalId: String, + region: Option[String] +) extends AwsAuthMode { private lazy val _provider: AwsCredentialsProvider = { // we need to perform operations on STS using the credentials provided from the baseAuthName val stsBuilder = StsClient.builder region.foreach(str => stsBuilder.region(Region.of(str))) - baseAuthObj match{ + baseAuthObj match { case Some(auth) => stsBuilder.credentialsProvider(auth.provider()) case _ => throw new RuntimeException(s"Base auth configuration required for assume role") } @@ -179,7 +178,7 @@ final case class AssumeRoleMode(override val name: String, .roleArn(roleArn) .durationSeconds(3600) .roleSessionName("cromwell") - if (! externalId.isEmpty) assumeRoleBuilder.externalId(externalId) + if (!externalId.isEmpty) assumeRoleBuilder.externalId(externalId) // this provider is one that will handle refreshing the assume-role creds when needed val p = StsAssumeRoleCredentialsProvider.builder @@ -195,23 +194,21 @@ final case class AssumeRoleMode(override val name: String, // start a background thread to perform the refresh override def provider(): AwsCredentialsProvider = _provider - private var baseAuthObj : Option[AwsAuthMode] = None + private var baseAuthObj: Option[AwsAuthMode] = None - def assign(baseAuth: AwsAuthMode) : Unit = { + def assign(baseAuth: AwsAuthMode): Unit = baseAuthObj match { case None => baseAuthObj = Some(baseAuth) case _ => throw new RuntimeException(s"Base auth object has already been assigned") } - } // We want to allow our tests access to the value // of the baseAuthObj - def baseAuthentication() : AwsAuthMode = { + def baseAuthentication(): AwsAuthMode = baseAuthObj match { case Some(o) => o case _ => throw new RuntimeException(s"Base auth object has not been set") } - } } class OptionLookupException(val key: String, cause: Throwable) extends RuntimeException(key, cause) diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala index 8ab07f37064..8caf8aa3fb1 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala @@ -60,13 +60,13 @@ object S3Storage { builder.build } - def s3Client(provider: AwsCredentialsProvider, region: Option[Region]): S3Client = { + def s3Client(provider: AwsCredentialsProvider, region: Option[Region]): S3Client = s3Client(s3Configuration(), provider, region) - } def s3Configuration(accelerateModeEnabled: Boolean = false, dualstackEnabled: Boolean = false, - pathStyleAccessEnabled: Boolean = false): S3Configuration = { + pathStyleAccessEnabled: Boolean = false + ): S3Configuration = { @nowarn("msg=method dualstackEnabled in trait Builder is deprecated") val builder = S3Configuration.builder diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala index 200b162c614..d3d66e1bafc 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala @@ -34,9 +34,11 @@ case object AzureCredentials { .authorityHost(azureProfile.getEnvironment.getActiveDirectoryEndpoint) def getAccessToken(identityClientId: Option[String] = None): ErrorOr[String] = { - val credentials = identityClientId.foldLeft(defaultCredentialBuilder) { - (builder, clientId) => builder.managedIdentityClientId(clientId) - }.build() + val credentials = identityClientId + .foldLeft(defaultCredentialBuilder) { (builder, clientId) => + builder.managedIdentityClientId(clientId) + } + .build() Try( credentials diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala index 09cf5f3869d..dd379ed3564 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala @@ -14,6 +14,7 @@ import scala.jdk.CollectionConverters.IterableHasAsScala import scala.util.{Failure, Success, Try} object AzureUtils { + /** * Generates a BlobContainerClient that can interact with the specified container. Authenticates using the local azure client running on the same machine. * @param blobContainer Name of the blob container. Looks something like "my-blob-container". @@ -21,10 +22,16 @@ object AzureUtils { * @param subscription Azure subscription. A globally unique identifier. If not provided, a default subscription will be used. * @return A blob container client capable of interacting with the specified container. */ - def buildContainerClientFromLocalEnvironment(blobContainer: String, azureEndpoint: String, subscription : Option[String]): Try[BlobContainerClient] = { + def buildContainerClientFromLocalEnvironment(blobContainer: String, + azureEndpoint: String, + subscription: Option[String] + ): Try[BlobContainerClient] = { def parseURI(string: String): Try[URI] = Try(URI.create(UrlEscapers.urlFragmentEscaper().escape(string))) - def parseStorageAccount(uri: URI): Try[String] = uri.getHost.split("\\.").find(_.nonEmpty) - .map(Success(_)).getOrElse(Failure(new Exception("Could not parse storage account"))) + def parseStorageAccount(uri: URI): Try[String] = uri.getHost + .split("\\.") + .find(_.nonEmpty) + .map(Success(_)) + .getOrElse(Failure(new Exception("Could not parse storage account"))) val azureProfile = new AzureProfile(AzureEnvironment.AZURE) @@ -32,29 +39,37 @@ object AzureUtils { .authorityHost(azureProfile.getEnvironment.getActiveDirectoryEndpoint) .build - def authenticateWithSubscription(sub: String) = AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withSubscription(sub) + def authenticateWithSubscription(sub: String) = + AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withSubscription(sub) - def authenticateWithDefaultSubscription = AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withDefaultSubscription() + def authenticateWithDefaultSubscription = + AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withDefaultSubscription() def azure = subscription.map(authenticateWithSubscription(_)).getOrElse(authenticateWithDefaultSubscription) - def findAzureStorageAccount(storageAccountName: String) = azure.storageAccounts.list.asScala.find(_.name.equals(storageAccountName)) - .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found."))) + def findAzureStorageAccount(storageAccountName: String) = azure.storageAccounts.list.asScala + .find(_.name.equals(storageAccountName)) + .map(Success(_)) + .getOrElse(Failure(new Exception("Azure Storage Account not found."))) - def buildBlobContainerClient(credential: StorageSharedKeyCredential, endpointURL: String, blobContainerName: String): BlobContainerClient = { + def buildBlobContainerClient(credential: StorageSharedKeyCredential, + endpointURL: String, + blobContainerName: String + ): BlobContainerClient = new BlobContainerClientBuilder() .credential(credential) .endpoint(endpointURL) .containerName(blobContainerName) .buildClient() - } def generateBlobContainerClient: Try[BlobContainerClient] = for { uri <- parseURI(azureEndpoint) configuredAccount <- parseStorageAccount(uri) azureAccount <- findAzureStorageAccount(configuredAccount) keys = azureAccount.getKeys.asScala - key <- keys.headOption.fold[Try[StorageAccountKey]](Failure(new Exception("Storage account has no keys")))(Success(_)) + key <- keys.headOption.fold[Try[StorageAccountKey]](Failure(new Exception("Storage account has no keys")))( + Success(_) + ) first = key.value sskc = new StorageSharedKeyCredential(configuredAccount, first) bcc = buildBlobContainerClient(sskc, azureEndpoint, blobContainer) diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/GoogleConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/GoogleConfiguration.scala index 2b4a183c121..999a9c2867a 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/GoogleConfiguration.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/GoogleConfiguration.scala @@ -18,14 +18,13 @@ import org.slf4j.LoggerFactory final case class GoogleConfiguration private (applicationName: String, authsByName: Map[String, GoogleAuthMode]) { - def auth(name: String): ErrorOr[GoogleAuthMode] = { + def auth(name: String): ErrorOr[GoogleAuthMode] = authsByName.get(name) match { case None => val knownAuthNames = authsByName.keys.mkString(", ") s"`google` configuration stanza does not contain an auth named '$name'. Known auth names: $knownAuthNames".invalidNel case Some(a) => a.validNel } - } } object GoogleConfiguration { @@ -37,7 +36,8 @@ object GoogleConfiguration { def withCustomTimeouts(httpRequestInitializer: HttpRequestInitializer, connectionTimeout: FiniteDuration = DefaultConnectionTimeout, - readTimeout: FiniteDuration = DefaultReadTimeout): HttpRequestInitializer = { + readTimeout: FiniteDuration = DefaultReadTimeout + ): HttpRequestInitializer = new HttpRequestInitializer() { @throws[IOException] override def initialize(httpRequest: HttpRequest): Unit = { @@ -47,7 +47,6 @@ object GoogleConfiguration { () } } - } private val log = LoggerFactory.getLogger("GoogleConfiguration") @@ -59,20 +58,27 @@ object GoogleConfiguration { val googleConfig = config.getConfig("google") - val appName = validate { googleConfig.as[String]("application-name") } + val appName = validate(googleConfig.as[String]("application-name")) def buildAuth(authConfig: Config): ErrorOr[GoogleAuthMode] = { def serviceAccountAuth(authConfig: Config, name: String): ErrorOr[GoogleAuthMode] = validate { (authConfig.getAs[String]("pem-file"), authConfig.getAs[String]("json-file")) match { - case (Some(pem), None) => ServiceAccountMode(name, PemFileFormat(authConfig.as[String]("service-account-id"), pem)) + case (Some(pem), None) => + ServiceAccountMode(name, PemFileFormat(authConfig.as[String]("service-account-id"), pem)) case (None, Some(json)) => ServiceAccountMode(name, JsonFileFormat(json)) - case (None, None) => throw new ConfigException.Generic(s"""No credential configuration was found for service account "$name". See reference.conf under the google.auth, service-account section for supported credential formats.""") - case (Some(_), Some(_)) => throw new ConfigException.Generic(s"""Both a pem file and a json file were supplied for service account "$name" in the configuration file. Only one credential file can be supplied for the same service account. Please choose between the two.""") + case (None, None) => + throw new ConfigException.Generic( + s"""No credential configuration was found for service account "$name". See reference.conf under the google.auth, service-account section for supported credential formats.""" + ) + case (Some(_), Some(_)) => + throw new ConfigException.Generic( + s"""Both a pem file and a json file were supplied for service account "$name" in the configuration file. Only one credential file can be supplied for the same service account. Please choose between the two.""" + ) } } - def userAccountAuth(authConfig: Config, name: String): ErrorOr[GoogleAuthMode] = validate { + def userAccountAuth(authConfig: Config, name: String): ErrorOr[GoogleAuthMode] = validate { UserMode(name, authConfig.as[String]("secrets-file")) } diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthMode.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthMode.scala index e850b53807a..52118303262 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthMode.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthMode.scala @@ -34,9 +34,8 @@ object GoogleAuthMode { type CredentialsValidation = Credentials => Unit private[auth] val NoCredentialsValidation = mouse.ignore _ - private def noOptionLookup(string: String): Nothing = { + private def noOptionLookup(string: String): Nothing = throw new UnsupportedOperationException(s"cannot lookup $string") - } lazy val jsonFactory: GsonFactory = GsonFactory.getDefaultInstance lazy val httpTransport: HttpTransport = GoogleNetHttpTransport.newTrustedTransport @@ -46,33 +45,29 @@ object GoogleAuthMode { val DockerCredentialsEncryptionKeyNameKey = "docker_credentials_key_name" val DockerCredentialsTokenKey = "docker_credentials_token" - def checkReadable(file: File): Unit = { + def checkReadable(file: File): Unit = if (!file.isReadable) throw new FileNotFoundException(s"File $file does not exist or is not readable") - } - def isFatal(ex: Throwable): Boolean = { + def isFatal(ex: Throwable): Boolean = ex match { case http: HttpResponseException => // Using HttpURLConnection fields as com.google.api.client.http.HttpStatusCodes doesn't have Bad Request (400) http.getStatusCode == HTTP_UNAUTHORIZED || - http.getStatusCode == HTTP_FORBIDDEN || - http.getStatusCode == HTTP_BAD_REQUEST + http.getStatusCode == HTTP_FORBIDDEN || + http.getStatusCode == HTTP_BAD_REQUEST case _: OptionLookupException => true case _ => false } - } - def extract(options: OptionLookup, key: String): String = { + def extract(options: OptionLookup, key: String): String = Try(options(key)) match { case Success(result) => result case Failure(throwable) => throw new OptionLookupException(key, throwable) } - } /** Used for both checking that the credential is valid and creating a fresh credential. */ - private def refreshCredentials(credentials: Credentials): Unit = { + private def refreshCredentials(credentials: Credentials): Unit = credentials.refresh() - } } sealed trait GoogleAuthMode extends LazyLogging { @@ -87,25 +82,22 @@ sealed trait GoogleAuthMode extends LazyLogging { * Alias for credentials(GoogleAuthMode.NoOptionLookup, scopes). * Only valid for credentials that are NOT externally provided, such as ApplicationDefault. */ - def credentials(scopes: Iterable[String]): OAuth2Credentials = { + def credentials(scopes: Iterable[String]): OAuth2Credentials = credentials(GoogleAuthMode.NoOptionLookup, scopes) - } /** * Alias for credentials(GoogleAuthMode.NoOptionLookup, Nil). * Only valid for credentials that are NOT externally provided and do not need scopes, such as ApplicationDefault. */ - private[auth] def credentials(): OAuth2Credentials = { + private[auth] def credentials(): OAuth2Credentials = credentials(GoogleAuthMode.NoOptionLookup, Nil) - } /** * Alias for credentials(options, Nil). * Only valid for credentials that are NOT externally provided and do not need scopes, such as ApplicationDefault. */ - private[auth] def credentials(options: OptionLookup): OAuth2Credentials = { + private[auth] def credentials(options: OptionLookup): OAuth2Credentials = credentials(options, Nil) - } /** * Enables swapping out credential validation for various testing purposes ONLY. @@ -116,7 +108,8 @@ sealed trait GoogleAuthMode extends LazyLogging { private[auth] var credentialsValidation: CredentialsValidation = refreshCredentials protected def validateCredentials[A <: GoogleCredentials](credential: A, - scopes: Iterable[String]): GoogleCredentials = { + scopes: Iterable[String] + ): GoogleCredentials = { val scopedCredentials = credential.createScoped(scopes.asJavaCollection) Try(credentialsValidation(scopedCredentials)) match { case Failure(ex) => throw new RuntimeException(s"Google credentials are invalid: ${ex.getMessage}", ex) @@ -126,9 +119,8 @@ sealed trait GoogleAuthMode extends LazyLogging { } case class MockAuthMode(override val name: String) extends GoogleAuthMode { - override def credentials(unusedOptions: OptionLookup, unusedScopes: Iterable[String]): NoCredentials = { + override def credentials(unusedOptions: OptionLookup, unusedScopes: Iterable[String]): NoCredentials = NoCredentials.getInstance - } } object ServiceAccountMode { @@ -143,35 +135,29 @@ object ServiceAccountMode { } -final case class ServiceAccountMode(override val name: String, - fileFormat: CredentialFileFormat) - extends GoogleAuthMode { +final case class ServiceAccountMode(override val name: String, fileFormat: CredentialFileFormat) + extends GoogleAuthMode { private val credentialsFile = File(fileFormat.file) checkReadable(credentialsFile) - private lazy val serviceAccountCredentials: ServiceAccountCredentials = { + private lazy val serviceAccountCredentials: ServiceAccountCredentials = fileFormat match { case PemFileFormat(accountId, _) => logger.warn("The PEM file format will be deprecated in the upcoming Cromwell version. Please use JSON instead.") ServiceAccountCredentials.fromPkcs8(accountId, accountId, credentialsFile.contentAsString, null, null) case _: JsonFileFormat => ServiceAccountCredentials.fromStream(credentialsFile.newInputStream) } - } - override def credentials(unusedOptions: OptionLookup, - scopes: Iterable[String]): GoogleCredentials = { + override def credentials(unusedOptions: OptionLookup, scopes: Iterable[String]): GoogleCredentials = validateCredentials(serviceAccountCredentials, scopes) - } } final case class UserServiceAccountMode(override val name: String) extends GoogleAuthMode { - private def extractServiceAccount(options: OptionLookup): String = { + private def extractServiceAccount(options: OptionLookup): String = extract(options, UserServiceAccountKey) - } - private def credentialStream(options: OptionLookup): InputStream = { + private def credentialStream(options: OptionLookup): InputStream = new ByteArrayInputStream(extractServiceAccount(options).getBytes(StandardCharsets.UTF_8)) - } override def credentials(options: OptionLookup, scopes: Iterable[String]): GoogleCredentials = { val newCredentials = ServiceAccountCredentials.fromStream(credentialStream(options)) @@ -179,7 +165,6 @@ final case class UserServiceAccountMode(override val name: String) extends Googl } } - final case class UserMode(override val name: String, secretsPath: String) extends GoogleAuthMode { private lazy val secretsStream = { @@ -190,9 +175,8 @@ final case class UserMode(override val name: String, secretsPath: String) extend private lazy val userCredentials: UserCredentials = UserCredentials.fromStream(secretsStream) - override def credentials(unusedOptions: OptionLookup, scopes: Iterable[String]): GoogleCredentials = { + override def credentials(unusedOptions: OptionLookup, scopes: Iterable[String]): GoogleCredentials = validateCredentials(userCredentials, scopes) - } } object ApplicationDefaultMode { @@ -200,10 +184,8 @@ object ApplicationDefaultMode { } final case class ApplicationDefaultMode(name: String) extends GoogleAuthMode { - override def credentials(unusedOptions: OptionLookup, - scopes: Iterable[String]): GoogleCredentials = { + override def credentials(unusedOptions: OptionLookup, scopes: Iterable[String]): GoogleCredentials = validateCredentials(applicationDefaultCredentials, scopes) - } } sealed trait ClientSecrets { diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/gcs/GcsStorage.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/gcs/GcsStorage.scala index 29ed842377d..6ae6e88542c 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/gcs/GcsStorage.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/gcs/GcsStorage.scala @@ -18,9 +18,11 @@ object GcsStorage { val HttpTransport = GoogleNetHttpTransport.newTrustedTransport val DefaultCloudStorageConfiguration = { - val UploadBufferBytes = ConfigFactory.load().as[Option[Int]]("google.upload-buffer-bytes").getOrElse(MediaHttpUploader.MINIMUM_CHUNK_SIZE) + val UploadBufferBytes = + ConfigFactory.load().as[Option[Int]]("google.upload-buffer-bytes").getOrElse(MediaHttpUploader.MINIMUM_CHUNK_SIZE) - CloudStorageConfiguration.builder() + CloudStorageConfiguration + .builder() .blockSize(UploadBufferBytes) .permitEmptyPathComponents(true) .stripPrefixSlash(true) @@ -28,23 +30,24 @@ object GcsStorage { .build() } - def gcsStorage(applicationName: String, - storageOptions: StorageOptions): Storage = { - new Storage.Builder(HttpTransport, + def gcsStorage(applicationName: String, storageOptions: StorageOptions): Storage = + new Storage.Builder( + HttpTransport, JsonFactory, - GoogleConfiguration.withCustomTimeouts(TransportOptions.getHttpRequestInitializer(storageOptions))) + GoogleConfiguration.withCustomTimeouts(TransportOptions.getHttpRequestInitializer(storageOptions)) + ) .setApplicationName(applicationName) .build() - } - def gcsStorage(applicationName: String, credentials: Credentials, retrySettings: RetrySettings): Storage = { + def gcsStorage(applicationName: String, credentials: Credentials, retrySettings: RetrySettings): Storage = gcsStorage(applicationName, gcsStorageOptions(credentials, retrySettings)) - } def gcsStorageOptions(credentials: Credentials, retrySettings: RetrySettings, - project: Option[String] = None): StorageOptions = { - val storageOptionsBuilder = StorageOptions.newBuilder() + project: Option[String] = None + ): StorageOptions = { + val storageOptionsBuilder = StorageOptions + .newBuilder() .setTransportOptions(TransportOptions) .setCredentials(credentials) diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/http/GoogleHttpTransportOptions.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/http/GoogleHttpTransportOptions.scala index 13faef01e0c..f328756cf67 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/http/GoogleHttpTransportOptions.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/http/GoogleHttpTransportOptions.scala @@ -4,7 +4,8 @@ import com.google.cloud.http.HttpTransportOptions import scala.concurrent.duration._ object GoogleHttpTransportOptions { - val TransportOptions = HttpTransportOptions.newBuilder() + val TransportOptions = HttpTransportOptions + .newBuilder() .setReadTimeout(3.minutes.toMillis.toInt) .build() } diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/AwsConfigurationSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/AwsConfigurationSpec.scala index 6ead310c925..27c66305466 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/AwsConfigurationSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/AwsConfigurationSpec.scala @@ -39,7 +39,6 @@ import cromwell.cloudsupport.aws.auth.{AssumeRoleMode, CustomKeyMode, DefaultMod import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class AwsConfigurationSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "AwsConfiguration" @@ -47,38 +46,38 @@ class AwsConfigurationSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat it should "parse all manner of well-formed auths" in { val righteousAwsConfig = s""" - |aws { - | application-name = "cromwell" - | - | auths = [ - | { - | name = "default" - | scheme = "default" - | }, - | { - | name = "custom-keys" - | scheme = "custom_keys" - | access-key = "access_key_id" - | secret-key = "secret_key" - | }, - | { - | name = "assume-role-based-on-another-with-external" - | scheme = "assume_role" - | base-auth = "default" - | role-arn = "my-role-arn" - | external-id = "my-external-id" - | }, - | { - | name = "assume-role-based-on-another" - | scheme = "assume_role" - | base-auth = "default" - | role-arn = "my-role-arn" - | } - | ] - | - | region = "region" - |} - | + |aws { + | application-name = "cromwell" + | + | auths = [ + | { + | name = "default" + | scheme = "default" + | }, + | { + | name = "custom-keys" + | scheme = "custom_keys" + | access-key = "access_key_id" + | secret-key = "secret_key" + | }, + | { + | name = "assume-role-based-on-another-with-external" + | scheme = "assume_role" + | base-auth = "default" + | role-arn = "my-role-arn" + | external-id = "my-external-id" + | }, + | { + | name = "assume-role-based-on-another" + | scheme = "assume_role" + | base-auth = "default" + | role-arn = "my-role-arn" + | } + | ] + | + | region = "region" + |} + | """.stripMargin val conf = AwsConfiguration(ConfigFactory.parseString(righteousAwsConfig)) @@ -146,8 +145,8 @@ class AwsConfigurationSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat val conf = AwsConfiguration(ConfigFactory.parseString(config)) conf.auth("name-botched") should be( - "`aws` configuration stanza does not contain an auth named 'name-botched'. Known auth names: name-default" - .invalidNel) + "`aws` configuration stanza does not contain an auth named 'name-botched'. Known auth names: name-default".invalidNel + ) } it should "not parse a configuration stanza without applicationName" in { diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala index 5311714e89c..e50b24d928f 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala @@ -36,7 +36,6 @@ import org.scalatest.matchers.should.Matchers import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider import software.amazon.awssdk.regions.Region - class S3StorageSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "S3Storage" diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/GoogleConfigurationSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/GoogleConfigurationSpec.scala index 95a2380034a..7ed9162a6d1 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/GoogleConfigurationSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/GoogleConfigurationSpec.scala @@ -14,7 +14,6 @@ import cromwell.cloudsupport.gcp.auth._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class GoogleConfigurationSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "GoogleConfiguration" @@ -25,39 +24,39 @@ class GoogleConfigurationSpec extends AnyFlatSpec with CromwellTimeoutSpec with val righteousGoogleConfig = s""" - |google { - | application-name = "cromwell" - | - | auths = [ - | { - | name = "name-default" - | scheme = "application_default" - | }, - | { - | name = "name-user" - | scheme = "user_account" - | user = "me" - | secrets-file = "${pemMockFile.pathAsString}" - | data-store-dir = "/where/the/data/at" - | }, - | { - | name = "name-pem-service" - | scheme = "service_account" - | service-account-id = "my-google-account" - | pem-file = "${pemMockFile.pathAsString}" - | }, - | { - | name = "name-json-service" - | scheme = "service_account" - | json-file = "${jsonMockFile.pathAsString}" - | }, - | { - | name = "name-user-service-account" - | scheme = "user_service_account" - | } - | ] - |} - | + |google { + | application-name = "cromwell" + | + | auths = [ + | { + | name = "name-default" + | scheme = "application_default" + | }, + | { + | name = "name-user" + | scheme = "user_account" + | user = "me" + | secrets-file = "${pemMockFile.pathAsString}" + | data-store-dir = "/where/the/data/at" + | }, + | { + | name = "name-pem-service" + | scheme = "service_account" + | service-account-id = "my-google-account" + | pem-file = "${pemMockFile.pathAsString}" + | }, + | { + | name = "name-json-service" + | scheme = "service_account" + | json-file = "${jsonMockFile.pathAsString}" + | }, + | { + | name = "name-user-service-account" + | scheme = "user_service_account" + | } + | ] + |} + | """.stripMargin val gconf = GoogleConfiguration(ConfigFactory.parseString(righteousGoogleConfig)) @@ -125,16 +124,16 @@ class GoogleConfigurationSpec extends AnyFlatSpec with CromwellTimeoutSpec with val googleConfiguration = GoogleConfiguration(ConfigFactory.parseString(config)) googleConfiguration.auth("name-botched") should be( - "`google` configuration stanza does not contain an auth named 'name-botched'. Known auth names: name-default" - .invalidNel) + "`google` configuration stanza does not contain an auth named 'name-botched'. Known auth names: name-default".invalidNel + ) } it should "create an initializer with custom timeouts" in { val transport = new MockHttpTransport() - val initializer = GoogleConfiguration.withCustomTimeouts(request => { + val initializer = GoogleConfiguration.withCustomTimeouts { request => request.getHeaders.set("custom_init", "ok") () - }) + } val factory = transport.createRequestFactory(initializer) val request = factory.buildGetRequest(new GenericUrl(new URL("http://example.com"))) request.getConnectTimeout should be(180000) diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ApplicationDefaultModeSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ApplicationDefaultModeSpec.scala index bb661ce9742..dde1f82a3a5 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ApplicationDefaultModeSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ApplicationDefaultModeSpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class ApplicationDefaultModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "ApplicationDefaultMode" diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthModeSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthModeSpec.scala index 368ce5d2472..bf77f65850d 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthModeSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthModeSpec.scala @@ -10,14 +10,12 @@ import org.scalatest.prop.TableDrivenPropertyChecks import scala.util.{Failure, Try} - class GoogleAuthModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with TableDrivenPropertyChecks { behavior of "GoogleAuthMode" - private def mockHttpResponseException(statusCode: Int): HttpResponseException = { + private def mockHttpResponseException(statusCode: Int): HttpResponseException = new HttpResponseException.Builder(statusCode, "mock message", new HttpHeaders).build() - } private val testedExceptions = Table( ("description", "exception", "isFatal"), @@ -55,14 +53,13 @@ object GoogleAuthModeSpec extends ServiceAccountTestSupport { () } - lazy val userCredentialsContents: String = { + lazy val userCredentialsContents: String = toJson( "type" -> "authorized_user", "client_id" -> "the_id", "client_secret" -> "the_secret", "refresh_token" -> "the_token" ) - } lazy val refreshTokenOptions: OptionLookup = Map("refresh_token" -> "the_refresh_token") lazy val userServiceAccountOptions: OptionLookup = Map("user_service_account_json" -> serviceAccountJsonContents) diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/MockAuthModeSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/MockAuthModeSpec.scala index 591da3354a7..b0c89e368c5 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/MockAuthModeSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/MockAuthModeSpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class MockAuthModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "MockAuthMode" diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountModeSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountModeSpec.scala index 94831b1410b..b10a823a6ca 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountModeSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountModeSpec.scala @@ -30,7 +30,7 @@ class ServiceAccountModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with M .write(serviceAccountPemContents) val serviceAccountMode = ServiceAccountMode( "service-account", - ServiceAccountMode.PemFileFormat("the_account_id", pemMockFile.pathAsString), + ServiceAccountMode.PemFileFormat("the_account_id", pemMockFile.pathAsString) ) val exception = intercept[RuntimeException](serviceAccountMode.credentials()) exception.getMessage should startWith("Google credentials are invalid: ") @@ -53,7 +53,7 @@ class ServiceAccountModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with M val exception = intercept[FileNotFoundException] { ServiceAccountMode( "service-account", - ServiceAccountMode.PemFileFormat("the_account_id", pemMockFile.pathAsString), + ServiceAccountMode.PemFileFormat("the_account_id", pemMockFile.pathAsString) ) } exception.getMessage should fullyMatch regex "File .*/service-account..*.pem does not exist or is not readable" @@ -79,7 +79,7 @@ class ServiceAccountModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with M .write(serviceAccountPemContents) val serviceAccountMode = ServiceAccountMode( "service-account", - ServiceAccountMode.PemFileFormat("the_account_id", pemMockFile.pathAsString), + ServiceAccountMode.PemFileFormat("the_account_id", pemMockFile.pathAsString) ) serviceAccountMode.credentialsValidation = GoogleAuthMode.NoCredentialsValidation val credentials = serviceAccountMode.credentials() diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountTestSupport.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountTestSupport.scala index 1624674b875..a87df58e21d 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountTestSupport.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountTestSupport.scala @@ -23,7 +23,7 @@ trait ServiceAccountTestSupport { // Hide me from git secrets false positives private val theStringThatShallNotBeNamed = List("private", "key").mkString("_") - def serviceAccountJsonContents: String = { + def serviceAccountJsonContents: String = toJson( "type" -> "service_account", "client_id" -> "the_account_id", @@ -31,7 +31,6 @@ trait ServiceAccountTestSupport { theStringThatShallNotBeNamed -> serviceAccountPemContents, s"${theStringThatShallNotBeNamed}_id" -> "the_key_id" ) - } def toJson(contents: (String, String)*): String = { // Generator doesn't matter as long as it generates JSON. Using `jsonFactory` to get an extra line hit of coverage. @@ -40,10 +39,9 @@ trait ServiceAccountTestSupport { val generator = factory.createJsonGenerator(writer) generator.enablePrettyPrint() generator.writeStartObject() - contents foreach { - case (key, value) => - generator.writeFieldName(key) - generator.writeString(value) + contents foreach { case (key, value) => + generator.writeFieldName(key) + generator.writeString(value) } generator.writeEndObject() generator.close() diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserModeSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserModeSpec.scala index cfc22b0ca9b..95a64ee1e6c 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserModeSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserModeSpec.scala @@ -7,7 +7,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class UserModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "UserMode" diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserServiceAccountModeSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserServiceAccountModeSpec.scala index 70a1f470d32..092072f7511 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserServiceAccountModeSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserServiceAccountModeSpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class UserServiceAccountModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "UserServiceAccountMode" diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/gcs/GcsStorageSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/gcs/GcsStorageSpec.scala index 558c9b74c4a..1f673acec67 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/gcs/GcsStorageSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/gcs/GcsStorageSpec.scala @@ -6,7 +6,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class GcsStorageSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "GcsStorage" @@ -19,7 +18,8 @@ class GcsStorageSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers } it should "build gcs storage" in { - val configuration = GcsStorage.gcsStorage("gcs-storage-spec", NoCredentials.getInstance(), RetrySettings.newBuilder().build()) + val configuration = + GcsStorage.gcsStorage("gcs-storage-spec", NoCredentials.getInstance(), RetrySettings.newBuilder().build()) configuration.getApplicationName should be("gcs-storage-spec") } diff --git a/codegen_java/project/Artifactory.scala b/codegen_java/project/Artifactory.scala index a13c9cc8c21..6d385ffbbc7 100644 --- a/codegen_java/project/Artifactory.scala +++ b/codegen_java/project/Artifactory.scala @@ -1,4 +1,4 @@ object Artifactory { val artifactoryHost = "broadinstitute.jfrog.io" val artifactory = s"https://$artifactoryHost/broadinstitute/" -} \ No newline at end of file +} diff --git a/codegen_java/project/Publishing.scala b/codegen_java/project/Publishing.scala index 6c3de9881fc..880cf40f1ff 100644 --- a/codegen_java/project/Publishing.scala +++ b/codegen_java/project/Publishing.scala @@ -2,10 +2,10 @@ import sbt.Keys._ import sbt._ import Artifactory._ - object Publishing { +object Publishing { private val buildTimestamp = System.currentTimeMillis() / 1000 - private def artifactoryResolver(isSnapshot: Boolean): Resolver = { + private def artifactoryResolver(isSnapshot: Boolean): Resolver = { val repoType = if (isSnapshot) "snapshot" else "release" val repoUrl = s"${artifactory}libs-$repoType-local;build.timestamp=$buildTimestamp" @@ -13,15 +13,15 @@ import Artifactory._ repoName at repoUrl } - private val artifactoryCredentials: Credentials = { + private val artifactoryCredentials: Credentials = { val username = sys.env.getOrElse("ARTIFACTORY_USERNAME", "") val password = sys.env.getOrElse("ARTIFACTORY_PASSWORD", "") Credentials("Artifactory Realm", artifactoryHost, username, password) } - val publishSettings: Seq[Setting[_]] = - //we only publish to libs-release-local because of a bug in sbt that makes snapshots take - //priority over the local package cache. see here: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 + val publishSettings: Seq[Setting[_]] = + // we only publish to libs-release-local because of a bug in sbt that makes snapshots take + // priority over the local package cache. see here: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 Seq( publishTo := Option(artifactoryResolver(false)), Compile / publishArtifact := true, @@ -29,9 +29,9 @@ import Artifactory._ credentials += artifactoryCredentials ) - val noPublishSettings: Seq[Setting[_]] = + val noPublishSettings: Seq[Setting[_]] = Seq( publish := {}, publishLocal := {} ) -} \ No newline at end of file +} diff --git a/codegen_java/project/Version.scala b/codegen_java/project/Version.scala index 875a80e2e10..be7c57bc527 100644 --- a/codegen_java/project/Version.scala +++ b/codegen_java/project/Version.scala @@ -1,20 +1,20 @@ import scala.sys.process._ - object Version { +object Version { - def createVersion(baseVersion: String) = { - def getLastCommitFromGit = { s"""git rev-parse --short HEAD""" !! } + def createVersion(baseVersion: String) = { + def getLastCommitFromGit = s"""git rev-parse --short HEAD""" !! - // either specify git hash as an env var or derive it + // either specify git hash as an env var or derive it // if building from the broadinstitute/scala-baseimage docker image use env var // (scala-baseimage doesn't have git in it) - val lastCommit = sys.env.getOrElse("GIT_HASH", getLastCommitFromGit ).trim() + val lastCommit = sys.env.getOrElse("GIT_HASH", getLastCommitFromGit).trim() val version = baseVersion + "-" + lastCommit - // The project isSnapshot string passed in via command line settings, if desired. + // The project isSnapshot string passed in via command line settings, if desired. val isSnapshot = sys.props.getOrElse("project.isSnapshot", "true").toBoolean - // For now, obfuscate SNAPSHOTs from sbt's developers: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 + // For now, obfuscate SNAPSHOTs from sbt's developers: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 if (isSnapshot) s"$version-SNAP" else version } -} \ No newline at end of file +} diff --git a/common/src/main/scala/common/collections/EnhancedCollections.scala b/common/src/main/scala/common/collections/EnhancedCollections.scala index beede1f9510..e1eb983029c 100644 --- a/common/src/main/scala/common/collections/EnhancedCollections.scala +++ b/common/src/main/scala/common/collections/EnhancedCollections.scala @@ -15,7 +15,10 @@ object EnhancedCollections { * After trying and failing to do this myself, I got this to work by copying the answer from here: * https://stackoverflow.com/questions/29886246/scala-filter-by-type */ - implicit class EnhancedIterableOps[T2, Repr[x] <: IterableOps[x, Repr, Repr[x]]](val iterableOps: IterableOps[T2, Repr, Repr[T2]]) extends AnyVal { + implicit class EnhancedIterableOps[T2, Repr[x] <: IterableOps[x, Repr, Repr[x]]]( + val iterableOps: IterableOps[T2, Repr, Repr[T2]] + ) extends AnyVal { + /** * Lets you filter a collection by type. * @@ -58,27 +61,29 @@ object EnhancedCollections { def takeWhileWeighted[W](maxWeight: W, weightFunction: A => W, maxHeadLength: Option[Int], - strict: Boolean = false) - (implicit n: Numeric[W], c: Ordering[W]): DeQueued[A] = { + strict: Boolean = false + )(implicit n: Numeric[W], c: Ordering[W]): DeQueued[A] = { import n._ @tailrec - def takeWhileWeightedRec(tail: Queue[A], head: Vector[A], weight: W): (Vector[A], Queue[A]) = { + def takeWhileWeightedRec(tail: Queue[A], head: Vector[A], weight: W): (Vector[A], Queue[A]) = // Stay under maxHeadLength if it's specified if (maxHeadLength.exists(head.size >= _)) head -> tail - else tail.dequeueOption - .map({ - // Compute the dequeued element's weight - case (element, dequeued) => (element, weightFunction(element), dequeued) - }) match { - // If the element's weight is > maxWeight and strict is true, drop the element - case Some((_, elementWeight, dequeued)) if c.gteq(elementWeight, maxWeight) && strict => takeWhileWeightedRec(dequeued, head, weight) - // If we're under the max weight, add the element to the head and recurse - case Some((element, elementWeight, dequeued)) if c.lteq(elementWeight + weight, maxWeight) => takeWhileWeightedRec(dequeued, head :+ element, weight + elementWeight) - // Otherwise stop here (make sure to return the original queue so we don't lose the last dequeued element) - case _ => head -> tail - } - } + else + tail.dequeueOption + .map { + // Compute the dequeued element's weight + case (element, dequeued) => (element, weightFunction(element), dequeued) + } match { + // If the element's weight is > maxWeight and strict is true, drop the element + case Some((_, elementWeight, dequeued)) if c.gteq(elementWeight, maxWeight) && strict => + takeWhileWeightedRec(dequeued, head, weight) + // If we're under the max weight, add the element to the head and recurse + case Some((element, elementWeight, dequeued)) if c.lteq(elementWeight + weight, maxWeight) => + takeWhileWeightedRec(dequeued, head :+ element, weight + elementWeight) + // Otherwise stop here (make sure to return the original queue so we don't lose the last dequeued element) + case _ => head -> tail + } if (queue.isEmpty || maxHeadLength.contains(0)) DeQueued(Vector.empty, queue) // If strict is enabled, we should never return a head with a weight > maxWeight. So start from the original queue and drop elements over maxWeight if necessary @@ -88,13 +93,17 @@ object EnhancedCollections { } // Otherwise to ensure we don't deadlock, start the recursion with the head of the queue, this way even if it's over maxWeight it'll return a single element head else { - val (head, tail) = takeWhileWeightedRec(queue.tail, queue.headOption.toVector, queue.headOption.map(weightFunction).getOrElse(n.zero)) + val (head, tail) = takeWhileWeightedRec(queue.tail, + queue.headOption.toVector, + queue.headOption.map(weightFunction).getOrElse(n.zero) + ) DeQueued(head, tail) } } } implicit class EnhancedMapLike[A, +B, +This <: Map[A, B]](val mapLike: Map[A, B]) { + /** * 'safe' in that unlike the implementation hiding behind `MapLike#mapValues` this is strict. i.e. this will only * evaluate the supplied function once on each value and at the time this method is called. @@ -104,17 +113,15 @@ object EnhancedCollections { /** * Based on scalaz's intersectWith, applies `f` to values of keys found in this `mapLike` and map */ - def intersectWith[C, D](map: Map[A, C])(f: (B, C) => D): Map[A, D] = { + def intersectWith[C, D](map: Map[A, C])(f: (B, C) => D): Map[A, D] = mapLike collect { case (mapLikeKey, mapLikeValue) if map.contains(mapLikeKey) => mapLikeKey -> f(mapLikeValue, map(mapLikeKey)) } - } } implicit class EnhancedNonEmptyList[A](val nel: NonEmptyList[A]) extends AnyVal { - def foreach(f: A => Unit): Unit = { + def foreach(f: A => Unit): Unit = nel.toList foreach f - } } } diff --git a/common/src/main/scala/common/collections/Table.scala b/common/src/main/scala/common/collections/Table.scala index 57c1b3f4a57..3f8b262bfb1 100644 --- a/common/src/main/scala/common/collections/Table.scala +++ b/common/src/main/scala/common/collections/Table.scala @@ -3,6 +3,7 @@ package common.collections import scala.collection.immutable object Table { + /** * Instantiates an empty table */ @@ -22,7 +23,7 @@ object Table { * @tparam V type of the value */ case class Table[R, C, V](table: Map[R, Map[C, V]]) { - + /** * Returns true if the table contains a value at row / column */ @@ -51,18 +52,16 @@ case class Table[R, C, V](table: Map[R, Map[C, V]]) { /** * Add a value at row / column */ - def add(row: R, column: C, value: V): Table[R, C, V] = { + def add(row: R, column: C, value: V): Table[R, C, V] = this.copy( table = table.updated(row, table.getOrElse(row, Map.empty).updated(column, value)) ) - } /** * Add all values */ - def addAll(values: Iterable[(R, C, V)]): Table[R, C, V] = { + def addAll(values: Iterable[(R, C, V)]): Table[R, C, V] = values.foldLeft(this)(_.addTriplet(_)) - } /** * Add a value as a triplet diff --git a/common/src/main/scala/common/collections/WeightedQueue.scala b/common/src/main/scala/common/collections/WeightedQueue.scala index 1ba5d869a64..fecb857db9b 100644 --- a/common/src/main/scala/common/collections/WeightedQueue.scala +++ b/common/src/main/scala/common/collections/WeightedQueue.scala @@ -4,9 +4,8 @@ import common.collections.EnhancedCollections._ import scala.collection.immutable.Queue object WeightedQueue { - def empty[T, W](weightFunction: T => W)(implicit n: Numeric[W]) = { + def empty[T, W](weightFunction: T => W)(implicit n: Numeric[W]) = WeightedQueue(Queue.empty[T], weightFunction, n.zero) - } } /** @@ -14,27 +13,24 @@ object WeightedQueue { * In addition to the queue, a weight function is provided that provides the weight W of an element T. * The total weight of the queue is accessible, as well as a method to take the head of the queue based on a max weight value. */ -final case class WeightedQueue[T, W](innerQueue: Queue[T], - private val weightFunction: T => W, - weight: W)(implicit n: Numeric[W]) { +final case class WeightedQueue[T, W](innerQueue: Queue[T], private val weightFunction: T => W, weight: W)(implicit + n: Numeric[W] +) { import n._ - def enqueue(element: T): WeightedQueue[T, W] = { + def enqueue(element: T): WeightedQueue[T, W] = this.copy(innerQueue = innerQueue.enqueue(element), weight = weight + weightFunction(element)) - } def dequeue: (T, WeightedQueue[T, W]) = { val (element, tail) = innerQueue.dequeue element -> this.copy(innerQueue = tail, weight = weight - weightFunction(element)) } - def dequeueOption: Option[(T, WeightedQueue[T, W])] = { - innerQueue.dequeueOption map { - case (element, tail) => - element -> this.copy(innerQueue = tail, weight = weight - weightFunction(element)) + def dequeueOption: Option[(T, WeightedQueue[T, W])] = + innerQueue.dequeueOption map { case (element, tail) => + element -> this.copy(innerQueue = tail, weight = weight - weightFunction(element)) } - } - + def behead(maxWeight: W, maxLength: Option[Int] = None, strict: Boolean = false): (Vector[T], WeightedQueue[T, W]) = { val DeQueued(head, tail) = innerQueue.takeWhileWeighted(maxWeight, weightFunction, maxLength, strict) head -> this.copy(innerQueue = tail, weight = weight - head.map(weightFunction).sum) diff --git a/common/src/main/scala/common/exception/ExceptionAggregation.scala b/common/src/main/scala/common/exception/ExceptionAggregation.scala index bd7f030331c..94059e94f50 100644 --- a/common/src/main/scala/common/exception/ExceptionAggregation.scala +++ b/common/src/main/scala/common/exception/ExceptionAggregation.scala @@ -8,12 +8,11 @@ import common.exception.Aggregation._ import scala.annotation.tailrec object Aggregation { - def formatMessageWithList(message: String, list: Iterable[String]) = { + def formatMessageWithList(message: String, list: Iterable[String]) = if (list.nonEmpty) { val messages = s"\n${list.mkString("\n")}" s"$message:$messages" } else message - } def flattenThrowable(throwable: Throwable) = { @tailrec @@ -57,12 +56,12 @@ trait ThrowableAggregation extends MessageAggregation { override def errorMessages = throwables map buildMessage private def buildMessage(t: Throwable): String = t match { - // The message for file not found exception only contains the file name, so add the actual reason + // The message for file not found exception only contains the file name, so add the actual reason case _: FileNotFoundException | _: NoSuchFileException => s"File not found ${t.getMessage}" case aggregation: ThrowableAggregation => formatMessageWithList(aggregation.exceptionContext, aggregation.throwables.map(buildMessage).map("\t" + _)) case other => - val cause = Option(other.getCause) map { c => s"\n\t${buildMessage(c)}" } getOrElse "" + val cause = Option(other.getCause) map { c => s"\n\t${buildMessage(c)}" } getOrElse "" s"${other.getMessage}$cause" } } @@ -70,6 +69,14 @@ trait ThrowableAggregation extends MessageAggregation { /** * Generic convenience case class for aggregated exceptions. */ -case class AggregatedException(exceptionContext: String, throwables: Iterable[Throwable]) extends Exception with ThrowableAggregation -case class AggregatedMessageException(exceptionContext: String, errorMessages: Iterable[String]) extends Exception with MessageAggregation -case class CompositeException(exceptionContext: String, throwables: Iterable[Throwable], override val errorMessages: Iterable[String]) extends Exception with ThrowableAggregation +case class AggregatedException(exceptionContext: String, throwables: Iterable[Throwable]) + extends Exception + with ThrowableAggregation +case class AggregatedMessageException(exceptionContext: String, errorMessages: Iterable[String]) + extends Exception + with MessageAggregation +case class CompositeException(exceptionContext: String, + throwables: Iterable[Throwable], + override val errorMessages: Iterable[String] +) extends Exception + with ThrowableAggregation diff --git a/common/src/main/scala/common/exception/package.scala b/common/src/main/scala/common/exception/package.scala index dccd49f2643..48d3dd43fb3 100644 --- a/common/src/main/scala/common/exception/package.scala +++ b/common/src/main/scala/common/exception/package.scala @@ -4,7 +4,6 @@ import cats.effect.IO package object exception { - def toIO[A](option: Option[A], errorMsg: String): IO[A] = { + def toIO[A](option: Option[A], errorMsg: String): IO[A] = IO.fromEither(option.toRight(new RuntimeException(errorMsg))) - } } diff --git a/common/src/main/scala/common/numeric/IntegerUtil.scala b/common/src/main/scala/common/numeric/IntegerUtil.scala index 4c33c5ace64..e24e478d400 100644 --- a/common/src/main/scala/common/numeric/IntegerUtil.scala +++ b/common/src/main/scala/common/numeric/IntegerUtil.scala @@ -2,14 +2,13 @@ package common.numeric object IntegerUtil { - private def ordinal(int: Int): String = { + private def ordinal(int: Int): String = int match { case 1 => "st" case 2 => "nd" case 3 => "rd" case _ => "th" } - } implicit class IntEnhanced(val value: Int) extends AnyVal { def toOrdinal: String = value match { @@ -19,9 +18,8 @@ object IntegerUtil { s"$v$suffix" } - def isBetweenInclusive(min: Int, max: Int): Boolean = { + def isBetweenInclusive(min: Int, max: Int): Boolean = min <= value && value <= max - } } } diff --git a/common/src/main/scala/common/transforms/package.scala b/common/src/main/scala/common/transforms/package.scala index 028f426d54a..112fb22d682 100644 --- a/common/src/main/scala/common/transforms/package.scala +++ b/common/src/main/scala/common/transforms/package.scala @@ -12,30 +12,42 @@ package object transforms { object CheckedAtoB { def apply[A, B](implicit runner: CheckedAtoB[A, B]): CheckedAtoB[A, B] = runner def fromCheck[A, B](run: A => Checked[B]): CheckedAtoB[A, B] = Kleisli(run) - def fromCheck[A, B](context: String)(run: A => Checked[B]): CheckedAtoB[A, B] = Kleisli(runCheckWithContext(run, _ => context)) - def fromCheck[A, B](context: A => String)(run: A => Checked[B]): CheckedAtoB[A, B] = Kleisli(runCheckWithContext(run, context)) + def fromCheck[A, B](context: String)(run: A => Checked[B]): CheckedAtoB[A, B] = Kleisli( + runCheckWithContext(run, _ => context) + ) + def fromCheck[A, B](context: A => String)(run: A => Checked[B]): CheckedAtoB[A, B] = Kleisli( + runCheckWithContext(run, context) + ) def fromErrorOr[A, B](run: A => ErrorOr[B]): CheckedAtoB[A, B] = Kleisli(runThenCheck(run)) - def fromErrorOr[A, B](context: String)(run: A => ErrorOr[B]): CheckedAtoB[A, B] = Kleisli(runErrorOrWithContext(run, _ => context)) - def fromErrorOr[A, B](context: A => String)(run: A => ErrorOr[B]): CheckedAtoB[A, B] = Kleisli(runErrorOrWithContext(run, context)) - private def runThenCheck[A, B](run: A => ErrorOr[B]): A => Checked[B] = (a: A) => { run(a).toEither } - private def runErrorOrWithContext[A, B](run: A => ErrorOr[B], context: A => String): A => Checked[B] = (a: A) => { run(a).toEither.contextualizeErrors(context(a)) } - private def runCheckWithContext[A, B](run: A => Checked[B], context: A => String): A => Checked[B] = (a: A) => { run(a).contextualizeErrors(context(a)) } + def fromErrorOr[A, B](context: String)(run: A => ErrorOr[B]): CheckedAtoB[A, B] = Kleisli( + runErrorOrWithContext(run, _ => context) + ) + def fromErrorOr[A, B](context: A => String)(run: A => ErrorOr[B]): CheckedAtoB[A, B] = Kleisli( + runErrorOrWithContext(run, context) + ) + private def runThenCheck[A, B](run: A => ErrorOr[B]): A => Checked[B] = (a: A) => run(a).toEither + private def runErrorOrWithContext[A, B](run: A => ErrorOr[B], context: A => String): A => Checked[B] = (a: A) => + run(a).toEither.contextualizeErrors(context(a)) + private def runCheckWithContext[A, B](run: A => Checked[B], context: A => String): A => Checked[B] = (a: A) => + run(a).contextualizeErrors(context(a)) - def firstSuccess[A, B](options: List[CheckedAtoB[A, B]], operationName: String): CheckedAtoB[A, B] = Kleisli[Checked, A, B] { a => - if (options.isEmpty) { - s"Unable to $operationName: No import resolvers provided".invalidNelCheck - } else { - val firstAttempt = options.head.run(a) - options.tail.foldLeft[Checked[B]](firstAttempt) { (currentResult, nextOption) => - currentResult match { - case v: Right[_, _] => v - case Left(currentErrors) => nextOption.run(a) match { + def firstSuccess[A, B](options: List[CheckedAtoB[A, B]], operationName: String): CheckedAtoB[A, B] = + Kleisli[Checked, A, B] { a => + if (options.isEmpty) { + s"Unable to $operationName: No import resolvers provided".invalidNelCheck + } else { + val firstAttempt = options.head.run(a) + options.tail.foldLeft[Checked[B]](firstAttempt) { (currentResult, nextOption) => + currentResult match { case v: Right[_, _] => v - case Left(newErrors) => Left(currentErrors ++ newErrors.toList) + case Left(currentErrors) => + nextOption.run(a) match { + case v: Right[_, _] => v + case Left(newErrors) => Left(currentErrors ++ newErrors.toList) + } } } } } - } } } diff --git a/common/src/main/scala/common/util/Backoff.scala b/common/src/main/scala/common/util/Backoff.scala index f748785eb10..747508083aa 100644 --- a/common/src/main/scala/common/util/Backoff.scala +++ b/common/src/main/scala/common/util/Backoff.scala @@ -3,8 +3,10 @@ package common.util import scala.concurrent.duration.FiniteDuration trait Backoff { + /** Next interval in millis */ def backoffMillis: Long + /** Get the next instance of backoff. This should be called after every call to backoffMillis */ def next: Backoff } diff --git a/common/src/main/scala/common/util/IORetry.scala b/common/src/main/scala/common/util/IORetry.scala index a8a7a63a209..db983fa7711 100644 --- a/common/src/main/scala/common/util/IORetry.scala +++ b/common/src/main/scala/common/util/IORetry.scala @@ -7,7 +7,7 @@ import scala.util.control.NonFatal object IORetry { def noOpOnRetry[S]: (Throwable, S) => S = (_, s) => s - + object StatefulIoError { def noop[S] = new StatefulIoError[S] { override def toThrowable(state: S, throwable: Throwable) = throwable @@ -35,8 +35,8 @@ object IORetry { backoff: Backoff, isRetryable: Throwable => Boolean = throwableToTrue, isInfinitelyRetryable: Throwable => Boolean = throwableToFalse, - onRetry: (Throwable, S) => S = noOpOnRetry[S]) - (implicit timer: Timer[IO], statefulIoException: StatefulIoError[S]): IO[A] = { + onRetry: (Throwable, S) => S = noOpOnRetry[S] + )(implicit timer: Timer[IO], statefulIoException: StatefulIoError[S]): IO[A] = { lazy val delay = backoff.backoffMillis.millis def fail(throwable: Throwable) = IO.raiseError(statefulIoException.toThrowable(state, throwable)) @@ -49,10 +49,16 @@ object IORetry { if (retriesLeft.forall(_ > 0)) { for { _ <- IO.sleep(delay) - retried <- withRetry(io, onRetry(throwable, state), retriesLeft, backoff.next, isRetryable, isInfinitelyRetryable, onRetry) + retried <- withRetry(io, + onRetry(throwable, state), + retriesLeft, + backoff.next, + isRetryable, + isInfinitelyRetryable, + onRetry + ) } yield retried - } - else fail(throwable) + } else fail(throwable) case fatal => throw fatal } diff --git a/common/src/main/scala/common/util/IntrospectableLazy.scala b/common/src/main/scala/common/util/IntrospectableLazy.scala index e578f49c1a1..7fe05dd1f4c 100644 --- a/common/src/main/scala/common/util/IntrospectableLazy.scala +++ b/common/src/main/scala/common/util/IntrospectableLazy.scala @@ -21,20 +21,21 @@ object IntrospectableLazy { } -class IntrospectableLazy[A] private(f: => A) { +class IntrospectableLazy[A] private (f: => A) { private var option: Option[A] = None - def apply(): A = { + def apply(): A = option match { case Some(a) => a case None => - synchronized { option match { - case Some(a) => a - case None => val a = f; option = Some(a); a - }} + synchronized { + option match { + case Some(a) => a + case None => val a = f; option = Some(a); a + } + } } - } def exists: Boolean = option.isDefined diff --git a/common/src/main/scala/common/util/StringUtil.scala b/common/src/main/scala/common/util/StringUtil.scala index 044ef890dea..2ddce884112 100644 --- a/common/src/main/scala/common/util/StringUtil.scala +++ b/common/src/main/scala/common/util/StringUtil.scala @@ -64,11 +64,10 @@ object StringUtil { */ def relativeDirectory: String = string.ensureNoLeadingSlash.ensureSlashed - def elided(limit: Int): String = { + def elided(limit: Int): String = if (string.length > limit) { s"(elided) ${string.take(limit)}..." } else string - } /** * Removes userInfo and sensitive query parts from strings that are RFC 2396 URIs. @@ -84,10 +83,9 @@ object StringUtil { * - the StringUtilSpec for current expectations * - https://stackoverflow.com/questions/4571346/how-to-encode-url-to-avoid-special-characters-in-java#answer-4571518 */ - def maskSensitiveUri: String = { + def maskSensitiveUri: String = Try(new URI(string)) .map(_.maskSensitive.toASCIIString) .getOrElse(string) - } } } diff --git a/common/src/main/scala/common/util/TerminalUtil.scala b/common/src/main/scala/common/util/TerminalUtil.scala index cb7c980aecb..33e30f612df 100644 --- a/common/src/main/scala/common/util/TerminalUtil.scala +++ b/common/src/main/scala/common/util/TerminalUtil.scala @@ -1,15 +1,15 @@ package common.util object TerminalUtil { - def highlight(colorCode:Int, string:String) = s"\u001B[38;5;${colorCode}m$string\u001B[0m" + def highlight(colorCode: Int, string: String) = s"\u001B[38;5;${colorCode}m$string\u001B[0m" def mdTable(rows: Seq[Seq[String]], header: Seq[String]): String = { - def maxWidth(lengths: Seq[Seq[Int]], column: Int) = lengths.map { length => length(column) }.max - val widths = (rows :+ header).map { row => row.map { s => s.length } } - val maxWidths = widths.head.indices.map { column => maxWidth(widths, column) } - val tableHeader = header.indices.map { i => header(i).padTo(maxWidths(i), ' ').mkString("") }.mkString("|") - val tableDivider = header.indices.map { i => "-" * maxWidths(i) }.mkString("|") + def maxWidth(lengths: Seq[Seq[Int]], column: Int) = lengths.map(length => length(column)).max + val widths = (rows :+ header).map(row => row.map(s => s.length)) + val maxWidths = widths.head.indices.map(column => maxWidth(widths, column)) + val tableHeader = header.indices.map(i => header(i).padTo(maxWidths(i), ' ').mkString("")).mkString("|") + val tableDivider = header.indices.map(i => "-" * maxWidths(i)).mkString("|") val tableRows = rows.map { row => - val mdRow = row.indices.map { i => row(i).padTo(maxWidths(i), ' ').mkString("") }.mkString("|") + val mdRow = row.indices.map(i => row(i).padTo(maxWidths(i), ' ').mkString("")).mkString("|") s"|$mdRow|" } s"|$tableHeader|\n|$tableDivider|\n${tableRows.mkString("\n")}\n" diff --git a/common/src/main/scala/common/util/TimeUtil.scala b/common/src/main/scala/common/util/TimeUtil.scala index dbd94c99b80..8c64dd341ce 100644 --- a/common/src/main/scala/common/util/TimeUtil.scala +++ b/common/src/main/scala/common/util/TimeUtil.scala @@ -4,6 +4,7 @@ import java.time.format.DateTimeFormatter import java.time.{OffsetDateTime, ZoneOffset} object TimeUtil { + /** * Instead of "one of" the valid ISO-8601 formats, standardize on this one: * https://github.com/openjdk/jdk/blob/jdk8-b120/jdk/src/share/classes/java/time/OffsetDateTime.java#L1886 @@ -11,12 +12,15 @@ object TimeUtil { private val Iso8601MillisecondsFormat = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSSXXXXX") implicit class EnhancedOffsetDateTime(val offsetDateTime: OffsetDateTime) extends AnyVal { + /** * Discards the original timezone and shifts the time to UTC, then returns the ISO-8601 formatted string with * exactly three digits of milliseconds. */ - def toUtcMilliString: String = Option(offsetDateTime).map( - _.atZoneSameInstant(ZoneOffset.UTC).format(Iso8601MillisecondsFormat) - ).orNull + def toUtcMilliString: String = Option(offsetDateTime) + .map( + _.atZoneSameInstant(ZoneOffset.UTC).format(Iso8601MillisecondsFormat) + ) + .orNull } } diff --git a/common/src/main/scala/common/util/TryUtil.scala b/common/src/main/scala/common/util/TryUtil.scala index e8edf01f8e5..02a90a5e983 100644 --- a/common/src/main/scala/common/util/TryUtil.scala +++ b/common/src/main/scala/common/util/TryUtil.scala @@ -22,14 +22,13 @@ object TryUtil { def stringifyFailures[T](possibleFailures: Iterable[Try[T]]): Iterable[String] = possibleFailures.collect { case failure: Failure[T] => stringifyFailure(failure) } - private def sequenceIterable[T](tries: Iterable[Try[_]], unbox: () => T, prefixErrorMessage: String): Try[T] = { + private def sequenceIterable[T](tries: Iterable[Try[_]], unbox: () => T, prefixErrorMessage: String): Try[T] = tries collect { case f: Failure[_] => f } match { case failures if failures.nonEmpty => val exceptions = failures.toSeq.map(_.exception) Failure(AggregatedException(prefixErrorMessage, exceptions.toList)) case _ => Success(unbox()) } - } def sequence[T](tries: Seq[Try[T]], prefixErrorMessage: String = ""): Try[Seq[T]] = { def unbox = tries map { _.get } diff --git a/common/src/main/scala/common/util/UriUtil.scala b/common/src/main/scala/common/util/UriUtil.scala index eb8c125768c..b2e4a07bced 100644 --- a/common/src/main/scala/common/util/UriUtil.scala +++ b/common/src/main/scala/common/util/UriUtil.scala @@ -15,20 +15,19 @@ object UriUtil { * - the StringUtilSpec for current expectations * - https://stackoverflow.com/questions/4571346/how-to-encode-url-to-avoid-special-characters-in-java#answer-4571518 */ - def maskSensitive: URI = { + def maskSensitive: URI = Try { - new URI( - uri.getScheme, - null, // Remove all userInfo - uri.getHost, - uri.getPort, - uri.getPath, - Option(uri.getQuery).map(maskSensitiveQuery).orNull, - uri.getFragment, - ) + new URI( + uri.getScheme, + null, // Remove all userInfo + uri.getHost, + uri.getPort, + uri.getPath, + Option(uri.getQuery).map(maskSensitiveQuery).orNull, + uri.getFragment + ) } - .getOrElse(uri) - } + .getOrElse(uri) } private def maskSensitiveQuery(query: String): String = { @@ -85,11 +84,16 @@ object UriUtil { private val SensitiveKeyParts = List( "credential", - "signature", + "signature" + ) + + private val SensitiveKeys = + List( + "sig" ) private def isSensitiveKey(name: String): Boolean = { val lower = name.toLowerCase - SensitiveKeyParts.exists(lower.contains(_)) + SensitiveKeyParts.exists(lower.contains(_)) || SensitiveKeys.exists(lower.equals(_)) } } diff --git a/common/src/main/scala/common/util/VersionUtil.scala b/common/src/main/scala/common/util/VersionUtil.scala index 3ddea0750d5..fcbffca77a4 100644 --- a/common/src/main/scala/common/util/VersionUtil.scala +++ b/common/src/main/scala/common/util/VersionUtil.scala @@ -35,19 +35,17 @@ object VersionUtil { * @param default What to return when the version cannot be found. The parameter passed is the `projectName`. * @return The version from the conf or the default */ - def getVersion(projectName: String, default: String => String = defaultMessage): String = { + def getVersion(projectName: String, default: String => String = defaultMessage): String = ConfigFactory .load(versionConf(projectName)) .as[Option[String]](versionProperty(projectName)) .getOrElse(default(projectName)) - } /** * Instead of returning a version, states that the version conf will be generated by sbt. */ - def defaultMessage(projectName: String): String = { + def defaultMessage(projectName: String): String = s"${versionConf(projectName)}-to-be-generated-by-sbt" - } /** * A regex compatible with the dependency constants in project/Dependencies.scala. @@ -62,7 +60,7 @@ object VersionUtil { * @return The dependency version from project/Dependencies.scala * @throws RuntimeException If the dependency cannot be found */ - def sbtDependencyVersion(dependencyName: String)(projectName: String): String = { + def sbtDependencyVersion(dependencyName: String)(projectName: String): String = try { val dependencies = Paths.get("project/Dependencies.scala").toAbsolutePath val lines = Files.readAllLines(dependencies).asScala @@ -79,6 +77,5 @@ object VersionUtil { e ) } - } } diff --git a/common/src/main/scala/common/validation/ErrorOr.scala b/common/src/main/scala/common/validation/ErrorOr.scala index 4be2aa633c9..3afdfcc5fb7 100644 --- a/common/src/main/scala/common/validation/ErrorOr.scala +++ b/common/src/main/scala/common/validation/ErrorOr.scala @@ -12,9 +12,8 @@ import scala.util.Try object ErrorOr { type ErrorOr[+A] = Validated[NonEmptyList[String], A] - def apply[A](f: => A): ErrorOr[A] = { + def apply[A](f: => A): ErrorOr[A] = Try(f).toErrorOr - } implicit class EnhancedErrorOr[A](val eoa: ErrorOr[A]) extends AnyVal { def contextualizeErrors(s: => String): ErrorOr[A] = eoa.leftMap { errors => @@ -31,17 +30,17 @@ object ErrorOr { } implicit class ShortCircuitingFlatMap[A](val fa: ErrorOr[A]) extends AnyVal { + /** * Not consistent with `Applicative#ap` but useful in for comprehensions. * * @see http://typelevel.org/cats/tut/validated.html#of-flatmaps-and-xors */ - def flatMap[B](f: A => ErrorOr[B]): ErrorOr[B] = { + def flatMap[B](f: A => ErrorOr[B]): ErrorOr[B] = fa match { case Valid(v) => ErrorOr(f(v)).flatten case i @ Invalid(_) => i } - } } implicit class MapErrorOrRhs[A, B](val m: Map[A, ErrorOr[B]]) extends AnyVal { @@ -49,8 +48,8 @@ object ErrorOr { } implicit class MapTraversal[A, B](val m: Map[A, B]) extends AnyVal { - def traverse[C,D](f: ((A,B)) => ErrorOr[(C,D)]): ErrorOr[Map[C,D]] = m.toList.traverse(f).map(_.toMap) - def traverseValues[C](f: B => ErrorOr[C]): ErrorOr[Map[A,C]] = m.traverse { case (a, b) => f(b).map(c => (a,c)) } + def traverse[C, D](f: ((A, B)) => ErrorOr[(C, D)]): ErrorOr[Map[C, D]] = m.toList.traverse(f).map(_.toMap) + def traverseValues[C](f: B => ErrorOr[C]): ErrorOr[Map[A, C]] = m.traverse { case (a, b) => f(b).map(c => (a, c)) } } // Note! See the bottom of this file for a generator function for 2 through 22 of these near-identical ShortCircuitingFlatMapTupleNs... @@ -67,83 +66,346 @@ object ErrorOr { def flatMapN[T_OUT](f3: (A, B, C) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t3.tupled flatMap f3.tupled } - implicit class ShortCircuitingFlatMapTuple4[A, B, C, D](val t4: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D])) extends AnyVal { + implicit class ShortCircuitingFlatMapTuple4[A, B, C, D](val t4: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D])) + extends AnyVal { def flatMapN[T_OUT](f4: (A, B, C, D) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t4.tupled flatMap f4.tupled } - implicit class ShortCircuitingFlatMapTuple5[A, B, C, D, E](val t5: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E])) extends AnyVal { + implicit class ShortCircuitingFlatMapTuple5[A, B, C, D, E]( + val t5: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E]) + ) extends AnyVal { def flatMapN[T_OUT](f5: (A, B, C, D, E) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t5.tupled flatMap f5.tupled } - implicit class ShortCircuitingFlatMapTuple6[A, B, C, D, E, F](val t6: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F])) extends AnyVal { + implicit class ShortCircuitingFlatMapTuple6[A, B, C, D, E, F]( + val t6: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F]) + ) extends AnyVal { def flatMapN[T_OUT](f6: (A, B, C, D, E, F) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t6.tupled flatMap f6.tupled } - implicit class ShortCircuitingFlatMapTuple7[A, B, C, D, E, F, G](val t7: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G])) extends AnyVal { + implicit class ShortCircuitingFlatMapTuple7[A, B, C, D, E, F, G]( + val t7: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G]) + ) extends AnyVal { def flatMapN[T_OUT](f7: (A, B, C, D, E, F, G) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t7.tupled flatMap f7.tupled } - implicit class ShortCircuitingFlatMapTuple8[A, B, C, D, E, F, G, H](val t8: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H])) extends AnyVal { + implicit class ShortCircuitingFlatMapTuple8[A, B, C, D, E, F, G, H]( + val t8: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H]) + ) extends AnyVal { def flatMapN[T_OUT](f8: (A, B, C, D, E, F, G, H) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t8.tupled flatMap f8.tupled } - implicit class ShortCircuitingFlatMapTuple9[A, B, C, D, E, F, G, H, I](val t9: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I])) extends AnyVal { + implicit class ShortCircuitingFlatMapTuple9[A, B, C, D, E, F, G, H, I]( + val t9: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I]) + ) extends AnyVal { def flatMapN[T_OUT](f9: (A, B, C, D, E, F, G, H, I) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t9.tupled flatMap f9.tupled } - implicit class ShortCircuitingFlatMapTuple10[A, B, C, D, E, F, G, H, I, J](val t10: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J])) extends AnyVal { - def flatMapN[T_OUT](f10: (A, B, C, D, E, F, G, H, I, J) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t10.tupled flatMap f10.tupled + implicit class ShortCircuitingFlatMapTuple10[A, B, C, D, E, F, G, H, I, J]( + val t10: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f10: (A, B, C, D, E, F, G, H, I, J) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t10.tupled flatMap f10.tupled } - implicit class ShortCircuitingFlatMapTuple11[A, B, C, D, E, F, G, H, I, J, K](val t11: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K])) extends AnyVal { - def flatMapN[T_OUT](f11: (A, B, C, D, E, F, G, H, I, J, K) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t11.tupled flatMap f11.tupled + implicit class ShortCircuitingFlatMapTuple11[A, B, C, D, E, F, G, H, I, J, K]( + val t11: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f11: (A, B, C, D, E, F, G, H, I, J, K) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t11.tupled flatMap f11.tupled } - implicit class ShortCircuitingFlatMapTuple12[A, B, C, D, E, F, G, H, I, J, K, L](val t12: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L])) extends AnyVal { - def flatMapN[T_OUT](f12: (A, B, C, D, E, F, G, H, I, J, K, L) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t12.tupled flatMap f12.tupled + implicit class ShortCircuitingFlatMapTuple12[A, B, C, D, E, F, G, H, I, J, K, L]( + val t12: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f12: (A, B, C, D, E, F, G, H, I, J, K, L) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t12.tupled flatMap f12.tupled } - implicit class ShortCircuitingFlatMapTuple13[A, B, C, D, E, F, G, H, I, J, K, L, M](val t13: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M])) extends AnyVal { - def flatMapN[T_OUT](f13: (A, B, C, D, E, F, G, H, I, J, K, L, M) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t13.tupled flatMap f13.tupled + implicit class ShortCircuitingFlatMapTuple13[A, B, C, D, E, F, G, H, I, J, K, L, M]( + val t13: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f13: (A, B, C, D, E, F, G, H, I, J, K, L, M) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t13.tupled flatMap f13.tupled } - implicit class ShortCircuitingFlatMapTuple14[A, B, C, D, E, F, G, H, I, J, K, L, M, N](val t14: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N])) extends AnyVal { - def flatMapN[T_OUT](f14: (A, B, C, D, E, F, G, H, I, J, K, L, M, N) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t14.tupled flatMap f14.tupled + implicit class ShortCircuitingFlatMapTuple14[A, B, C, D, E, F, G, H, I, J, K, L, M, N]( + val t14: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f14: (A, B, C, D, E, F, G, H, I, J, K, L, M, N) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t14.tupled flatMap f14.tupled } - implicit class ShortCircuitingFlatMapTuple15[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O](val t15: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O])) extends AnyVal { - def flatMapN[T_OUT](f15: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t15.tupled flatMap f15.tupled + implicit class ShortCircuitingFlatMapTuple15[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O]( + val t15: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f15: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t15.tupled flatMap f15.tupled } - implicit class ShortCircuitingFlatMapTuple16[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P](val t16: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P])) extends AnyVal { - def flatMapN[T_OUT](f16: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t16.tupled flatMap f16.tupled + implicit class ShortCircuitingFlatMapTuple16[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P]( + val t16: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f16: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t16.tupled flatMap f16.tupled } - implicit class ShortCircuitingFlatMapTuple17[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q](val t17: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P], ErrorOr[Q])) extends AnyVal { - def flatMapN[T_OUT](f17: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t17.tupled flatMap f17.tupled + implicit class ShortCircuitingFlatMapTuple17[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q]( + val t17: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P], + ErrorOr[Q] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f17: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t17.tupled flatMap f17.tupled } - implicit class ShortCircuitingFlatMapTuple18[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R](val t18: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P], ErrorOr[Q], ErrorOr[R])) extends AnyVal { - def flatMapN[T_OUT](f18: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t18.tupled flatMap f18.tupled + implicit class ShortCircuitingFlatMapTuple18[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R]( + val t18: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P], + ErrorOr[Q], + ErrorOr[R] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f18: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t18.tupled flatMap f18.tupled } - implicit class ShortCircuitingFlatMapTuple19[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S](val t19: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P], ErrorOr[Q], ErrorOr[R], ErrorOr[S])) extends AnyVal { - def flatMapN[T_OUT](f19: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t19.tupled flatMap f19.tupled + implicit class ShortCircuitingFlatMapTuple19[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S]( + val t19: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P], + ErrorOr[Q], + ErrorOr[R], + ErrorOr[S] + ) + ) extends AnyVal { + def flatMapN[T_OUT]( + f19: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) => ErrorOr[T_OUT] + ): ErrorOr[T_OUT] = t19.tupled flatMap f19.tupled } - implicit class ShortCircuitingFlatMapTuple20[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T](val t20: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P], ErrorOr[Q], ErrorOr[R], ErrorOr[S], ErrorOr[T])) extends AnyVal { - def flatMapN[T_OUT](f20: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t20.tupled flatMap f20.tupled + implicit class ShortCircuitingFlatMapTuple20[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T]( + val t20: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P], + ErrorOr[Q], + ErrorOr[R], + ErrorOr[S], + ErrorOr[T] + ) + ) extends AnyVal { + def flatMapN[T_OUT]( + f20: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) => ErrorOr[T_OUT] + ): ErrorOr[T_OUT] = t20.tupled flatMap f20.tupled } - implicit class ShortCircuitingFlatMapTuple21[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U](val t21: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P], ErrorOr[Q], ErrorOr[R], ErrorOr[S], ErrorOr[T], ErrorOr[U])) extends AnyVal { - def flatMapN[T_OUT](f21: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t21.tupled flatMap f21.tupled + implicit class ShortCircuitingFlatMapTuple21[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U]( + val t21: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P], + ErrorOr[Q], + ErrorOr[R], + ErrorOr[S], + ErrorOr[T], + ErrorOr[U] + ) + ) extends AnyVal { + def flatMapN[T_OUT]( + f21: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) => ErrorOr[T_OUT] + ): ErrorOr[T_OUT] = t21.tupled flatMap f21.tupled } - implicit class ShortCircuitingFlatMapTuple22[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V](val t22: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P], ErrorOr[Q], ErrorOr[R], ErrorOr[S], ErrorOr[T], ErrorOr[U], ErrorOr[V])) extends AnyVal { - def flatMapN[T_OUT](f22: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t22.tupled flatMap f22.tupled + implicit class ShortCircuitingFlatMapTuple22[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V]( + val t22: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P], + ErrorOr[Q], + ErrorOr[R], + ErrorOr[S], + ErrorOr[T], + ErrorOr[U], + ErrorOr[V] + ) + ) extends AnyVal { + def flatMapN[T_OUT]( + f22: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) => ErrorOr[T_OUT] + ): ErrorOr[T_OUT] = t22.tupled flatMap f22.tupled } object ErrorOrGen { + /** * Because maintaining 22 near-identical functions is a bore... * This function can regenerate them if we need to make changes. @@ -168,8 +430,7 @@ object ErrorOr { | ${line2(n)} |} | - |""".stripMargin + |""".stripMargin } } } - diff --git a/common/src/main/scala/common/validation/IOChecked.scala b/common/src/main/scala/common/validation/IOChecked.scala index f838e5ceb32..62e61f87eb4 100644 --- a/common/src/main/scala/common/validation/IOChecked.scala +++ b/common/src/main/scala/common/validation/IOChecked.scala @@ -1,6 +1,5 @@ package common.validation - import cats.arrow.FunctionK import cats.data.EitherT.fromEither import cats.data.{EitherT, NonEmptyList, ValidatedNel} @@ -22,6 +21,7 @@ object IOChecked { * The monad transformer allows to flatMap over the value while keeping the IO effect as well as the list of potential errors */ type IOChecked[A] = EitherT[IO, NonEmptyList[String], A] + /** * Fixes the left type of Either to Throwable * This is useful when calling on IO[A].attempt, transforming it to IO[ Either[Throwable, A] ] == IO[ Attempt[A] ] @@ -40,22 +40,22 @@ object IOChecked { */ implicit val eitherThrowableApplicative = new Applicative[Attempt] { override def pure[A](x: A) = Right(x) - override def ap[A, B](ff: Attempt[A => B])(fa: Attempt[A]): Attempt[B] = { + override def ap[A, B](ff: Attempt[A => B])(fa: Attempt[A]): Attempt[B] = (fa, ff) match { - // Both have a list or error messages, combine them in a single one - case (Left(t1: MessageAggregation), Left(t2: MessageAggregation)) => Left(AggregatedMessageException("", t1.errorMessages ++ t2.errorMessages)) - // Only one of them is a MessageAggregation, combined the errors and the other exception in a CompositeException + // Both have a list or error messages, combine them in a single one + case (Left(t1: MessageAggregation), Left(t2: MessageAggregation)) => + Left(AggregatedMessageException("", t1.errorMessages ++ t2.errorMessages)) + // Only one of them is a MessageAggregation, combined the errors and the other exception in a CompositeException case (Left(t1: MessageAggregation), Left(t2)) => Left(CompositeException("", List(t2), t1.errorMessages)) case (Left(t2), Left(t1: MessageAggregation)) => Left(CompositeException("", List(t2), t1.errorMessages)) - // None of them is a MessageAggregation, combine the 2 throwables in an AggregatedException + // None of them is a MessageAggregation, combine the 2 throwables in an AggregatedException case (Left(t1), Left(t2)) => Left(AggregatedException("", List(t1, t2))) - // Success case, apply f on v + // Success case, apply f on v case (Right(v), Right(f)) => Right(f(v)) - // Default failure case, just keep the failure + // Default failure case, just keep the failure case (Left(t1), _) => Left(t1) case (_, Left(t1)) => Left(t1) } - } } /** @@ -100,7 +100,7 @@ object IOChecked { * We now have an IO[ Either[NonEmptyList[String], A] ] which we can wrap into an IOChecked with EitherT.apply */ override def sequential: FunctionK[IOCheckedPar, IOChecked] = - new FunctionK[IOCheckedPar, IOChecked] { + new FunctionK[IOCheckedPar, IOChecked] { def apply[A](fa: IOCheckedPar[A]): IOChecked[A] = EitherT { IO.Par.unwrap(fa: IO.Par[Attempt[A]]) flatMap { case Left(t: MessageAggregation) => IO.pure(Left(NonEmptyList.fromListUnsafe(t.errorMessages.toList))) @@ -138,7 +138,7 @@ object IOChecked { def error[A](error: String, tail: String*): IOChecked[A] = EitherT.leftT { NonEmptyList.of(error, tail: _*) } - + def pure[A](value: A): IOChecked[A] = EitherT.pure(value) def goIOChecked[A](f: => A): IOChecked[A] = Try(f).toIOChecked @@ -150,12 +150,11 @@ object IOChecked { implicit class EnhancedIOChecked[A](val p: IOChecked[A]) extends AnyVal { import cats.syntax.either._ - def toChecked: Checked[A] = { + def toChecked: Checked[A] = Try(p.value.unsafeRunSync()) match { case Success(r) => r case Failure(f) => NonEmptyList.one(f.getMessage).asLeft } - } def toErrorOr: ErrorOr[A] = toChecked.toValidated @@ -163,12 +162,11 @@ object IOChecked { def unsafe(context: String) = unsafeToEither().unsafe(context) - def contextualizeErrors(context: String): IOChecked[A] = { - p.leftMap({ errors => + def contextualizeErrors(context: String): IOChecked[A] = + p.leftMap { errors => val total = errors.size errors.zipWithIndex map { case (e, i) => s"Failed to $context (reason ${i + 1} of $total): $e" } - }) - } + } } implicit class TryIOChecked[A](val t: Try[A]) extends AnyVal { @@ -176,9 +174,8 @@ object IOChecked { } implicit class FutureIOChecked[A](val future: Future[A]) extends AnyVal { - def toIOChecked(implicit cs: ContextShift[IO]): IOChecked[A] = { + def toIOChecked(implicit cs: ContextShift[IO]): IOChecked[A] = IO.fromFuture(IO(future)).to[IOChecked] - } } implicit class ErrorOrIOChecked[A](val e: ErrorOr[A]) extends AnyVal { @@ -198,9 +195,8 @@ object IOChecked { } implicit class OptionIOChecked[A](val o: Option[A]) extends AnyVal { - def toIOChecked(errorMessage: String): IOChecked[A] = { + def toIOChecked(errorMessage: String): IOChecked[A] = EitherT.fromOption(o, NonEmptyList.of(errorMessage)) - } } type IOCheckedValidated[A] = IO[ValidatedNel[String, A]] diff --git a/common/src/main/scala/common/validation/Validation.scala b/common/src/main/scala/common/validation/Validation.scala index 4099eff8ac6..af568909b5b 100644 --- a/common/src/main/scala/common/validation/Validation.scala +++ b/common/src/main/scala/common/validation/Validation.scala @@ -46,12 +46,12 @@ object Validation { case Failure(f) => defaultThrowableToString(f).invalidNel } - implicit class ValidationOps[B,A](val v: ValidatedNel[B, A]) { - //Convert this into a future by folding over the state and returning the corresponding Future terminal state. + implicit class ValidationOps[B, A](val v: ValidatedNel[B, A]) { + // Convert this into a future by folding over the state and returning the corresponding Future terminal state. def toFuture(f: NonEmptyList[B] => Throwable) = - v fold( - //Use f to turn the failure list into a Throwable, then fail a future with it. - //Function composition lets us ignore the actual argument of the error list + v fold ( + // Use f to turn the failure list into a Throwable, then fail a future with it. + // Function composition lets us ignore the actual argument of the error list Future.failed _ compose f, Future.successful ) @@ -59,31 +59,30 @@ object Validation { implicit class TryValidation[A](val t: Try[A]) extends AnyVal { def toErrorOr: ErrorOr[A] = toErrorOr(defaultThrowableToString) - def toErrorOr(throwableToStringFunction: ThrowableToStringFunction): ErrorOr[A] = { + def toErrorOr(throwableToStringFunction: ThrowableToStringFunction): ErrorOr[A] = Validated.fromTry(t).leftMap(throwableToStringFunction).toValidatedNel[String, A] - } def toErrorOrWithContext(context: String): ErrorOr[A] = toErrorOrWithContext(context, defaultThrowableToString) - def toErrorOrWithContext(context: String, - throwableToStringFunction: ThrowableToStringFunction): ErrorOr[A] = toChecked(throwableToStringFunction) - .contextualizeErrors(context) - .leftMap({contextualizedErrors => - if (t.failed.isFailure) { - val errors = new StringWriter - t.failed.get.printStackTrace(new PrintWriter(errors)) - contextualizedErrors.::(s"Stacktrace: ${errors.toString}") - } else contextualizedErrors - }) - .toValidated + def toErrorOrWithContext(context: String, throwableToStringFunction: ThrowableToStringFunction): ErrorOr[A] = + toChecked(throwableToStringFunction) + .contextualizeErrors(context) + .leftMap { contextualizedErrors => + if (t.failed.isFailure) { + val errors = new StringWriter + t.failed.get.printStackTrace(new PrintWriter(errors)) + contextualizedErrors.::(s"Stacktrace: ${errors.toString}") + } else contextualizedErrors + } + .toValidated def toChecked: Checked[A] = toChecked(defaultThrowableToString) - def toChecked(throwableToStringFunction: ThrowableToStringFunction): Checked[A] = { - Either.fromTry(t).leftMap { ex => NonEmptyList.one(throwableToStringFunction(ex)) } - } + def toChecked(throwableToStringFunction: ThrowableToStringFunction): Checked[A] = + Either.fromTry(t).leftMap(ex => NonEmptyList.one(throwableToStringFunction(ex))) - def toCheckedWithContext(context: String): Checked[A] = toErrorOrWithContext(context, defaultThrowableToString).toEither - def toCheckedWithContext(context: String, - throwableToStringFunction: ThrowableToStringFunction): Checked[A] = toErrorOrWithContext(context, throwableToStringFunction).toEither + def toCheckedWithContext(context: String): Checked[A] = + toErrorOrWithContext(context, defaultThrowableToString).toEither + def toCheckedWithContext(context: String, throwableToStringFunction: ThrowableToStringFunction): Checked[A] = + toErrorOrWithContext(context, throwableToStringFunction).toEither } implicit class ValidationTry[A](val e: ErrorOr[A]) extends AnyVal { @@ -109,12 +108,10 @@ object Validation { } implicit class OptionValidation[A](val o: Option[A]) extends AnyVal { - def toErrorOr(errorMessage: => String): ErrorOr[A] = { + def toErrorOr(errorMessage: => String): ErrorOr[A] = Validated.fromOption(o, NonEmptyList.of(errorMessage)) - } - def toChecked(errorMessage: => String): Checked[A] = { + def toChecked(errorMessage: => String): Checked[A] = Either.fromOption(o, NonEmptyList.of(errorMessage)) - } } } diff --git a/common/src/test/scala/common/assertion/CaseClassAssertions.scala b/common/src/test/scala/common/assertion/CaseClassAssertions.scala index 4cc191df2f6..f273898de1c 100644 --- a/common/src/test/scala/common/assertion/CaseClassAssertions.scala +++ b/common/src/test/scala/common/assertion/CaseClassAssertions.scala @@ -5,10 +5,9 @@ import org.scalatest.matchers.should.Matchers object CaseClassAssertions extends Matchers { implicit class ComparableCaseClass[A <: Product](actualA: A) { // Assumes that expectedA and actualA are the same type. If we don't subtype case classes, that should hold... - def shouldEqualFieldwise(expectedA: A): Unit = { + def shouldEqualFieldwise(expectedA: A): Unit = (0 until actualA.productArity) foreach { i => actualA.productElement(i) should be(expectedA.productElement(i)) } - } } } diff --git a/common/src/test/scala/common/assertion/ErrorOrAssertions.scala b/common/src/test/scala/common/assertion/ErrorOrAssertions.scala index 02a5e594bb4..00c3ab4fcfc 100644 --- a/common/src/test/scala/common/assertion/ErrorOrAssertions.scala +++ b/common/src/test/scala/common/assertion/ErrorOrAssertions.scala @@ -6,7 +6,6 @@ import common.validation.ErrorOr.ErrorOr import org.scalatest.Assertion import org.scalatest.matchers.should.Matchers - object ErrorOrAssertions { implicit class ErrorOrWithAssertions[A](errorOr: ErrorOr[A]) extends Matchers { def shouldBeValid(other: A): Assertion = errorOr match { diff --git a/common/src/test/scala/common/collections/TableSpec.scala b/common/src/test/scala/common/collections/TableSpec.scala index 7000e151df4..581fcf466e8 100644 --- a/common/src/test/scala/common/collections/TableSpec.scala +++ b/common/src/test/scala/common/collections/TableSpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class TableSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "Table" @@ -21,10 +20,14 @@ class TableSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "fill a table" in { - Table.fill(List( - ("a", "b", "c"), - ("d", "e", "f") - )).table shouldBe Map( + Table + .fill( + List( + ("a", "b", "c"), + ("d", "e", "f") + ) + ) + .table shouldBe Map( "a" -> Map("b" -> "c"), "d" -> Map("e" -> "f") ) @@ -48,7 +51,7 @@ class TableSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { someTable.rowOptional("a") shouldBe Some(Map("b" -> "c")) someTable.rowOptional("b") shouldBe None } - + it should "implement row" in { someTable.row("a") shouldBe Map("b" -> "c") someTable.row("b") shouldBe empty @@ -87,7 +90,7 @@ class TableSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { it should "implement addTriplet" in { someTable.addTriplet( - ("0", "1", "2") + ("0", "1", "2") ) shouldBe Table( Map( "a" -> Map("b" -> "c"), diff --git a/common/src/test/scala/common/collections/WeightedQueueSpec.scala b/common/src/test/scala/common/collections/WeightedQueueSpec.scala index 5d29b54949e..ba9cd81d908 100644 --- a/common/src/test/scala/common/collections/WeightedQueueSpec.scala +++ b/common/src/test/scala/common/collections/WeightedQueueSpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class WeightedQueueSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "WeightedQueue" @@ -39,7 +38,8 @@ class WeightedQueueSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche it should "behead the queue" in { // A queue of strings for which the weight is the number of char in the string val q = WeightedQueue.empty[String, Int](_.length) - val q2 = q.enqueue("hello") + val q2 = q + .enqueue("hello") .enqueue("hola") .enqueue("bonjour") val (head, q3) = q2.behead(10) diff --git a/common/src/test/scala/common/exception/ExceptionAggregationSpec.scala b/common/src/test/scala/common/exception/ExceptionAggregationSpec.scala index cd425b626f0..071b7836ceb 100644 --- a/common/src/test/scala/common/exception/ExceptionAggregationSpec.scala +++ b/common/src/test/scala/common/exception/ExceptionAggregationSpec.scala @@ -7,7 +7,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers - class ExceptionAggregationSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with Matchers { "MessageAggregation" should "aggregate messages" in { diff --git a/common/src/test/scala/common/mock/MockImplicits.scala b/common/src/test/scala/common/mock/MockImplicits.scala index 3e90e183d8e..424dbfbdfde 100644 --- a/common/src/test/scala/common/mock/MockImplicits.scala +++ b/common/src/test/scala/common/mock/MockImplicits.scala @@ -11,31 +11,27 @@ trait MockImplicits { * https://github.com/etorreborre/specs2/commit/6d56660e70980b5958e6c4ed8fd4158bf1cecf70#diff-a2627f56c432e4bc37f36bc56e13852225813aa604918471b61ec2080462d722 */ implicit class MockEnhanced[A](methodCall: A) { - def returns(result: A): OngoingStubbing[A] = { + def returns(result: A): OngoingStubbing[A] = Mockito.when(methodCall).thenReturn(result) - } - def answers(function: Any => A): OngoingStubbing[A] = { - Mockito.when(methodCall) thenAnswer { - invocationOnMock => { - val args = invocationOnMock.getArguments - // The DSL behavior of the below is directly taken with thanks from the link above. - args.size match { - case 0 => - function match { - case function0: Function0[_] => - function0.apply().asInstanceOf[A] - case _ => - function.apply(invocationOnMock.getMock) - } - case 1 => - function(args(0)) - case _ => - function(args) - } + def answers(function: Any => A): OngoingStubbing[A] = + Mockito.when(methodCall) thenAnswer { invocationOnMock => + val args = invocationOnMock.getArguments + // The DSL behavior of the below is directly taken with thanks from the link above. + args.size match { + case 0 => + function match { + case function0: Function0[_] => + function0.apply().asInstanceOf[A] + case _ => + function.apply(invocationOnMock.getMock) + } + case 1 => + function(args(0)) + case _ => + function(args) } } - } def responds(f: Any => A): OngoingStubbing[A] = answers(f) } diff --git a/common/src/test/scala/common/mock/MockSugar.scala b/common/src/test/scala/common/mock/MockSugar.scala index 27a4ae87b55..6a92bd2fd47 100644 --- a/common/src/test/scala/common/mock/MockSugar.scala +++ b/common/src/test/scala/common/mock/MockSugar.scala @@ -2,7 +2,7 @@ package common.mock import org.mockito.{ArgumentCaptor, Mockito} -import scala.reflect.{ClassTag, classTag} +import scala.reflect.{classTag, ClassTag} /** * Yet another scala wrapper around Mockito. @@ -37,12 +37,11 @@ trait MockSugar extends MockImplicits { * * Note: if you run into issues with `mock` then try [[mockWithDefaults]]. */ - def mock[A: ClassTag]: A = { + def mock[A: ClassTag]: A = Mockito.mock( classTag[A].runtimeClass.asInstanceOf[Class[A]], - Mockito.withSettings().defaultAnswer(Mockito.RETURNS_SMART_NULLS), + Mockito.withSettings().defaultAnswer(Mockito.RETURNS_SMART_NULLS) ) - } /** * Creates a mock returning default values instead of Smart Nulls. @@ -56,16 +55,14 @@ trait MockSugar extends MockImplicits { * * An alternative workaround was to use `Mockito.doReturn(retVal).when(mockObj).someMethod`. */ - def mockWithDefaults[A: ClassTag]: A = { + def mockWithDefaults[A: ClassTag]: A = Mockito.mock( classTag[A].runtimeClass.asInstanceOf[Class[A]], - Mockito.withSettings().defaultAnswer(Mockito.RETURNS_DEFAULTS), + Mockito.withSettings().defaultAnswer(Mockito.RETURNS_DEFAULTS) ) - } - def capture[A: ClassTag]: ArgumentCaptor[A] = { + def capture[A: ClassTag]: ArgumentCaptor[A] = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) - } } object MockSugar extends MockSugar diff --git a/common/src/test/scala/common/numeric/IntegerUtilSpec.scala b/common/src/test/scala/common/numeric/IntegerUtilSpec.scala index b06af539626..48fc9ed3b3b 100644 --- a/common/src/test/scala/common/numeric/IntegerUtilSpec.scala +++ b/common/src/test/scala/common/numeric/IntegerUtilSpec.scala @@ -5,18 +5,33 @@ import common.numeric.IntegerUtil.IntEnhanced import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class IntegerUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { it should "return ordinal String for any Int" in { - val numbers = List(0, 1, 2, 3, 4, - 10, 11, 12, 13, 14, - 20, 21, 22, 23, 24, - 100, 101, 102, 103, 104) map { _.toOrdinal } + val numbers = List(0, 1, 2, 3, 4, 10, 11, 12, 13, 14, 20, 21, 22, 23, 24, 100, 101, 102, 103, 104) map { + _.toOrdinal + } - val expected = List("0th", "1st", "2nd", "3rd", "4th", - "10th", "11th", "12th", "13th", "14th", - "20th", "21st", "22nd", "23rd", "24th", - "100th", "101st", "102nd", "103rd", "104th") + val expected = List("0th", + "1st", + "2nd", + "3rd", + "4th", + "10th", + "11th", + "12th", + "13th", + "14th", + "20th", + "21st", + "22nd", + "23rd", + "24th", + "100th", + "101st", + "102nd", + "103rd", + "104th" + ) numbers should contain theSameElementsInOrderAs expected } diff --git a/common/src/test/scala/common/util/IntrospectableLazySpec.scala b/common/src/test/scala/common/util/IntrospectableLazySpec.scala index 337e193ab8f..91243135ab6 100644 --- a/common/src/test/scala/common/util/IntrospectableLazySpec.scala +++ b/common/src/test/scala/common/util/IntrospectableLazySpec.scala @@ -22,17 +22,19 @@ class IntrospectableLazySpec extends AnyFlatSpec with CromwellTimeoutSpec with M 4 } - val myLazy = lazily { lazyContents } + val myLazy = lazily(lazyContents) assert(lazyInstantiations == 0) assert(!myLazy.exists) // Fails without `synchronized { ... }` Await.result(Future.sequence( - Seq.fill(100)(Future { - myLazy() shouldBe 4 - }) - ), 1.seconds) + Seq.fill(100)(Future { + myLazy() shouldBe 4 + }) + ), + 1.seconds + ) assert(lazyInstantiations == 1) assert(myLazy.exists) diff --git a/common/src/test/scala/common/util/IoRetrySpec.scala b/common/src/test/scala/common/util/IoRetrySpec.scala index 4a5fc582eac..2ec5c56aba4 100644 --- a/common/src/test/scala/common/util/IoRetrySpec.scala +++ b/common/src/test/scala/common/util/IoRetrySpec.scala @@ -9,7 +9,6 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.ExecutionContext import scala.concurrent.duration._ - class IoRetrySpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { implicit val timer = IO.timer(ExecutionContext.global) implicit val ioError = new StatefulIoError[Int] { @@ -28,7 +27,8 @@ class IoRetrySpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } val incrementOnRetry: (Throwable, Int) => Int = (_, s) => s + 1 - val io = IORetry.withRetry(work, 1, Option(3), backoff = Backoff.staticBackoff(10.millis), onRetry = incrementOnRetry) + val io = + IORetry.withRetry(work, 1, Option(3), backoff = Backoff.staticBackoff(10.millis), onRetry = incrementOnRetry) val statefulException = the[Exception] thrownBy io.unsafeRunSync() statefulException.getCause shouldBe exception statefulException.getMessage shouldBe "Attempted 3 times" diff --git a/common/src/test/scala/common/util/StringUtilSpec.scala b/common/src/test/scala/common/util/StringUtilSpec.scala index 3d7b8db92d1..f71d79819cb 100644 --- a/common/src/test/scala/common/util/StringUtilSpec.scala +++ b/common/src/test/scala/common/util/StringUtilSpec.scala @@ -17,11 +17,13 @@ class StringUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers // With the elided string, we stop processing early and are able to produce a nice, short string without ever // touching the later elements: - fooOfBars.toPrettyElidedString(1000) should be("""Foo( - | bar = "long long list", - | list = List( - | "blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0", - | "blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah...""".stripMargin) + fooOfBars.toPrettyElidedString(1000) should be( + """Foo( + | bar = "long long list", + | list = List( + | "blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0", + | "blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah...""".stripMargin + ) } @@ -36,84 +38,84 @@ class StringUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers ( "mask user info", "https://user:pass@example.com/path/to/file", - "https://example.com/path/to/file", + "https://example.com/path/to/file" ), ( "mask the entire query if no known sensitive query params are found", s"https://example.com/path/to/file?my_new_hidden_param=$InputToBeMasked", - "https://example.com/path/to/file?masked", + "https://example.com/path/to/file?masked" ), ( "mask credential params", s"https://example.com/path/to/file?my_credential_param=$InputToBeMasked&my_other_param=ok", - s"https://example.com/path/to/file?my_credential_param=$OutputMasked&my_other_param=ok", + s"https://example.com/path/to/file?my_credential_param=$OutputMasked&my_other_param=ok" ), ( "mask signature params", s"https://example.com/path/to/file?my_signature_param=$InputToBeMasked&my_other_param=ok", - s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=ok", + s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=ok" ), ( "mask encoded signature params", s"https://example.com/path/to/file?my_sign%61ture_param=$InputToBeMasked&my_other_param=ok", - s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=ok", + s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=ok" ), ( // There is a note in the docs for common.util.StringUtil.EnhancedString.maskSensitiveUri about this behavior "mask uris with encoded parameters", s"https://example.com/path/to/file?my_signature_param=$InputToBeMasked&my_other_param=%26%2F%3F", - s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=&/?", + s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=&/?" ), ( "mask uris with parameters without values", s"https://example.com/path/to/file?my_signature_param=$InputToBeMasked&my_other_param", - s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param", + s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param" ), ( "mask uris with parameters values containing equal signs", s"https://example.com/path/to/file?my_signature_param=$InputToBeMasked&my_other_param=with=equal", - s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=with=equal", + s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=with=equal" ), ( "not mask the fragment", s"https://example.com?my_signature_param=$InputToBeMasked#nofilter", - s"https://example.com?my_signature_param=$OutputMasked#nofilter", + s"https://example.com?my_signature_param=$OutputMasked#nofilter" ), ( "not mask the port number", s"https://example.com:1234?my_signature_param=$InputToBeMasked", - s"https://example.com:1234?my_signature_param=$OutputMasked", + s"https://example.com:1234?my_signature_param=$OutputMasked" ), ( // via: https://cr.openjdk.java.net/~dfuchs/writeups/updating-uri/ "not mask a RFC 3986 specific uri", s"urn:isbn:096139210?my_credential_param=$InputToBeMasked", - s"urn:isbn:096139210?my_credential_param=$InputToBeMasked", + s"urn:isbn:096139210?my_credential_param=$InputToBeMasked" ), ( // via: https://bvdp-saturn-dev.appspot.com/#workspaces/general-dev-billing-account/DRS%20and%20Signed%20URL%20Development%20-%20Dev/notebooks/launch/drs_signed_url_flow_kids_dev.ipynb "not mask a DRS CIB URI", "drs://dg.F82A1A:371b834f-a896-42e6-b1d1-9fad96891f33", - "drs://dg.F82A1A:371b834f-a896-42e6-b1d1-9fad96891f33", + "drs://dg.F82A1A:371b834f-a896-42e6-b1d1-9fad96891f33" ), ( // via: https://bvdp-saturn-dev.appspot.com/#workspaces/general-dev-billing-account/DRS%20and%20Signed%20URL%20Development%20-%20Dev/notebooks/launch/drs_signed_url_flow_kids_dev.ipynb "mask an AWS Signed URL", s"https://example-redacted-but-not-masked.s3.amazonaws.com/$InputRedacted.CNVs.p.value.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=$InputToBeMasked&X-Amz-Date=20210504T200819Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&user_id=122&username=$InputRedacted&X-Amz-Signature=$InputToBeMasked", - s"https://example-redacted-but-not-masked.s3.amazonaws.com/$InputRedacted.CNVs.p.value.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=$OutputMasked&X-Amz-Date=20210504T200819Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&user_id=122&username=$InputRedacted&X-Amz-Signature=$OutputMasked", + s"https://example-redacted-but-not-masked.s3.amazonaws.com/$InputRedacted.CNVs.p.value.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=$OutputMasked&X-Amz-Date=20210504T200819Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&user_id=122&username=$InputRedacted&X-Amz-Signature=$OutputMasked" ), ( // via: https://bvdp-saturn-dev.appspot.com/#workspaces/general-dev-billing-account/DRS%20and%20Signed%20URL%20Development%20-%20Dev/notebooks/launch/drs_signed_url_flow_bdcat_dev.ipynb "mask a GCS V2 Signed URL", s"https://storage.googleapis.com/$InputRedacted/testfile.txt?GoogleAccessId=$InputRedacted&Expires=1614119022&Signature=$InputToBeMasked&userProject=$InputRedacted", - s"https://storage.googleapis.com/$InputRedacted/testfile.txt?GoogleAccessId=$InputRedacted&Expires=1614119022&Signature=$OutputMasked&userProject=$InputRedacted", + s"https://storage.googleapis.com/$InputRedacted/testfile.txt?GoogleAccessId=$InputRedacted&Expires=1614119022&Signature=$OutputMasked&userProject=$InputRedacted" ), ( // via: gsutil signurl $HOME/.config/gcloud/legacy_credentials/cromwell@broad-dsde-cromwell-dev.iam.gserviceaccount.com/adc.json gs://cloud-cromwell-dev/some/gumby.png "mask a GCS V4 Signed URL", s"https://storage.googleapis.com/cloud-cromwell-dev/some/gumby.png?x-goog-signature=$InputToBeMasked&x-goog-algorithm=GOOG4-RSA-SHA256&x-goog-credential=$InputToBeMasked&x-goog-date=20210505T042119Z&x-goog-expires=3600&x-goog-signedheaders=host", - s"https://storage.googleapis.com/cloud-cromwell-dev/some/gumby.png?x-goog-signature=$OutputMasked&x-goog-algorithm=GOOG4-RSA-SHA256&x-goog-credential=$OutputMasked&x-goog-date=20210505T042119Z&x-goog-expires=3600&x-goog-signedheaders=host", - ), + s"https://storage.googleapis.com/cloud-cromwell-dev/some/gumby.png?x-goog-signature=$OutputMasked&x-goog-algorithm=GOOG4-RSA-SHA256&x-goog-credential=$OutputMasked&x-goog-date=20210505T042119Z&x-goog-expires=3600&x-goog-signedheaders=host" + ) ) forAll(maskSensitiveUriTests) { (description, input, expected) => @@ -128,7 +130,7 @@ object StringUtilSpec { final case class Foo(bar: String, list: List[Bar]) final class Bar(index: Int) { - private def longLine(i: Int) = "\"" + s"blah$i" * 100 + "\"" + private def longLine(i: Int) = "\"" + s"blah$i" * 100 + "\"" override def toString: String = if (index < 2) { longLine(index) } else { diff --git a/common/src/test/scala/common/util/TerminalUtilSpec.scala b/common/src/test/scala/common/util/TerminalUtilSpec.scala index c8fda6627e4..1c68b9f17fc 100644 --- a/common/src/test/scala/common/util/TerminalUtilSpec.scala +++ b/common/src/test/scala/common/util/TerminalUtilSpec.scala @@ -5,7 +5,6 @@ import common.util.TerminalUtil._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class TerminalUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "TerminalUtil" diff --git a/common/src/test/scala/common/util/TryUtilSpec.scala b/common/src/test/scala/common/util/TryUtilSpec.scala index d7b8739337b..0bb62f9ed33 100644 --- a/common/src/test/scala/common/util/TryUtilSpec.scala +++ b/common/src/test/scala/common/util/TryUtilSpec.scala @@ -9,7 +9,6 @@ import org.scalatest.matchers.should.Matchers import scala.util.{Failure, Success, Try} import org.scalatest.enablers.Emptiness._ - class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "TryUtil" @@ -80,15 +79,15 @@ class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "sequence successful keys and successful values" in { - val result: Try[Map[String, String]] = sequenceKeyValues( - Map(Success("success key") -> Success("success value")), "prefix") + val result: Try[Map[String, String]] = + sequenceKeyValues(Map(Success("success key") -> Success("success value")), "prefix") result.isSuccess should be(true) result.get.toList should contain theSameElementsAs Map("success key" -> "success value") } it should "sequence successful keys and failed values" in { - val result: Try[Map[String, String]] = sequenceKeyValues( - Map(Success("success key") -> Failure(new RuntimeException("failed value"))), "prefix") + val result: Try[Map[String, String]] = + sequenceKeyValues(Map(Success("success key") -> Failure(new RuntimeException("failed value"))), "prefix") result.isFailure should be(true) result.failed.get should be(an[AggregatedException]) val exception = result.failed.get.asInstanceOf[AggregatedException] @@ -98,8 +97,8 @@ class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "sequence failed keys and successful values" in { - val result: Try[Map[String, String]] = sequenceKeyValues( - Map(Failure(new RuntimeException("failed key")) -> Success("success value")), "prefix") + val result: Try[Map[String, String]] = + sequenceKeyValues(Map(Failure(new RuntimeException("failed key")) -> Success("success value")), "prefix") result.isFailure should be(true) result.failed.get should be(an[AggregatedException]) val exception = result.failed.get.asInstanceOf[AggregatedException] @@ -110,7 +109,9 @@ class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { it should "sequence failed keys and failed values" in { val result: Try[Map[String, String]] = sequenceKeyValues( - Map(Failure(new RuntimeException("failed key")) -> Failure(new RuntimeException("failed value"))), "prefix") + Map(Failure(new RuntimeException("failed key")) -> Failure(new RuntimeException("failed value"))), + "prefix" + ) result.isFailure should be(true) result.failed.get should be(an[AggregatedException]) val exception = result.failed.get.asInstanceOf[AggregatedException] @@ -127,8 +128,8 @@ class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "sequence a successful key with a failed value" in { - val result: Try[(String, String)] = sequenceTuple( - (Success("success key"), Failure(new RuntimeException("failed value"))), "prefix") + val result: Try[(String, String)] = + sequenceTuple((Success("success key"), Failure(new RuntimeException("failed value"))), "prefix") result.isFailure should be(true) result.failed.get should be(an[AggregatedException]) val exception = result.failed.get.asInstanceOf[AggregatedException] @@ -138,8 +139,8 @@ class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "sequence a failed key with a successful value" in { - val result: Try[(String, String)] = sequenceTuple( - (Failure(new RuntimeException("failed key")), Success("success value")), "prefix") + val result: Try[(String, String)] = + sequenceTuple((Failure(new RuntimeException("failed key")), Success("success value")), "prefix") result.isFailure should be(true) result.failed.get should be(an[AggregatedException]) val exception = result.failed.get.asInstanceOf[AggregatedException] @@ -149,8 +150,10 @@ class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "sequence a failed key with a failed value" in { - val result: Try[(String, String)] = sequenceTuple( - (Failure(new RuntimeException("failed key")), Failure(new RuntimeException("failed value"))), "prefix") + val result: Try[(String, String)] = + sequenceTuple((Failure(new RuntimeException("failed key")), Failure(new RuntimeException("failed value"))), + "prefix" + ) result.isFailure should be(true) result.failed.get should be(an[AggregatedException]) val exception = result.failed.get.asInstanceOf[AggregatedException] diff --git a/common/src/test/scala/common/util/VersionUtilSpec.scala b/common/src/test/scala/common/util/VersionUtilSpec.scala index 50aab84bad6..5d58d2f0852 100644 --- a/common/src/test/scala/common/util/VersionUtilSpec.scala +++ b/common/src/test/scala/common/util/VersionUtilSpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class VersionUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "VersionUtil" @@ -22,12 +21,12 @@ class VersionUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers it should "getVersion with the default" in { val version = VersionUtil.getVersion("made-up-artifact") - version should be ("made-up-artifact-version.conf-to-be-generated-by-sbt") + version should be("made-up-artifact-version.conf-to-be-generated-by-sbt") } it should "getVersion with a default override" in { val version = VersionUtil.getVersion("made-up-artifact", _ => "default override") - version should be ("default override") + version should be("default override") } it should "defaultMessage" in { @@ -38,7 +37,7 @@ class VersionUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers val expected = intercept[RuntimeException](VersionUtil.sbtDependencyVersion("madeUp")("made-up-project")) expected.getMessage should fullyMatch regex "Did not parse a version for 'madeUpV' from .*/project/Dependencies.scala " + - "\\(This occurred after made-up-project-version.conf was not found.\\)" + "\\(This occurred after made-up-project-version.conf was not found.\\)" } it should "pass sbtDependencyVersion check for typesafeConfig" in { diff --git a/common/src/test/scala/common/validation/CheckedSpec.scala b/common/src/test/scala/common/validation/CheckedSpec.scala index c9b5a387666..23281fccc0b 100644 --- a/common/src/test/scala/common/validation/CheckedSpec.scala +++ b/common/src/test/scala/common/validation/CheckedSpec.scala @@ -6,10 +6,9 @@ import common.validation.Checked._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class CheckedSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "Checked" - + it should "provide helper methods" in { 5.validNelCheck shouldBe Right(5) "argh".invalidNelCheck[Int] shouldBe Left(NonEmptyList.one("argh")) diff --git a/common/src/test/scala/common/validation/ErrorOrSpec.scala b/common/src/test/scala/common/validation/ErrorOrSpec.scala index ac0ebfd957b..55a7d004cc3 100644 --- a/common/src/test/scala/common/validation/ErrorOrSpec.scala +++ b/common/src/test/scala/common/validation/ErrorOrSpec.scala @@ -8,7 +8,6 @@ import common.validation.ErrorOr._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class ErrorOrSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "ErrorOr" @@ -44,11 +43,33 @@ class ErrorOrSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } val DivBy0Error: String = "Divide by 0!" - def errorOrDiv(v1: Int, v2: Int): ErrorOr[Double] = if (v2 != 0) { Valid(v1.toDouble / v2.toDouble) } else { DivBy0Error.invalidNel } - def errorOrDiv(v1: Double, v2: Int): ErrorOr[Double] = if (v2 != 0) { Valid(v1.toDouble / v2.toDouble) } else { DivBy0Error.invalidNel } - def errorOrSelect(v1: Int, v2: Int, v3: Int, v4: Int, v5: Int, v6: Int, v7: Int, - v8: Int, v9: Int, v10: Int, v11: Int, v12: Int, v13: Int, v14: Int, - v15: Int, v16: Int, v17: Int, v18: Int, v19: Int, v20: Int, v21: Int, v22: Int): ErrorOr[Int] = Valid(v4 + v6 + v22) + def errorOrDiv(v1: Int, v2: Int): ErrorOr[Double] = if (v2 != 0) { Valid(v1.toDouble / v2.toDouble) } + else { DivBy0Error.invalidNel } + def errorOrDiv(v1: Double, v2: Int): ErrorOr[Double] = if (v2 != 0) { Valid(v1.toDouble / v2.toDouble) } + else { DivBy0Error.invalidNel } + def errorOrSelect(v1: Int, + v2: Int, + v3: Int, + v4: Int, + v5: Int, + v6: Int, + v7: Int, + v8: Int, + v9: Int, + v10: Int, + v11: Int, + v12: Int, + v13: Int, + v14: Int, + v15: Int, + v16: Int, + v17: Int, + v18: Int, + v19: Int, + v20: Int, + v21: Int, + v22: Int + ): ErrorOr[Int] = Valid(v4 + v6 + v22) val valid0 = Valid(0) val valid1 = Valid(1) @@ -63,13 +84,29 @@ class ErrorOrSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "flatMapN 22-tuples into a Valid" in { - (valid0, valid1, valid2, - valid0, valid1, valid2, - valid0, valid1, valid2, - valid0, valid1, valid2, - valid0, valid1, valid2, - valid0, valid1, valid2, - valid0, valid1, valid2, valid0) flatMapN errorOrSelect should be(Valid(0 + 2 + 0)) + (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0 + ) flatMapN errorOrSelect should be(Valid(0 + 2 + 0)) } it should "flatMapN 1-tuples into a Valid string" in { @@ -139,82 +176,211 @@ class ErrorOrSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "flatMapN 13-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0) - .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) + val result = + (valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0) + .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("0120120120120")) } it should "flatMapN 14-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1) - .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) + val result = + (valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1) + .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("01201201201201")) } it should "flatMapN 15-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("012012012012012")) } it should "flatMapN 16-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("0120120120120120")) } it should "flatMapN 17-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0, valid1) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("01201201201201201")) } it should "flatMapN 18-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0, valid1, valid2) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("012012012012012012")) } it should "flatMapN 19-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0, valid1, valid2, valid0) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("0120120120120120120")) } it should "flatMapN 20-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("01201201201201201201")) } it should "flatMapN 21-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("012012012012012012012")) } it should "flatMapN 22-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0).flatMapN( - Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0 + ).flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("0120120120120120120120")) } @@ -228,7 +394,8 @@ class ErrorOrSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { | def flatMapN[T_OUT](f1: (A) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t1.tupled flatMap f1.tupled |} | - |""".stripMargin) + |""".stripMargin + ) result } diff --git a/common/src/test/scala/common/validation/ValidationSpec.scala b/common/src/test/scala/common/validation/ValidationSpec.scala index 543e33d7573..8a1c72a922e 100644 --- a/common/src/test/scala/common/validation/ValidationSpec.scala +++ b/common/src/test/scala/common/validation/ValidationSpec.scala @@ -14,7 +14,6 @@ import common.mock.MockSugar import scala.util.{Failure, Success} - class ValidationSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with MockSugar { behavior of "Validation" diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 6ec05cf6025..25fd0e6ac87 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -118,7 +118,8 @@ system { max-concurrent-workflows = 5000 # Cromwell will launch up to N submitted workflows at a time, regardless of how many open workflow slots exist - max-workflow-launch-count = 50 + # Deviating from 1 is not recommended for multi-runner setups due to possible deadlocks. [BW-962] + max-workflow-launch-count = 1 # Workflows will be grouped by the value of the specified field in their workflow options. # @@ -417,7 +418,7 @@ docker { throttle { number-of-requests = 1000 per = 60 seconds - } + } num-threads = 10 } google { @@ -523,6 +524,8 @@ backend { } } +# Note: When adding a new actor that uses service registry pattern make sure that the new actor handles the graceful +# shutdown command ('ShutdownCommand'). See https://github.com/broadinstitute/cromwell/issues/2575 services { KeyValue { class = "cromwell.services.keyvalue.impl.SqlKeyValueServiceActor" @@ -570,6 +573,15 @@ services { # Default - run within the Cromwell JVM class = "cromwell.services.womtool.impl.WomtoolServiceInCromwellActor" } + GithubAuthVending { + class = "cromwell.services.auth.impl.GithubAuthVendingActor" + config { + enabled = false + auth.azure = false + # Set this to the service that Cromwell should retrieve Github access token associated with user's token. + # ecm.base-url = "" + } + } } include required(classpath("reference_database.inc.conf")) diff --git a/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala b/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala index 3000b30e8df..8c53b38c36d 100644 --- a/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala +++ b/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala @@ -14,7 +14,9 @@ object DockerCredentials { object DockerCredentialUsernameAndPassword { private val tokenStringFormat = raw"([^:]*):(.*)".r - def unapply(arg: DockerCredentials): Option[(String, String)] = Try(new String(Base64.getDecoder.decode(arg.token))).toOption match { + def unapply(arg: DockerCredentials): Option[(String, String)] = Try( + new String(Base64.getDecoder.decode(arg.token)) + ).toOption match { case Some(tokenStringFormat(username, password)) => Some((username, password)) case _ => None } diff --git a/core/src/main/scala/cromwell/core/ConfigUtil.scala b/core/src/main/scala/cromwell/core/ConfigUtil.scala index 0fd5002ffa8..5adf56ec0a4 100644 --- a/core/src/main/scala/cromwell/core/ConfigUtil.scala +++ b/core/src/main/scala/cromwell/core/ConfigUtil.scala @@ -8,7 +8,7 @@ import com.typesafe.config.{Config, ConfigException, ConfigValue} import org.slf4j.LoggerFactory import scala.jdk.CollectionConverters._ -import scala.reflect.{ClassTag, classTag} +import scala.reflect.{classTag, ClassTag} object ConfigUtil { @@ -20,12 +20,12 @@ object ConfigUtil { /** * For keys that are in the configuration but not in the reference keySet, log a warning. */ - def warnNotRecognized(keySet: Set[String], context: String) = { + def warnNotRecognized(keySet: Set[String], context: String) = keys.diff(keySet) match { - case warnings if warnings.nonEmpty => validationLogger.warn(s"Unrecognized configuration key(s) for $context: ${warnings.mkString(", ")}") + case warnings if warnings.nonEmpty => + validationLogger.warn(s"Unrecognized configuration key(s) for $context: ${warnings.mkString(", ")}") case _ => } - } /** * Validates that the value for this key is a well formed URL. @@ -34,15 +34,15 @@ object ConfigUtil { new URL(config.getString(url)) } - def validateString(key: String): ValidatedNel[String, String] = try { + def validateString(key: String): ValidatedNel[String, String] = try config.getString(key).validNel - } catch { + catch { case _: ConfigException.Missing => s"Could not find key: $key".invalidNel } - def validateConfig(key: String): ValidatedNel[String, Config] = try { + def validateConfig(key: String): ValidatedNel[String, Config] = try config.getConfig(key).validNel - } catch { + catch { case _: ConfigException.Missing => s"Could not find key: $key".invalidNel case _: ConfigException.WrongType => s"key $key cannot be parsed to a Config".invalidNel } @@ -50,6 +50,7 @@ object ConfigUtil { } implicit class EnhancedValidation[I <: AnyRef](val value: I) extends AnyVal { + /** * Validates this value by applying validationFunction to it and returning a Validation: * Returns successNel upon success. @@ -58,9 +59,9 @@ object ConfigUtil { * @tparam O return type of validationFunction * @tparam E Restricts the subtype of Exception that should be caught during validation */ - def validateAny[O, E <: Exception: ClassTag](validationFunction: I => O): ValidatedNel[String, O] = try { + def validateAny[O, E <: Exception: ClassTag](validationFunction: I => O): ValidatedNel[String, O] = try validationFunction(value).validNel - } catch { + catch { case e if classTag[E].runtimeClass.isInstance(e) => e.getMessage.invalidNel } } diff --git a/core/src/main/scala/cromwell/core/DockerConfiguration.scala b/core/src/main/scala/cromwell/core/DockerConfiguration.scala index e765eec80fe..3f2dfa5c6f8 100644 --- a/core/src/main/scala/cromwell/core/DockerConfiguration.scala +++ b/core/src/main/scala/cromwell/core/DockerConfiguration.scala @@ -18,23 +18,30 @@ object DockerConfiguration { lazy val instance: DockerConfiguration = { if (dockerHashLookupConfig.hasPath("gcr-api-queries-per-100-seconds")) { - logger.warn("'docker.hash-lookup.gcr-api-queries-per-100-seconds' is no longer supported, use 'docker.hash-lookup.google.throttle' instead (see reference.conf)") + logger.warn( + "'docker.hash-lookup.gcr-api-queries-per-100-seconds' is no longer supported, use 'docker.hash-lookup.google.throttle' instead (see reference.conf)" + ) } - val enabled = validate { dockerHashLookupConfig.as[Boolean]("enabled") } - val cacheEntryTtl = validate { dockerHashLookupConfig.as[FiniteDuration]("cache-entry-ttl") } - val cacheSize = validate { dockerHashLookupConfig.as[Long]("cache-size") } - val method: ErrorOr[DockerHashLookupMethod] = validate { dockerHashLookupConfig.as[String]("method") } map { + val enabled = validate(dockerHashLookupConfig.as[Boolean]("enabled")) + val cacheEntryTtl = validate(dockerHashLookupConfig.as[FiniteDuration]("cache-entry-ttl")) + val cacheSize = validate(dockerHashLookupConfig.as[Long]("cache-size")) + val method: ErrorOr[DockerHashLookupMethod] = validate(dockerHashLookupConfig.as[String]("method")) map { case "local" => DockerLocalLookup case "remote" => DockerRemoteLookup case other => throw new IllegalArgumentException(s"Unrecognized docker hash lookup method: $other") } - val sizeCompressionFactor = validate { dockerHashLookupConfig.as[Double]("size-compression-factor") } - val maxTimeBetweenRetries = validate { dockerHashLookupConfig.as[FiniteDuration]("max-time-between-retries") } - val maxRetries = validate { dockerHashLookupConfig.as[Int]("max-retries") } + val sizeCompressionFactor = validate(dockerHashLookupConfig.as[Double]("size-compression-factor")) + val maxTimeBetweenRetries = validate(dockerHashLookupConfig.as[FiniteDuration]("max-time-between-retries")) + val maxRetries = validate(dockerHashLookupConfig.as[Int]("max-retries")) val dockerConfiguration = (enabled, - cacheEntryTtl, cacheSize, method, - sizeCompressionFactor, maxTimeBetweenRetries, maxRetries) mapN DockerConfiguration.apply + cacheEntryTtl, + cacheSize, + method, + sizeCompressionFactor, + maxTimeBetweenRetries, + maxRetries + ) mapN DockerConfiguration.apply dockerConfiguration match { case Valid(conf) => conf @@ -44,14 +51,14 @@ object DockerConfiguration { } case class DockerConfiguration( - enabled: Boolean, - cacheEntryTtl: FiniteDuration, - cacheSize: Long, - method: DockerHashLookupMethod, - sizeCompressionFactor: Double, - maxTimeBetweenRetries: FiniteDuration, - maxRetries: Int - ) + enabled: Boolean, + cacheEntryTtl: FiniteDuration, + cacheSize: Long, + method: DockerHashLookupMethod, + sizeCompressionFactor: Double, + maxTimeBetweenRetries: FiniteDuration, + maxRetries: Int +) sealed trait DockerHashLookupMethod diff --git a/core/src/main/scala/cromwell/core/Encryption.scala b/core/src/main/scala/cromwell/core/Encryption.scala index 0258d4511e3..4e05693b003 100644 --- a/core/src/main/scala/cromwell/core/Encryption.scala +++ b/core/src/main/scala/cromwell/core/Encryption.scala @@ -31,15 +31,18 @@ case object Aes256Cbc { cipher } - final def validateLength(arrayName: String, array: Array[Byte], expectedBitLength: Int): Try[Unit] = { + final def validateLength(arrayName: String, array: Array[Byte], expectedBitLength: Int): Try[Unit] = if (array.length * 8 == expectedBitLength) { Success(()) } else { - Failure(new IllegalArgumentException(s"$arrayName size (${array.length * 8} bits) did not match the required length $expectedBitLength")) + Failure( + new IllegalArgumentException( + s"$arrayName size (${array.length * 8} bits) did not match the required length $expectedBitLength" + ) + ) } - } - final def encrypt(plainText: Array[Byte], secretKey: SecretKey): Try[EncryptedBytes] = { + final def encrypt(plainText: Array[Byte], secretKey: SecretKey): Try[EncryptedBytes] = validateLength("Secret key", secretKey.key, keySize) map { _ => val iv = new Array[Byte](blockSize / 8) ranGen.nextBytes(iv) @@ -47,15 +50,15 @@ case object Aes256Cbc { val cipher = init(Cipher.ENCRYPT_MODE, secretKey.key, iv) EncryptedBytes(cipher.doFinal(plainText), iv) } - } - final def decrypt(encryptedBytes: EncryptedBytes, secretKey: SecretKey): Try[Array[Byte]] = { + final def decrypt(encryptedBytes: EncryptedBytes, secretKey: SecretKey): Try[Array[Byte]] = for { _ <- validateLength("Secret key", secretKey.key, keySize) _ <- validateLength("Initialization vector", encryptedBytes.initializationVector, blockSize) - bytes = init(Cipher.DECRYPT_MODE, secretKey.key, encryptedBytes.initializationVector).doFinal(encryptedBytes.cipherText) + bytes = init(Cipher.DECRYPT_MODE, secretKey.key, encryptedBytes.initializationVector).doFinal( + encryptedBytes.cipherText + ) } yield bytes - } } final case class EncryptedBytes(cipherText: Array[Byte], initializationVector: Array[Byte]) { @@ -71,4 +74,4 @@ object EncryptedBytes { final case class SecretKey(key: Array[Byte]) object SecretKey { def apply(base64KeyString: String): SecretKey = SecretKey(Base64.decodeBase64(base64KeyString)) -} \ No newline at end of file +} diff --git a/core/src/main/scala/cromwell/core/ExecutionIndex.scala b/core/src/main/scala/cromwell/core/ExecutionIndex.scala index 4f04179db0c..926fad13c11 100644 --- a/core/src/main/scala/cromwell/core/ExecutionIndex.scala +++ b/core/src/main/scala/cromwell/core/ExecutionIndex.scala @@ -23,8 +23,7 @@ object ExecutionIndex { } implicit val ExecutionIndexOrdering = new Ordering[ExecutionIndex] { - override def compare(x: ExecutionIndex, y: ExecutionIndex): Int = { + override def compare(x: ExecutionIndex, y: ExecutionIndex): Int = x.fromIndex.compareTo(y.fromIndex) - } } } diff --git a/core/src/main/scala/cromwell/core/ExecutionStatus.scala b/core/src/main/scala/cromwell/core/ExecutionStatus.scala index 3d4016d90a5..fa2ece67ab5 100644 --- a/core/src/main/scala/cromwell/core/ExecutionStatus.scala +++ b/core/src/main/scala/cromwell/core/ExecutionStatus.scala @@ -2,7 +2,8 @@ package cromwell.core object ExecutionStatus extends Enumeration { type ExecutionStatus = Value - val NotStarted, WaitingForQueueSpace, QueuedInCromwell, Starting, Running, Aborting, Failed, RetryableFailure, Done, Bypassed, Aborted, Unstartable = Value + val NotStarted, WaitingForQueueSpace, QueuedInCromwell, Starting, Running, Aborting, Failed, RetryableFailure, Done, + Bypassed, Aborted, Unstartable = Value val TerminalStatuses = Set(Failed, Done, Aborted, Bypassed, Unstartable) val TerminalOrRetryableStatuses = TerminalStatuses + RetryableFailure val NonTerminalStatuses = values.diff(TerminalOrRetryableStatuses) @@ -24,7 +25,7 @@ object ExecutionStatus extends Enumeration { case Done => 11 } } - + implicit class EnhancedExecutionStatus(val status: ExecutionStatus) extends AnyVal { def isTerminal: Boolean = TerminalStatuses contains status diff --git a/core/src/main/scala/cromwell/core/HogGroup.scala b/core/src/main/scala/cromwell/core/HogGroup.scala index 87bcad5d0c9..c49b84a9ecc 100644 --- a/core/src/main/scala/cromwell/core/HogGroup.scala +++ b/core/src/main/scala/cromwell/core/HogGroup.scala @@ -17,16 +17,16 @@ object HogGroup { if (config.hasPath("system.hog-safety.workflow-option")) { val hogGroupField = config.getString("system.hog-safety.workflow-option") - (options, workflowId) => { + (options, workflowId) => options.get(hogGroupField) match { case Success(hg) => HogGroup(hg) case Failure(_) => HogGroup(workflowId.shortString) } - } - } else { - (_, workflowId) => HogGroup(workflowId.shortString) + } else { (_, workflowId) => + HogGroup(workflowId.shortString) } } - def decide(workflowOptions: WorkflowOptions, workflowId: WorkflowId): HogGroup = HogGroupDeciderFunction.apply(workflowOptions, workflowId) + def decide(workflowOptions: WorkflowOptions, workflowId: WorkflowId): HogGroup = + HogGroupDeciderFunction.apply(workflowOptions, workflowId) } diff --git a/core/src/main/scala/cromwell/core/JobKey.scala b/core/src/main/scala/cromwell/core/JobKey.scala index e5f990aa433..99118dc2235 100644 --- a/core/src/main/scala/cromwell/core/JobKey.scala +++ b/core/src/main/scala/cromwell/core/JobKey.scala @@ -13,6 +13,6 @@ trait JobKey { import ExecutionIndex.IndexEnhancedIndex s"${getClass.getSimpleName}_${node.getClass.getSimpleName}_${node.fullyQualifiedName}:${index.fromIndex}:$attempt" } - - def isShard = index.isDefined + + def isShard = index.isDefined } diff --git a/core/src/main/scala/cromwell/core/MonitoringCompanionActor.scala b/core/src/main/scala/cromwell/core/MonitoringCompanionActor.scala index 53f0e4de6fc..8932b3f6836 100644 --- a/core/src/main/scala/cromwell/core/MonitoringCompanionActor.scala +++ b/core/src/main/scala/cromwell/core/MonitoringCompanionActor.scala @@ -9,21 +9,21 @@ import scala.language.postfixOps object MonitoringCompanionActor { sealed trait MonitoringCompanionCommand - private [core] case object AddWork extends MonitoringCompanionCommand - private [core] case object RemoveWork extends MonitoringCompanionCommand - private [core] def props(actorToMonitor: ActorRef) = Props(new MonitoringCompanionActor(actorToMonitor)) + private[core] case object AddWork extends MonitoringCompanionCommand + private[core] case object RemoveWork extends MonitoringCompanionCommand + private[core] def props(actorToMonitor: ActorRef) = Props(new MonitoringCompanionActor(actorToMonitor)) } -private [core] class MonitoringCompanionActor(actorToMonitor: ActorRef) extends Actor with ActorLogging { +private[core] class MonitoringCompanionActor(actorToMonitor: ActorRef) extends Actor with ActorLogging { private var workCount: Int = 0 - + override def receive = { case AddWork => workCount += 1 case RemoveWork => workCount -= 1 case ShutdownCommand if workCount <= 0 => context stop actorToMonitor context stop self - case ShutdownCommand => + case ShutdownCommand => log.info(s"{} is still processing {} messages", actorToMonitor.path.name, workCount) context.system.scheduler.scheduleOnce(1 second, self, ShutdownCommand)(context.dispatcher) () @@ -33,12 +33,12 @@ private [core] class MonitoringCompanionActor(actorToMonitor: ActorRef) extends trait MonitoringCompanionHelper { this: Actor => private val monitoringActor = context.actorOf(MonitoringCompanionActor.props(self)) private var shuttingDown: Boolean = false - + def addWork() = monitoringActor ! AddWork def removeWork() = monitoringActor ! RemoveWork val monitoringReceive: Receive = { - case ShutdownCommand if !shuttingDown => + case ShutdownCommand if !shuttingDown => shuttingDown = true monitoringActor ! ShutdownCommand case ShutdownCommand => // Ignore if we're already shutting down diff --git a/core/src/main/scala/cromwell/core/WorkflowId.scala b/core/src/main/scala/cromwell/core/WorkflowId.scala index feb0ee601a9..f2444738fed 100644 --- a/core/src/main/scala/cromwell/core/WorkflowId.scala +++ b/core/src/main/scala/cromwell/core/WorkflowId.scala @@ -8,19 +8,17 @@ sealed trait WorkflowId { override def toString = id.toString def shortString = id.toString.split("-")(0) - def toRoot: RootWorkflowId = { + def toRoot: RootWorkflowId = this match { case root: RootWorkflowId => root case _ => RootWorkflowId(id) } - } - def toPossiblyNotRoot: PossiblyNotRootWorkflowId = { + def toPossiblyNotRoot: PossiblyNotRootWorkflowId = this match { case possiblyNotRoot: PossiblyNotRootWorkflowId => possiblyNotRoot case _ => PossiblyNotRootWorkflowId(id) } - } } object WorkflowId { diff --git a/core/src/main/scala/cromwell/core/WorkflowOptions.scala b/core/src/main/scala/cromwell/core/WorkflowOptions.scala index 91a7c30bbfe..cbdb1201986 100644 --- a/core/src/main/scala/cromwell/core/WorkflowOptions.scala +++ b/core/src/main/scala/cromwell/core/WorkflowOptions.scala @@ -55,7 +55,7 @@ object WorkflowOptions { case object FinalWorkflowLogDir extends WorkflowOption("final_workflow_log_dir") case object FinalCallLogsDir extends WorkflowOption("final_call_logs_dir") case object FinalWorkflowOutputsDir extends WorkflowOption("final_workflow_outputs_dir") - case object UseRelativeOutputPaths extends WorkflowOption(name="use_relative_output_paths") + case object UseRelativeOutputPaths extends WorkflowOption(name = "use_relative_output_paths") // Misc. case object DefaultRuntimeOptions extends WorkflowOption("default_runtime_attributes") @@ -70,23 +70,28 @@ object WorkflowOptions { private lazy val defaultRuntimeOptionKey: String = DefaultRuntimeOptions.name private lazy val validObjectKeys: Set[String] = Set(DefaultRuntimeOptions.name, "google_labels") - def encryptField(value: JsString): Try[JsObject] = { + def encryptField(value: JsString): Try[JsObject] = Aes256Cbc.encrypt(value.value.getBytes("utf-8"), SecretKey(EncryptionKey)) match { - case Success(encryptedValue) => Success(JsObject(Map( - "iv" -> JsString(encryptedValue.base64Iv), - "ciphertext" -> JsString(encryptedValue.base64CipherText) - ))) + case Success(encryptedValue) => + Success( + JsObject( + Map( + "iv" -> JsString(encryptedValue.base64Iv), + "ciphertext" -> JsString(encryptedValue.base64CipherText) + ) + ) + ) case Failure(ex) => Failure(ex) } - } - def decryptField(obj: JsObject): Try[String] = { + def decryptField(obj: JsObject): Try[String] = (obj.fields.get("iv"), obj.fields.get("ciphertext")) match { case (Some(iv: JsString), Some(ciphertext: JsString)) => - Aes256Cbc.decrypt(EncryptedBytes(ciphertext.value, iv.value), SecretKey(WorkflowOptions.EncryptionKey)).map(new String(_, "utf-8")) + Aes256Cbc + .decrypt(EncryptedBytes(ciphertext.value, iv.value), SecretKey(WorkflowOptions.EncryptionKey)) + .map(new String(_, "utf-8")) case _ => Failure(new RuntimeException(s"JsObject must have 'iv' and 'ciphertext' fields to decrypt: $obj")) } - } def isEncryptedField(jsValue: JsValue): Boolean = jsValue match { case obj: JsObject if obj.fields.keys.exists(_ == "iv") && obj.fields.keys.exists(_ == "ciphertext") => true @@ -102,7 +107,8 @@ object WorkflowOptions { case (k, v: JsNumber) => k -> Success(v) case (k, v) if isEncryptedField(v) => k -> Success(v) case (k, v: JsArray) => k -> Success(v) - case (k, v) => k -> Failure(new UnsupportedOperationException(s"Unsupported key/value pair in WorkflowOptions: $k -> $v")) + case (k, v) => + k -> Failure(new UnsupportedOperationException(s"Unsupported key/value pair in WorkflowOptions: $k -> $v")) } encrypted.values collect { case f: Failure[_] => f } match { @@ -117,7 +123,7 @@ object WorkflowOptions { case Success(x) => Failure(new UnsupportedOperationException(s"Expecting JSON object, got $x")) } - def fromMap(m: Map[String, String]) = fromJsonObject(JsObject(m map { case (k, v) => k -> JsString(v)})) + def fromMap(m: Map[String, String]) = fromJsonObject(JsObject(m map { case (k, v) => k -> JsString(v) })) private def getAsJson(key: String, jsObject: JsObject) = jsObject.fields.get(key) match { case Some(jsStr: JsString) => Success(jsStr) @@ -157,7 +163,7 @@ case class WorkflowOptions(jsObject: JsObject) { } def getVectorOfStrings(key: String): ErrorOr[Option[Vector[String]]] = jsObject.fields.get(key) match { - case Some(jsArr: JsArray) => Option(jsArr.elements collect { case e: JsString => e.value } ).validNel + case Some(jsArr: JsArray) => Option(jsArr.elements collect { case e: JsString => e.value }).validNel case Some(jsVal: JsValue) => s"Unsupported JsValue as JsArray: $jsVal".invalidNel case _ => None.validNel } @@ -167,8 +173,14 @@ case class WorkflowOptions(jsObject: JsObject) { } lazy val defaultRuntimeOptions = jsObject.fields.get(defaultRuntimeOptionKey) match { - case Some(jsObj: JsObject) => TryUtil.sequenceMap(jsObj.fields map { case (k, _) => k -> WorkflowOptions.getAsJson(k, jsObj) }) - case Some(jsVal) => Failure(new IllegalArgumentException(s"Unsupported JsValue for $defaultRuntimeOptionKey: $jsVal. Expected a JSON object.")) + case Some(jsObj: JsObject) => + TryUtil.sequenceMap(jsObj.fields map { case (k, _) => k -> WorkflowOptions.getAsJson(k, jsObj) }) + case Some(jsVal) => + Failure( + new IllegalArgumentException( + s"Unsupported JsValue for $defaultRuntimeOptionKey: $jsVal. Expected a JSON object." + ) + ) case None => Failure(OptionNotFoundException(s"Cannot find definition for default runtime attributes")) } diff --git a/core/src/main/scala/cromwell/core/WorkflowProcessingEvents.scala b/core/src/main/scala/cromwell/core/WorkflowProcessingEvents.scala index ab80bf78724..e2db6df75c8 100644 --- a/core/src/main/scala/cromwell/core/WorkflowProcessingEvents.scala +++ b/core/src/main/scala/cromwell/core/WorkflowProcessingEvents.scala @@ -36,4 +36,8 @@ object WorkflowProcessingEvents { val ProcessingEventsKey = "workflowProcessingEvents" } -case class WorkflowProcessingEvent(cromwellId: String, description: String, timestamp: OffsetDateTime, cromwellVersion: String) +case class WorkflowProcessingEvent(cromwellId: String, + description: String, + timestamp: OffsetDateTime, + cromwellVersion: String +) diff --git a/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala b/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala index ee936b86819..aa1faf89542 100644 --- a/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala +++ b/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala @@ -22,16 +22,15 @@ sealed trait WorkflowSourceFilesCollection { def importsZipFileOption: Option[Array[Byte]] = this match { case _: WorkflowSourceFilesWithoutImports => None - case w: WorkflowSourceFilesWithDependenciesZip => Option(w.importsZip) // i.e. Some(importsZip) if our wiring is correct + case w: WorkflowSourceFilesWithDependenciesZip => + Option(w.importsZip) // i.e. Some(importsZip) if our wiring is correct } - def setOptions(workflowOptions: WorkflowOptions) = { - + def setOptions(workflowOptions: WorkflowOptions) = this match { case w: WorkflowSourceFilesWithoutImports => w.copy(workflowOptions = workflowOptions) case w: WorkflowSourceFilesWithDependenciesZip => w.copy(workflowOptions = workflowOptions) } - } } trait HasWorkflowIdAndSources { @@ -51,7 +50,8 @@ object WorkflowSourceFilesCollection { importsFile: Option[Array[Byte]], workflowOnHold: Boolean, warnings: Seq[String], - requestedWorkflowId: Option[WorkflowId]): WorkflowSourceFilesCollection = importsFile match { + requestedWorkflowId: Option[WorkflowId] + ): WorkflowSourceFilesCollection = importsFile match { case Some(imports) => WorkflowSourceFilesWithDependenciesZip( workflowSource = workflowSource, @@ -65,7 +65,8 @@ object WorkflowSourceFilesCollection { importsZip = imports, workflowOnHold = workflowOnHold, warnings = warnings, - requestedWorkflowId = requestedWorkflowId) + requestedWorkflowId = requestedWorkflowId + ) case None => WorkflowSourceFilesWithoutImports( workflowSource = workflowSource, @@ -78,7 +79,8 @@ object WorkflowSourceFilesCollection { labelsJson = labelsJson, workflowOnHold = workflowOnHold, warnings = warnings, - requestedWorkflowId = requestedWorkflowId) + requestedWorkflowId = requestedWorkflowId + ) } } @@ -92,7 +94,8 @@ final case class WorkflowSourceFilesWithoutImports(workflowSource: Option[Workfl labelsJson: WorkflowJson, workflowOnHold: Boolean = false, warnings: Seq[String], - requestedWorkflowId: Option[WorkflowId]) extends WorkflowSourceFilesCollection + requestedWorkflowId: Option[WorkflowId] +) extends WorkflowSourceFilesCollection final case class WorkflowSourceFilesWithDependenciesZip(workflowSource: Option[WorkflowSource], workflowUrl: Option[WorkflowUrl], @@ -105,9 +108,9 @@ final case class WorkflowSourceFilesWithDependenciesZip(workflowSource: Option[W importsZip: Array[Byte], workflowOnHold: Boolean = false, warnings: Seq[String], - requestedWorkflowId: Option[WorkflowId]) extends WorkflowSourceFilesCollection { - override def toString = { + requestedWorkflowId: Option[WorkflowId] +) extends WorkflowSourceFilesCollection { + override def toString = s"WorkflowSourceFilesWithDependenciesZip($workflowSource, $workflowUrl, $workflowType, $workflowTypeVersion," + s""" $inputsJson, ${workflowOptions.asPrettyJson}, $labelsJson, <>, $warnings)""" - } } diff --git a/core/src/main/scala/cromwell/core/WorkflowState.scala b/core/src/main/scala/cromwell/core/WorkflowState.scala index db4bddfedc5..26294ff648b 100644 --- a/core/src/main/scala/cromwell/core/WorkflowState.scala +++ b/core/src/main/scala/cromwell/core/WorkflowState.scala @@ -2,7 +2,6 @@ package cromwell.core import cats.Semigroup - sealed trait WorkflowState { def isTerminal: Boolean protected def ordinal: Int @@ -10,10 +9,18 @@ sealed trait WorkflowState { } object WorkflowState { - lazy val WorkflowStateValues = Seq(WorkflowOnHold, WorkflowSubmitted, WorkflowRunning, WorkflowFailed, WorkflowSucceeded, WorkflowAborting, WorkflowAborted) + lazy val WorkflowStateValues = Seq(WorkflowOnHold, + WorkflowSubmitted, + WorkflowRunning, + WorkflowFailed, + WorkflowSucceeded, + WorkflowAborting, + WorkflowAborted + ) - def withName(str: String): WorkflowState = WorkflowStateValues.find(_.toString.equalsIgnoreCase(str)).getOrElse( - throw new NoSuchElementException(s"No such WorkflowState: $str")) + def withName(str: String): WorkflowState = WorkflowStateValues + .find(_.toString.equalsIgnoreCase(str)) + .getOrElse(throw new NoSuchElementException(s"No such WorkflowState: $str")) implicit val WorkflowStateSemigroup = new Semigroup[WorkflowState] { override def combine(f1: WorkflowState, f2: WorkflowState): WorkflowState = f1.combine(f2) @@ -22,7 +29,7 @@ object WorkflowState { implicit val WorkflowStateOrdering = Ordering.by { self: WorkflowState => self.ordinal } } -case object WorkflowOnHold extends WorkflowState{ +case object WorkflowOnHold extends WorkflowState { override val toString: String = "On Hold" override val isTerminal = false override val ordinal = 0 diff --git a/core/src/main/scala/cromwell/core/actor/BatchActor.scala b/core/src/main/scala/cromwell/core/actor/BatchActor.scala index 5988dd822d2..9b002f4fcf5 100644 --- a/core/src/main/scala/cromwell/core/actor/BatchActor.scala +++ b/core/src/main/scala/cromwell/core/actor/BatchActor.scala @@ -14,7 +14,6 @@ import scala.concurrent.Future import scala.concurrent.duration.{Duration, FiniteDuration} import scala.util.{Failure, Success} - /** A collection of state, data, and message types to support BatchActor. */ object BatchActor { type BatchData[C] = WeightedQueue[C, Int] @@ -45,8 +44,9 @@ object BatchActor { * It is backed by a WeightedQueue which makes it possible to decouple the number of messages received from * the effective "weight" of the queue. */ -abstract class BatchActor[C](val flushRate: FiniteDuration, - val batchSize: Int) extends FSM[BatchActorState, BatchData[C]] with Timers { +abstract class BatchActor[C](val flushRate: FiniteDuration, val batchSize: Int) + extends FSM[BatchActorState, BatchData[C]] + with Timers { private var shuttingDown: Boolean = false implicit val ec = context.dispatcher @@ -63,7 +63,8 @@ abstract class BatchActor[C](val flushRate: FiniteDuration, protected def routed: Boolean = false override def preStart(): Unit = { - if (logOnStartUp) log.info("{} configured to flush with batch size {} and process rate {}.", name, batchSize, flushRate) + if (logOnStartUp) + log.info("{} configured to flush with batch size {} and process rate {}.", name, batchSize, flushRate) if (flushRate != Duration.Zero) { timers.startPeriodicTimer(ScheduledFlushKey, ScheduledProcessAction, flushRate) } @@ -133,10 +134,9 @@ abstract class BatchActor[C](val flushRate: FiniteDuration, */ protected def process(data: NonEmptyVector[C]): Future[Int] - private def processIfBatchSizeReached(data: BatchData[C]) = { + private def processIfBatchSizeReached(data: BatchData[C]) = if (data.weight >= batchSize) processHead(data) else goto(WaitingToProcess) using data - } private def processHead(data: BatchData[C]) = if (data.innerQueue.nonEmpty) { val (head, newQueue) = data.behead(batchSize) @@ -159,8 +159,8 @@ abstract class BatchActor[C](val flushRate: FiniteDuration, goto(Processing) using newQueue } else - // This goto is important, even if we're already in WaitingToProcess we want to trigger the onTransition below - // to check if it's time to shutdown + // This goto is important, even if we're already in WaitingToProcess we want to trigger the onTransition below + // to check if it's time to shutdown goto(WaitingToProcess) using data onTransition { diff --git a/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala b/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala index d8b0cbe0716..85376bedb37 100644 --- a/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala +++ b/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala @@ -15,14 +15,14 @@ object RobustClientHelper { } trait RobustClientHelper { this: Actor with ActorLogging => - private [actor] implicit val robustActorHelperEc = context.dispatcher + implicit private[actor] val robustActorHelperEc = context.dispatcher private var backoff: Option[Backoff] = None // package private for testing - private [core] var timeouts = Map.empty[Any, (Cancellable, FiniteDuration)] + private[core] var timeouts = Map.empty[Any, (Cancellable, FiniteDuration)] - protected def initialBackoff(): Backoff = SimpleExponentialBackoff(5.seconds, 20.minutes, 2D) + protected def initialBackoff(): Backoff = SimpleExponentialBackoff(5.seconds, 20.minutes, 2d) def robustReceive: Receive = { case BackPressure(request) => @@ -33,39 +33,38 @@ trait RobustClientHelper { this: Actor with ActorLogging => case RequestTimeout(request, to) => onTimeout(request, to) } - private final def newTimer(msg: Any, to: ActorRef, in: FiniteDuration) = { + final private def newTimer(msg: Any, to: ActorRef, in: FiniteDuration) = context.system.scheduler.scheduleOnce(in, to, msg)(robustActorHelperEc, self) - } def robustSend(msg: Any, to: ActorRef, timeout: FiniteDuration = DefaultRequestLostTimeout): Unit = { to ! msg addTimeout(msg, to, timeout) } - private final def addTimeout(command: Any, to: ActorRef, timeout: FiniteDuration) = { + final private def addTimeout(command: Any, to: ActorRef, timeout: FiniteDuration) = { val cancellable = newTimer(RequestTimeout(command, to), self, timeout) timeouts = timeouts + (command -> (cancellable -> timeout)) } - protected final def hasTimeout(command: Any) = timeouts.get(command).isDefined + final protected def hasTimeout(command: Any) = timeouts.get(command).isDefined - protected final def cancelTimeout(command: Any) = { + final protected def cancelTimeout(command: Any) = { timeouts.get(command) foreach { case (cancellable, _) => cancellable.cancel() } timeouts = timeouts - command } - private final def resetTimeout(command: Any, to: ActorRef) = { + final private def resetTimeout(command: Any, to: ActorRef) = { val timeout = timeouts.get(command) map { _._2 } cancelTimeout(command) timeout foreach { addTimeout(command, to, _) } } - private [actor] final def generateBackpressureTime: FiniteDuration = { - val effectiveBackoff = backoff.getOrElse({ + final private[actor] def generateBackpressureTime: FiniteDuration = { + val effectiveBackoff = backoff.getOrElse { val firstBackoff = initialBackoff() backoff = Option(firstBackoff) firstBackoff - }) + } val backoffTime = effectiveBackoff.backoffMillis backoff = Option(effectiveBackoff.next) backoffTime.millis diff --git a/core/src/main/scala/cromwell/core/actor/StreamActorHelper.scala b/core/src/main/scala/cromwell/core/actor/StreamActorHelper.scala index d03853e5585..023a01dc450 100644 --- a/core/src/main/scala/cromwell/core/actor/StreamActorHelper.scala +++ b/core/src/main/scala/cromwell/core/actor/StreamActorHelper.scala @@ -13,8 +13,8 @@ import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} object StreamActorHelper { - private [actor] case class StreamFailed(failure: Throwable) - private [actor] case object StreamCompleted + private[actor] case class StreamFailed(failure: Throwable) + private[actor] case object StreamCompleted class ActorRestartException(throwable: Throwable) extends RuntimeException(throwable) } @@ -25,71 +25,68 @@ trait StreamActorHelper[T <: StreamContext] { this: Actor with ActorLogging => implicit def materializer: ActorMaterializer private val decider: Supervision.Decider = _ => Supervision.Resume - - private val replySink = Sink.foreach[(Any, T)] { - case (response, commandContext) => - val reply = commandContext.clientContext map { (_, response) } getOrElse response - commandContext.replyTo ! reply + + private val replySink = Sink.foreach[(Any, T)] { case (response, commandContext) => + val reply = commandContext.clientContext map { (_, response) } getOrElse response + commandContext.replyTo ! reply } protected def actorReceive: Receive - + protected def streamSource: Source[(Any, T), SourceQueueWithComplete[T]] override def receive = streamReceive.orElse(actorReceive) - + protected def onBackpressure(scale: Option[Double] = None): Unit = {} - private [actor] lazy val stream = { + private[actor] lazy val stream = streamSource .to(replySink) .withAttributes(ActorAttributes.supervisionStrategy(decider)) .run() - } - override def preStart(): Unit = { + override def preStart(): Unit = stream.watchCompletion() onComplete { case Success(_) => self ! StreamCompleted case Failure(failure) => self ! StreamFailed(failure) } - } def sendToStream(commandContext: T) = { val enqueue = stream offer commandContext map { case Enqueued => EnqueueResponse(Enqueued, commandContext) case other => EnqueueResponse(other, commandContext) - } recoverWith { - case t => Future.successful(FailedToEnqueue(t, commandContext)) + } recoverWith { case t => + Future.successful(FailedToEnqueue(t, commandContext)) } pipe(enqueue) to self () } - + private def backpressure(commandContext: StreamContext) = { - val originalRequest = commandContext.clientContext map { _ -> commandContext.request } getOrElse commandContext.request + val originalRequest = commandContext.clientContext map { + _ -> commandContext.request + } getOrElse commandContext.request commandContext.replyTo ! BackPressure(originalRequest) onBackpressure() } private def streamReceive: Receive = { - case ShutdownCommand => + case ShutdownCommand => stream.complete() case EnqueueResponse(Enqueued, _: T @unchecked) => // Good ! - case EnqueueResponse(_, commandContext) => backpressure(commandContext) case FailedToEnqueue(_, commandContext) => backpressure(commandContext) - - case StreamCompleted => + + case StreamCompleted => context stop self - case StreamFailed(failure) => + case StreamFailed(failure) => restart(failure) } /** Throw the exception to force the actor to restart so it can be back in business * IMPORTANT: Make sure the supervision strategy for this actor is Restart */ - private def restart(throwable: Throwable) = { + private def restart(throwable: Throwable) = throw new ActorRestartException(throwable) - } } diff --git a/core/src/main/scala/cromwell/core/actor/ThrottlerActor.scala b/core/src/main/scala/cromwell/core/actor/ThrottlerActor.scala index c1128897445..6ddbb5f8b18 100644 --- a/core/src/main/scala/cromwell/core/actor/ThrottlerActor.scala +++ b/core/src/main/scala/cromwell/core/actor/ThrottlerActor.scala @@ -15,7 +15,7 @@ import scala.concurrent.duration._ abstract class ThrottlerActor[C] extends BatchActor[C](Duration.Zero, 1) { override protected def logOnStartUp = false override def weightFunction(command: C) = 1 - override final def process(data: NonEmptyVector[C]): Future[Int] = { + final override def process(data: NonEmptyVector[C]): Future[Int] = // This ShouldNotBePossible™ but in case it happens, instead of dropping elements process them all anyway // Explanation: batch size is 1 which means as soon as we receive 1 element, the process method should be called. // Because the BatchActor calls the process method with vector of elements which total weight is batch size, and because @@ -25,6 +25,5 @@ abstract class ThrottlerActor[C] extends BatchActor[C](Duration.Zero, 1) { log.error("{} is throttled and is not supposed to process more than one element at a time !", self.path.name) data.toVector.traverse(processHead).map(_.length) } else processHead(data.head).map(_ => 1) - } def processHead(head: C): Future[Int] } diff --git a/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala b/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala index 23d4f396a2c..08527dbcf1d 100644 --- a/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala +++ b/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala @@ -1,6 +1,7 @@ package cromwell.core.callcaching sealed trait CallCachingMode { + /** * Return an equivalent of this call caching mode with READ disabled. */ @@ -19,11 +20,14 @@ case object CallCachingOff extends CallCachingMode { override val withoutWrite = this } -case class CallCachingActivity(readWriteMode: ReadWriteMode, options: CallCachingOptions = CallCachingOptions()) extends CallCachingMode { +case class CallCachingActivity(readWriteMode: ReadWriteMode, options: CallCachingOptions = CallCachingOptions()) + extends CallCachingMode { override val readFromCache = readWriteMode.r override val writeToCache = readWriteMode.w - override lazy val withoutRead: CallCachingMode = if (!writeToCache) CallCachingOff else this.copy(readWriteMode = WriteCache) - override lazy val withoutWrite: CallCachingMode = if (!readFromCache) CallCachingOff else this.copy(readWriteMode = ReadCache) + override lazy val withoutRead: CallCachingMode = + if (!writeToCache) CallCachingOff else this.copy(readWriteMode = WriteCache) + override lazy val withoutWrite: CallCachingMode = + if (!readFromCache) CallCachingOff else this.copy(readWriteMode = ReadCache) override val toString = readWriteMode.toString } @@ -35,4 +39,6 @@ case object ReadCache extends ReadWriteMode { override val w = false } case object WriteCache extends ReadWriteMode { override val r = false } case object ReadAndWriteCache extends ReadWriteMode -final case class CallCachingOptions(invalidateBadCacheResults: Boolean = true, workflowOptionCallCachePrefixes: Option[Vector[String]] = None) +final case class CallCachingOptions(invalidateBadCacheResults: Boolean = true, + workflowOptionCallCachePrefixes: Option[Vector[String]] = None +) diff --git a/core/src/main/scala/cromwell/core/callcaching/HashResultMessage.scala b/core/src/main/scala/cromwell/core/callcaching/HashResultMessage.scala index 920702280b0..a78c7f20b6a 100644 --- a/core/src/main/scala/cromwell/core/callcaching/HashResultMessage.scala +++ b/core/src/main/scala/cromwell/core/callcaching/HashResultMessage.scala @@ -2,7 +2,6 @@ package cromwell.core.callcaching import cromwell.core.callcaching.HashKey.KeySeparator - object HashKey { private val KeySeparator = ": " def apply(keyComponents: String*) = new HashKey(true, keyComponents.toList) diff --git a/core/src/main/scala/cromwell/core/core.scala b/core/src/main/scala/cromwell/core/core.scala index 68edbaacbe7..2535a58f5c6 100644 --- a/core/src/main/scala/cromwell/core/core.scala +++ b/core/src/main/scala/cromwell/core/core.scala @@ -7,7 +7,6 @@ import mouse.boolean._ import scala.concurrent.duration.FiniteDuration - case class StandardPaths(output: Path, error: Path) case class CallContext(root: Path, standardPaths: StandardPaths, isDocker: Boolean) @@ -27,26 +26,35 @@ object CromwellFatalException { class CromwellFatalException(val exception: Throwable) extends Exception(exception) with CromwellFatalExceptionMarker case class CromwellAggregatedException(throwables: Seq[Throwable], exceptionContext: String = "") - extends Exception with ThrowableAggregation + extends Exception + with ThrowableAggregation case class CacheConfig(concurrency: Int, size: Long, ttl: FiniteDuration) import net.ceedubs.ficus.Ficus._ object CacheConfig { + /** * From an optional `Config` entry and specified defaults, always return a `CacheConfig` object. */ - def config(caching: Option[Config], defaultConcurrency: Int, defaultSize: Long, defaultTtl: FiniteDuration): CacheConfig = { + def config(caching: Option[Config], + defaultConcurrency: Int, + defaultSize: Long, + defaultTtl: FiniteDuration + ): CacheConfig = caching flatMap { c => optionalConfig(c, defaultConcurrency = defaultConcurrency, defaultSize = defaultSize, defaultTtl = defaultTtl) } getOrElse CacheConfig(concurrency = defaultConcurrency, size = defaultSize, ttl = defaultTtl) - } /** * From a non-optional `Config` and specified defaults, if caching is enabled return a `CacheConfig` object wrapped in a `Some`, * otherwise return `None`. */ - def optionalConfig(caching: Config, defaultConcurrency: Int, defaultSize: Long, defaultTtl: FiniteDuration): Option[CacheConfig] = { + def optionalConfig(caching: Config, + defaultConcurrency: Int, + defaultSize: Long, + defaultTtl: FiniteDuration + ): Option[CacheConfig] = { val cachingEnabled = caching.getOrElse("enabled", false) cachingEnabled.option( diff --git a/core/src/main/scala/cromwell/core/filesystem/CromwellFileSystems.scala b/core/src/main/scala/cromwell/core/filesystem/CromwellFileSystems.scala index caeb56509fd..ac2fe473572 100644 --- a/core/src/main/scala/cromwell/core/filesystem/CromwellFileSystems.scala +++ b/core/src/main/scala/cromwell/core/filesystem/CromwellFileSystems.scala @@ -24,23 +24,29 @@ import scala.util.{Failure, Try} */ class CromwellFileSystems(globalConfig: Config) { // Validate the configuration and creates a Map of PathBuilderFactory constructors, along with their optional singleton config - private [filesystem] val factoryBuilders: Map[String, (Constructor[_], Option[AnyRef])] = if (globalConfig.hasPath("filesystems")) { - val rawConfigSet = globalConfig.getObject("filesystems").entrySet.asScala - val configMap = rawConfigSet.toList.map({ entry => entry.getKey -> entry.getValue }) - val constructorMap = configMap.traverse[ErrorOr, (String, (Constructor[_], Option[AnyRef]))]({ - case (key, fsConfig: ConfigObject) => processFileSystem(key, fsConfig) - case (key, _) => s"Invalid filesystem configuration for $key".invalidNel - }).map(_.toMap) - - constructorMap.unsafe("Failed to initialize Cromwell filesystems") - } else Map.empty + private[filesystem] val factoryBuilders: Map[String, (Constructor[_], Option[AnyRef])] = + if (globalConfig.hasPath("filesystems")) { + val rawConfigSet = globalConfig.getObject("filesystems").entrySet.asScala + val configMap = rawConfigSet.toList.map(entry => entry.getKey -> entry.getValue) + val constructorMap = configMap + .traverse[ErrorOr, (String, (Constructor[_], Option[AnyRef]))] { + case (key, fsConfig: ConfigObject) => processFileSystem(key, fsConfig) + case (key, _) => s"Invalid filesystem configuration for $key".invalidNel + } + .map(_.toMap) + + constructorMap.unsafe("Failed to initialize Cromwell filesystems") + } else Map.empty val supportedFileSystems: Iterable[String] = factoryBuilders.keys // Generate the appropriate constructor and optional singleton instance for a filesystem - private def processFileSystem(key: String, fsConfig: ConfigObject): ErrorOr[(String, (Constructor[_], Option[AnyRef]))] = { + private def processFileSystem(key: String, + fsConfig: ConfigObject + ): ErrorOr[(String, (Constructor[_], Option[AnyRef]))] = { // This is the (optional) singleton instance shared by all factory instances - val singletonInstance: Checked[Option[AnyRef]] = fsConfig.toConfig.getAs[Config]("global") + val singletonInstance: Checked[Option[AnyRef]] = fsConfig.toConfig + .getAs[Config]("global") .map(c => instantiateSingletonConfig(key, c).toValidated) .sequence[ErrorOr, AnyRef] .toEither @@ -57,70 +63,88 @@ class CromwellFileSystems(globalConfig: Config) { } // Instantiates the singleton config for a filesystem - private def instantiateSingletonConfig(filesystem: String, config: Config): Checked[AnyRef] = { + private def instantiateSingletonConfig(filesystem: String, config: Config): Checked[AnyRef] = for { constructor <- createConstructor(filesystem, config, List(classOf[Config])) instanceConfig = config.getAs[Config]("config").getOrElse(ConfigFactory.empty) instance <- Try(constructor.newInstance(instanceConfig)).toChecked - cast <- instance.cast[AnyRef].toChecked(s"The filesystem global configuration class for $filesystem is not a Java Object") + cast <- instance + .cast[AnyRef] + .toChecked(s"The filesystem global configuration class for $filesystem is not a Java Object") } yield cast - } // Create a constructor from a configuration object - private def createConstructor(key: String, config: Config, parameterTypes: List[Class[_]]): Checked[Constructor[_]] = for { - clazz <- config.as[Option[String]]("class").toChecked(s"Filesystem configuration $key doesn't have a class field") - constructor <- createConstructor(key, clazz, parameterTypes) - } yield constructor + private def createConstructor(key: String, config: Config, parameterTypes: List[Class[_]]): Checked[Constructor[_]] = + for { + clazz <- config.as[Option[String]]("class").toChecked(s"Filesystem configuration $key doesn't have a class field") + constructor <- createConstructor(key, clazz, parameterTypes) + } yield constructor // Getting a constructor from a class name - private def createConstructor(filesystem: String, className: String, parameterTypes: List[Class[_]]): Checked[Constructor[_]] = Try ( + private def createConstructor(filesystem: String, + className: String, + parameterTypes: List[Class[_]] + ): Checked[Constructor[_]] = Try( Class.forName(className).getConstructor(parameterTypes: _*) - ).recoverWith({ - case e: ClassNotFoundException => Failure( - new RuntimeException(s"Class $className for filesystem $filesystem cannot be found in the class path.", e) - ) - case e: NoSuchMethodException => Failure( - new RuntimeException(s"Class $className for filesystem $filesystem does not have the required constructor signature: (${parameterTypes.map(_.getCanonicalName).mkString(", ")})", e) - ) - }).toChecked + ).recoverWith { + case e: ClassNotFoundException => + Failure( + new RuntimeException(s"Class $className for filesystem $filesystem cannot be found in the class path.", e) + ) + case e: NoSuchMethodException => + Failure( + new RuntimeException( + s"Class $className for filesystem $filesystem does not have the required constructor signature: (${parameterTypes.map(_.getCanonicalName).mkString(", ")})", + e + ) + ) + }.toChecked // Instantiate a PathBuilderFactory from its constructor and instance config - private def instantiate(name: String, constructor: Constructor[_], instanceConfig: Config, global: Option[AnyRef]): Checked[PathBuilderFactory] = { + private def instantiate(name: String, + constructor: Constructor[_], + instanceConfig: Config, + global: Option[AnyRef] + ): Checked[PathBuilderFactory] = for { instance <- global match { case Some(g) => Try(constructor.newInstance(globalConfig, instanceConfig, g)).toChecked case None => Try(constructor.newInstance(globalConfig, instanceConfig)).toChecked } - cast <- instance.cast[PathBuilderFactory].toChecked(s"The filesystem class for $name is not an instance of PathBuilderFactory") + cast <- instance + .cast[PathBuilderFactory] + .toChecked(s"The filesystem class for $name is not an instance of PathBuilderFactory") } yield cast - } // Look for a constructor in the map of known filesystems private def getConstructor(fileSystemName: String): Checked[(Constructor[_], Option[AnyRef])] = factoryBuilders .get(fileSystemName) - .toChecked(s"Cannot find a filesystem with name $fileSystemName in the configuration. Available filesystems: ${factoryBuilders.keySet.mkString(", ")}") + .toChecked( + s"Cannot find a filesystem with name $fileSystemName in the configuration. Available filesystems: ${factoryBuilders.keySet + .mkString(", ")}" + ) /** * Try to find a configured filesystem with the given name and build a PathFactory for it * @param name name of the filesystem * @param instanceConfig filesystem specific configuration for this instance of the factory to build */ - def buildFactory(name: String, instanceConfig: Config): Checked[PathBuilderFactory] = { + def buildFactory(name: String, instanceConfig: Config): Checked[PathBuilderFactory] = if (DefaultPathBuilderFactory.name.equalsIgnoreCase(name)) DefaultPathBuilderFactory.validNelCheck - else for { - constructorAndGlobal <- getConstructor(name) - factory <- instantiate(name, constructorAndGlobal._1, instanceConfig, constructorAndGlobal._2) - } yield factory - } + else + for { + constructorAndGlobal <- getConstructor(name) + factory <- instantiate(name, constructorAndGlobal._1, instanceConfig, constructorAndGlobal._2) + } yield factory /** * Given a filesystems config, build the PathBuilderFactories */ - def factoriesFromConfig(filesystemsConfig: Config): Checked[Map[String, PathBuilderFactory]] = { + def factoriesFromConfig(filesystemsConfig: Config): Checked[Map[String, PathBuilderFactory]] = if (filesystemsConfig.hasPath("filesystems")) { // Iterate over the config entries under the "filesystems" config val rawConfigSet = filesystemsConfig.getObject("filesystems").entrySet().asScala - val configMap = rawConfigSet.toList.map({ entry => entry.getKey -> entry.getValue }) + val configMap = rawConfigSet.toList.map(entry => entry.getKey -> entry.getValue) import net.ceedubs.ficus.Ficus._ def isFilesystemEnabled(configObject: ConfigObject): Boolean = { @@ -141,7 +165,6 @@ class CromwellFileSystems(globalConfig: Config) { case (key, _) => s"Invalid filesystem backend configuration for $key".invalidNel } map { _.toMap } toEither } else Map.empty[String, PathBuilderFactory].validNelCheck - } } object CromwellFileSystems { diff --git a/core/src/main/scala/cromwell/core/io/AsyncIo.scala b/core/src/main/scala/cromwell/core/io/AsyncIo.scala index 435058d942c..535c7e1a8a4 100644 --- a/core/src/main/scala/cromwell/core/io/AsyncIo.scala +++ b/core/src/main/scala/cromwell/core/io/AsyncIo.scala @@ -23,7 +23,8 @@ object AsyncIo { */ class AsyncIo(ioEndpoint: ActorRef, ioCommandBuilder: IoCommandBuilder) { private def asyncCommand[A](commandTry: Try[IoCommand[A]], - timeout: FiniteDuration = AsyncIo.defaultTimeout): Future[A] = { + timeout: FiniteDuration = AsyncIo.defaultTimeout + ): Future[A] = commandTry match { case Failure(throwable) => Future.failed(throwable) @@ -32,46 +33,36 @@ class AsyncIo(ioEndpoint: ActorRef, ioCommandBuilder: IoCommandBuilder) { ioEndpoint ! commandWithPromise commandWithPromise.promise.future } - } /** * IMPORTANT: This loads the entire content of the file into memory ! * Only use for small files ! */ - def contentAsStringAsync(path: Path, maxBytes: Option[Int], failOnOverflow: Boolean): Future[String] = { + def contentAsStringAsync(path: Path, maxBytes: Option[Int], failOnOverflow: Boolean): Future[String] = asyncCommand(ioCommandBuilder.contentAsStringCommand(path, maxBytes, failOnOverflow)) - } - def writeAsync(path: Path, content: String, options: OpenOptions, compressPayload: Boolean = false): Future[Unit] = { + def writeAsync(path: Path, content: String, options: OpenOptions, compressPayload: Boolean = false): Future[Unit] = asyncCommand(ioCommandBuilder.writeCommand(path, content, options, compressPayload)) - } - def sizeAsync(path: Path): Future[Long] = { + def sizeAsync(path: Path): Future[Long] = asyncCommand(ioCommandBuilder.sizeCommand(path)) - } - def hashAsync(path: Path): Future[String] = { + def hashAsync(path: Path): Future[String] = asyncCommand(ioCommandBuilder.hashCommand(path)) - } - def deleteAsync(path: Path, swallowIoExceptions: Boolean = false): Future[Unit] = { + def deleteAsync(path: Path, swallowIoExceptions: Boolean = false): Future[Unit] = asyncCommand(ioCommandBuilder.deleteCommand(path, swallowIoExceptions)) - } - def existsAsync(path: Path): Future[Boolean] = { + def existsAsync(path: Path): Future[Boolean] = asyncCommand(ioCommandBuilder.existsCommand(path)) - } - def readLinesAsync(path: Path): Future[Iterable[String]] = { + def readLinesAsync(path: Path): Future[Iterable[String]] = asyncCommand(ioCommandBuilder.readLines(path)) - } - def isDirectory(path: Path): Future[Boolean] = { + def isDirectory(path: Path): Future[Boolean] = asyncCommand(ioCommandBuilder.isDirectoryCommand(path)) - } - def copyAsync(src: Path, dest: Path): Future[Unit] = { + def copyAsync(src: Path, dest: Path): Future[Unit] = // Allow for a much larger timeout for copies, as large files can take a while (even on gcs, if they are in different locations...) asyncCommand(ioCommandBuilder.copyCommand(src, dest), AsyncIo.copyTimeout) - } } diff --git a/core/src/main/scala/cromwell/core/io/AsyncIoFunctions.scala b/core/src/main/scala/cromwell/core/io/AsyncIoFunctions.scala index 6b4a7763d2a..c7bb7927a00 100644 --- a/core/src/main/scala/cromwell/core/io/AsyncIoFunctions.scala +++ b/core/src/main/scala/cromwell/core/io/AsyncIoFunctions.scala @@ -3,6 +3,7 @@ package cromwell.core.io import wom.expression.IoFunctionSet trait AsyncIoFunctions { this: IoFunctionSet => + /** * Used to perform io functions asynchronously through the ioActorProxy */ diff --git a/core/src/main/scala/cromwell/core/io/CorePathFunctionSet.scala b/core/src/main/scala/cromwell/core/io/CorePathFunctionSet.scala index cf79d06cd2b..59eed184e45 100644 --- a/core/src/main/scala/cromwell/core/io/CorePathFunctionSet.scala +++ b/core/src/main/scala/cromwell/core/io/CorePathFunctionSet.scala @@ -8,7 +8,9 @@ import wom.expression.{IoFunctionSet, PathFunctionSet} import scala.util.Try class WorkflowCorePathFunctionSet(override val pathBuilders: PathBuilders) extends PathFunctionSet with PathFactory { - private def fail(name: String) = throw new UnsupportedOperationException(s"$name is not implemented at the workflow level") + private def fail(name: String) = throw new UnsupportedOperationException( + s"$name is not implemented at the workflow level" + ) override def sibling(of: String, path: String): String = buildPath(of).sibling(path).pathAsString override def isAbsolute(path: String): Boolean = Try(buildPath(path)).map(_.isAbsolute).toOption.contains(true) override def name(path: String) = buildPath(path).name @@ -19,8 +21,10 @@ class WorkflowCorePathFunctionSet(override val pathBuilders: PathBuilders) exten override def stderr: String = fail("stderr") } -class CallCorePathFunctionSet(pathBuilders: PathBuilders, callContext: CallContext) extends WorkflowCorePathFunctionSet(pathBuilders) { - override def relativeToHostCallRoot(path: String) = if (isAbsolute(path)) path else callContext.root.resolve(path).pathAsString +class CallCorePathFunctionSet(pathBuilders: PathBuilders, callContext: CallContext) + extends WorkflowCorePathFunctionSet(pathBuilders) { + override def relativeToHostCallRoot(path: String) = + if (isAbsolute(path)) path else callContext.root.resolve(path).pathAsString override def stdout = callContext.standardPaths.output.pathAsString override def stderr = callContext.standardPaths.error.pathAsString } @@ -29,7 +33,6 @@ trait WorkflowCorePathFunctions extends { this: IoFunctionSet with PathFactory = override lazy val pathFunctions = new WorkflowCorePathFunctionSet(pathBuilders) } - trait CallCorePathFunctions extends { this: IoFunctionSet with PathFactory => def callContext: CallContext override lazy val pathFunctions = new CallCorePathFunctionSet(pathBuilders, callContext) diff --git a/core/src/main/scala/cromwell/core/io/DefaultIoCommand.scala b/core/src/main/scala/cromwell/core/io/DefaultIoCommand.scala index bb5e5eec973..2faf49696ac 100644 --- a/core/src/main/scala/cromwell/core/io/DefaultIoCommand.scala +++ b/core/src/main/scala/cromwell/core/io/DefaultIoCommand.scala @@ -5,13 +5,13 @@ import cromwell.core.io.IoContentAsStringCommand.IoReadOptions import cromwell.core.path.Path object DefaultIoCommand { - case class DefaultIoCopyCommand(override val source: Path, - override val destination: Path, - ) extends IoCopyCommand(source, destination) { + case class DefaultIoCopyCommand(override val source: Path, override val destination: Path) + extends IoCopyCommand(source, destination) { override def commandDescription: String = s"DefaultIoCopyCommand source '$source' destination '$destination'" } - case class DefaultIoContentAsStringCommand(override val file: Path, override val options: IoReadOptions) extends IoContentAsStringCommand(file, options) { + case class DefaultIoContentAsStringCommand(override val file: Path, override val options: IoReadOptions) + extends IoContentAsStringCommand(file, options) { override def commandDescription: String = s"DefaultIoContentAsStringCommand file '$file' options '$options'" } @@ -22,18 +22,24 @@ object DefaultIoCommand { case class DefaultIoWriteCommand(override val file: Path, override val content: String, override val openOptions: OpenOptions, - override val compressPayload: Boolean) extends IoWriteCommand( - file, content, openOptions, compressPayload - ) { + override val compressPayload: Boolean + ) extends IoWriteCommand( + file, + content, + openOptions, + compressPayload + ) { override def commandDescription: String = s"DefaultIoWriteCommand file '$file' content length " + s"'${content.length}' openOptions '$openOptions' compressPayload '$compressPayload'" } - case class DefaultIoDeleteCommand(override val file: Path, - override val swallowIOExceptions: Boolean) extends IoDeleteCommand( - file, swallowIOExceptions - ) { - override def commandDescription: String = s"DefaultIoDeleteCommand file '$file' swallowIOExceptions '$swallowIOExceptions'" + case class DefaultIoDeleteCommand(override val file: Path, override val swallowIOExceptions: Boolean) + extends IoDeleteCommand( + file, + swallowIOExceptions + ) { + override def commandDescription: String = + s"DefaultIoDeleteCommand file '$file' swallowIOExceptions '$swallowIOExceptions'" } case class DefaultIoHashCommand(override val file: Path) extends IoHashCommand(file) { diff --git a/core/src/main/scala/cromwell/core/io/IoAck.scala b/core/src/main/scala/cromwell/core/io/IoAck.scala index 430b30db792..41066791fe3 100644 --- a/core/src/main/scala/cromwell/core/io/IoAck.scala +++ b/core/src/main/scala/cromwell/core/io/IoAck.scala @@ -8,6 +8,7 @@ import scala.util.{Failure, Success, Try} * @tparam T type of the returned value if success */ sealed trait IoAck[T] { + /** * Original command */ @@ -20,13 +21,12 @@ case class IoSuccess[T](command: IoCommand[T], result: T) extends IoAck[T] { } object IoFailAck { - def unapply(any: Any): Option[(IoCommand[_], Throwable)] = { + def unapply(any: Any): Option[(IoCommand[_], Throwable)] = any match { case f: IoFailAck[_] => Option((f.command, f.failure)) case _ => None } - } } trait IoFailAck[T] extends IoAck[T] { @@ -36,5 +36,7 @@ trait IoFailAck[T] extends IoAck[T] { /** Failure of an unspecified variety. */ case class IoFailure[T](command: IoCommand[T], override val failure: Throwable) extends IoFailAck[T] + /** Specifically read forbidden failure. */ -case class IoReadForbiddenFailure[T](command: IoCommand[T], override val failure: Throwable, forbiddenPath: String) extends IoFailAck[T] +case class IoReadForbiddenFailure[T](command: IoCommand[T], override val failure: Throwable, forbiddenPath: String) + extends IoFailAck[T] diff --git a/core/src/main/scala/cromwell/core/io/IoClientHelper.scala b/core/src/main/scala/cromwell/core/io/IoClientHelper.scala index 26dd0732d33..169e55a496d 100644 --- a/core/src/main/scala/cromwell/core/io/IoClientHelper.scala +++ b/core/src/main/scala/cromwell/core/io/IoClientHelper.scala @@ -13,9 +13,9 @@ trait IoClientHelper extends RobustClientHelper { this: Actor with ActorLogging def ioActor: ActorRef lazy val defaultIoTimeout = RobustClientHelper.DefaultRequestLostTimeout - + protected def config = ConfigFactory.load().as[Config]("system.io.backpressure-backoff") - + override protected def initialBackoff(): Backoff = SimpleExponentialBackoff(config) protected def ioResponseReceive: Receive = { @@ -26,19 +26,16 @@ trait IoClientHelper extends RobustClientHelper { this: Actor with ActorLogging cancelTimeout(context -> ack.command) receive.apply(context -> ack) } - + def ioReceive = robustReceive orElse ioResponseReceive - - def sendIoCommand(ioCommand: IoCommand[_]) = { + + def sendIoCommand(ioCommand: IoCommand[_]) = sendIoCommandWithCustomTimeout(ioCommand, defaultIoTimeout) - } - def sendIoCommandWithCustomTimeout(ioCommand: IoCommand[_], timeout: FiniteDuration) = { + def sendIoCommandWithCustomTimeout(ioCommand: IoCommand[_], timeout: FiniteDuration) = robustSend(ioCommand, ioActor, timeout) - } - def sendIoCommandWithContext[T](ioCommand: IoCommand[_], context: T, timeout: FiniteDuration = defaultIoTimeout) = { + def sendIoCommandWithContext[T](ioCommand: IoCommand[_], context: T, timeout: FiniteDuration = defaultIoTimeout) = robustSend(context -> ioCommand, ioActor, timeout) - } } diff --git a/core/src/main/scala/cromwell/core/io/IoCommand.scala b/core/src/main/scala/cromwell/core/io/IoCommand.scala index 7d71f9bd307..9db50bc2c87 100644 --- a/core/src/main/scala/cromwell/core/io/IoCommand.scala +++ b/core/src/main/scala/cromwell/core/io/IoCommand.scala @@ -23,7 +23,7 @@ object IoCommand { .setInitialIntervalMillis((1 second).toMillis.toInt) .setMaxIntervalMillis((5 minutes).toMillis.toInt) .setMultiplier(3L) - .setRandomizationFactor(0.2D) + .setRandomizationFactor(0.2d) .setMaxElapsedTimeMillis((10 minutes).toMillis.toInt) .build() @@ -47,7 +47,7 @@ trait IoCommand[+T] { def logIOMsgOverLimit(message: => String): Unit = { val millis: Long = java.time.Duration.between(creation, OffsetDateTime.now).toMillis if (millis > IoCommand.IOCommandWarnLimit.toMillis) { - val seconds = millis / 1000D + val seconds = millis / 1000d /* For now we decided to log this as INFO. In future if needed, we can update this to WARN. @@ -59,8 +59,10 @@ trait IoCommand[+T] { (https://github.com/broadinstitute/firecloud-develop/blob/c77e0f371be0aac545e204f1a134cc6f8ef3c301/run-context/live/configs/cromwell/app.env.ctmpl#L42-L51) - Logback manual (http://logback.qos.ch/manual/index.html) */ - IoCommand.logger.info(f"(IO-$uuid) '$message' is over 5 minutes. It was running for " + - f"$seconds%,.3f seconds. IO command description: '$commandDescription'") + IoCommand.logger.info( + f"(IO-$uuid) '$message' is over 5 minutes. It was running for " + + f"$seconds%,.3f seconds. IO command description: '$commandDescription'" + ) } } @@ -82,7 +84,9 @@ trait IoCommand[+T] { } def failReadForbidden[S >: T](failure: Throwable, forbiddenPath: String): IoReadForbiddenFailure[S] = { - logIOMsgOverLimit(s"IOCommand.failReadForbidden '${failure.toPrettyElidedString(limit = 1000)}' path '$forbiddenPath'") + logIOMsgOverLimit( + s"IOCommand.failReadForbidden '${failure.toPrettyElidedString(limit = 1000)}' path '$forbiddenPath'" + ) IoReadForbiddenFailure(this, failure, forbiddenPath) } @@ -118,7 +122,9 @@ object IoContentAsStringCommand { /** * Read file as a string (load the entire content in memory) */ -abstract class IoContentAsStringCommand(val file: Path, val options: IoReadOptions = IoReadOptions(None, failOnOverflow = false)) extends SingleFileIoCommand[String] { +abstract class IoContentAsStringCommand(val file: Path, + val options: IoReadOptions = IoReadOptions(None, failOnOverflow = false) +) extends SingleFileIoCommand[String] { override def toString = s"read content of ${file.pathAsString}" override lazy val name = "read" } @@ -138,7 +144,8 @@ abstract class IoSizeCommand(val file: Path) extends SingleFileIoCommand[Long] { abstract class IoWriteCommand(val file: Path, val content: String, val openOptions: OpenOptions, - val compressPayload: Boolean) extends SingleFileIoCommand[Unit] { + val compressPayload: Boolean +) extends SingleFileIoCommand[Unit] { override def toString = s"write to ${file.pathAsString}" override lazy val name = "write" } diff --git a/core/src/main/scala/cromwell/core/io/IoCommandBuilder.scala b/core/src/main/scala/cromwell/core/io/IoCommandBuilder.scala index 43a6f5864b0..c4f5da49959 100644 --- a/core/src/main/scala/cromwell/core/io/IoCommandBuilder.scala +++ b/core/src/main/scala/cromwell/core/io/IoCommandBuilder.scala @@ -26,13 +26,11 @@ abstract class PartialIoCommandBuilder { } object IoCommandBuilder { - def apply(partialBuilders: PartialIoCommandBuilder*): IoCommandBuilder = { + def apply(partialBuilders: PartialIoCommandBuilder*): IoCommandBuilder = new IoCommandBuilder(partialBuilders.toList) - } - def apply: IoCommandBuilder = { + def apply: IoCommandBuilder = new IoCommandBuilder(List.empty) - } } /** @@ -49,56 +47,58 @@ class IoCommandBuilder(partialBuilders: List[PartialIoCommandBuilder] = List.emp // Find the first partialBuilder for which the partial function is defined, or use the default private def buildOrDefault[A, B](builder: PartialIoCommandBuilder => PartialFunction[A, Try[B]], params: A, - default: => B): Try[B] = { - partialBuilders.to(LazyList).map(builder(_).lift(params)).collectFirst({ - case Some(command) => command - }).getOrElse(Try(default)) - } + default: => B + ): Try[B] = + partialBuilders + .to(LazyList) + .map(builder(_).lift(params)) + .collectFirst { case Some(command) => + command + } + .getOrElse(Try(default)) def contentAsStringCommand(path: Path, maxBytes: Option[Int], - failOnOverflow: Boolean): Try[IoContentAsStringCommand] = { - buildOrDefault(_.contentAsStringCommand, (path, maxBytes, failOnOverflow), DefaultIoContentAsStringCommand(path, IoReadOptions(maxBytes, failOnOverflow))) - } + failOnOverflow: Boolean + ): Try[IoContentAsStringCommand] = + buildOrDefault(_.contentAsStringCommand, + (path, maxBytes, failOnOverflow), + DefaultIoContentAsStringCommand(path, IoReadOptions(maxBytes, failOnOverflow)) + ) def writeCommand(path: Path, content: String, options: OpenOptions, - compressPayload: Boolean = false): Try[IoWriteCommand] = { - buildOrDefault(_.writeCommand, (path, content, options, compressPayload), DefaultIoWriteCommand(path, content, options, compressPayload)) - } - - def sizeCommand(path: Path): Try[IoSizeCommand] = { + compressPayload: Boolean = false + ): Try[IoWriteCommand] = + buildOrDefault(_.writeCommand, + (path, content, options, compressPayload), + DefaultIoWriteCommand(path, content, options, compressPayload) + ) + + def sizeCommand(path: Path): Try[IoSizeCommand] = buildOrDefault(_.sizeCommand, path, DefaultIoSizeCommand(path)) - } - def deleteCommand(path: Path, swallowIoExceptions: Boolean = true): Try[IoDeleteCommand] = { + def deleteCommand(path: Path, swallowIoExceptions: Boolean = true): Try[IoDeleteCommand] = buildOrDefault(_.deleteCommand, (path, swallowIoExceptions), DefaultIoDeleteCommand(path, swallowIoExceptions)) - } - def copyCommand(src: Path, dest: Path): Try[IoCopyCommand] = { + def copyCommand(src: Path, dest: Path): Try[IoCopyCommand] = buildOrDefault(_.copyCommand, (src, dest), DefaultIoCopyCommand(src, dest)) - } - def hashCommand(file: Path): Try[IoHashCommand] = { + def hashCommand(file: Path): Try[IoHashCommand] = buildOrDefault(_.hashCommand, file, DefaultIoHashCommand(file)) - } - def touchCommand(file: Path): Try[IoTouchCommand] = { + def touchCommand(file: Path): Try[IoTouchCommand] = buildOrDefault(_.touchCommand, file, DefaultIoTouchCommand(file)) - } - def existsCommand(file: Path): Try[IoExistsCommand] = { + def existsCommand(file: Path): Try[IoExistsCommand] = buildOrDefault(_.existsCommand, file, DefaultIoExistsCommand(file)) - } - def isDirectoryCommand(file: Path): Try[IoIsDirectoryCommand] = { + def isDirectoryCommand(file: Path): Try[IoIsDirectoryCommand] = buildOrDefault(_.isDirectoryCommand, file, DefaultIoIsDirectoryCommand(file)) - } - def readLines(file: Path): Try[IoReadLinesCommand] = { + def readLines(file: Path): Try[IoReadLinesCommand] = buildOrDefault(_.readLinesCommand, file, DefaultIoReadLinesCommand(file)) - } } /** diff --git a/core/src/main/scala/cromwell/core/io/IoPromiseProxyActor.scala b/core/src/main/scala/cromwell/core/io/IoPromiseProxyActor.scala index dd8f44464f3..e9a8a8934bc 100644 --- a/core/src/main/scala/cromwell/core/io/IoPromiseProxyActor.scala +++ b/core/src/main/scala/cromwell/core/io/IoPromiseProxyActor.scala @@ -25,17 +25,15 @@ object IoPromiseProxyActor { class IoPromiseProxyActor(override val ioActor: ActorRef) extends Actor with ActorLogging with IoClientHelper { override def receive = ioReceive orElse actorReceive - def actorReceive: Receive = { - case withPromise: IoCommandWithPromise[_] => - sendIoCommandWithContext(withPromise.ioCommand, withPromise.promise, withPromise.timeout) + def actorReceive: Receive = { case withPromise: IoCommandWithPromise[_] => + sendIoCommandWithContext(withPromise.ioCommand, withPromise.promise, withPromise.timeout) } - override protected def ioResponseReceive: Receive = { - case (promise: Promise[_], ack: IoAck[Any] @unchecked) => - cancelTimeout(promise -> ack.command) - // This is not typesafe and assumes the Promise context is of the same type as the IoAck response. - promise.asInstanceOf[Promise[Any]].complete(ack.toTry) - () + override protected def ioResponseReceive: Receive = { case (promise: Promise[_], ack: IoAck[Any] @unchecked) => + cancelTimeout(promise -> ack.command) + // This is not typesafe and assumes the Promise context is of the same type as the IoAck response. + promise.asInstanceOf[Promise[Any]].complete(ack.toTry) + () } override def onTimeout(message: Any, to: ActorRef): Unit = message match { diff --git a/core/src/main/scala/cromwell/core/io/Throttle.scala b/core/src/main/scala/cromwell/core/io/Throttle.scala index 88246f1d88f..aaa5b5d30f5 100644 --- a/core/src/main/scala/cromwell/core/io/Throttle.scala +++ b/core/src/main/scala/cromwell/core/io/Throttle.scala @@ -11,11 +11,10 @@ case class Throttle(elements: Int, per: FiniteDuration, maximumBurst: Int) { } object Throttle { - implicit val throttleOptionValueReader: ValueReader[Option[Throttle]] = (config: Config, path: String) => { + implicit val throttleOptionValueReader: ValueReader[Option[Throttle]] = (config: Config, path: String) => config.getAs[Config](path) map { throttleConfig => val elements = throttleConfig.as[Int]("number-of-requests") val per = throttleConfig.as[FiniteDuration]("per") Throttle(elements, per, elements) } - } } diff --git a/core/src/main/scala/cromwell/core/labels/Label.scala b/core/src/main/scala/cromwell/core/labels/Label.scala index 759300da5b1..616298840b7 100644 --- a/core/src/main/scala/cromwell/core/labels/Label.scala +++ b/core/src/main/scala/cromwell/core/labels/Label.scala @@ -12,21 +12,19 @@ object Label { val MaxLabelLength = 255 val LabelExpectationsMessage = s"A Label key must be non-empty." - def validateLabelKey(s: String): ErrorOr[String] = { + def validateLabelKey(s: String): ErrorOr[String] = (s.length >= 1, s.length <= MaxLabelLength) match { case (true, true) => s.validNel case (false, _) => s"Invalid label: `$s` can't be empty".invalidNel case (_, false) => s"Invalid label: `$s` is ${s.length} characters. The maximum is $MaxLabelLength.".invalidNel } - } - def validateLabelValue(s: String): ErrorOr[String] = { + def validateLabelValue(s: String): ErrorOr[String] = if (s.length <= MaxLabelLength) { s.validNel } else { s"Invalid label: `$s` is ${s.length} characters. The maximum is $MaxLabelLength.".invalidNel } - } def validateLabel(key: String, value: String): ErrorOr[Label] = { val validatedKey = validateLabelKey(key) @@ -35,7 +33,6 @@ object Label { (validatedKey, validatedValue) mapN Label.apply } - def apply(key: String, value: String) = { + def apply(key: String, value: String) = new Label(key, value) {} - } } diff --git a/core/src/main/scala/cromwell/core/labels/Labels.scala b/core/src/main/scala/cromwell/core/labels/Labels.scala index 5499fa5b2e8..148a0a7926d 100644 --- a/core/src/main/scala/cromwell/core/labels/Labels.scala +++ b/core/src/main/scala/cromwell/core/labels/Labels.scala @@ -19,13 +19,11 @@ case class Labels(value: Vector[Label]) { } object Labels { - def apply(values: (String, String)*): Labels = { + def apply(values: (String, String)*): Labels = Labels(values.toVector map (Label.apply _).tupled) - } - def validateMapOfLabels(labels: Map[String, String]): ErrorOr[Labels] = { + def validateMapOfLabels(labels: Map[String, String]): ErrorOr[Labels] = labels.toVector traverse { Label.validateLabel _ }.tupled map Labels.apply - } def empty = Labels(Vector.empty) } diff --git a/core/src/main/scala/cromwell/core/logging/EnhancedDateConverter.scala b/core/src/main/scala/cromwell/core/logging/EnhancedDateConverter.scala index dcf62bc0b28..47eb951199a 100644 --- a/core/src/main/scala/cromwell/core/logging/EnhancedDateConverter.scala +++ b/core/src/main/scala/cromwell/core/logging/EnhancedDateConverter.scala @@ -25,9 +25,9 @@ class EnhancedDateConverter extends DateConverter { cachingDateFormatterProtected = Option(getFirstOption) match { case Some(CoreConstants.ISO8601_STR) | None => new CachingDateFormatter(CoreConstants.ISO8601_PATTERN) case Some(datePattern) => - try { + try new CachingDateFormatter(datePattern) - } catch { + catch { case e: IllegalArgumentException => addWarn("Could not instantiate SimpleDateFormat with pattern " + datePattern, e) // default to the ISO8601 format @@ -35,8 +35,7 @@ class EnhancedDateConverter extends DateConverter { } } // if the option list contains a TZ option, then set it. - Option(getOptionList) - .toList + Option(getOptionList).toList .flatMap(_.asScala) .drop(1) .headOption diff --git a/core/src/main/scala/cromwell/core/logging/EnhancedSlf4jLogger.scala b/core/src/main/scala/cromwell/core/logging/EnhancedSlf4jLogger.scala index 0999ec18055..6f9e5f2fdfb 100644 --- a/core/src/main/scala/cromwell/core/logging/EnhancedSlf4jLogger.scala +++ b/core/src/main/scala/cromwell/core/logging/EnhancedSlf4jLogger.scala @@ -3,6 +3,7 @@ package cromwell.core.logging import akka.event.slf4j.Slf4jLogger class EnhancedSlf4jLogger extends Slf4jLogger { + /** * Format the timestamp as a simple long. Allows the akkaTimestamp to be retrieved later from the MDC by custom * converters. diff --git a/core/src/main/scala/cromwell/core/logging/JavaLoggingBridge.scala b/core/src/main/scala/cromwell/core/logging/JavaLoggingBridge.scala index 1dc10fab46d..beff484f941 100644 --- a/core/src/main/scala/cromwell/core/logging/JavaLoggingBridge.scala +++ b/core/src/main/scala/cromwell/core/logging/JavaLoggingBridge.scala @@ -8,6 +8,7 @@ import org.slf4j.bridge.SLF4JBridgeHandler import scala.jdk.CollectionConverters._ object JavaLoggingBridge { + /** * Replace java.util.logging with SLF4J while ensuring Logback is configured with a LevelChangePropogator. * diff --git a/core/src/main/scala/cromwell/core/logging/JobLogger.scala b/core/src/main/scala/cromwell/core/logging/JobLogger.scala index 37ab6cfa2da..4af851124cb 100644 --- a/core/src/main/scala/cromwell/core/logging/JobLogger.scala +++ b/core/src/main/scala/cromwell/core/logging/JobLogger.scala @@ -28,14 +28,14 @@ class JobLogger(loggerName: String, rootWorkflowIdForLogging: RootWorkflowId, jobTag: String, akkaLogger: Option[LoggingAdapter] = None, - otherLoggers: Set[Logger] = Set.empty[Logger]) - extends WorkflowLogger( - loggerName = loggerName, - workflowId = workflowIdForLogging, - rootWorkflowId = rootWorkflowIdForLogging, - akkaLogger = akkaLogger, - otherLoggers = otherLoggers - ) { + otherLoggers: Set[Logger] = Set.empty[Logger] +) extends WorkflowLogger( + loggerName = loggerName, + workflowId = workflowIdForLogging, + rootWorkflowId = rootWorkflowIdForLogging, + akkaLogger = akkaLogger, + otherLoggers = otherLoggers + ) { override def tag = s"$loggerName [UUID(${workflowIdForLogging.shortString})$jobTag]" } diff --git a/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala b/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala index 5af71c7099c..3f9044aeaa3 100644 --- a/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala +++ b/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala @@ -23,9 +23,8 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { * * https://github.com/qos-ch/slf4j/blob/v_1.7.30/slf4j-simple/src/main/java/org/slf4j/impl/SimpleLogger.java#L293-L295 */ - private def format(msg: String, throwable: Throwable): String = { + private def format(msg: String, throwable: Throwable): String = format(msg) + "\n" + ExceptionUtils.getStackTrace(throwable) - } /** * Passes a formatted string to akka similar to slf4j's SimpleLogger @@ -113,7 +112,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { lazy val formatted: String = format(pattern) varargsAkkaLog(Logging.ErrorLevel, pattern, arguments) - slf4jLoggers.foreach(_.error(formatted, arguments:_*)) + slf4jLoggers.foreach(_.error(formatted, arguments: _*)) } override def error(pattern: String, arg: Any): Unit = { @@ -130,10 +129,9 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.error(formatted, arg1, arg2: Any)) } - def error(t: Throwable, pattern: String, arguments: Any*): Unit = { + def error(t: Throwable, pattern: String, arguments: Any*): Unit = // slf4j extracts the last variable argument as a throwable. error(pattern, (arguments :+ t).map(_.asInstanceOf[AnyRef]): _*) - } override def debug(msg: String): Unit = { lazy val formatted: String = format(msg) @@ -153,7 +151,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { lazy val formatted: String = format(pattern) varargsAkkaLog(Logging.DebugLevel, pattern, arguments) - slf4jLoggers.foreach(_.debug(formatted, arguments:_*)) + slf4jLoggers.foreach(_.debug(formatted, arguments: _*)) } override def debug(pattern: String, argument: Any): Unit = { @@ -170,25 +168,20 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.debug(formatted, arg1, arg2: Any)) } - override def trace(msg: String): Unit = { + override def trace(msg: String): Unit = slf4jLoggers.foreach(_.trace(format(msg))) - } - override def trace(msg: String, t: Throwable): Unit = { + override def trace(msg: String, t: Throwable): Unit = slf4jLoggers.foreach(_.trace(format(msg), t)) - } - override def trace(pattern: String, arguments: AnyRef*): Unit = { - slf4jLoggers.foreach(_.trace(format(pattern), arguments:_*)) - } + override def trace(pattern: String, arguments: AnyRef*): Unit = + slf4jLoggers.foreach(_.trace(format(pattern), arguments: _*)) - override def trace(pattern: String, arg: Any): Unit = { + override def trace(pattern: String, arg: Any): Unit = slf4jLoggers.foreach(_.trace(format(pattern), arg)) - } - override def trace(pattern: String, arg1: Any, arg2: Any): Unit = { + override def trace(pattern: String, arg1: Any, arg2: Any): Unit = slf4jLoggers.foreach(_.trace(format(pattern), arg1, arg2: Any)) - } override def info(msg: String): Unit = { lazy val formatted: String = format(msg) @@ -208,7 +201,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { lazy val formatted: String = format(pattern) varargsAkkaLog(Logging.InfoLevel, pattern, arguments) - slf4jLoggers.foreach(_.info(formatted, arguments:_*)) + slf4jLoggers.foreach(_.info(formatted, arguments: _*)) } override def info(pattern: String, arg: Any): Unit = { @@ -225,14 +218,24 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.info(formatted, arg1, arg2: Any)) } - override def isErrorEnabled: Boolean = throw new UnsupportedOperationException("This logger wraps an arbitrary set of loggers that can each have a different level enabled.") + override def isErrorEnabled: Boolean = throw new UnsupportedOperationException( + "This logger wraps an arbitrary set of loggers that can each have a different level enabled." + ) - override def isInfoEnabled: Boolean = throw new UnsupportedOperationException("This logger wraps an arbitrary set of loggers that can each have a different level enabled.") + override def isInfoEnabled: Boolean = throw new UnsupportedOperationException( + "This logger wraps an arbitrary set of loggers that can each have a different level enabled." + ) - override def isDebugEnabled: Boolean = throw new UnsupportedOperationException("This logger wraps an arbitrary set of loggers that can each have a different level enabled.") + override def isDebugEnabled: Boolean = throw new UnsupportedOperationException( + "This logger wraps an arbitrary set of loggers that can each have a different level enabled." + ) - override def isTraceEnabled: Boolean = throw new UnsupportedOperationException("This logger wraps an arbitrary set of loggers that can each have a different level enabled.") + override def isTraceEnabled: Boolean = throw new UnsupportedOperationException( + "This logger wraps an arbitrary set of loggers that can each have a different level enabled." + ) - override def isWarnEnabled: Boolean = throw new UnsupportedOperationException("This logger wraps an arbitrary set of loggers that can each have a different level enabled.") + override def isWarnEnabled: Boolean = throw new UnsupportedOperationException( + "This logger wraps an arbitrary set of loggers that can each have a different level enabled." + ) } diff --git a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala index 94028b39407..d404cb36336 100644 --- a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala +++ b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala @@ -49,41 +49,40 @@ object WorkflowLogger { https://github.com/qos-ch/logback/commit/77128a003a7fd7e8bd7a6ddb12da7a65cf296593#diff-f8cd32379a53986c2e70e2abe86fa0faR145 */ private def makeSynchronizedFileLogger(path: Path, level: Level, ctx: LoggerContext, name: String): Logger = - ctx.synchronized { - Option(ctx.exists(name)) match { - case Some(existingLogger) => existingLogger - case None => - val encoder = new PatternLayoutEncoder() - encoder.setPattern("%date %-5level - %msg%n") - encoder.setContext(ctx) - encoder.start() - - val appender = new FileAppender[ILoggingEvent]() - appender.setFile(path.pathAsString) - appender.setEncoder(encoder) - appender.setName(name) - appender.setContext(ctx) - appender.start() - - val fileLogger = ctx.getLogger(name) - fileLogger.addAppender(appender) - fileLogger.setAdditive(false) - fileLogger.setLevel(level) - fileLogger + ctx.synchronized { + Option(ctx.exists(name)) match { + case Some(existingLogger) => existingLogger + case None => + val encoder = new PatternLayoutEncoder() + encoder.setPattern("%date %-5level - %msg%n") + encoder.setContext(ctx) + encoder.start() + + val appender = new FileAppender[ILoggingEvent]() + appender.setFile(path.pathAsString) + appender.setEncoder(encoder) + appender.setName(name) + appender.setContext(ctx) + appender.start() + + val fileLogger = ctx.getLogger(name) + fileLogger.addAppender(appender) + fileLogger.setAdditive(false) + fileLogger.setLevel(level) + fileLogger + } } - } case class WorkflowLogConfiguration(dir: Path, temporary: Boolean) private val conf = ConfigFactory.load() - val workflowLogConfiguration: Option[WorkflowLogConfiguration] = { + val workflowLogConfiguration: Option[WorkflowLogConfiguration] = for { workflowConfig <- conf.as[Option[Config]]("workflow-options") dir <- workflowConfig.as[Option[String]]("workflow-log-dir") if !dir.isEmpty temporary <- workflowConfig.as[Option[Boolean]]("workflow-log-temporary") orElse Option(true) } yield WorkflowLogConfiguration(DefaultPathBuilder.get(dir).toAbsolutePath, temporary) - } val isEnabled = workflowLogConfiguration.isDefined val isTemporary = workflowLogConfiguration exists { @@ -111,8 +110,8 @@ class WorkflowLogger(loggerName: String, workflowId: PossiblyNotRootWorkflowId, rootWorkflowId: RootWorkflowId, override val akkaLogger: Option[LoggingAdapter], - otherLoggers: Set[Logger] = Set.empty[Logger]) - extends LoggerWrapper { + otherLoggers: Set[Logger] = Set.empty[Logger] +) extends LoggerWrapper { override def getName = loggerName @@ -137,7 +136,8 @@ class WorkflowLogger(loggerName: String, import WorkflowLogger._ lazy val workflowLogPath = workflowLogConfiguration.map(workflowLogConfigurationActual => - workflowLogConfigurationActual.dir.createPermissionedDirectories() / s"workflow.$rootWorkflowId.log") + workflowLogConfigurationActual.dir.createPermissionedDirectories() / s"workflow.$rootWorkflowId.log" + ) lazy val fileLogger = workflowLogPath match { case Some(path) => makeFileLogger(path, Level.toLevel(sys.props.getOrElse("LOG_LEVEL", "debug"))) diff --git a/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala b/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala index f0a6464f12e..ba300da9b83 100644 --- a/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala +++ b/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala @@ -27,9 +27,9 @@ trait BetterFileMethods { import BetterFileMethods._ - private final def newPath(file: better.files.File): Path = newPath(file.path) + final private def newPath(file: better.files.File): Path = newPath(file.path) - private final def newPathOrNull(file: better.files.File): Path = Option(file).map(newPath).orNull + final private def newPathOrNull(file: better.files.File): Path = Option(file).map(newPath).orNull final def toJava: JFile = betterFile.toJava @@ -44,8 +44,10 @@ trait BetterFileMethods { final def extension: Option[String] = betterFile.extension - final def extension(includeDot: Boolean = true, includeAll: Boolean = false, - toLowerCase: Boolean = true): Option[String] = + final def extension(includeDot: Boolean = true, + includeAll: Boolean = false, + toLowerCase: Boolean = true + ): Option[String] = betterFile.extension(includeDot, includeAll, toLowerCase) final def hasExtension: Boolean = betterFile.hasExtension @@ -60,16 +62,17 @@ trait BetterFileMethods { final def /(child: String): Path = newPath(betterFile./(child)) - final def createChild(child: String, asDirectory: Boolean = false) - (implicit attributes: Attributes = Attributes.default, - linkOptions: LinkOptions = LinkOptions.default): Path = + final def createChild(child: String, asDirectory: Boolean = false)(implicit + attributes: Attributes = Attributes.default, + linkOptions: LinkOptions = LinkOptions.default + ): Path = newPath(betterFile.createChild(child, asDirectory)(attributes, linkOptions)) - final def createIfNotExists(asDirectory: Boolean = false, createParents: Boolean = false) - (implicit attributes: Attributes = Attributes.default, - linkOptions: LinkOptions = LinkOptions.default): Path = { + final def createIfNotExists(asDirectory: Boolean = false, createParents: Boolean = false)(implicit + attributes: Attributes = Attributes.default, + linkOptions: LinkOptions = LinkOptions.default + ): Path = newPath(betterFile.createIfNotExists(asDirectory, createParents)(attributes, linkOptions)) - } final def exists(implicit linkOptions: LinkOptions = LinkOptions.default): Boolean = betterFile.exists(linkOptions) @@ -108,15 +111,17 @@ trait BetterFileMethods { final def lines(implicit charset: Charset = DefaultCharset): Iterable[String] = betterFile.lines(charset) - final def lineIterator(implicit charset: Charset= DefaultCharset): Iterator[String] = betterFile.lineIterator(charset) + final def lineIterator(implicit charset: Charset = DefaultCharset): Iterator[String] = + betterFile.lineIterator(charset) - final def tokens(splitter: StringSplitter = StringSplitter.Default) - (implicit charset: Charset = DefaultCharset): Iterator[String] = + final def tokens(splitter: StringSplitter = StringSplitter.Default)(implicit + charset: Charset = DefaultCharset + ): Iterator[String] = betterFile.tokens(splitter)(charset) final def contentAsString(implicit charset: Charset = DefaultCharset): String = betterFile.contentAsString(charset) - final def `!`(implicit charset: Charset= DefaultCharset): String = betterFile.contentAsString(charset) + final def `!`(implicit charset: Charset = DefaultCharset): String = betterFile.contentAsString(charset) final def printLines(lines: Iterator[Any])(implicit openOptions: OpenOptions = OpenOptions.append): this.type = { betterFile.printLines(lines)(openOptions) @@ -173,33 +178,37 @@ trait BetterFileMethods { this } - final def writeText(text: String)(implicit openOptions: OpenOptions = OpenOptions.default, - charset: Charset = DefaultCharset): this.type = { + final def writeText( + text: String + )(implicit openOptions: OpenOptions = OpenOptions.default, charset: Charset = DefaultCharset): this.type = { betterFile.writeText(text)(openOptions, charset) this } - final def write(text: String) - (implicit openOptions: OpenOptions = OpenOptions.default, - charset: Charset = DefaultCharset): this.type = { + final def write( + text: String + )(implicit openOptions: OpenOptions = OpenOptions.default, charset: Charset = DefaultCharset): this.type = { betterFile.write(text)(openOptions, charset) this } - final def overwrite(text: String)(implicit openOptions: OpenOptions = OpenOptions.default, - charset: Charset = DefaultCharset): this.type = { + final def overwrite( + text: String + )(implicit openOptions: OpenOptions = OpenOptions.default, charset: Charset = DefaultCharset): this.type = { betterFile.overwrite(text)(openOptions, charset) this } - final def <(text: String)(implicit openOptions: OpenOptions = OpenOptions.default, - charset: Charset = DefaultCharset): this.type = { + final def <( + text: String + )(implicit openOptions: OpenOptions = OpenOptions.default, charset: Charset = DefaultCharset): this.type = { betterFile.write(text)(openOptions, charset) this } - final def `>:`(text: String)(implicit openOptions: OpenOptions = OpenOptions.default, - charset: Charset = DefaultCharset): this.type = { + final def `>:`( + text: String + )(implicit openOptions: OpenOptions = OpenOptions.default, charset: Charset = DefaultCharset): this.type = { betterFile.write(text)(openOptions, charset) this } @@ -221,12 +230,16 @@ trait BetterFileMethods { final def bufferedReader(implicit charset: Charset = DefaultCharset): Dispose[BufferedReader] = betterFile.bufferedReader(charset) - final def newBufferedWriter(implicit charset: Charset = DefaultCharset, - openOptions: OpenOptions = OpenOptions.default): BufferedWriter = + final def newBufferedWriter(implicit + charset: Charset = DefaultCharset, + openOptions: OpenOptions = OpenOptions.default + ): BufferedWriter = betterFile.newBufferedWriter(charset, openOptions) - final def bufferedWriter(implicit charset: Charset = DefaultCharset, - openOptions: OpenOptions = OpenOptions.default): Dispose[BufferedWriter] = + final def bufferedWriter(implicit + charset: Charset = DefaultCharset, + openOptions: OpenOptions = OpenOptions.default + ): Dispose[BufferedWriter] = betterFile.bufferedWriter(charset, openOptions) final def newFileReader: FileReader = betterFile.newFileReader @@ -237,12 +250,14 @@ trait BetterFileMethods { final def fileWriter(append: Boolean = false): Dispose[FileWriter] = betterFile.fileWriter(append) - final def newPrintWriter(autoFlush: Boolean = false) - (implicit openOptions: OpenOptions = OpenOptions.default): PrintWriter = + final def newPrintWriter(autoFlush: Boolean = false)(implicit + openOptions: OpenOptions = OpenOptions.default + ): PrintWriter = betterFile.newPrintWriter(autoFlush) - final def printWriter(autoFlush: Boolean = false) - (implicit openOptions: OpenOptions = OpenOptions.default): Dispose[PrintWriter] = + final def printWriter(autoFlush: Boolean = false)(implicit + openOptions: OpenOptions = OpenOptions.default + ): Dispose[PrintWriter] = betterFile.printWriter(autoFlush) final def newInputStream(implicit openOptions: OpenOptions = OpenOptions.default): InputStream = @@ -251,12 +266,14 @@ trait BetterFileMethods { final def inputStream(implicit openOptions: OpenOptions = OpenOptions.default): Dispose[InputStream] = betterFile.inputStream(openOptions) - final def newScanner(splitter: StringSplitter = StringSplitter.Default) - (implicit charset: Charset = DefaultCharset): Scanner = + final def newScanner(splitter: StringSplitter = StringSplitter.Default)(implicit + charset: Charset = DefaultCharset + ): Scanner = betterFile.newScanner(splitter)(charset) - final def scanner(splitter: StringSplitter = StringSplitter.Default) - (implicit charset: Charset = DefaultCharset): Dispose[Scanner] = + final def scanner(splitter: StringSplitter = StringSplitter.Default)(implicit + charset: Charset = DefaultCharset + ): Dispose[Scanner] = betterFile.scanner(splitter)(charset) final def newOutputStream(implicit openOptions: OpenOptions = OpenOptions.default): OutputStream = @@ -265,19 +282,25 @@ trait BetterFileMethods { final def outputStream(implicit openOptions: OpenOptions = OpenOptions.default): Dispose[OutputStream] = betterFile.outputStream(openOptions) - final def newFileChannel(implicit openOptions: OpenOptions = OpenOptions.default, - attributes: Attributes = Attributes.default): FileChannel = + final def newFileChannel(implicit + openOptions: OpenOptions = OpenOptions.default, + attributes: Attributes = Attributes.default + ): FileChannel = betterFile.newFileChannel(openOptions, attributes) - final def fileChannel(implicit openOptions: OpenOptions = OpenOptions.default, - attributes: Attributes = Attributes.default): Dispose[FileChannel] = + final def fileChannel(implicit + openOptions: OpenOptions = OpenOptions.default, + attributes: Attributes = Attributes.default + ): Dispose[FileChannel] = betterFile.fileChannel(openOptions, attributes) - final def newAsynchronousFileChannel(implicit openOptions: OpenOptions = OpenOptions.default): - AsynchronousFileChannel = betterFile.newAsynchronousFileChannel(openOptions) + final def newAsynchronousFileChannel(implicit + openOptions: OpenOptions = OpenOptions.default + ): AsynchronousFileChannel = betterFile.newAsynchronousFileChannel(openOptions) - final def asynchronousFileChannel(implicit openOptions: OpenOptions = OpenOptions.default): - Dispose[AsynchronousFileChannel] = betterFile.asynchronousFileChannel(openOptions) + final def asynchronousFileChannel(implicit + openOptions: OpenOptions = OpenOptions.default + ): Dispose[AsynchronousFileChannel] = betterFile.asynchronousFileChannel(openOptions) final def digest(algorithmName: String): Array[Byte] = { val messageDigest = MessageDigest.getInstance(algorithmName) @@ -307,8 +330,11 @@ trait BetterFileMethods { final def isHidden: Boolean = betterFile.isHidden - final def isLocked(mode: RandomAccessMode, position: Long = 0L, size: Long = Long.MaxValue, - isShared: Boolean = false): Boolean = betterFile.isLocked(mode, position, size, isShared) + final def isLocked(mode: RandomAccessMode, + position: Long = 0L, + size: Long = Long.MaxValue, + isShared: Boolean = false + ): Boolean = betterFile.isLocked(mode, position, size, isShared) final def isReadLocked(position: Long = 0L, size: Long = Long.MaxValue, isShared: Boolean = false): Boolean = betterFile.isReadLocked(position, size, isShared) @@ -359,7 +385,7 @@ trait BetterFileMethods { } // Conflicts with the legacy cromwell.core.path.Obsolete.PathMethodAliases.getFileName(). Uncomment when that's gone. - //final def apply(permission: PosixFilePermission): Boolean = betterFile.apply(permission) + // final def apply(permission: PosixFilePermission): Boolean = betterFile.apply(permission) final def isOwnerReadable: Boolean = betterFile.isOwnerReadable @@ -416,9 +442,9 @@ trait BetterFileMethods { this } - final def touch(time: Instant = Instant.now()) - (implicit attributes: Attributes = Attributes.default, - linkOptions: LinkOptions = LinkOptions.default): this.type = { + final def touch( + time: Instant = Instant.now() + )(implicit attributes: Attributes = Attributes.default, linkOptions: LinkOptions = LinkOptions.default): this.type = { betterFile.touch(time)(attributes, linkOptions) this } @@ -443,14 +469,16 @@ trait BetterFileMethods { destination } - final def symbolicLinkTo(destination: Path) - (implicit attributes: Attributes = Attributes.default): destination.type = { + final def symbolicLinkTo( + destination: Path + )(implicit attributes: Attributes = Attributes.default): destination.type = { betterFile.symbolicLinkTo(destination.betterFile)(attributes) destination } - final def linkTo(destination: Path, symbolic: Boolean = false) - (implicit attributes: Attributes = Attributes.default): destination.type = { + final def linkTo(destination: Path, symbolic: Boolean = false)(implicit + attributes: Attributes = Attributes.default + ): destination.type = { betterFile.linkTo(destination.betterFile, symbolic)(attributes) destination } @@ -477,13 +505,16 @@ trait BetterFileMethods { this } - final def zipTo(destination: Path, compressionLevel: Int = Deflater.DEFAULT_COMPRESSION) - (implicit charset: Charset = DefaultCharset): destination.type = { + final def zipTo(destination: Path, compressionLevel: Int = Deflater.DEFAULT_COMPRESSION)(implicit + charset: Charset = DefaultCharset + ): destination.type = { betterFile.zipTo(destination.betterFile, compressionLevel)(charset) destination } - final def zip(compressionLevel: Int = Deflater.DEFAULT_COMPRESSION)(implicit charset: Charset = DefaultCharset): Path = + final def zip(compressionLevel: Int = Deflater.DEFAULT_COMPRESSION)(implicit + charset: Charset = DefaultCharset + ): Path = newPath(betterFile.zip(compressionLevel)(charset)) final def unzipTo(destination: Path)(implicit charset: Charset = DefaultCharset): destination.type = { @@ -515,9 +546,9 @@ object BetterFileMethods { def cwd: Path = pwd - val `..`: Path => Path = _.parent + val `..` : Path => Path = _.parent - val `.`: Path => Path = identity + val `.` : Path => Path = identity implicit class FileDsl(file: Path) { def /(f: Path => Path): Path = f(file) @@ -561,7 +592,8 @@ object BetterFileMethods { def chgrp(group: String, file: Path): Path = file.setGroup(group) - def chmod(permissions: String, file: Path): Path = file.setPermissions(PosixFilePermissions.fromString(permissions).asScala.toSet) + def chmod(permissions: String, file: Path): Path = + file.setPermissions(PosixFilePermissions.fromString(permissions).asScala.toSet) def chmod_+(permission: PosixFilePermission, file: Path): Path = file.addPermission(permission) @@ -572,10 +604,12 @@ object BetterFileMethods { def unzip(zipFile: Path)(destination: Path)(implicit charset: Charset = DefaultCharset): destination.type = zipFile.unzipTo(destination)(charset) - def zip(files: better.files.File*)(destination: better.files.File, compressionLevel: Int = Deflater.DEFAULT_COMPRESSION) - (implicit charset: Charset = DefaultCharset): destination.type = { + def zip( + files: better.files.File* + )(destination: better.files.File, compressionLevel: Int = Deflater.DEFAULT_COMPRESSION)(implicit + charset: Charset = DefaultCharset + ): destination.type = destination.zipIn(files.iterator, compressionLevel)(charset) - } } type PathMatcherSyntax = better.files.File.PathMatcherSyntax diff --git a/core/src/main/scala/cromwell/core/path/CustomRetryParams.scala b/core/src/main/scala/cromwell/core/path/CustomRetryParams.scala index 13f577a1718..5136d7baa20 100644 --- a/core/src/main/scala/cromwell/core/path/CustomRetryParams.scala +++ b/core/src/main/scala/cromwell/core/path/CustomRetryParams.scala @@ -11,7 +11,7 @@ object CustomRetryParams { val Default = CustomRetryParams( timeout = Duration.Inf, maxRetries = Option(3), - backoff = SimpleExponentialBackoff(1 seconds, 3 seconds, 1.5D), + backoff = SimpleExponentialBackoff(1 seconds, 3 seconds, 1.5d), isTransient = throwableToFalse, isFatal = throwableToFalse ) @@ -23,4 +23,5 @@ case class CustomRetryParams(timeout: Duration, maxRetries: Option[Int], backoff: Backoff, isTransient: Throwable => Boolean, - isFatal: Throwable => Boolean) + isFatal: Throwable => Boolean +) diff --git a/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala b/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala index 4bbefe71d0a..f478d04aa67 100644 --- a/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala +++ b/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala @@ -19,7 +19,6 @@ case object DefaultPathBuilder extends PathBuilder { val uri = URI.create(UrlEscapers.urlFragmentEscaper().escape(pathAsString)) Option(uri.getScheme) match { case Some("file") | None => - if (pathAsString.startsWith("file://")) { // NOTE: Legacy support for old paths generated as URIs by the old .toRealString val host = Option(uri.getHost) getOrElse "" @@ -44,15 +43,14 @@ case object DefaultPathBuilder extends PathBuilder { def createTempDirectory(prefix: String): DefaultPath = DefaultPath(java.nio.file.Files.createTempDirectory(prefix)) - def createTempFile(prefix: String = "", suffix: String = "", parent: Option[Path] = None): Path = { + def createTempFile(prefix: String = "", suffix: String = "", parent: Option[Path] = None): Path = parent match { case Some(dir) => dir.createTempFile(prefix, suffix) case _ => DefaultPath(java.nio.file.Files.createTempFile(prefix, suffix)) } - } } -case class DefaultPath private[path](nioPath: NioPath) extends Path { +case class DefaultPath private[path] (nioPath: NioPath) extends Path { override protected def newPath(nioPath: NioPath): DefaultPath = DefaultPath(nioPath) override def pathAsString: String = nioPath.toString diff --git a/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala b/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala index 20a2afdd5ce..f986dcfa1a5 100644 --- a/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala +++ b/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala @@ -7,7 +7,8 @@ import cromwell.core.path.PathBuilderFactory.PriorityDefault import scala.concurrent.{ExecutionContext, Future} case object DefaultPathBuilderFactory extends PathBuilderFactory { - override def withOptions(options: WorkflowOptions)(implicit actorSystem: ActorSystem, ec: ExecutionContext) = Future.successful(DefaultPathBuilder) + override def withOptions(options: WorkflowOptions)(implicit actorSystem: ActorSystem, ec: ExecutionContext) = + Future.successful(DefaultPathBuilder) val name = "local" val tuple = name -> this diff --git a/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala b/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala index ffe862c2df3..e455867e5e6 100644 --- a/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala +++ b/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala @@ -1,6 +1,6 @@ package cromwell.core.path -import java.io.{BufferedInputStream, BufferedReader, ByteArrayOutputStream, IOException, InputStream, InputStreamReader} +import java.io.{BufferedInputStream, BufferedReader, ByteArrayOutputStream, InputStream, InputStreamReader, IOException} import java.nio.file.{FileAlreadyExistsException, Files} import java.nio.file.attribute.{PosixFilePermission, PosixFilePermissions} import java.util.zip.GZIPOutputStream @@ -33,13 +33,11 @@ trait EvenBetterPathMethods { final def plusSuffix(suffix: String): Path = swapSuffix("", suffix) - final def swapSuffix(oldSuffix: String, newSuffix: String): Path = { + final def swapSuffix(oldSuffix: String, newSuffix: String): Path = sibling(s"${name.stripSuffix(oldSuffix)}$newSuffix") - } - final def createTempFile(prefix: String = "", suffix: String = ""): Path = { + final def createTempFile(prefix: String = "", suffix: String = ""): Path = newPath(java.nio.file.Files.createTempFile(nioPathPrivate, prefix, suffix)) - } def chmod(permissions: String): this.type = { setPermissions(PosixFilePermissions.fromString(permissions).asScala.toSet) @@ -49,18 +47,16 @@ trait EvenBetterPathMethods { // betterFile.symbolicLink calls Files.readSymbolicLink, but then implicitly converts the java.nio.Path returned to a better.File // which calls toAbsolutePath. Consequently, if the path was relative, the current directory is used to make it absolute. // This is not the desired behavior to be able to follow relative symbolic links, so bypass better files method and directly use the java one. - final def symbolicLinkRelative: Option[Path] = { + final def symbolicLinkRelative: Option[Path] = if (betterFile.isSymbolicLink) { Option(newPath(Files.readSymbolicLink(betterFile.path))) } else None - } - final def followSymbolicLinks: Path = { + final def followSymbolicLinks: Path = symbolicLinkRelative match { case Some(target) => parent.resolve(target.followSymbolicLinks) case None => this } - } final def createPermissionedDirectories(): this.type = { if (!exists) { @@ -73,8 +69,7 @@ trait EvenBetterPathMethods { addPermission(PosixFilePermission.OTHERS_READ) addPermission(PosixFilePermission.OTHERS_WRITE) addPermission(PosixFilePermission.OTHERS_EXECUTE) - } - catch { + } catch { // Race condition that's particularly likely with scatters. Ignore. case _: FileAlreadyExistsException => // The GCS filesystem does not support setting permissions and will throw an `UnsupportedOperationException`. @@ -105,7 +100,9 @@ trait EvenBetterPathMethods { byteStream.toByteArray } - def writeContent(content: String)(openOptions: OpenOptions, codec: Codec, compressPayload: Boolean)(implicit ec: ExecutionContext): this.type = { + def writeContent( + content: String + )(openOptions: OpenOptions, codec: Codec, compressPayload: Boolean)(implicit ec: ExecutionContext): this.type = { locally(ec) val contentByteArray = content.getBytes(codec.charSet) writeByteArray { @@ -113,8 +110,8 @@ trait EvenBetterPathMethods { }(openOptions) } - private def fileIoErrorPf[A]: PartialFunction[Throwable, Try[A]] = { - case ex: Throwable => Failure(new IOException(s"Could not read from ${this.pathAsString}: ${ex.getMessage}", ex)) + private def fileIoErrorPf[A]: PartialFunction[Throwable, Try[A]] = { case ex: Throwable => + Failure(new IOException(s"Could not read from ${this.pathAsString}: ${ex.getMessage}", ex)) } /** @@ -122,35 +119,36 @@ trait EvenBetterPathMethods { * The input stream will be closed when this method returns, which means the f function * cannot leak an open stream. */ - def withReader[A](f: BufferedReader => A)(implicit ec: ExecutionContext): A = { + def withReader[A](f: BufferedReader => A)(implicit ec: ExecutionContext): A = // Use an input reader to convert the byte stream to character stream. Buffered reader for efficiency. - tryWithResource(() => new BufferedReader(new InputStreamReader(this.mediaInputStream, Codec.UTF8.name)))(f).recoverWith(fileIoErrorPf).get - } + tryWithResource(() => new BufferedReader(new InputStreamReader(this.mediaInputStream, Codec.UTF8.name)))(f) + .recoverWith(fileIoErrorPf) + .get /** * InputStream's read method reads bytes, whereas InputStreamReader's read method reads characters. * BufferedInputStream can be used to read bytes directly from input stream, without conversion to characters. */ - def withBufferedStream[A](f: BufferedInputStream => A)(implicit ec: ExecutionContext): A = { + def withBufferedStream[A](f: BufferedInputStream => A)(implicit ec: ExecutionContext): A = tryWithResource(() => new BufferedInputStream(this.mediaInputStream))(f).recoverWith(fileIoErrorPf).get - } /** * Returns an Array[Byte] from a Path. Limit the array size to "limit" byte if defined. * @throws IOException if failOnOverflow is true and the file is larger than limit */ - def limitFileContent(limit: Option[Int], failOnOverflow: Boolean)(implicit ec: ExecutionContext): Array[Byte] = withBufferedStream { bufferedStream => - val bytesIterator = Iterator.continually(bufferedStream.read).takeWhile(_ != -1).map(_.toByte) - // Take 1 more than the limit so that we can look at the size and know if it's overflowing - val bytesArray = limit.map(l => bytesIterator.take(l + 1)).getOrElse(bytesIterator).toArray - - limit match { - case Some(l) if failOnOverflow && bytesArray.length > l => - throw new IOException(s"File $this is larger than requested maximum of $l Bytes.") - case Some(l) => bytesArray.take(l) - case _ => bytesArray + def limitFileContent(limit: Option[Int], failOnOverflow: Boolean)(implicit ec: ExecutionContext): Array[Byte] = + withBufferedStream { bufferedStream => + val bytesIterator = Iterator.continually(bufferedStream.read).takeWhile(_ != -1).map(_.toByte) + // Take 1 more than the limit so that we can look at the size and know if it's overflowing + val bytesArray = limit.map(l => bytesIterator.take(l + 1)).getOrElse(bytesIterator).toArray + + limit match { + case Some(l) if failOnOverflow && bytesArray.length > l => + throw new IOException(s"File $this is larger than requested maximum of $l Bytes.") + case Some(l) => bytesArray.take(l) + case _ => bytesArray + } } - } /** * Reads the first limitBytes of a file and makes a String. Prepend with an annotation at the start (to say that this is the diff --git a/core/src/main/scala/cromwell/core/path/JavaWriterImplicits.scala b/core/src/main/scala/cromwell/core/path/JavaWriterImplicits.scala index cc1b7f40dde..2b5e36f2086 100644 --- a/core/src/main/scala/cromwell/core/path/JavaWriterImplicits.scala +++ b/core/src/main/scala/cromwell/core/path/JavaWriterImplicits.scala @@ -4,6 +4,7 @@ import java.io.Writer object JavaWriterImplicits { implicit class FlushingAndClosingWriter(writer: Writer) { + /** Convenience method to flush and close in one shot. */ def flushAndClose() = { writer.flush() diff --git a/core/src/main/scala/cromwell/core/path/NioPathMethods.scala b/core/src/main/scala/cromwell/core/path/NioPathMethods.scala index 42c17ca3222..3ce92dddd93 100644 --- a/core/src/main/scala/cromwell/core/path/NioPathMethods.scala +++ b/core/src/main/scala/cromwell/core/path/NioPathMethods.scala @@ -35,9 +35,9 @@ trait NioPathMethods { final def getNameCount: Int = nioPathPrivate.getNameCount /* This method cannot be used safely because it could fail for valid GcsPaths that are not valid URIs - * See https://github.com/GoogleCloudPlatform/google-cloud-java/issues/1343 + * See https://github.com/GoogleCloudPlatform/google-cloud-java/issues/1343 */ - //final def toUri: URI = nioPathPrivate.toUri + // final def toUri: URI = nioPathPrivate.toUri final def compareTo(other: Path): Int = nioPathPrivate.compareTo(other.nioPathPrivate) @@ -73,5 +73,5 @@ trait NioPathMethods { * Default implementation assumes symlinks are supported, and that toRealPath may return a valid path. * This implementation may be overridden for NIO implementations that do not support symbolic links (For example the Azure NIO library) */ - def getSymlinkSafePath(options: LinkOption*): Path = toRealPath(options: _*) + def getSymlinkSafePath(options: LinkOption*): Path = toRealPath(options: _*) } diff --git a/core/src/main/scala/cromwell/core/path/Obsolete.scala b/core/src/main/scala/cromwell/core/path/Obsolete.scala index 77fd32b54a7..78c346ce09d 100644 --- a/core/src/main/scala/cromwell/core/path/Obsolete.scala +++ b/core/src/main/scala/cromwell/core/path/Obsolete.scala @@ -47,16 +47,14 @@ object Obsolete { val File = ObsoleteFile object ObsoleteFile { - def newTemporaryDirectory(prefix: String = ""): DefaultPath = { + def newTemporaryDirectory(prefix: String = ""): DefaultPath = DefaultPath(better.files.File.newTemporaryDirectory(prefix).path) - } - def newTemporaryFile(prefix: String = "", suffix: String = "", parent: Option[Path] = None): Path = { + def newTemporaryFile(prefix: String = "", suffix: String = "", parent: Option[Path] = None): Path = parent match { case Some(dir) => dir.createTempFile(prefix, suffix) case _ => DefaultPathBuilder.createTempFile(prefix, suffix) } - } def apply(path: String, fragments: String*) = DefaultPath(better.files.File(path, fragments: _*).path) diff --git a/core/src/main/scala/cromwell/core/path/PathBuilder.scala b/core/src/main/scala/cromwell/core/path/PathBuilder.scala index 371c9e98157..fecfaebd507 100644 --- a/core/src/main/scala/cromwell/core/path/PathBuilder.scala +++ b/core/src/main/scala/cromwell/core/path/PathBuilder.scala @@ -40,6 +40,7 @@ trait PreResolvePathBuilder extends PathBuilder { * @see [[cromwell.core.path.EvenBetterPathMethods]] */ trait Path extends PathObjectMethods with NioPathMethods with BetterFileMethods with EvenBetterPathMethods { + /** * A reference to the underlying nioPath, used to create new java.nio.Path's that will then be sent to newPath * for wrapping. @@ -132,11 +133,11 @@ trait Path extends PathObjectMethods with NioPathMethods with BetterFileMethods def pathWithoutScheme: String // Used by various extension traits within this scala package - private[path] final def nioPathPrivate: NioPath = nioPath + final private[path] def nioPathPrivate: NioPath = nioPath // Used within BetterFileMethods - private[path] final def betterFile: better.files.File = nioPathPrivate + final private[path] def betterFile: better.files.File = nioPathPrivate // Some Path methods return null. - private[path] final def newPathOrNull(nioPath: NioPath) = Option(nioPath).map(newPath).orNull + final private[path] def newPathOrNull(nioPath: NioPath) = Option(nioPath).map(newPath).orNull } diff --git a/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala b/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala index 79adaae4d6b..cbc750f9f6f 100644 --- a/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala +++ b/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala @@ -11,15 +11,18 @@ import scala.concurrent.{ExecutionContext, Future} object PathBuilderFactory { // Given a list of factories, instantiates the corresponding path builders - def instantiatePathBuilders(factories: List[PathBuilderFactory], workflowOptions: WorkflowOptions)(implicit as: ActorSystem): Future[List[PathBuilder]] = { + def instantiatePathBuilders(factories: List[PathBuilderFactory], workflowOptions: WorkflowOptions)(implicit + as: ActorSystem + ): Future[List[PathBuilder]] = { implicit val ec: ExecutionContext = as.dispatchers.lookup(Dispatcher.IoDispatcher) val sortedFactories = factories.sortBy(_.priority) sortedFactories.traverse(_.withOptions(workflowOptions)) } - val PriorityBlob = 100 // High priority to evaluate first, because blob files may inadvertently match other filesystems + val PriorityBlob = + 100 // High priority to evaluate first, because blob files may inadvertently match other filesystems val PriorityStandard = 1000 - val PriorityDefault = 10000 // "Default" is a fallback, evaluate last + val PriorityDefault = 10000 // "Default" is a fallback, evaluate last } /** diff --git a/core/src/main/scala/cromwell/core/path/PathCopier.scala b/core/src/main/scala/cromwell/core/path/PathCopier.scala index b9352cb2082..6848ab3d24b 100644 --- a/core/src/main/scala/cromwell/core/path/PathCopier.scala +++ b/core/src/main/scala/cromwell/core/path/PathCopier.scala @@ -18,7 +18,7 @@ object PathCopier { val tokens2 = string2.split(regexIncludingSlashes) val matchingTokens: Array[(String, String)] = tokens1.zip(tokens2).takeWhile(Function.tupled(_ == _)) - val matchingPrefix = matchingTokens.map({ case (str, _) => str }).mkString + val matchingPrefix = matchingTokens.map { case (str, _) => str }.mkString string2.stripPrefix(matchingPrefix).replaceAll("^/+", "") } @@ -39,13 +39,12 @@ object PathCopier { /** * Copies from source to destination. NOTE: Copies are not atomic, and may create a partial copy. */ - def copy(sourceFilePath: Path, destinationFilePath: Path): Try[Unit] = { + def copy(sourceFilePath: Path, destinationFilePath: Path): Try[Unit] = Try { Option(destinationFilePath.parent).foreach(_.createDirectories()) sourceFilePath.copyTo(destinationFilePath, overwrite = true) () - } recoverWith { - case ex => Failure(new IOException(s"Failed to copy $sourceFilePath to $destinationFilePath", ex)) + } recoverWith { case ex => + Failure(new IOException(s"Failed to copy $sourceFilePath to $destinationFilePath", ex)) } - } } diff --git a/core/src/main/scala/cromwell/core/path/PathFactory.scala b/core/src/main/scala/cromwell/core/path/PathFactory.scala index a9e074afcc7..b2777b0fc83 100644 --- a/core/src/main/scala/cromwell/core/path/PathFactory.scala +++ b/core/src/main/scala/cromwell/core/path/PathFactory.scala @@ -15,6 +15,7 @@ import scala.util.{Failure, Success, Try} * Convenience trait delegating to the PathFactory singleton */ trait PathFactory { + /** * Path builders to be applied (in order) to attempt to build a Path from a string. */ @@ -43,11 +44,13 @@ object PathFactory { private def findFirstSuccess(string: String, allPathBuilders: PathBuilders, restPathBuilders: PathBuilders, - failures: Vector[String]): ErrorOr[Path] = restPathBuilders match { - case Nil => NonEmptyList.fromList(failures.toList) match { - case Some(errors) => Invalid(errors) - case None => s"Could not parse '$string' to path. No PathBuilders were provided".invalidNel - } + failures: Vector[String] + ): ErrorOr[Path] = restPathBuilders match { + case Nil => + NonEmptyList.fromList(failures.toList) match { + case Some(errors) => Invalid(errors) + case None => s"Could not parse '$string' to path. No PathBuilders were provided".invalidNel + } case pb :: rest => pb.build(string, allPathBuilders) match { case Success(path) => @@ -64,7 +67,8 @@ object PathFactory { def buildPath(string: String, pathBuilders: PathBuilders, preMapping: String => String = identity[String], - postMapping: Path => Path = identity[Path]): Path = { + postMapping: Path => Path = identity[Path] + ): Path = { lazy val pathBuilderNames: String = pathBuilders map { _.name } mkString ", " @@ -77,12 +81,12 @@ object PathFactory { path match { case Valid(v) => v case Invalid(errors) => - throw PathParsingException( - s"""Could not build the path "$string". It may refer to a filesystem not supported by this instance of Cromwell.""" + - s" Supported filesystems are: $pathBuilderNames." + - s" Failures: ${errors.toList.mkString(System.lineSeparator, System.lineSeparator, System.lineSeparator)}" + - s" Please refer to the documentation for more information on how to configure filesystems: http://cromwell.readthedocs.io/en/develop/backends/HPC/#filesystems" - ) + throw PathParsingException( + s"""Could not build the path "$string". It may refer to a filesystem not supported by this instance of Cromwell.""" + + s" Supported filesystems are: $pathBuilderNames." + + s" Failures: ${errors.toList.mkString(System.lineSeparator, System.lineSeparator, System.lineSeparator)}" + + s" Please refer to the documentation for more information on how to configure filesystems: http://cromwell.readthedocs.io/en/develop/backends/HPC/#filesystems" + ) } } } diff --git a/core/src/main/scala/cromwell/core/path/PathObjectMethods.scala b/core/src/main/scala/cromwell/core/path/PathObjectMethods.scala index d109faa797c..ab3d981316c 100644 --- a/core/src/main/scala/cromwell/core/path/PathObjectMethods.scala +++ b/core/src/main/scala/cromwell/core/path/PathObjectMethods.scala @@ -8,12 +8,11 @@ trait PathObjectMethods { override def toString: String = pathAsString - override def equals(obj: Any) = { + override def equals(obj: Any) = obj match { case other: Path => nioPathPrivate == other.nioPathPrivate case _ => false } - } override def hashCode = nioPathPrivate.hashCode() } diff --git a/core/src/main/scala/cromwell/core/path/PathWriter.scala b/core/src/main/scala/cromwell/core/path/PathWriter.scala index 5a602810183..ee7717b6765 100644 --- a/core/src/main/scala/cromwell/core/path/PathWriter.scala +++ b/core/src/main/scala/cromwell/core/path/PathWriter.scala @@ -59,7 +59,7 @@ case class TailedWriter(path: Path, tailedSize: Int) extends PathWriter { * * @return a descriptive tail of the `path` and the last `tailedLines` written. */ - def tailString: String = { + def tailString: String = if (tailedLines.isEmpty) { s"Contents of $path were empty." } else if (isTailed) { @@ -67,5 +67,4 @@ case class TailedWriter(path: Path, tailedSize: Int) extends PathWriter { } else { s"Contents of $path:\n${tailedLines.mkString("\n")}" } - } } diff --git a/core/src/main/scala/cromwell/core/retry/GoogleBackoff.scala b/core/src/main/scala/cromwell/core/retry/GoogleBackoff.scala index 0a50b79e091..c901bc0d1ec 100644 --- a/core/src/main/scala/cromwell/core/retry/GoogleBackoff.scala +++ b/core/src/main/scala/cromwell/core/retry/GoogleBackoff.scala @@ -8,47 +8,59 @@ import net.ceedubs.ficus.Ficus._ import scala.concurrent.duration.{Duration, FiniteDuration} object InitialGapBackoff { - def apply(initialGap: FiniteDuration, initialInterval: FiniteDuration, maxInterval: FiniteDuration, multiplier: Double) = { - new InitialGapBackoff(initialGap, new ExponentialBackOff.Builder() - .setInitialIntervalMillis(initialInterval.toMillis.toInt) - .setMaxIntervalMillis(maxInterval.toMillis.toInt) - .setMultiplier(multiplier) - .setMaxElapsedTimeMillis(Int.MaxValue) - .build()) - } + def apply(initialGap: FiniteDuration, + initialInterval: FiniteDuration, + maxInterval: FiniteDuration, + multiplier: Double + ) = + new InitialGapBackoff( + initialGap, + new ExponentialBackOff.Builder() + .setInitialIntervalMillis(initialInterval.toMillis.toInt) + .setMaxIntervalMillis(maxInterval.toMillis.toInt) + .setMultiplier(multiplier) + .setMaxElapsedTimeMillis(Int.MaxValue) + .build() + ) } case class InitialGapBackoff(initialGapMillis: FiniteDuration, googleBackoff: ExponentialBackOff) extends Backoff { assert(initialGapMillis.compareTo(Duration.Zero) != 0, "Initial gap cannot be null, use SimpleBackoff instead.") override val backoffMillis = initialGapMillis.toMillis + /** Switch to a SimpleExponentialBackoff after the initial gap has been used */ override def next = new SimpleExponentialBackoff(googleBackoff) } object SimpleExponentialBackoff { - def apply(initialInterval: FiniteDuration, maxInterval: FiniteDuration, multiplier: Double, randomizationFactor: Double = ExponentialBackOff.DEFAULT_RANDOMIZATION_FACTOR) = { - new SimpleExponentialBackoff(new ExponentialBackOff.Builder() - .setInitialIntervalMillis(initialInterval.toMillis.toInt) - .setMaxIntervalMillis(maxInterval.toMillis.toInt) - .setMultiplier(multiplier) - .setMaxElapsedTimeMillis(Int.MaxValue) - .setRandomizationFactor(randomizationFactor) - .build()) - } - - def apply(config: Config): SimpleExponentialBackoff = { + def apply(initialInterval: FiniteDuration, + maxInterval: FiniteDuration, + multiplier: Double, + randomizationFactor: Double = ExponentialBackOff.DEFAULT_RANDOMIZATION_FACTOR + ) = + new SimpleExponentialBackoff( + new ExponentialBackOff.Builder() + .setInitialIntervalMillis(initialInterval.toMillis.toInt) + .setMaxIntervalMillis(maxInterval.toMillis.toInt) + .setMultiplier(multiplier) + .setMaxElapsedTimeMillis(Int.MaxValue) + .setRandomizationFactor(randomizationFactor) + .build() + ) + + def apply(config: Config): SimpleExponentialBackoff = SimpleExponentialBackoff( config.as[FiniteDuration]("min"), config.as[FiniteDuration]("max"), config.as[Double]("multiplier"), config.as[Double]("randomization-factor") ) - } } case class SimpleExponentialBackoff(googleBackoff: ExponentialBackOff) extends Backoff { override def backoffMillis = googleBackoff.nextBackOffMillis() + /** google ExponentialBackOff is mutable so we can keep returning the same instance */ override def next = this } diff --git a/core/src/main/scala/cromwell/core/retry/Retry.scala b/core/src/main/scala/cromwell/core/retry/Retry.scala index 5e5ba4fe6b1..a7c10cd48dd 100644 --- a/core/src/main/scala/cromwell/core/retry/Retry.scala +++ b/core/src/main/scala/cromwell/core/retry/Retry.scala @@ -34,11 +34,11 @@ object Retry extends StrictLogging { */ def withRetry[A](f: () => Future[A], maxRetries: Option[Int] = Option(10), - backoff: Backoff = SimpleExponentialBackoff(5 seconds, 10 seconds, 1.1D), + backoff: Backoff = SimpleExponentialBackoff(5 seconds, 10 seconds, 1.1d), isTransient: Throwable => Boolean = throwableToFalse, isFatal: Throwable => Boolean = throwableToFalse, - onRetry: Throwable => Unit = noopOnRetry) - (implicit actorSystem: ActorSystem): Future[A] = { + onRetry: Throwable => Unit = noopOnRetry + )(implicit actorSystem: ActorSystem): Future[A] = { // In the future we might want EC passed in separately but at the moment it caused more issues than it solved to do so implicit val ec: ExecutionContext = actorSystem.dispatcher val delay = backoff.backoffMillis.millis @@ -47,10 +47,18 @@ object Retry extends StrictLogging { case throwable if isFatal(throwable) => Future.failed(CromwellFatalException(throwable)) case throwable if !isFatal(throwable) => val retriesLeft = if (isTransient(throwable)) maxRetries else maxRetries map { _ - 1 } - + if (retriesLeft.forall(_ > 0)) { onRetry(throwable) - after(delay, actorSystem.scheduler)(withRetry(f, backoff = backoff.next, maxRetries = retriesLeft, isTransient = isTransient, isFatal = isFatal, onRetry = onRetry)) + after(delay, actorSystem.scheduler)( + withRetry(f, + backoff = backoff.next, + maxRetries = retriesLeft, + isTransient = isTransient, + isFatal = isFatal, + onRetry = onRetry + ) + ) } else { Future.failed(new CromwellFatalException(throwable)) } @@ -69,8 +77,8 @@ object Retry extends StrictLogging { */ def withRetryForTransactionRollback[A](f: () => Future[A], maxRetries: Int = 5, - backoff: Backoff = SimpleExponentialBackoff(5 seconds, 10 seconds, 1.1D)) - (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[A] = { + backoff: Backoff = SimpleExponentialBackoff(5 seconds, 10 seconds, 1.1d) + )(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[A] = { val delay = backoff.backoffMillis.millis f() recoverWith { @@ -85,4 +93,3 @@ object Retry extends StrictLogging { } } } - diff --git a/core/src/main/scala/cromwell/core/simpleton/WomValueBuilder.scala b/core/src/main/scala/cromwell/core/simpleton/WomValueBuilder.scala index bfae7232d64..d2387b5062a 100644 --- a/core/src/main/scala/cromwell/core/simpleton/WomValueBuilder.scala +++ b/core/src/main/scala/cromwell/core/simpleton/WomValueBuilder.scala @@ -8,7 +8,6 @@ import wom.values._ import scala.language.postfixOps - /** * Builds arbitrary `WomValues` from `WomValueSimpletons`. **/ @@ -63,28 +62,26 @@ object WomValueBuilder { private val MapElementPattern = raw"^:((?:\\[]\[:]|[^]\[:])+)(.*)".r // Group tuples by key using a Map with key type `K`. - private def group[K](tuples: Iterable[(K, SimpletonComponent)]): Map[K, Iterable[SimpletonComponent]] = { - tuples groupBy { case (i, _) => i } map { case (k, v) => k -> (v map { case (_, s) => s}) } - } + private def group[K](tuples: Iterable[(K, SimpletonComponent)]): Map[K, Iterable[SimpletonComponent]] = + tuples groupBy { case (i, _) => i } map { case (k, v) => k -> (v map { case (_, s) => s }) } // Returns a tuple of the index into the outermost array and a `SimpletonComponent` whose path reflects the "descent" // into the array. e.g. for a component // SimpletonComponent("[0][1]", v) this would return (0 -> SimpletonComponent("[1]", v)). - private def descendIntoArray(component: SimpletonComponent): (Int, SimpletonComponent) = { - component.path match { case ArrayElementPattern(index, more) => index.toInt -> component.copy(path = more)} - } + private def descendIntoArray(component: SimpletonComponent): (Int, SimpletonComponent) = + component.path match { case ArrayElementPattern(index, more) => index.toInt -> component.copy(path = more) } // Returns a tuple of the key into the outermost map and a `SimpletonComponent` whose path reflects the "descent" // into the map. e.g. for a component // SimpletonComponent(":bar:baz", v) this would return ("bar" -> SimpletonComponent(":baz", v)). // Map keys are treated as Strings by this method, the caller must ultimately do the appropriate coercion to the // actual map key type. - private def descendIntoMap(component: SimpletonComponent): (String, SimpletonComponent) = { - component.path match { case MapElementPattern(key, more) => key.unescapeMeta -> component.copy(path = more)} - } + private def descendIntoMap(component: SimpletonComponent): (String, SimpletonComponent) = + component.path match { case MapElementPattern(key, more) => key.unescapeMeta -> component.copy(path = more) } - private implicit class EnhancedSimpletonComponents(val components: Iterable[SimpletonComponent]) extends AnyVal { - def asArray: List[Iterable[SimpletonComponent]] = group(components map descendIntoArray).toList.sortBy(_._1).map(_._2) + implicit private class EnhancedSimpletonComponents(val components: Iterable[SimpletonComponent]) extends AnyVal { + def asArray: List[Iterable[SimpletonComponent]] = + group(components map descendIntoArray).toList.sortBy(_._1).map(_._2) def asMap: Map[String, Iterable[SimpletonComponent]] = group(components map descendIntoMap) def asPrimitive: WomValue = components.head.value def asString: String = asPrimitive.valueString @@ -92,51 +89,50 @@ object WomValueBuilder { private def toWomValue(outputType: WomType, components: Iterable[SimpletonComponent]): WomValue = { - - // Returns a tuple of the key into the pair (i.e. left or right) and a `SimpletonComponent` whose path reflects the "descent" // into the pair. e.g. for a component // SimpletonComponent(":left:foo", someValue) this would return (PairLeft -> SimpletonComponent(":baz", someValue)). sealed trait PairLeftOrRight case object PairLeft extends PairLeftOrRight case object PairRight extends PairLeftOrRight - def descendIntoPair(component: SimpletonComponent): (PairLeftOrRight, SimpletonComponent) = { + def descendIntoPair(component: SimpletonComponent): (PairLeftOrRight, SimpletonComponent) = component.path match { case MapElementPattern("left", more) => PairLeft -> component.copy(path = more) case MapElementPattern("right", more) => PairRight -> component.copy(path = more) } - } - def toWomFile(components: Iterable[SimpletonComponent]) = { + def toWomFile(components: Iterable[SimpletonComponent]) = // If there's just one simpleton, it's a primitive (file or directory) if (components.size == 1) components.asPrimitive else { // Otherwise make a map of the components and detect the type of file from the class field val groupedListing = components.asMap - def isClass(className: String) = { - groupedListing.get(ClassKey) - /* If the class field is in an array it will be prefixed with a ':', so check for that as well. - * e.g: secondaryFiles[0]:class -> "File" - * secondaryFiles[0]:value -> "file/path" - * would produce a Map( - * ":class" -> List(Simpleton("File")), - * ":value" -> List(Simpleton("file/path")) - * ) - */ - .orElse(groupedListing.get(s":$ClassKey")) - .map(_.asPrimitive.valueString) - .contains(className) - } + def isClass(className: String) = + groupedListing + .get(ClassKey) + /* If the class field is in an array it will be prefixed with a ':', so check for that as well. + * e.g: secondaryFiles[0]:class -> "File" + * secondaryFiles[0]:value -> "file/path" + * would produce a Map( + * ":class" -> List(Simpleton("File")), + * ":value" -> List(Simpleton("file/path")) + * ) + */ + .orElse(groupedListing.get(s":$ClassKey")) + .map(_.asPrimitive.valueString) + .contains(className) def isDirectory = isClass(WomValueSimpleton.DirectoryClass) def isFile = isClass(WomValueSimpleton.FileClass) if (isDirectory) toWomValue(WomMaybeListedDirectoryType, components) else if (isFile) toWomValue(WomMaybePopulatedFileType, components) - else throw new IllegalArgumentException(s"There is no WomFile that can be built from simpletons: ${groupedListing.toList.mkString(", ")}") + else + throw new IllegalArgumentException( + s"There is no WomFile that can be built from simpletons: ${groupedListing.toList.mkString(", ")}" + ) } - } outputType match { case _: WomPrimitiveType => @@ -151,19 +147,31 @@ object WomValueBuilder { WomArray(arrayType, components.asArray map { toWomValue(arrayType.memberType, _) }) case mapType: WomMapType => // map keys are guaranteed by WOM to be primitives, so the "coerceRawValue(..).get" is safe. - WomMap(mapType, components.asMap map { case (k, ss) => mapType.keyType.coerceRawValue(k).get -> toWomValue(mapType.valueType, ss) }) + WomMap(mapType, + components.asMap map { case (k, ss) => + mapType.keyType.coerceRawValue(k).get -> toWomValue(mapType.valueType, ss) + } + ) case pairType: WomPairType => - val groupedByLeftOrRight: Map[PairLeftOrRight, Iterable[SimpletonComponent]] = group(components map descendIntoPair) - WomPair(toWomValue(pairType.leftType, groupedByLeftOrRight(PairLeft)), toWomValue(pairType.rightType, groupedByLeftOrRight(PairRight))) + val groupedByLeftOrRight: Map[PairLeftOrRight, Iterable[SimpletonComponent]] = group( + components map descendIntoPair + ) + WomPair(toWomValue(pairType.leftType, groupedByLeftOrRight(PairLeft)), + toWomValue(pairType.rightType, groupedByLeftOrRight(PairRight)) + ) case WomObjectType => // map keys are guaranteed by WOM to be primitives, so the "coerceRawValue(..).get" is safe. val map: Map[String, WomValue] = components.asMap map { case (k, ss) => k -> toWomValue(WomAnyType, ss) } WomObject(map) case composite: WomCompositeType => val map: Map[String, WomValue] = components.asMap map { case (k, ss) => - val valueType = composite - .typeMap - .getOrElse(k, throw new RuntimeException(s"Field $k is not a declared field of composite type $composite. Cannot build a WomValue from the simpletons.")) + val valueType = composite.typeMap + .getOrElse( + k, + throw new RuntimeException( + s"Field $k is not a declared field of composite type $composite. Cannot build a WomValue from the simpletons." + ) + ) k -> toWomValue(valueType, ss) } WomObject.withTypeUnsafe(map, composite) @@ -171,8 +179,9 @@ object WomValueBuilder { val directoryValues = components.asMap val value = directoryValues.get("value").map(_.asString) - val listing = directoryValues.get("listing") - .map({ _.asArray.map(toWomFile).collect({ case womFile: WomFile => womFile }) }) + val listing = directoryValues + .get("listing") + .map(_.asArray.map(toWomFile).collect { case womFile: WomFile => womFile }) WomMaybeListedDirectory(value, listing) case WomMaybePopulatedFileType => @@ -183,9 +192,9 @@ object WomValueBuilder { val size = populatedValues.get("size").map(_.asString.toLong) val format = populatedValues.get("format").map(_.asString) val contents = populatedValues.get("contents").map(_.asString) - val secondaryFiles = populatedValues.get("secondaryFiles").toList.flatMap({ - _.asArray.map(toWomFile).collect({ case womFile: WomFile => womFile }) - }) + val secondaryFiles = populatedValues.get("secondaryFiles").toList.flatMap { + _.asArray.map(toWomFile).collect { case womFile: WomFile => womFile } + } WomMaybePopulatedFile( valueOption = value, @@ -234,23 +243,28 @@ object WomValueBuilder { */ private case class SimpletonComponent(path: String, value: WomValue) - def toJobOutputs(taskOutputs: Iterable[OutputPort], simpletons: Iterable[WomValueSimpleton]): CallOutputs = { + def toJobOutputs(taskOutputs: Iterable[OutputPort], simpletons: Iterable[WomValueSimpleton]): CallOutputs = CallOutputs(toWomValues(taskOutputs, simpletons)) - } - def toWomValues(taskOutputs: Iterable[OutputPort], simpletons: Iterable[WomValueSimpleton]): Map[OutputPort, WomValue] = { + def toWomValues(taskOutputs: Iterable[OutputPort], + simpletons: Iterable[WomValueSimpleton] + ): Map[OutputPort, WomValue] = { - def simpletonToComponent(name: String)(simpleton: WomValueSimpleton): SimpletonComponent = { + def simpletonToComponent(name: String)(simpleton: WomValueSimpleton): SimpletonComponent = SimpletonComponent(simpleton.simpletonKey.drop(name.length), simpleton.simpletonValue) - } // This is meant to "rehydrate" simpletonized WomValues back to WomValues. It is assumed that these WomValues were // "dehydrated" to WomValueSimpletons correctly. This code is not robust to corrupt input whatsoever. val types = taskOutputs map { o => o -> o.womType } toMap - val simpletonsByOutputName = simpletons groupBy { _.simpletonKey match { case IdentifierAndPathPattern(i, _) => i } } + val simpletonsByOutputName = simpletons groupBy { + _.simpletonKey match { case IdentifierAndPathPattern(i, _) => i } + } val simpletonComponentsByOutputName: Map[String, Iterable[SimpletonComponent]] = simpletonsByOutputName map { case (name, ss) => name -> (ss map simpletonToComponent(name)) } - types map { case (outputPort, outputType) => outputPort -> toWomValue(outputType, simpletonComponentsByOutputName.getOrElse(outputPort.internalName, Seq.empty))} + types map { case (outputPort, outputType) => + outputPort -> toWomValue(outputType, + simpletonComponentsByOutputName.getOrElse(outputPort.internalName, Seq.empty) + ) + } } } - diff --git a/core/src/main/scala/cromwell/core/simpleton/WomValueSimpleton.scala b/core/src/main/scala/cromwell/core/simpleton/WomValueSimpleton.scala index 01ec7ddf8c8..35e7accdf07 100644 --- a/core/src/main/scala/cromwell/core/simpleton/WomValueSimpleton.scala +++ b/core/src/main/scala/cromwell/core/simpleton/WomValueSimpleton.scala @@ -15,7 +15,7 @@ case class WomValueSimpleton(simpletonKey: String, simpletonValue: WomPrimitive) * `WomValueSimpleton`s are transformed back to `WomValue`s. */ object WomValueSimpleton { - + val ClassKey = "class" val DirectoryClass = "Directory" val FileClass = "File" @@ -35,23 +35,30 @@ object WomValueSimpleton { private def toNumberSimpleton(key: String)(value: Long) = WomValueSimpleton(key, WomInteger(value.toInt)) // Pass the simplifyMode down to recursive calls without having to sling the parameter around explicitly. - def simplify(name: String)(implicit simplifyMode: SimplifyMode = SimplifyMode(forCaching = false)): Iterable[WomValueSimpleton] = { + def simplify( + name: String + )(implicit simplifyMode: SimplifyMode = SimplifyMode(forCaching = false)): Iterable[WomValueSimpleton] = { def suffix(suffix: String) = s"$name:$suffix" val fileValueSimplifier: String => String => WomValueSimpleton = if (simplifyMode.forCaching) key => value => WomValueSimpleton(key, WomSingleFile(value)) else toStringSimpleton // What should this even do? Maybe just pick out the last bit of the path and store that as a String? val directoryValueSimplifier: String => String => WomValueSimpleton = - if (simplifyMode.forCaching) key => value => WomValueSimpleton(key, WomString(value.substring(value.lastIndexOf("/") + 1))) else toStringSimpleton + if (simplifyMode.forCaching) + key => value => WomValueSimpleton(key, WomString(value.substring(value.lastIndexOf("/") + 1))) + else toStringSimpleton womValue match { case prim: WomPrimitive => List(WomValueSimpleton(name, prim)) case opt: WomOptionalValue => opt.value.map(_.simplify(name)).getOrElse(Seq.empty) - case WomArray(_, arrayValue) => arrayValue.zipWithIndex flatMap { case (arrayItem, index) => arrayItem.simplify(s"$name[$index]") } - case WomMap(_, mapValue) => mapValue flatMap { case (key, value) => value.simplify(s"$name:${key.valueString.escapeMeta}") } + case WomArray(_, arrayValue) => + arrayValue.zipWithIndex flatMap { case (arrayItem, index) => arrayItem.simplify(s"$name[$index]") } + case WomMap(_, mapValue) => + mapValue flatMap { case (key, value) => value.simplify(s"$name:${key.valueString.escapeMeta}") } case WomPair(left, right) => left.simplify(s"$name:left") ++ right.simplify(s"$name:right") - case womObjectLike: WomObjectLike => womObjectLike.values flatMap { - case (key, value) => value.simplify(s"$name:${key.escapeMeta}") - } + case womObjectLike: WomObjectLike => + womObjectLike.values flatMap { case (key, value) => + value.simplify(s"$name:${key.escapeMeta}") + } case WomMaybeListedDirectory(valueOption, listingOption, _, _) => // This simpleton is not strictly part of the WomFile but is used to record the type of this WomValue so it can // be re-built appropriately in the WomValueBuilder @@ -82,10 +89,14 @@ object WomValueSimpleton { } implicit class WomValuesSimplifier(womValues: Map[String, WomValue]) { - def simplifyForCaching: Iterable[WomValueSimpleton] = womValues flatMap { case (name, value) => value.simplify(name)(simplifyMode = SimplifyMode(forCaching = true)) } + def simplifyForCaching: Iterable[WomValueSimpleton] = womValues flatMap { case (name, value) => + value.simplify(name)(simplifyMode = SimplifyMode(forCaching = true)) + } } implicit class WomValuesSimplifierPort(womValues: Map[OutputPort, WomValue]) { - def simplify: Iterable[WomValueSimpleton] = womValues flatMap { case (port, value) => value.simplify(port.internalName) } + def simplify: Iterable[WomValueSimpleton] = womValues flatMap { case (port, value) => + value.simplify(port.internalName) + } } } diff --git a/core/src/main/scala/cromwell/util/DatabaseUtil.scala b/core/src/main/scala/cromwell/util/DatabaseUtil.scala index d5033415df1..af1b7c176d2 100644 --- a/core/src/main/scala/cromwell/util/DatabaseUtil.scala +++ b/core/src/main/scala/cromwell/util/DatabaseUtil.scala @@ -16,7 +16,7 @@ object DatabaseUtil { } def withRetry[A](f: () => Future[A])(implicit actorSystem: ActorSystem): Future[A] = { - val RetryBackoff = SimpleExponentialBackoff(50 millis, 1 seconds, 1D) + val RetryBackoff = SimpleExponentialBackoff(50 millis, 1 seconds, 1d) Retry.withRetry(f, maxRetries = Option(10), backoff = RetryBackoff, isTransient = isTransient) } } diff --git a/core/src/main/scala/cromwell/util/GracefulShutdownHelper.scala b/core/src/main/scala/cromwell/util/GracefulShutdownHelper.scala index 5ed66fb5e9c..5a1ea5ef2d9 100644 --- a/core/src/main/scala/cromwell/util/GracefulShutdownHelper.scala +++ b/core/src/main/scala/cromwell/util/GracefulShutdownHelper.scala @@ -12,10 +12,10 @@ object GracefulShutdownHelper { trait GracefulShutdownHelper extends GracefulStopSupport { this: Actor with ActorLogging => private var shuttingDown: Boolean = false private var shutdownList: Set[ActorRef] = Set.empty - + def isShuttingDown: Boolean = shuttingDown - def waitForActorsAndShutdown(actorsLists: NonEmptyList[ActorRef]): Unit = { + def waitForActorsAndShutdown(actorsLists: NonEmptyList[ActorRef]): Unit = if (shuttingDown) { log.error("Programmer error, this actor has already initiated its shutdown. Only call this once per actor !") } else { @@ -23,12 +23,11 @@ trait GracefulShutdownHelper extends GracefulStopSupport { this: Actor with Acto shutdownList = actorsLists.toList.toSet shutdownList foreach context.watch shutdownList foreach { _ ! ShutdownCommand } - + context become { case Terminated(actor) if shuttingDown && shutdownList.contains(actor) => shutdownList = shutdownList - actor if (shutdownList.isEmpty) context stop self } } - } } diff --git a/core/src/main/scala/cromwell/util/JsonFormatting/WomValueJsonFormatter.scala b/core/src/main/scala/cromwell/util/JsonFormatting/WomValueJsonFormatter.scala index 811f9dce61f..1de0da45989 100644 --- a/core/src/main/scala/cromwell/util/JsonFormatting/WomValueJsonFormatter.scala +++ b/core/src/main/scala/cromwell/util/JsonFormatting/WomValueJsonFormatter.scala @@ -12,15 +12,15 @@ object WomValueJsonFormatter extends DefaultJsonProtocol { case f: WomFloat => JsNumber(f.value) case b: WomBoolean => JsBoolean(b.value) case f: WomSingleFile => JsString(f.value) - case o: WomObjectLike => new JsObject(o.values map {case(k, v) => k -> write(v)}) + case o: WomObjectLike => new JsObject(o.values map { case (k, v) => k -> write(v) }) case a: WomArray => new JsArray(a.value.map(write).toVector) - case m: WomMap => new JsObject(m.value map {case(k,v) => k.valueString -> write(v)}) + case m: WomMap => new JsObject(m.value map { case (k, v) => k.valueString -> write(v) }) case q: WomPair => new JsObject(Map("left" -> write(q.left), "right" -> write(q.right))) case WomOptionalValue(_, Some(innerValue)) => write(innerValue) case WomOptionalValue(_, None) => JsNull case WomCoproductValue(_, innerValue) => write(innerValue) case WomEnumerationValue(_, innerValue) => JsString(innerValue) - // handles WdlExpression + // handles WdlExpression case v: WomValue => JsString(v.toWomString) } @@ -31,7 +31,7 @@ object WomValueJsonFormatter extends DefaultJsonProtocol { // In addition, we make a lot of assumptions about what type of WomValue to create. Oh well... it should all fall out in the coercion (fingercrossed)! def read(value: JsValue): WomValue = value match { case JsObject(fields) => - val wdlFields: Map[WomValue, WomValue] = fields map {case (k, v) => WomString(k) -> read(v)} + val wdlFields: Map[WomValue, WomValue] = fields map { case (k, v) => WomString(k) -> read(v) } if (fields.isEmpty) WomMap(WomMapType(WomStringType, WomStringType), Map.empty[WomValue, WomValue]) else WomMap(WomMapType(wdlFields.head._1.womType, wdlFields.head._2.womType), wdlFields) case JsArray(vector) if vector.nonEmpty => WomArray(WomArrayType(read(vector.head).womType), vector map read) @@ -53,4 +53,3 @@ object WomSingleFileJsonFormatter extends DefaultJsonProtocol { } } } - diff --git a/core/src/main/scala/cromwell/util/PromiseActor.scala b/core/src/main/scala/cromwell/util/PromiseActor.scala index bd5efa5b0c4..efe3f0cb217 100644 --- a/core/src/main/scala/cromwell/util/PromiseActor.scala +++ b/core/src/main/scala/cromwell/util/PromiseActor.scala @@ -17,7 +17,10 @@ private class PromiseActor(promise: Promise[Any], sendTo: ActorRef, msg: Any) ex if (actorRef == sendTo) { promise.tryFailure(new RuntimeException("Promise-watched actor completed before sending back a message")) } else { - log.error("Spooky happenstances! A Terminated({}) message was sent to a private Promise actor which wasn't watching it!?", actorRef) + log.error( + "Spooky happenstances! A Terminated({}) message was sent to a private Promise actor which wasn't watching it!?", + actorRef + ) } context.stop(self) case success => @@ -27,6 +30,7 @@ private class PromiseActor(promise: Promise[Any], sendTo: ActorRef, msg: Any) ex } object PromiseActor { + /** * Sends a message to an actor and returns the future associated with the fullfilment of the reply * Can be used instead of the akka `ask` semantics, without any timeout @@ -42,11 +46,11 @@ object PromiseActor { promise.future } - def props(promise: Promise[Any], sendTo: ActorRef, msg: Any): Props = Props(new PromiseActor(promise, sendTo, msg)).withDispatcher(EngineDispatcher) + def props(promise: Promise[Any], sendTo: ActorRef, msg: Any): Props = + Props(new PromiseActor(promise, sendTo, msg)).withDispatcher(EngineDispatcher) implicit class EnhancedActorRef(val actorRef: ActorRef) extends AnyVal { - def askNoTimeout(message: Any)(implicit actorRefFactory: ActorRefFactory): Future[Any] = { + def askNoTimeout(message: Any)(implicit actorRefFactory: ActorRefFactory): Future[Any] = PromiseActor.askNoTimeout(message, actorRef) - } } } diff --git a/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala b/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala index 238cdd70573..50a43cf2987 100644 --- a/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala +++ b/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala @@ -8,13 +8,12 @@ trait StopAndLogSupervisor { this: Actor => protected def onFailure(actorRef: ActorRef, throwable: => Throwable): Unit final val stopAndLogStrategy: SupervisorStrategy = { - def stoppingDecider: Decider = { - case e: Exception => - onFailure(sender(), e) - Stop + def stoppingDecider: Decider = { case e: Exception => + onFailure(sender(), e) + Stop } OneForOneStrategy(loggingEnabled = false)(stoppingDecider) } - override final val supervisorStrategy = stopAndLogStrategy + final override val supervisorStrategy = stopAndLogStrategy } diff --git a/core/src/main/scala/cromwell/util/TryWithResource.scala b/core/src/main/scala/cromwell/util/TryWithResource.scala index fdbfbda1882..6909232150d 100644 --- a/core/src/main/scala/cromwell/util/TryWithResource.scala +++ b/core/src/main/scala/cromwell/util/TryWithResource.scala @@ -20,17 +20,17 @@ object TryWithResource { case x: Throwable => t = Option(x) throw x - } finally { + } finally resource foreach { r => - try { + try r.close() - } catch { - case y: Throwable => t match { - case Some(_t) => _t.addSuppressed(y) - case None => throw y - } + catch { + case y: Throwable => + t match { + case Some(_t) => _t.addSuppressed(y) + case None => throw y + } } } - } } } diff --git a/core/src/test/scala/cromwell/core/DockerCredentialsSpec.scala b/core/src/test/scala/cromwell/core/DockerCredentialsSpec.scala index 25418a88fa3..4a3dc8c4929 100644 --- a/core/src/test/scala/cromwell/core/DockerCredentialsSpec.scala +++ b/core/src/test/scala/cromwell/core/DockerCredentialsSpec.scala @@ -18,11 +18,11 @@ class DockerCredentialsSpec extends AnyFlatSpec with Matchers { val credentials: Any = new DockerCredentials(Base64.getEncoder.encodeToString(tokenString.getBytes()), None, None) credentials match { - case DockerCredentialUsernameAndPassword(u, p) => { + case DockerCredentialUsernameAndPassword(u, p) => u should be(expectedUsername) p should be(expectedPassword) - } - case _ => fail(s"Expected to decompose ${tokenString} into username=$expectedPassword and password=$expectedPassword") + case _ => + fail(s"Expected to decompose ${tokenString} into username=$expectedPassword and password=$expectedPassword") } } } diff --git a/core/src/test/scala/cromwell/core/LoadConfigSpec.scala b/core/src/test/scala/cromwell/core/LoadConfigSpec.scala index 0ac4947aa1c..b2008ef3aa0 100644 --- a/core/src/test/scala/cromwell/core/LoadConfigSpec.scala +++ b/core/src/test/scala/cromwell/core/LoadConfigSpec.scala @@ -8,7 +8,7 @@ import scala.concurrent.duration._ class LoadConfigSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "LoadConfig" - + it should "parse load config" in { LoadConfig.JobStoreReadThreshold shouldBe 10000 LoadConfig.JobStoreWriteThreshold shouldBe 10000 diff --git a/core/src/test/scala/cromwell/core/MockIoActor.scala b/core/src/test/scala/cromwell/core/MockIoActor.scala index 3c1831ae8d3..dd2e8d114ac 100644 --- a/core/src/test/scala/cromwell/core/MockIoActor.scala +++ b/core/src/test/scala/cromwell/core/MockIoActor.scala @@ -18,13 +18,14 @@ class MockIoActor(returnCode: String, stderrSize: Long) extends Actor { case command: IoSizeCommand => sender() ! IoSuccess(command, 0L) case command: IoContentAsStringCommand => sender() ! IoSuccess(command, "0") case command: IoExistsCommand => sender() ! IoSuccess(command, false) - - // With context + + // With context case (requestContext: Any, command: IoCopyCommand) => sender() ! (requestContext -> IoSuccess(command, ())) case (requestContext: Any, command: IoWriteCommand) => sender() ! (requestContext -> IoSuccess(command, ())) case (requestContext: Any, command: IoDeleteCommand) => sender() ! (requestContext -> IoSuccess(command, ())) case (requestContext: Any, command: IoSizeCommand) => sender() ! (requestContext -> IoSuccess(command, stderrSize)) - case (requestContext: Any, command: IoContentAsStringCommand) => sender() ! (requestContext -> IoSuccess(command, returnCode)) + case (requestContext: Any, command: IoContentAsStringCommand) => + sender() ! (requestContext -> IoSuccess(command, returnCode)) case (requestContext: Any, command: IoExistsCommand) => sender() ! (requestContext -> IoSuccess(command, false)) case withPromise: IoCommandWithPromise[_] => self ! ((withPromise.promise, withPromise.ioCommand)) diff --git a/core/src/test/scala/cromwell/core/SimpleIoActor.scala b/core/src/test/scala/cromwell/core/SimpleIoActor.scala index 589065758c5..4ca3d3c1bdf 100644 --- a/core/src/test/scala/cromwell/core/SimpleIoActor.scala +++ b/core/src/test/scala/cromwell/core/SimpleIoActor.scala @@ -14,49 +14,44 @@ object SimpleIoActor { } class SimpleIoActor extends Actor { - + override def receive: Receive = { case command: IoCopyCommand => - Try(command.source.copyTo(command.destination)) match { case Success(_) => sender() ! IoSuccess(command, ()) case Failure(failure) => sender() ! IoFailure(command, failure) } - + case command: IoWriteCommand => - Try(command.file.write(command.content)(command.openOptions, StandardCharsets.UTF_8)) match { case Success(_) => sender() ! IoSuccess(command, ()) case Failure(failure) => sender() ! IoFailure(command, failure) } - + case command: IoDeleteCommand => - Try(command.file.delete(command.swallowIOExceptions)) match { case Success(_) => sender() ! IoSuccess(command, ()) case Failure(failure) => sender() ! IoFailure(command, failure) } - + case command: IoSizeCommand => - Try(command.file.size) match { case Success(size) => sender() ! IoSuccess(command, size) case Failure(failure) => sender() ! IoFailure(command, failure) } - + case command: IoContentAsStringCommand => - Try(command.file.contentAsString) match { case Success(content) => sender() ! IoSuccess(command, content) case Failure(failure) => sender() ! IoFailure(command, failure) } - + case command: IoHashCommand => Try(command.file.md5) match { case Success(hash) => sender() ! IoSuccess(command, hash) case Failure(failure) => sender() ! IoFailure(command, failure) } - + case command: IoExistsCommand => Try(command.file.exists) match { case Success(exists) => sender() ! IoSuccess(command, exists) @@ -65,49 +60,42 @@ class SimpleIoActor extends Actor { // With context case (requestContext: Any, command: IoCopyCommand) => - Try(command.source.copyTo(command.destination, overwrite = true)) match { case Success(_) => sender() ! (requestContext -> IoSuccess(command, ())) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) } - - case (requestContext: Any, command: IoWriteCommand) => + case (requestContext: Any, command: IoWriteCommand) => Try(command.file.write(command.content)) match { case Success(_) => sender() ! (requestContext -> IoSuccess(command, ())) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) } - - case (requestContext: Any, command: IoDeleteCommand) => + case (requestContext: Any, command: IoDeleteCommand) => Try(command.file.delete(command.swallowIOExceptions)) match { case Success(_) => sender() ! (requestContext -> IoSuccess(command, ())) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) } - + case (requestContext: Any, command: IoSizeCommand) => - Try(command.file.size) match { case Success(size) => sender() ! (requestContext -> IoSuccess(command, size)) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) } - - case (requestContext: Any, command: IoContentAsStringCommand) => + case (requestContext: Any, command: IoContentAsStringCommand) => Try(command.file.contentAsString) match { case Success(content) => sender() ! (requestContext -> IoSuccess(command, content)) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) } - + case (requestContext: Any, command: IoHashCommand) => - Try(command.file.md5) match { case Success(hash) => sender() ! (requestContext -> IoSuccess(command, hash)) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) } case (requestContext: Any, command: IoExistsCommand) => - Try(command.file.exists) match { case Success(exists) => sender() ! (requestContext -> IoSuccess(command, exists)) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) diff --git a/core/src/test/scala/cromwell/core/TestKitSuite.scala b/core/src/test/scala/cromwell/core/TestKitSuite.scala index d9fe5c3d56c..8febaa3edd8 100644 --- a/core/src/test/scala/cromwell/core/TestKitSuite.scala +++ b/core/src/test/scala/cromwell/core/TestKitSuite.scala @@ -18,9 +18,8 @@ abstract class TestKitSuite extends TestKitBase with Suite with BeforeAndAfterAl implicit lazy val system: ActorSystem = ActorSystem(actorSystemName, actorSystemConfig) - override protected def afterAll(): Unit = { + override protected def afterAll(): Unit = shutdown() - } // 'BlackHoleActor' swallows messages without logging them (thus reduces log file overhead): val emptyActor: ActorRef = system.actorOf(TestActors.blackholeProps, "TestKitSuiteEmptyActor") diff --git a/core/src/test/scala/cromwell/core/WorkflowOptionsSpec.scala b/core/src/test/scala/cromwell/core/WorkflowOptionsSpec.scala index 22aa4577c63..dd4de0a4773 100644 --- a/core/src/test/scala/cromwell/core/WorkflowOptionsSpec.scala +++ b/core/src/test/scala/cromwell/core/WorkflowOptionsSpec.scala @@ -21,11 +21,11 @@ class WorkflowOptionsSpec extends Matchers with AnyWordSpecLike { WorkflowOptions.fromJsonObject(workflowOptionsJson) match { case Success(options) => options.get("key") shouldEqual Success("value") - options.get("bad_key") shouldBe a [Failure[_]] + options.get("bad_key") shouldBe a[Failure[_]] options.clearEncryptedValues.asPrettyJson shouldEqual """{ - | "key": "value" - |}""".stripMargin + | "key": "value" + |}""".stripMargin case _ => fail("Expecting workflow options to be parseable") } } diff --git a/core/src/test/scala/cromwell/core/actor/BatchActorSpec.scala b/core/src/test/scala/cromwell/core/actor/BatchActorSpec.scala index e426a349448..1fe0ca6658b 100644 --- a/core/src/test/scala/cromwell/core/actor/BatchActorSpec.scala +++ b/core/src/test/scala/cromwell/core/actor/BatchActorSpec.scala @@ -189,24 +189,23 @@ class BatchActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers wit } } - class BatchActorTest(processingTime: FiniteDuration = Duration.Zero, fail: Boolean = false) extends BatchActor[String](10.hours, 10) { + class BatchActorTest(processingTime: FiniteDuration = Duration.Zero, fail: Boolean = false) + extends BatchActor[String](10.hours, 10) { var processed: Vector[String] = Vector.empty - override def commandToData(snd: ActorRef) = { - case command: String => command + override def commandToData(snd: ActorRef) = { case command: String => + command } override protected def weightFunction(command: String) = command.length - override protected def process(data: NonEmptyVector[String]) = { + override protected def process(data: NonEmptyVector[String]) = if (processingTime != Duration.Zero) { processed = processed ++ data.toVector - val promise = Promise[Int]() - system.scheduler.scheduleOnce(processingTime) { promise.success(data.map(weightFunction).toVector.sum) } + val promise = Promise[Int]() + system.scheduler.scheduleOnce(processingTime)(promise.success(data.map(weightFunction).toVector.sum)) promise.future } else if (!fail) { processed = processed ++ data.toVector Future.successful(data.map(weightFunction).toVector.sum) - } - else Future.failed(new Exception("Oh nose ! (This is a test failure and is expected !)") with NoStackTrace) - } + } else Future.failed(new Exception("Oh nose ! (This is a test failure and is expected !)") with NoStackTrace) } } diff --git a/core/src/test/scala/cromwell/core/actor/RobustClientHelperSpec.scala b/core/src/test/scala/cromwell/core/actor/RobustClientHelperSpec.scala index 939a1aa1834..dc29807d061 100644 --- a/core/src/test/scala/cromwell/core/actor/RobustClientHelperSpec.scala +++ b/core/src/test/scala/cromwell/core/actor/RobustClientHelperSpec.scala @@ -14,39 +14,39 @@ import scala.language.postfixOps class RobustClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with ImplicitSender { behavior of "RobustClientHelper" - + it should "handle Backpressure responses" in { val remoteActor = TestProbe() val delegateActor = TestProbe() - + val margin = 2.second - val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2D, 0D) + val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2d, 0d) val noResponseTimeout = 10 seconds val testActor = TestActorRef(new TestActor(delegateActor.ref, backoff, noResponseTimeout)) - + val messageToSend = TestActor.TestMessage("hello") - - //send message + + // send message testActor.underlyingActor.sendMessage(messageToSend, remoteActor.ref) - + // remote actor receives message remoteActor.expectMsg(messageToSend) - + // remote actor sends a backpressure message remoteActor.reply(BackPressure(messageToSend)) - + // remote actor expects request again after backpressureTimeout remoteActor.expectMsg(1.second + margin, messageToSend) - + // remote actor replies remoteActor.reply("world") - + // delegate actor receives response delegateActor.expectMsg("world") - + // remote actor doesn't receives new messages remoteActor.expectNoMessage() - + // Wait long enough that to make sure that we won't receive a ServiceUnreachable message, meaning the timeout timer // has been cancelled. Note that it is the responsibility of the actor to cancel it, the RobustClientHelper does not // handle that part. @@ -57,7 +57,7 @@ class RobustClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matc val remoteActor = TestProbe() val delegateActor = TestProbe() - val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2D, 0D) + val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2d, 0d) val noResponseTimeout = 20 seconds val testActor = TestActorRef(new TestActor(delegateActor.ref, backoff, noResponseTimeout)) @@ -68,13 +68,13 @@ class RobustClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matc // remote actor receives message remoteActor.expectMsg(messageToSend) - + // remote actor replies remoteActor.reply("world") - + // delegate receives response delegateActor.expectMsg("world") - + // remote actor doesn't receives new messages remoteActor.expectNoMessage() delegateActor.expectNoMessage() @@ -84,7 +84,7 @@ class RobustClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matc val remoteActor = TestProbe() val delegateActor = TestProbe() - val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2D, 0D) + val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2d, 0d) val noResponseTimeout = 2 seconds val testActor = TestActorRef(new TestActor(delegateActor.ref, backoff, noResponseTimeout)) @@ -109,9 +109,9 @@ class RobustClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matc it should "reset timeout when backpressured is received" in { val remoteActor = TestProbe() val delegateActor = TestProbe() - + val margin = 1 second - val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2D, 0D) + val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2d, 0d) val noResponseTimeout = 3 seconds val testActor = TestActorRef(new TestActor(delegateActor.ref, backoff, noResponseTimeout)) @@ -141,32 +141,31 @@ class RobustClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matc delegateActor.expectNoMessage(4 seconds) } - private [actor] object TestActor { + private[actor] object TestActor { case class TestMessage(v: String) case object ServiceUnreachable } - private class TestActor(delegateTo: ActorRef, - backoff: Backoff, - noResponseTimeout: FiniteDuration) extends Actor with ActorLogging with RobustClientHelper { + private class TestActor(delegateTo: ActorRef, backoff: Backoff, noResponseTimeout: FiniteDuration) + extends Actor + with ActorLogging + with RobustClientHelper { override def initialBackoff(): Backoff = backoff context.become(robustReceive orElse receive) var messageSent: Any = _ - - override def receive: Receive = { - case message => - cancelTimeout(messageSent) - delegateTo ! message + + override def receive: Receive = { case message => + cancelTimeout(messageSent) + delegateTo ! message } - + def sendMessage(message: Any, to: ActorRef) = { messageSent = message robustSend(message, to, noResponseTimeout) } - override protected def onTimeout(message: Any, to: ActorRef): Unit = { + override protected def onTimeout(message: Any, to: ActorRef): Unit = delegateTo ! TestActor.ServiceUnreachable - } } } diff --git a/core/src/test/scala/cromwell/core/actor/StreamActorHelperSpec.scala b/core/src/test/scala/cromwell/core/actor/StreamActorHelperSpec.scala index a7d42d7ffdd..b45a5e6fba7 100644 --- a/core/src/test/scala/cromwell/core/actor/StreamActorHelperSpec.scala +++ b/core/src/test/scala/cromwell/core/actor/StreamActorHelperSpec.scala @@ -13,10 +13,9 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.ExecutionContext - class StreamActorHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with ImplicitSender { behavior of "StreamActorHelper" - + implicit val materializer = ActorMaterializer() it should "catch EnqueueResponse message" in { @@ -26,7 +25,7 @@ class StreamActorHelperSpec extends TestKitSuite with AnyFlatSpecLike with Match expectMsg("hello") system stop actor } - + it should "send a backpressure message when messages are dropped by the queue" in { val actor = TestActorRef(new TestStreamActor(1)) val command = new TestStreamActorCommand @@ -50,14 +49,19 @@ class StreamActorHelperSpec extends TestKitSuite with AnyFlatSpecLike with Match } } - private object TestStreamActor { class TestStreamActorCommand - case class TestStreamActorContext(request: TestStreamActorCommand, replyTo: ActorRef, override val clientContext: Option[Any]) extends StreamContext + case class TestStreamActorContext(request: TestStreamActorCommand, + replyTo: ActorRef, + override val clientContext: Option[Any] + ) extends StreamContext } -private class TestStreamActor(queueSize: Int)(implicit override val materializer: ActorMaterializer) extends Actor with ActorLogging with StreamActorHelper[TestStreamActorContext] { - +private class TestStreamActor(queueSize: Int)(implicit override val materializer: ActorMaterializer) + extends Actor + with ActorLogging + with StreamActorHelper[TestStreamActorContext] { + override protected def actorReceive: Receive = { case command: TestStreamActorCommand => val replyTo = sender() @@ -69,8 +73,9 @@ private class TestStreamActor(queueSize: Int)(implicit override val materializer sendToStream(commandContext) } - override protected val streamSource = Source.queue[TestStreamActorContext](queueSize, OverflowStrategy.dropNew) - .map{ ("hello", _) } + override protected val streamSource = Source + .queue[TestStreamActorContext](queueSize, OverflowStrategy.dropNew) + .map(("hello", _)) - override implicit def ec: ExecutionContext = context.dispatcher + implicit override def ec: ExecutionContext = context.dispatcher } diff --git a/core/src/test/scala/cromwell/core/callcaching/HashKeySpec.scala b/core/src/test/scala/cromwell/core/callcaching/HashKeySpec.scala index 82bc6bacb25..dbd8bd4b82c 100644 --- a/core/src/test/scala/cromwell/core/callcaching/HashKeySpec.scala +++ b/core/src/test/scala/cromwell/core/callcaching/HashKeySpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class HashKeySpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "HashKey" should "produce consistent key value" in { @@ -20,7 +19,7 @@ class HashKeySpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { HashKey("output", "String myOutput"), HashKey("runtime attribute", "docker") ) - + keys map { _.key } should contain theSameElementsAs Set( "command template", "backend name", @@ -34,5 +33,5 @@ class HashKeySpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "runtime attribute: docker" ) } - + } diff --git a/core/src/test/scala/cromwell/core/filesystem/CromwellFileSystemsSpec.scala b/core/src/test/scala/cromwell/core/filesystem/CromwellFileSystemsSpec.scala index 8e958cce518..caacfd54ed8 100644 --- a/core/src/test/scala/cromwell/core/filesystem/CromwellFileSystemsSpec.scala +++ b/core/src/test/scala/cromwell/core/filesystem/CromwellFileSystemsSpec.scala @@ -15,13 +15,12 @@ import scala.concurrent.ExecutionContext class CromwellFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "CromwellFileSystems" - val globalConfig = ConfigFactory.parseString( - """ - |filesystems { - | fs1.class = "cromwell.core.path.MockPathBuilderFactory" - | fs2.class = "cromwell.core.path.MockPathBuilderFactory" - | fs3.class = "cromwell.core.filesystem.MockNotPathBuilderFactory" - |} + val globalConfig = ConfigFactory.parseString(""" + |filesystems { + | fs1.class = "cromwell.core.path.MockPathBuilderFactory" + | fs2.class = "cromwell.core.path.MockPathBuilderFactory" + | fs3.class = "cromwell.core.filesystem.MockNotPathBuilderFactory" + |} """.stripMargin) val cromwellFileSystems = new CromwellFileSystems(globalConfig) @@ -29,12 +28,11 @@ class CromwellFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with it should "build factory builders and factories for valid configuration" in { cromwellFileSystems.factoryBuilders.keySet shouldBe Set("fs1", "fs2", "fs3") - val factoriesConfig = ConfigFactory.parseString( - """ - |filesystems { - | fs1.somekey = "somevalue" - | fs2.someotherkey = "someothervalue" - |} + val factoriesConfig = ConfigFactory.parseString(""" + |filesystems { + | fs1.somekey = "somevalue" + | fs2.someotherkey = "someothervalue" + |} """.stripMargin) val pathFactories = cromwellFileSystems.factoriesFromConfig(factoriesConfig) @@ -48,16 +46,16 @@ class CromwellFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with } it should "build singleton instance if specified" in { - val rootConf = ConfigFactory.parseString( - """ - |filesystems { - | fs1 { - | class = "cromwell.core.filesystem.MockPathBuilderFactoryCustomSingletonConfig" - | global { - | class = "cromwell.core.filesystem.MockSingletonConfig" - | } - | } - |} + val rootConf = + ConfigFactory.parseString(""" + |filesystems { + | fs1 { + | class = "cromwell.core.filesystem.MockPathBuilderFactoryCustomSingletonConfig" + | global { + | class = "cromwell.core.filesystem.MockSingletonConfig" + | } + | } + |} """.stripMargin) val cromwellFileSystems = new CromwellFileSystems(rootConf) @@ -69,21 +67,33 @@ class CromwellFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with val factory2 = cromwellFileSystems.buildFactory("fs1", ConfigFactory.empty) // The singleton configs should be the same for different factories - assert(factory1.toOption.get.asInstanceOf[MockPathBuilderFactoryCustomSingletonConfig].singletonConfig == - factory2.toOption.get.asInstanceOf[MockPathBuilderFactoryCustomSingletonConfig].singletonConfig) + assert( + factory1.toOption.get.asInstanceOf[MockPathBuilderFactoryCustomSingletonConfig].singletonConfig == + factory2.toOption.get.asInstanceOf[MockPathBuilderFactoryCustomSingletonConfig].singletonConfig + ) } List( - ("if the filesystem does not exist", "filesystems.fs4.key = value", NonEmptyList.one("Cannot find a filesystem with name fs4 in the configuration. Available filesystems: fs1, fs2, fs3")), - ("if the config is invalid", "filesystems.fs1 = true", NonEmptyList.one("Invalid filesystem backend configuration for fs1")), - ("the class is not a PathBuilderFactory", "filesystems.fs3.key = value", NonEmptyList.one("The filesystem class for fs3 is not an instance of PathBuilderFactory")) - ) foreach { - case (description, config, expected) => - it should s"fail to build factories $description" in { - val result = cromwellFileSystems.factoriesFromConfig(ConfigFactory.parseString(config)) - result.isLeft shouldBe true - result.swap.toOption.get shouldBe expected - } + ("if the filesystem does not exist", + "filesystems.fs4.key = value", + NonEmptyList.one( + "Cannot find a filesystem with name fs4 in the configuration. Available filesystems: fs1, fs2, fs3" + ) + ), + ("if the config is invalid", + "filesystems.fs1 = true", + NonEmptyList.one("Invalid filesystem backend configuration for fs1") + ), + ("the class is not a PathBuilderFactory", + "filesystems.fs3.key = value", + NonEmptyList.one("The filesystem class for fs3 is not an instance of PathBuilderFactory") + ) + ) foreach { case (description, config, expected) => + it should s"fail to build factories $description" in { + val result = cromwellFileSystems.factoriesFromConfig(ConfigFactory.parseString(config)) + result.isLeft shouldBe true + result.swap.toOption.get shouldBe expected + } } val classNotFoundException = AggregatedMessageException( @@ -93,7 +103,9 @@ class CromwellFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with val wrongSignatureException = AggregatedMessageException( "Failed to initialize Cromwell filesystems", - List("Class cromwell.core.filesystem.MockPathBuilderFactoryWrongSignature for filesystem fs1 does not have the required constructor signature: (com.typesafe.config.Config, com.typesafe.config.Config)") + List( + "Class cromwell.core.filesystem.MockPathBuilderFactoryWrongSignature for filesystem fs1 does not have the required constructor signature: (com.typesafe.config.Config, com.typesafe.config.Config)" + ) ) val invalidConfigException = AggregatedMessageException( @@ -110,13 +122,15 @@ class CromwellFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with ("is invalid", "filesystems.gcs = true", invalidConfigException), ("is missing class fields", "filesystems.fs1.notclass = hello", missingClassFieldException), ("can't find class", "filesystems.fs1.class = do.not.exists", classNotFoundException), - ("has invalid class signature", "filesystems.fs1.class = cromwell.core.filesystem.MockPathBuilderFactoryWrongSignature", wrongSignatureException) - ) foreach { - case (description, config, expected) => - it should s"fail if global filesystems config $description" in { - val ex = the[Exception] thrownBy { new CromwellFileSystems(ConfigFactory.parseString(config)) } - ex shouldBe expected - } + ("has invalid class signature", + "filesystems.fs1.class = cromwell.core.filesystem.MockPathBuilderFactoryWrongSignature", + wrongSignatureException + ) + ) foreach { case (description, config, expected) => + it should s"fail if global filesystems config $description" in { + val ex = the[Exception] thrownBy new CromwellFileSystems(ConfigFactory.parseString(config)) + ex shouldBe expected + } } } @@ -124,6 +138,10 @@ class MockPathBuilderFactoryWrongSignature() class MockNotPathBuilderFactory(globalConfig: Config, val instanceConfig: Config) class MockSingletonConfig(config: Config) -class MockPathBuilderFactoryCustomSingletonConfig(globalConfig: Config, val instanceConfig: Config, val singletonConfig: MockSingletonConfig) extends cromwell.core.path.PathBuilderFactory { - override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext) = throw new UnsupportedOperationException +class MockPathBuilderFactoryCustomSingletonConfig(globalConfig: Config, + val instanceConfig: Config, + val singletonConfig: MockSingletonConfig +) extends cromwell.core.path.PathBuilderFactory { + override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext) = + throw new UnsupportedOperationException } diff --git a/core/src/test/scala/cromwell/core/io/AsyncIoSpec.scala b/core/src/test/scala/cromwell/core/io/AsyncIoSpec.scala index 83521ea3432..07ad722ead2 100644 --- a/core/src/test/scala/cromwell/core/io/AsyncIoSpec.scala +++ b/core/src/test/scala/cromwell/core/io/AsyncIoSpec.scala @@ -95,7 +95,7 @@ class AsyncIoSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers { } // Honor swallow exception false - //noinspection RedundantDefaultArgument + // noinspection RedundantDefaultArgument recoverToSucceededIf[NoSuchFileException] { testActor.underlyingActor.asyncIo.deleteAsync(testPath, swallowIoExceptions = false) } @@ -103,8 +103,8 @@ class AsyncIoSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers { it should "handle command creation errors asynchronously" in { val partialIoCommandBuilder = new PartialIoCommandBuilder { - override def existsCommand: PartialFunction[Path, Try[IoExistsCommand]] = { - case _ => Failure(new Exception("everything's fine, I am an expected exists fail") with NoStackTrace) + override def existsCommand: PartialFunction[Path, Try[IoExistsCommand]] = { case _ => + Failure(new Exception("everything's fine, I am an expected exists fail") with NoStackTrace) } } val testActor = @@ -120,10 +120,11 @@ class AsyncIoSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers { private class AsyncIoTestActor(override val ioActor: ActorRef, override val ioCommandBuilder: IoCommandBuilder = DefaultIoCommandBuilder - ) extends Actor with ActorLogging with AsyncIoActorClient { + ) extends Actor + with ActorLogging + with AsyncIoActorClient { - override def receive: Receive = { - case _ => + override def receive: Receive = { case _ => } } diff --git a/core/src/test/scala/cromwell/core/io/IoClientHelperSpec.scala b/core/src/test/scala/cromwell/core/io/IoClientHelperSpec.scala index 06132ba152b..ed07aeb3fc5 100644 --- a/core/src/test/scala/cromwell/core/io/IoClientHelperSpec.scala +++ b/core/src/test/scala/cromwell/core/io/IoClientHelperSpec.scala @@ -21,10 +21,11 @@ class IoClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers it should "intercept IoAcks and cancel timers" in { val ioActorProbe = TestProbe() val delegateProbe = TestProbe() - val backoff = SimpleExponentialBackoff(100 seconds, 10.hours, 2D, 0D) + val backoff = SimpleExponentialBackoff(100 seconds, 10.hours, 2d, 0d) val noResponseTimeout = 3 seconds - val testActor = TestActorRef(new IoClientHelperTestActor(ioActorProbe.ref, delegateProbe.ref, backoff, noResponseTimeout)) + val testActor = + TestActorRef(new IoClientHelperTestActor(ioActorProbe.ref, delegateProbe.ref, backoff, noResponseTimeout)) val command = DefaultIoSizeCommand(mock[Path]) val response = IoSuccess(command, 5L) @@ -51,10 +52,11 @@ class IoClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers it should "intercept IoAcks and cancel timers for a command with context" in { val ioActorProbe = TestProbe() val delegateProbe = TestProbe() - val backoff = SimpleExponentialBackoff(100 seconds, 10.hours, 2D, 0D) + val backoff = SimpleExponentialBackoff(100 seconds, 10.hours, 2d, 0d) val noResponseTimeout = 3 seconds - val testActor = TestActorRef(new IoClientHelperTestActor(ioActorProbe.ref, delegateProbe.ref, backoff, noResponseTimeout)) + val testActor = + TestActorRef(new IoClientHelperTestActor(ioActorProbe.ref, delegateProbe.ref, backoff, noResponseTimeout)) val commandContext = "context" val command = DefaultIoSizeCommand(mock[Path]) @@ -71,7 +73,7 @@ class IoClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers // delegate should receive the response delegateProbe.expectMsgPF(1 second) { - case (contextReceived, responseReceived) if contextReceived == "context" && responseReceived == response => + case (contextReceived, responseReceived) if contextReceived == "context" && responseReceived == response => } // And nothing else, meaning the timeout timer has been cancelled @@ -84,9 +86,12 @@ class IoClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers private case object ServiceUnreachable private class IoClientHelperTestActor(override val ioActor: ActorRef, - delegateTo: ActorRef, - backoff: Backoff, - noResponseTimeout: FiniteDuration) extends Actor with ActorLogging with IoClientHelper { + delegateTo: ActorRef, + backoff: Backoff, + noResponseTimeout: FiniteDuration + ) extends Actor + with ActorLogging + with IoClientHelper { implicit val ioCommandBuilder: DefaultIoCommandBuilder.type = DefaultIoCommandBuilder @@ -94,21 +99,18 @@ class IoClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers context.become(ioReceive orElse receive) - override def receive: Receive = { - case message => delegateTo ! message + override def receive: Receive = { case message => + delegateTo ! message } - def sendMessage(command: IoCommand[_]): Unit = { + def sendMessage(command: IoCommand[_]): Unit = sendIoCommandWithCustomTimeout(command, noResponseTimeout) - } - def sendMessageWithContext(context: Any, command: IoCommand[_]): Unit = { + def sendMessageWithContext(context: Any, command: IoCommand[_]): Unit = sendIoCommandWithContext(command, context, noResponseTimeout) - } - override protected def onTimeout(message: Any, to: ActorRef): Unit = { + override protected def onTimeout(message: Any, to: ActorRef): Unit = delegateTo ! ServiceUnreachable - } } } diff --git a/core/src/test/scala/cromwell/core/labels/LabelSpec.scala b/core/src/test/scala/cromwell/core/labels/LabelSpec.scala index 6bf4e046ed2..bdb443a91ac 100644 --- a/core/src/test/scala/cromwell/core/labels/LabelSpec.scala +++ b/core/src/test/scala/cromwell/core/labels/LabelSpec.scala @@ -29,7 +29,7 @@ class LabelSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "11f2468c-39d6-4be3-85c8-32735c01e66b", "", "!@#$%^&*()_+={}[]:;'<>?,./`~", - "now valid 255 character value-at vero eosd accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias excepturi sint occaecati cupiditate non provident, similique sunt in culpa", + "now valid 255 character value-at vero eosd accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias excepturi sint occaecati cupiditate non provident, similique sunt in culpa" ) val badLabelKeys = List( diff --git a/core/src/test/scala/cromwell/core/logging/LoggerWrapperSpec.scala b/core/src/test/scala/cromwell/core/logging/LoggerWrapperSpec.scala index d40456e0128..d3d99289a35 100644 --- a/core/src/test/scala/cromwell/core/logging/LoggerWrapperSpec.scala +++ b/core/src/test/scala/cromwell/core/logging/LoggerWrapperSpec.scala @@ -13,8 +13,12 @@ import org.slf4j.Logger import org.slf4j.event.Level import common.mock.MockSugar -class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with MockSugar - with TableDrivenPropertyChecks { +class LoggerWrapperSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with MockSugar + with TableDrivenPropertyChecks { behavior of "LoggerWrapper" @@ -25,7 +29,6 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche "slf4jMessages", "akkaMessages" ), - ( "log error with no args", _.error("Hello {} {} {} {}"), @@ -80,7 +83,6 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche List(Slf4jMessage(Level.ERROR, List("tag: Hello {} {} {} {}", "arg1", exception))), List(AkkaMessage(Logging.ErrorLevel, s"tag: Hello arg1 {} {} {}", Option(exception))) ), - ( "log warn with no args", _.warn("Hello {} {} {} {}"), @@ -123,7 +125,6 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche List(Slf4jMessage(Level.WARN, List("tag: Hello {} {} {} {}", exception))), List(AkkaMessage(Logging.WarningLevel, s"tag: Hello {} {} {} {}\n$exceptionMessage")) ), - ( "log info with no args", _.info("Hello {} {} {} {}"), @@ -138,7 +139,7 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche ), ( "log info with one arg", - _.info("Hello {} {} {} {}", arg ="arg1"), + _.info("Hello {} {} {} {}", arg = "arg1"), List(Slf4jMessage(Level.INFO, List("tag: Hello {} {} {} {}", "arg1"))), List(AkkaMessage(Logging.InfoLevel, "tag: Hello arg1 {} {} {}")) ), @@ -166,7 +167,6 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche List(Slf4jMessage(Level.INFO, List("tag: Hello {} {} {} {}", exception))), List(AkkaMessage(Logging.InfoLevel, s"tag: Hello {} {} {} {}\n$exceptionMessage")) ), - ( "log debug with no args", _.debug("Hello {} {} {} {}"), @@ -181,7 +181,7 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche ), ( "log debug with one arg", - _.debug("Hello {} {} {} {}", argument ="arg1"), + _.debug("Hello {} {} {} {}", argument = "arg1"), List(Slf4jMessage(Level.DEBUG, List("tag: Hello {} {} {} {}", "arg1"))), List(AkkaMessage(Logging.DebugLevel, "tag: Hello arg1 {} {} {}")) ), @@ -209,7 +209,6 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche List(Slf4jMessage(Level.DEBUG, List("tag: Hello {} {} {} {}", exception))), List(AkkaMessage(Logging.DebugLevel, s"tag: Hello {} {} {} {}\n$exceptionMessage")) ), - ( "log trace with no args", _.trace("Hello {} {} {} {}"), @@ -260,40 +259,36 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche var actualAkkaMessages = List.empty[AkkaMessage] - def toList(arguments: Any): List[Any] = { + def toList(arguments: Any): List[Any] = arguments match { case array: Array[_] => toLastFlattened(array) case seq: Seq[_] => seq.toList case any => List(any) } - } /* - * Flatten the last element of the array if the last element is itself an array. - * - * org.mockito.ArgumentMatchers#anyVararg() is deprecated, but works, sending in an empty array in the tail - * position. If we tried to use org.mockito.ArgumentMatchers#any(), it ends up mocking the wrong overloaded - * method. At each logging level there are two methods with very similar signatures: - * - * cromwell.core.logging.LoggerWrapper.error(pattern: String, arguments: AnyRef*) - * cromwell.core.logging.LoggerWrapper.error(pattern: String, arg: Any) - * - * As is, the Any vs. AnyRef overloads are barely dodging the issue https://issues.scala-lang.org/browse/SI-2991. - */ - def toLastFlattened(array: Array[_]): List[Any] = { + * Flatten the last element of the array if the last element is itself an array. + * + * org.mockito.ArgumentMatchers#anyVararg() is deprecated, but works, sending in an empty array in the tail + * position. If we tried to use org.mockito.ArgumentMatchers#any(), it ends up mocking the wrong overloaded + * method. At each logging level there are two methods with very similar signatures: + * + * cromwell.core.logging.LoggerWrapper.error(pattern: String, arguments: AnyRef*) + * cromwell.core.logging.LoggerWrapper.error(pattern: String, arg: Any) + * + * As is, the Any vs. AnyRef overloads are barely dodging the issue https://issues.scala-lang.org/browse/SI-2991. + */ + def toLastFlattened(array: Array[_]): List[Any] = array.toList.reverse match { case (array: Array[_]) :: tail => tail.reverse ++ array.toList case other => other.reverse } - } - def updateSlf4jMessages(level: Level, arguments: Any): Unit = { + def updateSlf4jMessages(level: Level, arguments: Any): Unit = actualSlf4jMessages :+= Slf4jMessage(level, toList(arguments)) - } - def updateAkkaMessages(logLevel: LogLevel, message: String, causeOption: Option[Throwable] = None): Unit = { + def updateAkkaMessages(logLevel: LogLevel, message: String, causeOption: Option[Throwable] = None): Unit = actualAkkaMessages :+= AkkaMessage(logLevel, message, causeOption) - } val mockLogger = mock[Logger] @@ -333,25 +328,20 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche override val isInfoEnabled: Boolean = true override val isDebugEnabled: Boolean = true - override protected def notifyError(message: String): Unit = { + override protected def notifyError(message: String): Unit = updateAkkaMessages(Logging.ErrorLevel, message) - } - override protected def notifyError(cause: Throwable, message: String): Unit = { + override protected def notifyError(cause: Throwable, message: String): Unit = updateAkkaMessages(Logging.ErrorLevel, message, Option(cause)) - } - override protected def notifyWarning(message: String): Unit = { + override protected def notifyWarning(message: String): Unit = updateAkkaMessages(Logging.WarningLevel, message) - } - override protected def notifyInfo(message: String): Unit = { + override protected def notifyInfo(message: String): Unit = updateAkkaMessages(Logging.InfoLevel, message) - } - override protected def notifyDebug(message: String): Unit = { + override protected def notifyDebug(message: String): Unit = updateAkkaMessages(Logging.DebugLevel, message) - } } val wrapper = new LoggerWrapper { diff --git a/core/src/test/scala/cromwell/core/path/DefaultPathBuilderSpec.scala b/core/src/test/scala/cromwell/core/path/DefaultPathBuilderSpec.scala index 63a5a8604cf..5235f711063 100644 --- a/core/src/test/scala/cromwell/core/path/DefaultPathBuilderSpec.scala +++ b/core/src/test/scala/cromwell/core/path/DefaultPathBuilderSpec.scala @@ -6,7 +6,12 @@ import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers import org.scalatest.prop.Tables.Table -class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers with PathBuilderSpecUtils with TestFileUtil { +class DefaultPathBuilderSpec + extends Suite + with AnyFlatSpecLike + with Matchers + with PathBuilderSpecUtils + with TestFileUtil { private val pwd = BetterFileMethods.Cmds.pwd private val parentOption = Option(pwd.parent) @@ -57,7 +62,6 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi ) private def goodPaths = Seq( - // Normal paths, not normalized GoodPath( @@ -72,8 +76,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "world", getFileName = "world", getNameCount = 2, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a relative path", path = "hello/world", @@ -86,8 +90,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "world", getFileName = "world", getNameCount = 2, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a path with spaces", path = "/hello/world/with spaces", @@ -100,8 +104,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "with spaces", getFileName = "with spaces", getNameCount = 3, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path with encode spaces", path = "/hello/world/encoded%20spaces", @@ -114,8 +118,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "encoded%20spaces", getFileName = "encoded%20spaces", getNameCount = 3, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path with non-ascii characters", path = "/hello/world/with non ascii £€", @@ -128,7 +132,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "with non ascii £€", getFileName = "with non ascii £€", getNameCount = 3, - isAbsolute = true), + isAbsolute = true + ), // Special paths @@ -144,8 +149,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = pwdName, getFileName = "", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a path from /", path = "/", @@ -158,8 +163,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "", getFileName = null, getNameCount = 0, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path from .", path = ".", @@ -172,8 +177,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = pwdName, getFileName = ".", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a path from ..", path = "..", @@ -186,8 +191,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = parentName, getFileName = "..", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a path including .", path = "/hello/world/with/./dots", @@ -200,8 +205,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "dots", getFileName = "dots", getNameCount = 5, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path including ..", path = "/hello/world/with/../dots", @@ -214,7 +219,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "dots", getFileName = "dots", getNameCount = 5, - isAbsolute = true), + isAbsolute = true + ), // Normalized @@ -230,8 +236,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = pwdName, getFileName = "", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a normalized path from /", path = "/", @@ -244,8 +250,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "", getFileName = null, getNameCount = 0, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a normalized path from .", path = ".", @@ -258,8 +264,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = pwdName, getFileName = "", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a normalized path from ..", path = "..", @@ -272,8 +278,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = parentName, getFileName = "..", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a normalized path including a .", path = "/hello/world/with/./dots", @@ -286,8 +292,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "dots", getFileName = "dots", getNameCount = 4, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a normalized path including ..", path = "/hello/world/with/../dots", @@ -300,7 +306,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "dots", getFileName = "dots", getNameCount = 3, - isAbsolute = true), + isAbsolute = true + ), // URI @@ -316,8 +323,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "world", getFileName = "world", getNameCount = 2, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path from a file uri with encoded spaces", path = "file:///hello/world/encoded%20spaces", @@ -330,7 +337,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "encoded%20spaces", getFileName = "encoded%20spaces", getNameCount = 3, - isAbsolute = true) + isAbsolute = true + ) ) private def badPaths = Seq( diff --git a/core/src/test/scala/cromwell/core/path/PathBuilderFactorySpec.scala b/core/src/test/scala/cromwell/core/path/PathBuilderFactorySpec.scala index 7fea4f7b9f3..7119a62b1f6 100644 --- a/core/src/test/scala/cromwell/core/path/PathBuilderFactorySpec.scala +++ b/core/src/test/scala/cromwell/core/path/PathBuilderFactorySpec.scala @@ -12,24 +12,28 @@ import scala.concurrent.{ExecutionContext, Future} class PathBuilderFactorySpec extends TestKitSuite with AnyFlatSpecLike with ScalaFutures with Matchers { behavior of "PathBuilderFactory" implicit val ec = system.dispatcher - + it should "sort factories when instantiating path builders" in { val factory1 = new MockPathBuilderFactory(ConfigFactory.empty(), ConfigFactory.parseString("name=factory1")) val factory2 = new MockPathBuilderFactory(ConfigFactory.empty(), ConfigFactory.parseString("name=factory2")) PathBuilderFactory - .instantiatePathBuilders(List(DefaultPathBuilderFactory, factory1, factory2), WorkflowOptions.empty).map({ pathBuilders => - pathBuilders.last shouldBe DefaultPathBuilder - // check that the order of the other factories has not been changed - pathBuilders.map(_.name) shouldBe List("factory1", "factory2", DefaultPathBuilder.name) - }).futureValue + .instantiatePathBuilders(List(DefaultPathBuilderFactory, factory1, factory2), WorkflowOptions.empty) + .map { pathBuilders => + pathBuilders.last shouldBe DefaultPathBuilder + // check that the order of the other factories has not been changed + pathBuilders.map(_.name) shouldBe List("factory1", "factory2", DefaultPathBuilder.name) + } + .futureValue } } -class MockPathBuilderFactory(globalConfig: Config, val instanceConfig: Config) extends cromwell.core.path.PathBuilderFactory { - override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext) = Future.successful( - new PathBuilder { - override def name = instanceConfig.getString("name") - override def build(pathAsString: String) = throw new UnsupportedOperationException - } - ) +class MockPathBuilderFactory(globalConfig: Config, val instanceConfig: Config) + extends cromwell.core.path.PathBuilderFactory { + override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext) = + Future.successful( + new PathBuilder { + override def name = instanceConfig.getString("name") + override def build(pathAsString: String) = throw new UnsupportedOperationException + } + ) } diff --git a/core/src/test/scala/cromwell/core/path/PathBuilderSpecUtils.scala b/core/src/test/scala/cromwell/core/path/PathBuilderSpecUtils.scala index 92bfd77233f..324928d2cb4 100644 --- a/core/src/test/scala/cromwell/core/path/PathBuilderSpecUtils.scala +++ b/core/src/test/scala/cromwell/core/path/PathBuilderSpecUtils.scala @@ -17,7 +17,8 @@ case class GoodPath(description: String, name: String, getFileName: String, getNameCount: Int, - isAbsolute: Boolean) + isAbsolute: Boolean +) case class BadPath(description: String, path: String, exceptionMessage: String) @@ -29,7 +30,8 @@ trait PathBuilderSpecUtils { def truncateCommonRoots(builder: => PathBuilder, pathsToTruncate: TableFor3[String, String, String], - tag: Tag = PathBuilderSpecUtils.PathTest): Unit = { + tag: Tag = PathBuilderSpecUtils.PathTest + ): Unit = { behavior of s"PathCopier" it should "truncate common roots" taggedAs tag in { diff --git a/core/src/test/scala/cromwell/core/retry/BackoffSpec.scala b/core/src/test/scala/cromwell/core/retry/BackoffSpec.scala index c6a008feb02..e5d947da83d 100644 --- a/core/src/test/scala/cromwell/core/retry/BackoffSpec.scala +++ b/core/src/test/scala/cromwell/core/retry/BackoffSpec.scala @@ -18,11 +18,10 @@ class BackoffSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { .setInitialIntervalMillis(1.second.toMillis.toInt) .setMaxIntervalMillis(2.seconds.toMillis.toInt) .setMaxElapsedTimeMillis(Integer.MAX_VALUE) - .setRandomizationFactor(0D) + .setRandomizationFactor(0d) .build() ) - exponentialBackoff.backoffMillis shouldBe 3.seconds.toMillis exponentialBackoff.next.backoffMillis shouldBe 1.second.toMillis } @@ -33,7 +32,7 @@ class BackoffSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { .setInitialIntervalMillis(1.second.toMillis.toInt) .setMaxIntervalMillis(2.seconds.toMillis.toInt) .setMaxElapsedTimeMillis(Integer.MAX_VALUE) - .setRandomizationFactor(0D) + .setRandomizationFactor(0d) .build() ).backoffMillis shouldBe 1.second.toMillis } @@ -46,7 +45,7 @@ class BackoffSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { .setInitialIntervalMillis(1.second.toMillis.toInt) .setMaxIntervalMillis(2.seconds.toMillis.toInt) .setMaxElapsedTimeMillis(Integer.MAX_VALUE) - .setRandomizationFactor(0D) + .setRandomizationFactor(0d) .build() ) } @@ -57,16 +56,16 @@ class BackoffSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { Map[String, Any]( "min" -> "5 seconds", "max" -> "30 seconds", - "multiplier" -> 6D, - "randomization-factor" -> 0D + "multiplier" -> 6d, + "randomization-factor" -> 0d ).asJava ) val backoff = SimpleExponentialBackoff(config) backoff.googleBackoff.getCurrentIntervalMillis shouldBe 5.seconds.toMillis.toInt backoff.googleBackoff.getMaxIntervalMillis shouldBe 30.seconds.toMillis.toInt - backoff.googleBackoff.getMultiplier shouldBe 6D - backoff.googleBackoff.getRandomizationFactor shouldBe 0D + backoff.googleBackoff.getMultiplier shouldBe 6d + backoff.googleBackoff.getRandomizationFactor shouldBe 0d } } diff --git a/core/src/test/scala/cromwell/core/retry/RetrySpec.scala b/core/src/test/scala/cromwell/core/retry/RetrySpec.scala index 83516f3328b..134b1c1db25 100644 --- a/core/src/test/scala/cromwell/core/retry/RetrySpec.scala +++ b/core/src/test/scala/cromwell/core/retry/RetrySpec.scala @@ -16,7 +16,7 @@ class RetrySpec extends TestKitSuite with AnyFlatSpecLike with Matchers with Sca var counter: Int = n - def doIt(): Future[Int] = { + def doIt(): Future[Int] = if (counter == 0) Future.successful(9) else { @@ -24,23 +24,22 @@ class RetrySpec extends TestKitSuite with AnyFlatSpecLike with Matchers with Sca val ex = if (counter <= transients) new TransientException else new IllegalArgumentException("Failed") Future.failed(ex) } - } } - implicit val defaultPatience: PatienceConfig = PatienceConfig(timeout = Span(30, Seconds), interval = Span(100, Millis)) + implicit val defaultPatience: PatienceConfig = + PatienceConfig(timeout = Span(30, Seconds), interval = Span(100, Millis)) private def runRetry(retries: Int, work: MockWork, isTransient: Throwable => Boolean = Retry.throwableToFalse, - isFatal: Throwable => Boolean = Retry.throwableToFalse): Future[Int] = { - + isFatal: Throwable => Boolean = Retry.throwableToFalse + ): Future[Int] = withRetry( f = () => work.doIt(), maxRetries = Option(retries), isTransient = isTransient, isFatal = isFatal ) - } "Retry" should "retry a function until it works" in { val work = new MockWork(2) @@ -53,7 +52,7 @@ class RetrySpec extends TestKitSuite with AnyFlatSpecLike with Matchers with Sca it should "fail if it hits the max retry count" in { whenReady(runRetry(1, new MockWork(3)).failed) { x => - x shouldBe an [CromwellFatalException] + x shouldBe an[CromwellFatalException] } } @@ -61,18 +60,19 @@ class RetrySpec extends TestKitSuite with AnyFlatSpecLike with Matchers with Sca val work = new MockWork(3) whenReady(runRetry(3, work, isFatal = (t: Throwable) => t.isInstanceOf[IllegalArgumentException]).failed) { x => - x shouldBe an [CromwellFatalException] + x shouldBe an[CromwellFatalException] work.counter shouldBe 2 } val work2 = new MockWork(4, 2) val retry = runRetry(4, - work2, - isFatal = (t: Throwable) => t.isInstanceOf[IllegalArgumentException], - isTransient = (t: Throwable) => t.isInstanceOf[TransientException]) + work2, + isFatal = (t: Throwable) => t.isInstanceOf[IllegalArgumentException], + isTransient = (t: Throwable) => t.isInstanceOf[TransientException] + ) whenReady(retry.failed) { x => - x shouldBe an [CromwellFatalException] + x shouldBe an[CromwellFatalException] work2.counter shouldBe 3 } } diff --git a/core/src/test/scala/cromwell/core/simpleton/WomValueBuilderSpec.scala b/core/src/test/scala/cromwell/core/simpleton/WomValueBuilderSpec.scala index 8db76caab1d..fb347e43441 100644 --- a/core/src/test/scala/cromwell/core/simpleton/WomValueBuilderSpec.scala +++ b/core/src/test/scala/cromwell/core/simpleton/WomValueBuilderSpec.scala @@ -23,110 +23,165 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc case class SimpletonConversion(name: String, womValue: WomValue, simpletons: Seq[WomValueSimpleton]) val simpletonConversions = List( SimpletonConversion("foo", WomString("none"), List(WomValueSimpleton("foo", WomString("none")))), - SimpletonConversion("bar", WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))), List(WomValueSimpleton("bar[0]", WomInteger(1)), WomValueSimpleton("bar[1]", WomInteger(2)))), + SimpletonConversion( + "bar", + WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))), + List(WomValueSimpleton("bar[0]", WomInteger(1)), WomValueSimpleton("bar[1]", WomInteger(2))) + ), SimpletonConversion("empty_array", WomArray(WomArrayType(WomIntegerType), List.empty), List()), SimpletonConversion( "baz", - WomArray(WomArrayType(WomArrayType(WomIntegerType)), List( - WomArray(WomArrayType(WomIntegerType), List(WomInteger(0), WomInteger(1))), - WomArray(WomArrayType(WomIntegerType), List(WomInteger(2), WomInteger(3))))), - List(WomValueSimpleton("baz[0][0]", WomInteger(0)), WomValueSimpleton("baz[0][1]", WomInteger(1)), WomValueSimpleton("baz[1][0]", WomInteger(2)), WomValueSimpleton("baz[1][1]", WomInteger(3))) + WomArray( + WomArrayType(WomArrayType(WomIntegerType)), + List(WomArray(WomArrayType(WomIntegerType), List(WomInteger(0), WomInteger(1))), + WomArray(WomArrayType(WomIntegerType), List(WomInteger(2), WomInteger(3))) + ) + ), + List( + WomValueSimpleton("baz[0][0]", WomInteger(0)), + WomValueSimpleton("baz[0][1]", WomInteger(1)), + WomValueSimpleton("baz[1][0]", WomInteger(2)), + WomValueSimpleton("baz[1][1]", WomInteger(3)) + ) ), SimpletonConversion( "map", - WomMap(WomMapType(WomStringType, WomStringType), Map( - WomString("foo") -> WomString("foo"), - WomString("bar") -> WomString("bar"))), + WomMap(WomMapType(WomStringType, WomStringType), + Map(WomString("foo") -> WomString("foo"), WomString("bar") -> WomString("bar")) + ), List(WomValueSimpleton("map:foo", WomString("foo")), WomValueSimpleton("map:bar", WomString("bar"))) ), SimpletonConversion( "mapOfMaps", - WomMap(WomMapType(WomStringType, WomMapType(WomStringType, WomStringType)), Map( - WomString("foo") -> WomMap(WomMapType(WomStringType, WomStringType), Map(WomString("foo2") -> WomString("foo"))), - WomString("bar") ->WomMap(WomMapType(WomStringType, WomStringType), Map(WomString("bar2") -> WomString("bar"))))), - List(WomValueSimpleton("mapOfMaps:foo:foo2", WomString("foo")), WomValueSimpleton("mapOfMaps:bar:bar2", WomString("bar"))) + WomMap( + WomMapType(WomStringType, WomMapType(WomStringType, WomStringType)), + Map( + WomString("foo") -> WomMap(WomMapType(WomStringType, WomStringType), + Map(WomString("foo2") -> WomString("foo")) + ), + WomString("bar") -> WomMap(WomMapType(WomStringType, WomStringType), + Map(WomString("bar2") -> WomString("bar")) + ) + ) + ), + List(WomValueSimpleton("mapOfMaps:foo:foo2", WomString("foo")), + WomValueSimpleton("mapOfMaps:bar:bar2", WomString("bar")) + ) ), SimpletonConversion( "simplePair1", WomPair(WomInteger(1), WomString("hello")), - List(WomValueSimpleton("simplePair1:left", WomInteger(1)), WomValueSimpleton("simplePair1:right", WomString("hello"))) + List(WomValueSimpleton("simplePair1:left", WomInteger(1)), + WomValueSimpleton("simplePair1:right", WomString("hello")) + ) ), SimpletonConversion( "simplePair2", WomPair(WomString("left"), WomInteger(5)), - List(WomValueSimpleton("simplePair2:left", WomString("left")), WomValueSimpleton("simplePair2:right", WomInteger(5))) + List(WomValueSimpleton("simplePair2:left", WomString("left")), + WomValueSimpleton("simplePair2:right", WomInteger(5)) + ) ), SimpletonConversion( "pairOfPairs", - WomPair( - WomPair(WomInteger(1), WomString("one")), - WomPair(WomString("two"), WomInteger(2))), + WomPair(WomPair(WomInteger(1), WomString("one")), WomPair(WomString("two"), WomInteger(2))), List( WomValueSimpleton("pairOfPairs:left:left", WomInteger(1)), WomValueSimpleton("pairOfPairs:left:right", WomString("one")), WomValueSimpleton("pairOfPairs:right:left", WomString("two")), - WomValueSimpleton("pairOfPairs:right:right", WomInteger(2))) + WomValueSimpleton("pairOfPairs:right:right", WomInteger(2)) + ) ), SimpletonConversion( "pairOfArrayAndMap", WomPair( WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))), - WomMap(WomMapType(WomStringType, WomIntegerType), Map(WomString("left") -> WomInteger(100), WomString("right") -> WomInteger(200)))), + WomMap(WomMapType(WomStringType, WomIntegerType), + Map(WomString("left") -> WomInteger(100), WomString("right") -> WomInteger(200)) + ) + ), List( WomValueSimpleton("pairOfArrayAndMap:left[0]", WomInteger(1)), WomValueSimpleton("pairOfArrayAndMap:left[1]", WomInteger(2)), WomValueSimpleton("pairOfArrayAndMap:right:left", WomInteger(100)), - WomValueSimpleton("pairOfArrayAndMap:right:right", WomInteger(200))) + WomValueSimpleton("pairOfArrayAndMap:right:right", WomInteger(200)) + ) ), SimpletonConversion( "mapOfArrays", - WomMap(WomMapType(WomStringType, WomArrayType(WomIntegerType)), Map( - WomString("foo") -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(0), WomInteger(1))), - WomString("bar") -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(2), WomInteger(3))))), - List(WomValueSimpleton("mapOfArrays:foo[0]", WomInteger(0)), WomValueSimpleton("mapOfArrays:foo[1]", WomInteger(1)), - WomValueSimpleton("mapOfArrays:bar[0]", WomInteger(2)), WomValueSimpleton("mapOfArrays:bar[1]", WomInteger(3))) + WomMap( + WomMapType(WomStringType, WomArrayType(WomIntegerType)), + Map( + WomString("foo") -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(0), WomInteger(1))), + WomString("bar") -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(2), WomInteger(3))) + ) + ), + List( + WomValueSimpleton("mapOfArrays:foo[0]", WomInteger(0)), + WomValueSimpleton("mapOfArrays:foo[1]", WomInteger(1)), + WomValueSimpleton("mapOfArrays:bar[0]", WomInteger(2)), + WomValueSimpleton("mapOfArrays:bar[1]", WomInteger(3)) + ) ), SimpletonConversion( "escapology", - WomMap(WomMapType(WomStringType, WomStringType), Map( - WomString("foo[1]") -> WomString("foo"), - WomString("bar[[") -> WomString("bar"), - WomString("baz:qux") -> WomString("baz:qux"))), - List(WomValueSimpleton("escapology:foo\\[1\\]", WomString("foo")), + WomMap( + WomMapType(WomStringType, WomStringType), + Map(WomString("foo[1]") -> WomString("foo"), + WomString("bar[[") -> WomString("bar"), + WomString("baz:qux") -> WomString("baz:qux") + ) + ), + List( + WomValueSimpleton("escapology:foo\\[1\\]", WomString("foo")), WomValueSimpleton("escapology:bar\\[\\[", WomString("bar")), - WomValueSimpleton("escapology:baz\\:qux", WomString("baz:qux"))) + WomValueSimpleton("escapology:baz\\:qux", WomString("baz:qux")) + ) ), SimpletonConversion( "flat_object", - WomObject(Map( - "a" -> WomString("aardvark"), - "b" -> WomInteger(25), - "c" -> WomBoolean(false) - )), - List(WomValueSimpleton("flat_object:a", WomString("aardvark")), + WomObject( + Map( + "a" -> WomString("aardvark"), + "b" -> WomInteger(25), + "c" -> WomBoolean(false) + ) + ), + List( + WomValueSimpleton("flat_object:a", WomString("aardvark")), WomValueSimpleton("flat_object:b", WomInteger(25)), - WomValueSimpleton("flat_object:c", WomBoolean(false))) + WomValueSimpleton("flat_object:c", WomBoolean(false)) + ) ), SimpletonConversion( "object_with_array", - WomObject(Map( - "a" -> WomArray(WomArrayType(WomStringType), Seq(WomString("aardvark"), WomString("beetle"))) - )), + WomObject( + Map( + "a" -> WomArray(WomArrayType(WomStringType), Seq(WomString("aardvark"), WomString("beetle"))) + ) + ), List(WomValueSimpleton("object_with_array:a[0]", WomString("aardvark")), - WomValueSimpleton("object_with_array:a[1]", WomString("beetle"))) + WomValueSimpleton("object_with_array:a[1]", WomString("beetle")) + ) ), SimpletonConversion( "object_with_object", - WomObject(Map( - "a" -> WomObject(Map( - "aa" -> WomArray(WomArrayType(WomStringType), Seq(WomString("aardvark"), WomString("aaron"))), - "ab" -> WomArray(WomArrayType(WomStringType), Seq(WomString("abacus"), WomString("a bee"))) - )), - "b" -> WomObject(Map( - "ba" -> WomArray(WomArrayType(WomStringType), Seq(WomString("baa"), WomString("battle"))), - "bb" -> WomArray(WomArrayType(WomStringType), Seq(WomString("bbrrrr"), WomString("bb gun"))) - )) - )), + WomObject( + Map( + "a" -> WomObject( + Map( + "aa" -> WomArray(WomArrayType(WomStringType), Seq(WomString("aardvark"), WomString("aaron"))), + "ab" -> WomArray(WomArrayType(WomStringType), Seq(WomString("abacus"), WomString("a bee"))) + ) + ), + "b" -> WomObject( + Map( + "ba" -> WomArray(WomArrayType(WomStringType), Seq(WomString("baa"), WomString("battle"))), + "bb" -> WomArray(WomArrayType(WomStringType), Seq(WomString("bbrrrr"), WomString("bb gun"))) + ) + ) + ) + ), List( WomValueSimpleton("object_with_object:a:aa[0]", WomString("aardvark")), WomValueSimpleton("object_with_object:a:aa[1]", WomString("aaron")), @@ -135,58 +190,59 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc WomValueSimpleton("object_with_object:b:ba[0]", WomString("baa")), WomValueSimpleton("object_with_object:b:ba[1]", WomString("battle")), WomValueSimpleton("object_with_object:b:bb[0]", WomString("bbrrrr")), - WomValueSimpleton("object_with_object:b:bb[1]", WomString("bb gun")), + WomValueSimpleton("object_with_object:b:bb[1]", WomString("bb gun")) ) ), /* - * Wom object representing a directory listing - * - a single file - * - a "maybe populated file" with some properties (checksum etc..) and secondary files: - * - another single file - * - a directory listing a single file - * - an unlisted directory - * - a glob file - * - an unlisted directory - * - a glob file - * - * Note: glob files technically are never simpletonized but as WomFiles they *can* be + * Wom object representing a directory listing + * - a single file + * - a "maybe populated file" with some properties (checksum etc..) and secondary files: + * - another single file + * - a directory listing a single file + * - an unlisted directory + * - a glob file + * - an unlisted directory + * - a glob file + * + * Note: glob files technically are never simpletonized but as WomFiles they *can* be */ SimpletonConversion( "directory", WomMaybeListedDirectory( Option("outerValueName"), - Option(List( - WomSingleFile("outerSingleFile"), - WomMaybeListedDirectory(Option("innerValueName"), Option(List(WomSingleFile("innerSingleFile")))), - WomMaybePopulatedFile( - Option("populatedInnerValueName"), - Option("innerChecksum"), - Option(10L), - Option("innerFormat"), - Option("innerContents"), - List( - WomSingleFile("populatedInnerSingleFile"), - WomMaybeListedDirectory(Option("innerDirectoryValueName"), Option(List(WomSingleFile("innerDirectorySingleFile")))), - WomUnlistedDirectory("innerUnlistedDirectory"), - WomGlobFile("innerGlobFile") - ) - ), - WomUnlistedDirectory("outerUnlistedDirectory"), - WomGlobFile("outerGlobFile") - ))), + Option( + List( + WomSingleFile("outerSingleFile"), + WomMaybeListedDirectory(Option("innerValueName"), Option(List(WomSingleFile("innerSingleFile")))), + WomMaybePopulatedFile( + Option("populatedInnerValueName"), + Option("innerChecksum"), + Option(10L), + Option("innerFormat"), + Option("innerContents"), + List( + WomSingleFile("populatedInnerSingleFile"), + WomMaybeListedDirectory(Option("innerDirectoryValueName"), + Option(List(WomSingleFile("innerDirectorySingleFile"))) + ), + WomUnlistedDirectory("innerUnlistedDirectory"), + WomGlobFile("innerGlobFile") + ) + ), + WomUnlistedDirectory("outerUnlistedDirectory"), + WomGlobFile("outerGlobFile") + ) + ) + ), List( WomValueSimpleton("directory:class", WomString("Directory")), WomValueSimpleton("directory:value", WomString("outerValueName")), - WomValueSimpleton("directory:listing[0]", WomSingleFile("outerSingleFile")), - WomValueSimpleton("directory:listing[1]:class", WomString("Directory")), WomValueSimpleton("directory:listing[1]:value", WomString("innerValueName")), WomValueSimpleton("directory:listing[1]:listing[0]", WomSingleFile("innerSingleFile")), - WomValueSimpleton("directory:listing[2]:class", WomString("File")), WomValueSimpleton("directory:listing[2]:value", WomString("populatedInnerValueName")), - WomValueSimpleton("directory:listing[2]:checksum", WomString("innerChecksum")), WomValueSimpleton("directory:listing[2]:size", WomInteger(10)), WomValueSimpleton("directory:listing[2]:format", WomString("innerFormat")), @@ -194,10 +250,11 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc WomValueSimpleton("directory:listing[2]:secondaryFiles[0]", WomSingleFile("populatedInnerSingleFile")), WomValueSimpleton("directory:listing[2]:secondaryFiles[1]:class", WomString("Directory")), WomValueSimpleton("directory:listing[2]:secondaryFiles[1]:value", WomString("innerDirectoryValueName")), - WomValueSimpleton("directory:listing[2]:secondaryFiles[1]:listing[0]", WomSingleFile("innerDirectorySingleFile")), + WomValueSimpleton("directory:listing[2]:secondaryFiles[1]:listing[0]", + WomSingleFile("innerDirectorySingleFile") + ), WomValueSimpleton("directory:listing[2]:secondaryFiles[2]", WomUnlistedDirectory("innerUnlistedDirectory")), WomValueSimpleton("directory:listing[2]:secondaryFiles[3]", WomGlobFile("innerGlobFile")), - WomValueSimpleton("directory:listing[3]", WomUnlistedDirectory("outerUnlistedDirectory")), WomValueSimpleton("directory:listing[4]", WomGlobFile("outerGlobFile")) ) @@ -225,7 +282,9 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc it should "round trip everything together with no losses" in { - val wdlValues = (simpletonConversions map { case SimpletonConversion(name, womValue, _) => WomMocks.mockOutputPort(name, womValue.womType) -> womValue }).toMap + val wdlValues = (simpletonConversions map { case SimpletonConversion(name, womValue, _) => + WomMocks.mockOutputPort(name, womValue.womType) -> womValue + }).toMap val allSimpletons = simpletonConversions flatMap { case SimpletonConversion(_, _, simpletons) => simpletons } val actualSimpletons = wdlValues.simplify @@ -239,17 +298,23 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc // coerceable back into the original type: it should "decompose then reconstruct a map in an object into a coerceable value" in { - val aMap = WomMap(WomMapType(WomStringType, WomArrayType(WomStringType)), Map( - WomString("aa") -> WomArray(WomArrayType(WomStringType), Seq(WomString("aardvark"), WomString("aaron"))), - WomString("ab") -> WomArray(WomArrayType(WomStringType), Seq(WomString("abacus"), WomString("a bee"))) - )) + val aMap = WomMap( + WomMapType(WomStringType, WomArrayType(WomStringType)), + Map( + WomString("aa") -> WomArray(WomArrayType(WomStringType), Seq(WomString("aardvark"), WomString("aaron"))), + WomString("ab") -> WomArray(WomArrayType(WomStringType), Seq(WomString("abacus"), WomString("a bee"))) + ) + ) - val bMap = WomMap(WomMapType(WomStringType, WomArrayType(WomStringType)), Map( - WomString("ba") -> WomArray(WomArrayType(WomStringType), Seq(WomString("baa"), WomString("battle"))), - WomString("bb") -> WomArray(WomArrayType(WomStringType), Seq(WomString("bbrrrr"), WomString("bb gun"))) - )) + val bMap = WomMap( + WomMapType(WomStringType, WomArrayType(WomStringType)), + Map( + WomString("ba") -> WomArray(WomArrayType(WomStringType), Seq(WomString("baa"), WomString("battle"))), + WomString("bb") -> WomArray(WomArrayType(WomStringType), Seq(WomString("bbrrrr"), WomString("bb gun"))) + ) + ) - val initial = WomObject(Map("a" -> aMap, "b" -> bMap )) + val initial = WomObject(Map("a" -> aMap, "b" -> bMap)) val map = Map(WomMocks.mockOutputPort("map_in_object") -> initial) @@ -263,9 +328,10 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc WomValueSimpleton("map_in_object:b:ba[0]", WomString("baa")), WomValueSimpleton("map_in_object:b:ba[1]", WomString("battle")), WomValueSimpleton("map_in_object:b:bb[0]", WomString("bbrrrr")), - WomValueSimpleton("map_in_object:b:bb[1]", WomString("bb gun")), + WomValueSimpleton("map_in_object:b:bb[1]", WomString("bb gun")) ), - actualSimpletons) + actualSimpletons + ) // Reconstruct: val outputPort = WomMocks.mockOutputPort(OutputDefinition("map_in_object", initial.womType, IgnoredExpression)) @@ -283,7 +349,8 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc } private def assertSimpletonsEqual(expectedSimpletons: Iterable[WomValueSimpleton], - actualSimpletons: Iterable[WomValueSimpleton]): Unit = { + actualSimpletons: Iterable[WomValueSimpleton] + ): Unit = { // Sanity check, make sure we don't lose anything when we "toSet": actualSimpletons.toSet should contain theSameElementsAs actualSimpletons diff --git a/core/src/test/scala/cromwell/util/AkkaTestUtil.scala b/core/src/test/scala/cromwell/util/AkkaTestUtil.scala index ef1183e9a07..c237ee8c127 100644 --- a/core/src/test/scala/cromwell/util/AkkaTestUtil.scala +++ b/core/src/test/scala/cromwell/util/AkkaTestUtil.scala @@ -39,7 +39,9 @@ object AkkaTestUtil { class DeathTestActor extends Actor { protected def stoppingReceive: Actor.Receive = { case InternalStop => context.stop(self) - case ThrowException => throw new Exception("Don't panic, dear debugger! This was a deliberate exception for the test case.") with NoStackTrace + case ThrowException => + throw new Exception("Don't panic, dear debugger! This was a deliberate exception for the test case.") + with NoStackTrace } override def receive = stoppingReceive orElse Actor.ignoringBehavior } @@ -55,6 +57,6 @@ object AkkaTestUtil { def loggedReceive: Receive - override final def receive: Receive = logMessage orElse loggedReceive + final override def receive: Receive = logMessage orElse loggedReceive } } diff --git a/core/src/test/scala/cromwell/util/EncryptionSpec.scala b/core/src/test/scala/cromwell/util/EncryptionSpec.scala index d1d64f3725d..f9b91107c96 100644 --- a/core/src/test/scala/cromwell/util/EncryptionSpec.scala +++ b/core/src/test/scala/cromwell/util/EncryptionSpec.scala @@ -14,7 +14,8 @@ object EncryptionSpec { """| |Did you install the Java Cryptography Extension (JCE) Unlimited Strength Jurisdiction Policy Files? |http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html - |""".stripMargin) + |""".stripMargin + ) } class EncryptionSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { diff --git a/core/src/test/scala/cromwell/util/GracefulShutdownHelperSpec.scala b/core/src/test/scala/cromwell/util/GracefulShutdownHelperSpec.scala index 875ffd48e3d..214ca6c8454 100644 --- a/core/src/test/scala/cromwell/util/GracefulShutdownHelperSpec.scala +++ b/core/src/test/scala/cromwell/util/GracefulShutdownHelperSpec.scala @@ -10,17 +10,17 @@ import org.scalatest.matchers.should.Matchers class GracefulShutdownHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers { behavior of "GracefulShutdownHelper" - + it should "send ShutdownCommand to actors, wait for them to shutdown, then shut itself down" in { val testProbeA = TestProbe() val testProbeB = TestProbe() - + val testActor = system.actorOf(Props(new Actor with GracefulShutdownHelper with ActorLogging { - override def receive: Receive = { - case ShutdownCommand => waitForActorsAndShutdown(NonEmptyList.of(testProbeA.ref, testProbeB.ref)) + override def receive: Receive = { case ShutdownCommand => + waitForActorsAndShutdown(NonEmptyList.of(testProbeA.ref, testProbeB.ref)) } })) - + watch(testActor) testActor ! ShutdownCommand @@ -37,7 +37,7 @@ class GracefulShutdownHelperSpec extends TestKitSuite with AnyFlatSpecLike with expectNoMessage() system stop testProbeB.ref - + expectTerminated(testActor) } } diff --git a/core/src/test/scala/cromwell/util/SampleWdl.scala b/core/src/test/scala/cromwell/util/SampleWdl.scala index 6e790ca67be..8180049fc73 100644 --- a/core/src/test/scala/cromwell/util/SampleWdl.scala +++ b/core/src/test/scala/cromwell/util/SampleWdl.scala @@ -2,7 +2,12 @@ package cromwell.util import java.util.UUID import cromwell.core.path.{DefaultPath, DefaultPathBuilder, Path} -import cromwell.core.{WorkflowOptions, WorkflowSourceFilesCollection, WorkflowSourceFilesWithDependenciesZip, WorkflowSourceFilesWithoutImports} +import cromwell.core.{ + WorkflowOptions, + WorkflowSourceFilesCollection, + WorkflowSourceFilesWithDependenciesZip, + WorkflowSourceFilesWithoutImports +} import spray.json._ import wom.core.{ExecutableInputMap, WorkflowJson, WorkflowSource} import wom.values._ @@ -30,7 +35,8 @@ trait SampleWdl extends TestFileUtil { labels: String = "{}", workflowType: Option[String] = Option("WDL"), workflowTypeVersion: Option[String] = None, - workflowOnHold: Boolean = false): WorkflowSourceFilesCollection = { + workflowOnHold: Boolean = false + ): WorkflowSourceFilesCollection = importsZip match { case Some(zip) => WorkflowSourceFilesWithDependenciesZip( @@ -45,7 +51,8 @@ trait SampleWdl extends TestFileUtil { warnings = Vector.empty, workflowOnHold = workflowOnHold, importsZip = zip, - requestedWorkflowId = None) + requestedWorkflowId = None + ) case None => WorkflowSourceFilesWithoutImports( workflowSource = Option(workflowSource(runtime)), @@ -58,9 +65,9 @@ trait SampleWdl extends TestFileUtil { workflowTypeVersion = workflowTypeVersion, warnings = Vector.empty, workflowOnHold = workflowOnHold, - requestedWorkflowId = None) + requestedWorkflowId = None + ) } - } val rawInputs: ExecutableInputMap @@ -84,8 +91,8 @@ trait SampleWdl extends TestFileUtil { def write(x: Any): JsValue = x match { case n: Int => JsNumber(n) case s: String => JsString(s) - case b: Boolean => if(b) JsTrue else JsFalse - case s: Seq[Any] => JsArray(s map {_.toJson} toVector) + case b: Boolean => if (b) JsTrue else JsFalse + case s: Seq[Any] => JsArray(s map { _.toJson } toVector) case a: WomArray => write(a.value) case s: WomString => JsString(s.value) case i: WomInteger => JsNumber(i.value) @@ -111,20 +118,20 @@ object SampleWdl { object HelloWorld extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s""" - |task hello { - | String addressee - | command { - | echo "Hello $${addressee}!" - | } - | output { - | String salutation = read_string(stdout()) - | } - | RUNTIME - |} - | - |workflow wf_hello { - | call hello - |} + |task hello { + | String addressee + | command { + | echo "Hello $${addressee}!" + | } + | output { + | String salutation = read_string(stdout()) + | } + | RUNTIME + |} + | + |workflow wf_hello { + | call hello + |} """.stripMargin.replace("RUNTIME", runtime) val Addressee = "wf_hello.hello.addressee" @@ -157,35 +164,35 @@ object SampleWdl { object EmptyString extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s""" - |task hello { - | command { - | echo "Hello!" - | } - | output { - | String empty = "" - | } - | RUNTIME - |} - | - |task goodbye { - | String emptyInputString - | command { - | echo "$${emptyInputString}" - | } - | output { - | String empty = read_string(stdout()) - | } - | RUNTIME - |} - | - |workflow wf_hello { - | call hello - | call goodbye {input: emptyInputString=hello.empty } - | output { - | hello.empty - | goodbye.empty - | } - |} + |task hello { + | command { + | echo "Hello!" + | } + | output { + | String empty = "" + | } + | RUNTIME + |} + | + |task goodbye { + | String emptyInputString + | command { + | echo "$${emptyInputString}" + | } + | output { + | String empty = read_string(stdout()) + | } + | RUNTIME + |} + | + |workflow wf_hello { + | call hello + | call goodbye {input: emptyInputString=hello.empty } + | output { + | hello.empty + | goodbye.empty + | } + |} """.stripMargin.replace("RUNTIME", runtime) val rawInputs = Map.empty[String, Any] @@ -196,32 +203,31 @@ object SampleWdl { } object CoercionNotDefined extends SampleWdl { - override def workflowSource(runtime: String = ""): WorkflowSource = { + override def workflowSource(runtime: String = ""): WorkflowSource = s""" - |task summary { - | String bfile - | command { - | ~/plink --bfile $${bfile} --missing --hardy --out foo --allow-no-sex - | } - | output { - | File hwe = "foo.hwe" - | File log = "foo.log" - | File imiss = "foo.imiss" - | File lmiss = "foo.lmiss" - | } - | meta { - | author: "Jackie Goldstein" - | email: "jigold@broadinstitute.org" - | } - |} - | - |workflow test1 { - | call summary { - | input: bfile = bfile - | } - |} + |task summary { + | String bfile + | command { + | ~/plink --bfile $${bfile} --missing --hardy --out foo --allow-no-sex + | } + | output { + | File hwe = "foo.hwe" + | File log = "foo.log" + | File imiss = "foo.imiss" + | File lmiss = "foo.lmiss" + | } + | meta { + | author: "Jackie Goldstein" + | email: "jigold@broadinstitute.org" + | } + |} + | + |workflow test1 { + | call summary { + | input: bfile = bfile + | } + |} """.stripMargin - } override val rawInputs: ExecutableInputMap = Map("test1.bfile" -> "data/example1") } @@ -281,67 +287,66 @@ object SampleWdl { withPlaceholders.stripMargin.replace(outputSectionPlaceholder, outputsSection) } - val PatternKey ="three_step.cgrep.pattern" + val PatternKey = "three_step.cgrep.pattern" override lazy val rawInputs = Map(PatternKey -> "...") } object ThreeStep extends ThreeStepTemplate object ThreeStepWithOutputsSection extends ThreeStepTemplate { - override def workflowSource(runtime: String = ""): WorkflowJson = sourceString(outputsSection = - """ - |output { - | cgrep.count - | wc.count - |} + override def workflowSource(runtime: String = ""): WorkflowJson = sourceString(outputsSection = """ + |output { + | cgrep.count + | wc.count + |} """.stripMargin).replaceAll("RUNTIME", runtime) } object DeclarationsWorkflow extends SampleWdl { override def workflowSource(runtime: String): WorkflowSource = s""" - |task cat { - | File file - | String? flags - | String? flags2 # This should be a workflow input - | command { - | cat $${flags} $${flags2} $${file} - | } - | output { - | File procs = stdout() - | } - |} - | - |task cgrep { - | String str_decl - | String pattern - | File in_file - | command { - | grep '$${pattern}' $${in_file} | wc -l - | } - | output { - | Int count = read_int(stdout()) - | String str = str_decl - | } - |} - | - |workflow two_step { - | String flags_suffix - | String flags = "-" + flags_suffix - | String static_string = "foobarbaz" - | call cat { - | input: flags = flags - | } - | call cgrep { - | input: in_file = cat.procs - | } - |} + |task cat { + | File file + | String? flags + | String? flags2 # This should be a workflow input + | command { + | cat $${flags} $${flags2} $${file} + | } + | output { + | File procs = stdout() + | } + |} + | + |task cgrep { + | String str_decl + | String pattern + | File in_file + | command { + | grep '$${pattern}' $${in_file} | wc -l + | } + | output { + | Int count = read_int(stdout()) + | String str = str_decl + | } + |} + | + |workflow two_step { + | String flags_suffix + | String flags = "-" + flags_suffix + | String static_string = "foobarbaz" + | call cat { + | input: flags = flags + | } + | call cgrep { + | input: in_file = cat.procs + | } + |} """.stripMargin private val fileContents = s"""first line - |second line - |third line + |second line + |third line """.stripMargin override val rawInputs: ExecutableInputMap = Map( @@ -446,94 +451,94 @@ object SampleWdl { object ArrayIO extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s""" - |task serialize { - | Array[String] strs - | command { - | cat $${write_lines(strs)} - | } - | output { - | String contents = read_string(stdout()) - | } - | RUNTIME - |} - | - |workflow wf { - | Array[String] strings = ["str1", "str2", "str3"] - | call serialize { - | input: strs = strings - | } - |} + |task serialize { + | Array[String] strs + | command { + | cat $${write_lines(strs)} + | } + | output { + | String contents = read_string(stdout()) + | } + | RUNTIME + |} + | + |workflow wf { + | Array[String] strings = ["str1", "str2", "str3"] + | call serialize { + | input: strs = strings + | } + |} """.stripMargin.replace("RUNTIME", runtime) override val rawInputs: Map[String, Any] = Map.empty } class ScatterWdl extends SampleWdl { val tasks: String = s"""task A { - | command { - | echo -n -e "jeff\nchris\nmiguel\nthibault\nkhalid\nruchi" - | } - | RUNTIME - | output { - | Array[String] A_out = read_lines(stdout()) - | } - |} - | - |task B { - | String B_in - | command { - | python -c "print(len('$${B_in}'))" - | } - | RUNTIME - | output { - | Int B_out = read_int(stdout()) - | } - |} - | - |task C { - | Int C_in - | command { - | python -c "print($${C_in}*100)" - | } - | RUNTIME - | output { - | Int C_out = read_int(stdout()) - | } - |} - | - |task D { - | Array[Int] D_in - | command { - | python -c "print($${sep = '+' D_in})" - | } - | RUNTIME - | output { - | Int D_out = read_int(stdout()) - | } - |} - | - |task E { - | command { - | python -c "print(9)" - | } - | RUNTIME - | output { - | Int E_out = read_int(stdout()) - | } - |} + | command { + | echo -n -e "jeff\nchris\nmiguel\nthibault\nkhalid\nruchi" + | } + | RUNTIME + | output { + | Array[String] A_out = read_lines(stdout()) + | } + |} + | + |task B { + | String B_in + | command { + | python -c "print(len('$${B_in}'))" + | } + | RUNTIME + | output { + | Int B_out = read_int(stdout()) + | } + |} + | + |task C { + | Int C_in + | command { + | python -c "print($${C_in}*100)" + | } + | RUNTIME + | output { + | Int C_out = read_int(stdout()) + | } + |} + | + |task D { + | Array[Int] D_in + | command { + | python -c "print($${sep = '+' D_in})" + | } + | RUNTIME + | output { + | Int D_out = read_int(stdout()) + | } + |} + | + |task E { + | command { + | python -c "print(9)" + | } + | RUNTIME + | output { + | Int E_out = read_int(stdout()) + | } + |} """.stripMargin override def workflowSource(runtime: String = ""): WorkflowSource = s"""$tasks - | - |workflow w { - | call A - | scatter (item in A.A_out) { - | call B {input: B_in = item} - | call C {input: C_in = B.B_out} - | call E - | } - | call D {input: D_in = B.B_out} - |} + | + |workflow w { + | call A + | scatter (item in A.A_out) { + | call B {input: B_in = item} + | call C {input: C_in = B.B_out} + | call E + | } + | call D {input: D_in = B.B_out} + |} """.stripMargin.replace("RUNTIME", runtime) override lazy val rawInputs = Map.empty[String, String] @@ -542,21 +547,21 @@ object SampleWdl { object SimpleScatterWdl extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s"""task echo_int { - | Int int - | command {echo $${int}} - | output {Int out = read_int(stdout())} - | RUNTIME_PLACEHOLDER - |} - | - |workflow scatter0 { - | Array[Int] ints = [1,2,3,4,5] - | call echo_int as outside_scatter {input: int = 8000} - | scatter(i in ints) { - | call echo_int as inside_scatter { - | input: int = i - | } - | } - |} + | Int int + | command {echo $${int}} + | output {Int out = read_int(stdout())} + | RUNTIME_PLACEHOLDER + |} + | + |workflow scatter0 { + | Array[Int] ints = [1,2,3,4,5] + | call echo_int as outside_scatter {input: int = 8000} + | scatter(i in ints) { + | call echo_int as inside_scatter { + | input: int = i + | } + | } + |} """.stripMargin.replace("RUNTIME_PLACEHOLDER", runtime) override lazy val rawInputs = Map.empty[String, String] @@ -565,108 +570,108 @@ object SampleWdl { object SimpleScatterWdlWithOutputs extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s"""task echo_int { - | Int int - | command {echo $${int}} - | output {Int out = read_int(stdout())} - |} - | - |workflow scatter0 { - | Array[Int] ints = [1,2,3,4,5] - | call echo_int as outside_scatter {input: int = 8000} - | scatter(i in ints) { - | call echo_int as inside_scatter { - | input: int = i - | } - | } - | output { - | inside_scatter.* - | } - |} + | Int int + | command {echo $${int}} + | output {Int out = read_int(stdout())} + |} + | + |workflow scatter0 { + | Array[Int] ints = [1,2,3,4,5] + | call echo_int as outside_scatter {input: int = 8000} + | scatter(i in ints) { + | call echo_int as inside_scatter { + | input: int = i + | } + | } + | output { + | inside_scatter.* + | } + |} """.stripMargin override lazy val rawInputs = Map.empty[String, String] } case class PrepareScatterGatherWdl(salt: String = UUID.randomUUID().toString) extends SampleWdl { - override def workflowSource(runtime: String = ""): WorkflowSource = { + override def workflowSource(runtime: String = ""): WorkflowSource = s""" - |# - |# Goal here is to split up the input file into files of 1 line each (in the prepare) then in parallel call wc -w on each newly created file and count the words into another file then in the gather, sum the results of each parallel call to come up with - |# the word-count for the fil - |# - |# splits each line into a file with the name temp_?? (shuffle) - |task do_prepare { - | File input_file - | command { - | split -l 1 $${input_file} temp_ && ls -1 temp_?? > files.list - | } - | output { - | Array[File] split_files = read_lines("files.list") - | } - | RUNTIME - |} - |# count the number of words in the input file, writing the count to an output file overkill in this case, but simulates a real scatter-gather that would just return an Int (map) - |task do_scatter { - | String salt - | File input_file - | command { - | # $${salt} - | wc -w $${input_file} > output.txt - | } - | output { - | File count_file = "output.txt" - | } - | RUNTIME - |} - |# aggregate the results back together (reduce) - |task do_gather { - | Array[File] input_files - | command <<< - | cat $${sep = ' ' input_files} | awk '{s+=$$1} END {print s}' - | >>> - | output { - | Int sum = read_int(stdout()) - | } - | RUNTIME - |} - |workflow sc_test { - | call do_prepare - | scatter(f in do_prepare.split_files) { - | call do_scatter { - | input: input_file = f - | } - | } - | call do_gather { - | input: input_files = do_scatter.count_file - | } - |} + |# + |# Goal here is to split up the input file into files of 1 line each (in the prepare) then in parallel call wc -w on each newly created file and count the words into another file then in the gather, sum the results of each parallel call to come up with + |# the word-count for the fil + |# + |# splits each line into a file with the name temp_?? (shuffle) + |task do_prepare { + | File input_file + | command { + | split -l 1 $${input_file} temp_ && ls -1 temp_?? > files.list + | } + | output { + | Array[File] split_files = read_lines("files.list") + | } + | RUNTIME + |} + |# count the number of words in the input file, writing the count to an output file overkill in this case, but simulates a real scatter-gather that would just return an Int (map) + |task do_scatter { + | String salt + | File input_file + | command { + | # $${salt} + | wc -w $${input_file} > output.txt + | } + | output { + | File count_file = "output.txt" + | } + | RUNTIME + |} + |# aggregate the results back together (reduce) + |task do_gather { + | Array[File] input_files + | command <<< + | cat $${sep = ' ' input_files} | awk '{s+=$$1} END {print s}' + | >>> + | output { + | Int sum = read_int(stdout()) + | } + | RUNTIME + |} + |workflow sc_test { + | call do_prepare + | scatter(f in do_prepare.split_files) { + | call do_scatter { + | input: input_file = f + | } + | } + | call do_gather { + | input: input_files = do_scatter.count_file + | } + |} """.stripMargin.replace("RUNTIME", runtime) - } val contents: String = - """|the - |total number - |of words in this - |text file is 11 - |""".stripMargin + """|the + |total number + |of words in this + |text file is 11 + |""".stripMargin override lazy val rawInputs = Map( "sc_test.do_prepare.input_file" -> createCannedFile("scatter", contents).pathAsString, - "sc_test.do_scatter.salt" -> salt) + "sc_test.do_scatter.salt" -> salt + ) } object FileClobber extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s"""task read_line { - | File in - | command { cat $${in} } - | output { String out = read_string(stdout()) } - |} - | - |workflow two { - | call read_line as x - | call read_line as y - |} + | File in + | command { cat $${in} } + | output { String out = read_string(stdout()) } + |} + | + |workflow two { + | call read_line as x + | call read_line as y + |} """.stripMargin val tempDir1: DefaultPath = DefaultPathBuilder.createTempDirectory("FileClobber1") @@ -683,26 +688,26 @@ object SampleWdl { object FilePassingWorkflow extends SampleWdl { override def workflowSource(runtime: String): WorkflowSource = s"""task a { - | File in - | String out_name = "out" - | - | command { - | cat $${in} > $${out_name} - | } - | RUNTIME - | output { - | File out = "out" - | File out_interpolation = "$${out_name}" - | String contents = read_string("$${out_name}") - | } - |} - | - |workflow file_passing { - | File f - | - | call a {input: in = f} - | call a as b {input: in = a.out} - |} + | File in + | String out_name = "out" + | + | command { + | cat $${in} > $${out_name} + | } + | RUNTIME + | output { + | File out = "out" + | File out_interpolation = "$${out_name}" + | String contents = read_string("$${out_name}") + | } + |} + | + |workflow file_passing { + | File f + | + | call a {input: in = f} + | call a as b {input: in = a.out} + |} """.stripMargin.replace("RUNTIME", runtime) private val fileContents = s"foo bar baz" @@ -723,30 +728,30 @@ object SampleWdl { case class CallCachingWorkflow(salt: String) extends SampleWdl { override def workflowSource(runtime: String): WorkflowSource = s"""task a { - | File in - | String out_name = "out" - | String salt - | - | command { - | # $${salt} - | echo "Something" - | cat $${in} > $${out_name} - | } - | RUNTIME - | output { - | File out = "out" - | File out_interpolation = "$${out_name}" - | String contents = read_string("$${out_name}") - | Array[String] stdoutContent = read_lines(stdout()) - | } - |} - | - |workflow file_passing { - | File f - | - | call a {input: in = f} - | call a as b {input: in = a.out} - |} + | File in + | String out_name = "out" + | String salt + | + | command { + | # $${salt} + | echo "Something" + | cat $${in} > $${out_name} + | } + | RUNTIME + | output { + | File out = "out" + | File out_interpolation = "$${out_name}" + | String contents = read_string("$${out_name}") + | Array[String] stdoutContent = read_lines(stdout()) + | } + |} + | + |workflow file_passing { + | File f + | + | call a {input: in = f} + | call a as b {input: in = a.out} + |} """.stripMargin.replace("RUNTIME", runtime) private val fileContents = s"foo bar baz" @@ -761,29 +766,29 @@ object SampleWdl { object CallCachingHashingWdl extends SampleWdl { override def workflowSource(runtime: String): WorkflowSource = s"""task t { - | Int a - | Float b - | String c - | File d - | - | command { - | echo "$${a}" > a - | echo "$${b}" > b - | echo "$${c}" > c - | cat $${d} > d - | } - | output { - | Int w = read_int("a") + 2 - | Float x = read_float("b") - | String y = read_string("c") - | File z = "d" - | } - | RUNTIME - |} - | - |workflow w { - | call t - |} + | Int a + | Float b + | String c + | File d + | + | command { + | echo "$${a}" > a + | echo "$${b}" > b + | echo "$${c}" > c + | cat $${d} > d + | } + | output { + | Int w = read_int("a") + 2 + | Float x = read_float("b") + | String y = read_string("c") + | File z = "d" + | } + | RUNTIME + |} + | + |workflow w { + | call t + |} """.stripMargin.replace("RUNTIME", runtime) val tempDir: DefaultPath = DefaultPathBuilder.createTempDirectory("CallCachingHashingWdl") @@ -799,26 +804,26 @@ object SampleWdl { object ExpressionsInInputs extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s"""task echo { - | String inString - | command { - | echo $${inString} - | } - | - | output { - | String outString = read_string(stdout()) - | } - |} - | - |workflow wf { - | String a1 - | String a2 - | call echo { - | input: inString = a1 + " " + a2 - | } - | call echo as echo2 { - | input: inString = a1 + " " + echo.outString + " " + a2 - | } - |} + | String inString + | command { + | echo $${inString} + | } + | + | output { + | String outString = read_string(stdout()) + | } + |} + | + |workflow wf { + | String a1 + | String a2 + | call echo { + | input: inString = a1 + " " + a2 + | } + | call echo as echo2 { + | input: inString = a1 + " " + echo.outString + " " + a2 + | } + |} """.stripMargin override val rawInputs = Map( "wf.a1" -> WomString("hello"), @@ -830,61 +835,61 @@ object SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s""" task shouldCompleteFast { - | Int a - | command { - | echo "The number was: $${a}" - | } - | output { - | Int echo = a - | } - |} - | - |task shouldCompleteSlow { - | Int a - | command { - | echo "The number was: $${a}" - | # More than 1 so this should finish second - | sleep 2 - | } - | output { - | Int echo = a - | } - |} - | - |task failMeSlowly { - | Int a - | command { - | echo "The number was: $${a}" - | # Less than 2 so this should finish first - | sleep 1 - | ./NOOOOOO - | } - | output { - | Int echo = a - | } - |} - | - |task shouldNeverRun { - | Int a - | Int b - | command { - | echo "You can't fight in here - this is the war room $${a + b}" - | } - | output { - | Int echo = a - | } - |} - | - |workflow wf { - | call shouldCompleteFast as A { input: a = 5 } - | call shouldCompleteFast as B { input: a = 5 } - | - | call failMeSlowly as ohNOOOOOOOO { input: a = A.echo } - | call shouldCompleteSlow as C { input: a = B.echo } - | - | call shouldNeverRun as D { input: a = ohNOOOOOOOO.echo, b = C.echo } - | call shouldCompleteSlow as E { input: a = C.echo } - |} + | Int a + | command { + | echo "The number was: $${a}" + | } + | output { + | Int echo = a + | } + |} + | + |task shouldCompleteSlow { + | Int a + | command { + | echo "The number was: $${a}" + | # More than 1 so this should finish second + | sleep 2 + | } + | output { + | Int echo = a + | } + |} + | + |task failMeSlowly { + | Int a + | command { + | echo "The number was: $${a}" + | # Less than 2 so this should finish first + | sleep 1 + | ./NOOOOOO + | } + | output { + | Int echo = a + | } + |} + | + |task shouldNeverRun { + | Int a + | Int b + | command { + | echo "You can't fight in here - this is the war room $${a + b}" + | } + | output { + | Int echo = a + | } + |} + | + |workflow wf { + | call shouldCompleteFast as A { input: a = 5 } + | call shouldCompleteFast as B { input: a = 5 } + | + | call failMeSlowly as ohNOOOOOOOO { input: a = A.echo } + | call shouldCompleteSlow as C { input: a = B.echo } + | + | call shouldNeverRun as D { input: a = ohNOOOOOOOO.echo, b = C.echo } + | call shouldCompleteSlow as E { input: a = C.echo } + |} """.stripMargin val rawInputs = Map( diff --git a/core/src/test/scala/cromwell/util/TestFileUtil.scala b/core/src/test/scala/cromwell/util/TestFileUtil.scala index dbdad8a47ed..f13fb57891b 100644 --- a/core/src/test/scala/cromwell/util/TestFileUtil.scala +++ b/core/src/test/scala/cromwell/util/TestFileUtil.scala @@ -17,9 +17,8 @@ trait TestFileUtil { tempFile.write(contents) } - def createFile(name: String, dir: Path, contents: String): Path = { + def createFile(name: String, dir: Path, contents: String): Path = dir.createPermissionedDirectories()./(name).write(contents) - } } trait HashUtil extends TestFileUtil { @@ -36,6 +35,7 @@ trait HashUtil extends TestFileUtil { object ErrorOrUtil { implicit class EnhancedErrorOr[A](val value: ErrorOr[A]) extends AnyVal { + /** Extract a value from an `ErrorOr` box if the box is `Valid`, throw an exception if the box is `Invalid`. * For test code only. */ def get: A = value match { diff --git a/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala b/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala index 0764bdd9826..b15cdcebec7 100644 --- a/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala +++ b/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala @@ -11,31 +11,34 @@ class TryWithResourceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc behavior of "tryWithResource" it should "catch instantiation errors" in { - val triedMyBest = tryWithResource(() => if (1 == 1) throw InstantiationException else null) { _ => 5 } + val triedMyBest = tryWithResource(() => if (1 == 1) throw InstantiationException else null)(_ => 5) triedMyBest should be(Failure(InstantiationException)) } it should "close the closeable" in { val myCloseable = new MyCloseable - val triedMyBest = tryWithResource(() => myCloseable) { _.value } // Nothing special about 5... Just need to return something! + val triedMyBest = tryWithResource(() => myCloseable) { + _.value + } // Nothing special about 5... Just need to return something! triedMyBest should be(Success(5)) myCloseable.isClosed should be(true) } it should "catch errors and still close the closeable" in { val myCloseable = new MyCloseable - val triedMyBest = tryWithResource(() => myCloseable) { _.badValue } + val triedMyBest = tryWithResource(() => myCloseable)(_.badValue) triedMyBest should be(Failure(ReadValueException)) myCloseable.isClosed should be(true) } it should "be robust to failures in close methods" in { val myCloseable = new FailingCloseable - val triedMyBest = tryWithResource(() => myCloseable) { _.value } + val triedMyBest = tryWithResource(() => myCloseable)(_.value) triedMyBest should be(Failure(CloseCloseableException)) - val triedMyBest2 = tryWithResource(() => myCloseable) { _.badValue } + val triedMyBest2 = tryWithResource(() => myCloseable)(_.badValue) triedMyBest2 match { - case Failure(ReadValueException) => ReadValueException.getSuppressed.headOption should be(Some(CloseCloseableException)) + case Failure(ReadValueException) => + ReadValueException.getSuppressed.headOption should be(Some(CloseCloseableException)) case x => fail(s"$x was not equal to $ReadValueException") } } @@ -47,9 +50,8 @@ class MyCloseable extends AutoCloseable { val value = if (isClosed) throw ReadValueException else 5 // Ensures we aren't closed when .value is called def badValue = throw ReadValueException - override def close() = { + override def close() = isClosed = true - } } class FailingCloseable extends MyCloseable { diff --git a/core/src/test/scala/cromwell/util/WomMocks.scala b/core/src/test/scala/cromwell/util/WomMocks.scala index 2dc75385658..07111be17ce 100644 --- a/core/src/test/scala/cromwell/util/WomMocks.scala +++ b/core/src/test/scala/cromwell/util/WomMocks.scala @@ -6,44 +6,71 @@ import wom.RuntimeAttributes import wom.callable.Callable.OutputDefinition import wom.callable.{CallableTaskDefinition, CommandTaskDefinition, WorkflowDefinition} import wom.graph.GraphNodePort.{GraphNodeOutputPort, OutputPort} -import wom.graph.{Graph, CommandCallNode, WomIdentifier, WorkflowCallNode} +import wom.graph.{CommandCallNode, Graph, WomIdentifier, WorkflowCallNode} import wom.types.{WomStringType, WomType} import wom.values.WomValue object WomMocks { - val EmptyTaskDefinition = CallableTaskDefinition("emptyTask", Function.const(List.empty.validNel), RuntimeAttributes(Map.empty), - Map.empty, Map.empty, List.empty, List.empty, Set.empty, Map.empty, sourceLocation = None) + val EmptyTaskDefinition = CallableTaskDefinition( + "emptyTask", + Function.const(List.empty.validNel), + RuntimeAttributes(Map.empty), + Map.empty, + Map.empty, + List.empty, + List.empty, + Set.empty, + Map.empty, + sourceLocation = None + ) val EmptyWorkflowDefinition = mockWorkflowDefinition("emptyWorkflow") - def mockTaskCall(identifier: WomIdentifier, definition: CommandTaskDefinition = EmptyTaskDefinition) = { - CommandCallNode(identifier, definition, Set.empty, List.empty, Set.empty, (_, localName) => WomIdentifier(localName = localName), None) - } + def mockTaskCall(identifier: WomIdentifier, definition: CommandTaskDefinition = EmptyTaskDefinition) = + CommandCallNode(identifier, + definition, + Set.empty, + List.empty, + Set.empty, + (_, localName) => WomIdentifier(localName = localName), + None + ) - def mockWorkflowCall(identifier: WomIdentifier, definition: WorkflowDefinition = EmptyWorkflowDefinition) = { - WorkflowCallNode(identifier, definition, Set.empty, List.empty, Set.empty, (_, localName) => identifier.combine(localName), None) - } + def mockWorkflowCall(identifier: WomIdentifier, definition: WorkflowDefinition = EmptyWorkflowDefinition) = + WorkflowCallNode(identifier, + definition, + Set.empty, + List.empty, + Set.empty, + (_, localName) => identifier.combine(localName), + None + ) - def mockWorkflowDefinition(name: String) = { + def mockWorkflowDefinition(name: String) = WorkflowDefinition(name, Graph(Set.empty), Map.empty, Map.empty, None) - } - def mockTaskDefinition(name: String) = { - CallableTaskDefinition(name, Function.const(List.empty.validNel), RuntimeAttributes(Map.empty), - Map.empty, Map.empty, List.empty, List.empty, Set.empty, Map.empty, sourceLocation = None) - } + def mockTaskDefinition(name: String) = + CallableTaskDefinition( + name, + Function.const(List.empty.validNel), + RuntimeAttributes(Map.empty), + Map.empty, + Map.empty, + List.empty, + List.empty, + Set.empty, + Map.empty, + sourceLocation = None + ) - def mockOutputPort(name: String, womType: WomType = WomStringType): OutputPort = { + def mockOutputPort(name: String, womType: WomType = WomStringType): OutputPort = GraphNodeOutputPort(WomIdentifier(name, name), womType, null) - } - def mockOutputPort(outputDefinition: OutputDefinition): OutputPort = { + def mockOutputPort(outputDefinition: OutputDefinition): OutputPort = GraphNodeOutputPort(WomIdentifier(outputDefinition.name, outputDefinition.name), outputDefinition.womType, null) - } - def mockOutputExpectations(outputs: Map[String, WomValue]): CallOutputs = { - CallOutputs(outputs.map { - case (key, value) => WomMocks.mockOutputPort(key, value.womType) -> value + def mockOutputExpectations(outputs: Map[String, WomValue]): CallOutputs = + CallOutputs(outputs.map { case (key, value) => + WomMocks.mockOutputPort(key, value.womType) -> value }) - } } diff --git a/core/src/test/scala/cromwell/util/WomValueJsonFormatterSpec.scala b/core/src/test/scala/cromwell/util/WomValueJsonFormatterSpec.scala index 1c4cc15dfdd..69739d9868f 100644 --- a/core/src/test/scala/cromwell/util/WomValueJsonFormatterSpec.scala +++ b/core/src/test/scala/cromwell/util/WomValueJsonFormatterSpec.scala @@ -4,7 +4,7 @@ import common.assertion.CromwellTimeoutSpec import cromwell.util.JsonFormatting.WomValueJsonFormatter.WomValueJsonFormat import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import spray.json.{JsObject, enrichString} +import spray.json.{enrichString, JsObject} import wom.types._ import wom.values._ @@ -15,7 +15,7 @@ class WomValueJsonFormatterSpec extends AnyFlatSpec with CromwellTimeoutSpec wit it should "write WdlPair to left/right structured JsObject" in { val left = "sanders" val right = Vector("rubio", "carson", "cruz") - val wdlPair = WomPair(WomString(left), WomArray(WomArrayType(WomStringType), right.map { WomString(_) })) + val wdlPair = WomPair(WomString(left), WomArray(WomArrayType(WomStringType), right.map(WomString(_)))) val ExpectedJson: JsObject = """|{ | "left": "sanders", diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/CommandLineParser.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/CommandLineParser.scala index 3b7be2b38bd..2dbe3fa6c27 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/CommandLineParser.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/CommandLineParser.scala @@ -4,7 +4,6 @@ import common.util.VersionUtil import drs.localizer.CommandLineParser.AccessTokenStrategy._ import drs.localizer.CommandLineParser.Usage - class CommandLineParser extends scopt.OptionParser[CommandLineArguments](Usage) { lazy val localizerVersion: String = VersionUtil.getVersion("cromwell-drs-localizer") @@ -14,31 +13,30 @@ class CommandLineParser extends scopt.OptionParser[CommandLineArguments](Usage) head("cromwell-drs-localizer", localizerVersion) - arg[String]("drs-object-id").text("DRS object ID").optional(). - action((s, c) => - c.copy(drsObject = Option(s))) - arg[String]("container-path").text("Container path").optional(). - action((s, c) => - c.copy(containerPath = Option(s))) - arg[String]("requester-pays-project").text(s"Requester pays project (only valid with '$Google' auth strategy)").optional(). - action((s, c) => - c.copy(googleRequesterPaysProject = Option(s))) - opt[String]('m', "manifest-path").text("File path of manifest containing multiple files to localize"). - action((s, c) => - c.copy(manifestPath = Option(s))) - opt[String]('r', "requester-pays-project").text(s"Requester pays project (only valid with '$Google' auth strategy)").optional(). - action((s, c) => { + arg[String]("drs-object-id").text("DRS object ID").optional().action((s, c) => c.copy(drsObject = Option(s))) + arg[String]("container-path").text("Container path").optional().action((s, c) => c.copy(containerPath = Option(s))) + arg[String]("requester-pays-project") + .text(s"Requester pays project (only valid with '$Google' auth strategy)") + .optional() + .action((s, c) => c.copy(googleRequesterPaysProject = Option(s))) + opt[String]('m', "manifest-path") + .text("File path of manifest containing multiple files to localize") + .action((s, c) => c.copy(manifestPath = Option(s))) + opt[String]('r', "requester-pays-project") + .text(s"Requester pays project (only valid with '$Google' auth strategy)") + .optional() + .action { (s, c) => c.copy( googleRequesterPaysProject = Option(s), googleRequesterPaysProjectConflict = c.googleRequesterPaysProject.exists(_ != s) ) - }) - opt[String]('t', "access-token-strategy").text(s"Access token strategy, must be one of '$Azure' or '$Google' (default '$Google')"). - action((s, c) => - c.copy(accessTokenStrategy = Option(s.toLowerCase()))) - opt[String]('i', "identity-client-id").text("Azure identity client id"). - action((s, c) => - c.copy(azureIdentityClientId = Option(s))) + } + opt[String]('t', "access-token-strategy") + .text(s"Access token strategy, must be one of '$Azure' or '$Google' (default '$Google')") + .action((s, c) => c.copy(accessTokenStrategy = Option(s.toLowerCase()))) + opt[String]('i', "identity-client-id") + .text("Azure identity client id") + .action((s, c) => c.copy(azureIdentityClientId = Option(s))) checkConfig(c => if (c.googleRequesterPaysProjectConflict) failure("Requester pays project differs between positional argument and option flag") @@ -66,6 +64,7 @@ class CommandLineParser extends scopt.OptionParser[CommandLineArguments](Usage) } object CommandLineParser { + /** * These access token strategies are named simplistically as there is currently only one access token strategy being * used for each of these cloud vendors. But it is certainly possible that multiple strategies could come into use @@ -95,4 +94,5 @@ case class CommandLineArguments(accessTokenStrategy: Option[String] = Option(Goo googleRequesterPaysProject: Option[String] = None, azureIdentityClientId: Option[String] = None, manifestPath: Option[String] = None, - googleRequesterPaysProjectConflict: Boolean = false) + googleRequesterPaysProjectConflict: Boolean = false +) diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerDrsPathResolver.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerDrsPathResolver.scala deleted file mode 100644 index 944d418acbb..00000000000 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerDrsPathResolver.scala +++ /dev/null @@ -1,9 +0,0 @@ -package drs.localizer - -import cloud.nio.impl.drs.{DrsConfig, DrsCredentials, DrsPathResolver} -import common.validation.ErrorOr.ErrorOr - - -class DrsLocalizerDrsPathResolver(drsConfig: DrsConfig, drsCredentials: DrsCredentials) extends DrsPathResolver(drsConfig) { - override def getAccessToken: ErrorOr[String] = drsCredentials.getAccessToken -} diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala index 3d99538f614..64f36239db5 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala @@ -43,16 +43,19 @@ object DrsLocalizerMain extends IOApp with StrictLogging { // Default retry parameters for resolving a DRS url val defaultNumRetries: Int = 5 - val defaultBackoff: CloudNioBackoff = CloudNioSimpleExponentialBackoff( - initialInterval = 1 seconds, maxInterval = 60 seconds, multiplier = 2) + val defaultBackoff: CloudNioBackoff = + CloudNioSimpleExponentialBackoff(initialInterval = 1 seconds, maxInterval = 60 seconds, multiplier = 2) val defaultDownloaderFactory: DownloaderFactory = new DownloaderFactory { - override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): Downloader = + override def buildGcsUriDownloader(gcsPath: String, + serviceAccountJsonOption: Option[String], + downloadLoc: String, + requesterPaysProjectOption: Option[String] + ): Downloader = GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption) - override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = { + override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = BulkAccessUrlDownloader(urlsToDownload) - } } private def printUsage: IO[ExitCode] = { @@ -64,52 +67,55 @@ object DrsLocalizerMain extends IOApp with StrictLogging { * Helper function to read a CSV file as pairs of drsURL -> local download destination. * @param csvManifestPath Path to a CSV file where each row is something like: drs://asdf.ghj, path/to/my/directory */ - def loadCSVManifest(csvManifestPath: String): IO[List[UnresolvedDrsUrl]] = { + def loadCSVManifest(csvManifestPath: String): IO[List[UnresolvedDrsUrl]] = IO { val openFile = new File(csvManifestPath) val csvParser = CSVParser.parse(openFile, Charset.defaultCharset(), CSVFormat.DEFAULT) - try{ + try csvParser.getRecords.asScala.map(record => UnresolvedDrsUrl(record.get(0), record.get(1))).toList - } finally { + finally csvParser.close() - } } - } - def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials) : IO[ExitCode] = { - val urlList = (commandLineArguments.manifestPath, commandLineArguments.drsObject, commandLineArguments.containerPath) match { - case (Some(manifestPath), _, _) => { - loadCSVManifest(manifestPath) - } - case (_, Some(drsObject), Some(containerPath)) => { - IO.pure(List(UnresolvedDrsUrl(drsObject, containerPath))) - } - case(_,_,_) => { - throw new RuntimeException("Illegal command line arguments supplied to drs localizer.") + def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials): IO[ExitCode] = { + val urlList = + (commandLineArguments.manifestPath, commandLineArguments.drsObject, commandLineArguments.containerPath) match { + case (Some(manifestPath), _, _) => + loadCSVManifest(manifestPath) + case (_, Some(drsObject), Some(containerPath)) => + IO.pure(List(UnresolvedDrsUrl(drsObject, containerPath))) + case (_, _, _) => + throw new RuntimeException("Illegal command line arguments supplied to drs localizer.") } - } - val main = new DrsLocalizerMain(urlList, defaultDownloaderFactory, drsCredentials, commandLineArguments.googleRequesterPaysProject) + val main = new DrsLocalizerMain(urlList, + defaultDownloaderFactory, + drsCredentials, + commandLineArguments.googleRequesterPaysProject + ) main.resolveAndDownload().map(_.exitCode) - } + } /** * Helper function to decide which downloader to use based on data from the DRS response. * Throws a runtime exception if the DRS response is invalid. */ - def toValidatedUriType(accessUrl: Option[AccessUrl], gsUri: Option[String]): URIType = { + def toValidatedUriType(accessUrl: Option[AccessUrl], gsUri: Option[String]): URIType = // if both are provided, prefer using access urls (accessUrl, gsUri) match { case (Some(_), _) => - if(!accessUrl.get.url.startsWith("https://")) { throw new RuntimeException("Resolved Access URL does not start with https://")} + if (!accessUrl.get.url.startsWith("https://")) { + throw new RuntimeException("Resolved Access URL does not start with https://") + } URIType.ACCESS case (_, Some(_)) => - if(!gsUri.get.startsWith("gs://")) { throw new RuntimeException("Resolved Google URL does not start with gs://")} + if (!gsUri.get.startsWith("gs://")) { + throw new RuntimeException("Resolved Google URL does not start with gs://") + } URIType.GCS case (_, _) => throw new RuntimeException("DRS response did not contain any URLs") } - } - } +} object URIType extends Enumeration { type URIType = Value @@ -119,7 +125,8 @@ object URIType extends Enumeration { class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]], downloaderFactory: DownloaderFactory, drsCredentials: DrsCredentials, - requesterPaysProjectIdOption: Option[String]) extends StrictLogging { + requesterPaysProjectIdOption: Option[String] +) extends StrictLogging { /** * This will: @@ -132,18 +139,17 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]], val downloadResults = buildDownloaders().flatMap { downloaderList => downloaderList.map(downloader => downloader.download).traverse(identity) } - downloadResults.map{list => + downloadResults.map { list => list.find(result => result != DownloadSuccess).getOrElse(DownloadSuccess) } } - def getDrsPathResolver: IO[DrsLocalizerDrsPathResolver] = { + def getDrsPathResolver: IO[DrsPathResolver] = IO { val drsConfig = DrsConfig.fromEnv(sys.env) logger.info(s"Using ${drsConfig.drsResolverUrl} to resolve DRS Objects") - new DrsLocalizerDrsPathResolver(drsConfig, drsCredentials) + new DrsPathResolver(drsConfig, drsCredentials) } - } /** * After resolving all of the URLs, this sorts them into an "Access" or "GCS" bucket. @@ -151,80 +157,95 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]], * All google URLs will be downloaded individually in their own google downloader. * @return List of all downloaders required to fulfill the request. */ - def buildDownloaders() : IO[List[Downloader]] = { + def buildDownloaders(): IO[List[Downloader]] = resolveUrls(toResolveAndDownload).map { pendingDownloads => val accessUrls = pendingDownloads.filter(url => url.uriType == URIType.ACCESS) val googleUrls = pendingDownloads.filter(url => url.uriType == URIType.GCS) - val bulkDownloader: List[Downloader] = if (accessUrls.isEmpty) List() else List(buildBulkAccessUrlDownloader(accessUrls)) + val bulkDownloader: List[Downloader] = + if (accessUrls.isEmpty) List() else List(buildBulkAccessUrlDownloader(accessUrls)) val googleDownloaders: List[Downloader] = if (googleUrls.isEmpty) List() else buildGoogleDownloaders(googleUrls) bulkDownloader ++ googleDownloaders } - } - def buildGoogleDownloaders(resolvedGoogleUrls: List[ResolvedDrsUrl]) : List[Downloader] = { - resolvedGoogleUrls.map{url=> + def buildGoogleDownloaders(resolvedGoogleUrls: List[ResolvedDrsUrl]): List[Downloader] = + resolvedGoogleUrls.map { url => downloaderFactory.buildGcsUriDownloader( gcsPath = url.drsResponse.gsUri.get, serviceAccountJsonOption = url.drsResponse.googleServiceAccount.map(_.data.spaces2), downloadLoc = url.downloadDestinationPath, - requesterPaysProjectOption = requesterPaysProjectIdOption) + requesterPaysProjectOption = requesterPaysProjectIdOption + ) } - } - def buildBulkAccessUrlDownloader(resolvedUrls: List[ResolvedDrsUrl]) : Downloader = { + def buildBulkAccessUrlDownloader(resolvedUrls: List[ResolvedDrsUrl]): Downloader = downloaderFactory.buildBulkAccessUrlDownloader(resolvedUrls) - } /** * Runs a synchronous HTTP request to resolve the provided DRS URL with the provided resolver. */ - def resolveSingleUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrl] = { - val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl, DrsResolverField.Hashes) + def resolveSingleUrl(resolverObject: DrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrl] = { + val fields = NonEmptyList.of(DrsResolverField.GsUri, + DrsResolverField.GoogleServiceAccount, + DrsResolverField.AccessUrl, + DrsResolverField.Hashes + ) val drsResponse = resolverObject.resolveDrs(drsUrlToResolve.drsUrl, fields) - drsResponse.map(resp => ResolvedDrsUrl(resp, drsUrlToResolve.downloadDestinationPath, toValidatedUriType(resp.accessUrl, resp.gsUri))) + drsResponse.map(resp => + ResolvedDrsUrl(resp, drsUrlToResolve.downloadDestinationPath, toValidatedUriType(resp.accessUrl, resp.gsUri)) + ) } - - val defaultBackoff: CloudNioBackoff = CloudNioSimpleExponentialBackoff( - initialInterval = 10 seconds, maxInterval = 60 seconds, multiplier = 2) + val defaultBackoff: CloudNioBackoff = + CloudNioSimpleExponentialBackoff(initialInterval = 10 seconds, maxInterval = 60 seconds, multiplier = 2) /** * Runs synchronous HTTP requests to resolve all the DRS urls. */ - def resolveUrls(unresolvedUrls: IO[List[UnresolvedDrsUrl]]): IO[List[ResolvedDrsUrl]] = { + def resolveUrls(unresolvedUrls: IO[List[UnresolvedDrsUrl]]): IO[List[ResolvedDrsUrl]] = unresolvedUrls.flatMap { unresolvedList => getDrsPathResolver.flatMap { resolver => - unresolvedList.map { unresolvedUrl => - resolveWithRetries(resolver, unresolvedUrl, defaultNumRetries, Option(defaultBackoff)) - }.traverse(identity) + unresolvedList + .map { unresolvedUrl => + resolveWithRetries(resolver, unresolvedUrl, defaultNumRetries, Option(defaultBackoff)) + } + .traverse(identity) } } - } - def resolveWithRetries(resolverObject: DrsLocalizerDrsPathResolver, + def resolveWithRetries(resolverObject: DrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl, resolutionRetries: Int, backoff: Option[CloudNioBackoff], - resolutionAttempt: Int = 0) : IO[ResolvedDrsUrl] = { + resolutionAttempt: Int = 0 + ): IO[ResolvedDrsUrl] = { - def maybeRetryForResolutionFailure(t: Throwable): IO[ResolvedDrsUrl] = { + def maybeRetryForResolutionFailure(t: Throwable): IO[ResolvedDrsUrl] = if (resolutionAttempt < resolutionRetries) { backoff foreach { b => Thread.sleep(b.backoffMillis) } - logger.warn(s"Attempting retry $resolutionAttempt of $resolutionRetries drs resolution retries to resolve ${drsUrlToResolve.drsUrl}", t) - resolveWithRetries(resolverObject, drsUrlToResolve, resolutionRetries, backoff map { _.next }, resolutionAttempt+1) + logger.warn( + s"Attempting retry $resolutionAttempt of $resolutionRetries drs resolution retries to resolve ${drsUrlToResolve.drsUrl}", + t + ) + resolveWithRetries(resolverObject, + drsUrlToResolve, + resolutionRetries, + backoff map { _.next }, + resolutionAttempt + 1 + ) } else { - IO.raiseError(new RuntimeException(s"Exhausted $resolutionRetries resolution retries to resolve $drsUrlToResolve.drsUrl", t)) + IO.raiseError( + new RuntimeException(s"Exhausted $resolutionRetries resolution retries to resolve $drsUrlToResolve.drsUrl", t) + ) } - } resolveSingleUrl(resolverObject, drsUrlToResolve).redeemWith( recover = maybeRetryForResolutionFailure, bind = { - case f: FatalRetryDisposition => - IO.raiseError(new RuntimeException(s"Fatal error resolving DRS URL: $f")) - case _: RegularRetryDisposition => - resolveWithRetries(resolverObject, drsUrlToResolve, resolutionRetries, backoff, resolutionAttempt+1) - case o => IO.pure(o) - }) + case f: FatalRetryDisposition => + IO.raiseError(new RuntimeException(s"Fatal error resolving DRS URL: $f")) + case _: RegularRetryDisposition => + resolveWithRetries(resolverObject, drsUrlToResolve, resolutionRetries, backoff, resolutionAttempt + 1) + case o => IO.pure(o) + } + ) } } - diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala index 4668c5072ed..1fd5bc6bf60 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala @@ -1,7 +1,7 @@ package drs.localizer.downloaders import cats.effect.{ExitCode, IO} -import cloud.nio.impl.drs.{AccessUrl, DrsResolverResponse} +import cloud.nio.impl.drs.AccessUrl import com.typesafe.scalalogging.StrictLogging import java.nio.charset.StandardCharsets @@ -9,16 +9,21 @@ import java.nio.file.{Files, Path, Paths} import scala.sys.process.{Process, ProcessLogger} import scala.util.matching.Regex import drs.localizer.ResolvedDrsUrl +import spray.json.DefaultJsonProtocol.{listFormat, mapFormat, StringJsonFormat} +import spray.json._ + case class GetmResult(returnCode: Int, stderr: String) + /** * Getm is a python tool that is used to download resolved DRS uris quickly and in parallel. * This class builds a getm-manifest.json file that it uses for input, and builds/executes a shell command * to invoke the Getm tool, which is expected to already be installed in the local environment. * @param resolvedUrls */ -case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends Downloader with StrictLogging { +case class BulkAccessUrlDownloader(resolvedUrls: List[ResolvedDrsUrl]) extends Downloader with StrictLogging { val getmManifestPath: Path = Paths.get("getm-manifest.json") + /** * Write a json manifest to disk that looks like: * // [ @@ -39,53 +44,42 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends * @param resolvedUrls * @return Filepath of a getm-manifest.json that Getm can use to download multiple files in parallel. */ - def generateJsonManifest(resolvedUrls : List[ResolvedDrsUrl]): IO[Path] = { - def toJsonString(drsResponse: DrsResolverResponse, destinationFilepath: String): String = { - //NB: trailing comma is being removed in generateJsonManifest - val accessUrl: AccessUrl = drsResponse.accessUrl.getOrElse(AccessUrl("missing", None)) - drsResponse.hashes.map(_ => { - val checksum = GetmChecksum(drsResponse.hashes, accessUrl).value.getOrElse("error_calculating_checksum") - val checksumAlgorithm = GetmChecksum(drsResponse.hashes, accessUrl).getmAlgorithm - s""" { - | "url" : "${accessUrl.url}", - | "filepath" : "$destinationFilepath", - | "checksum" : "$checksum", - | "checksum-algorithm" : "$checksumAlgorithm" - | }, - |""".stripMargin - }).getOrElse( - s""" { - | "url" : "${accessUrl.url}", - | "filepath" : "$destinationFilepath" - | }, - |""".stripMargin - ) - } - IO { - var jsonString: String = "[\n" - for (resolvedUrl <- resolvedUrls) { - jsonString += toJsonString(resolvedUrl.drsResponse, resolvedUrl.downloadDestinationPath) - } - if(jsonString.contains(',')) { - //remove trailing comma from array elements, but don't crash on empty list. - jsonString = jsonString.substring(0, jsonString.lastIndexOf(",")) - } - jsonString += "\n]" - Files.write(getmManifestPath, jsonString.getBytes(StandardCharsets.UTF_8)) + def generateJsonManifest(resolvedUrls: List[ResolvedDrsUrl]): IO[Path] = { + def resolvedUrlToJsonMap(resolvedUrl: ResolvedDrsUrl): Map[String, String] = { + val accessUrl: AccessUrl = resolvedUrl.drsResponse.accessUrl.getOrElse(AccessUrl("missing", None)) + resolvedUrl.drsResponse.hashes + .map { _ => + val checksum = + GetmChecksum(resolvedUrl.drsResponse.hashes, accessUrl).value.getOrElse("error_calculating_checksum") + val checksumAlgorithm = GetmChecksum(resolvedUrl.drsResponse.hashes, accessUrl).getmAlgorithm + Map( + ("url", accessUrl.url), + ("filepath", resolvedUrl.downloadDestinationPath), + ("checksum", checksum), + ("checksum-algorithm", checksumAlgorithm) + ) + } + .getOrElse( + Map( + ("url", accessUrl.url), + ("filepath", resolvedUrl.downloadDestinationPath) + ) + ) } + + val jsonArray: String = resolvedUrls.map(resolved => resolvedUrlToJsonMap(resolved)).toJson.prettyPrint + IO(Files.write(getmManifestPath, jsonArray.getBytes(StandardCharsets.UTF_8))) } - def deleteJsonManifest() = { + def deleteJsonManifest() = Files.deleteIfExists(getmManifestPath) - } - def generateGetmCommand(pathToMainfestJson : Path) : String = { - s"""getm --manifest ${pathToMainfestJson.toString}""" - } - def runGetm: IO[GetmResult] = { - generateJsonManifest(resolvedUrls).flatMap{ manifestPath => + def generateGetmCommand(pathToMainfestJson: Path): String = + s"""timeout 24h getm --manifest ${pathToMainfestJson.toString} -vv""" + def runGetm: IO[GetmResult] = + generateJsonManifest(resolvedUrls).flatMap { manifestPath => val script = generateGetmCommand(manifestPath) - val copyCommand : Seq[String] = Seq("bash", "-c", script) + val copyCommand: Seq[String] = Seq("bash", "-c", script) logger.info(script) val copyProcess = Process(copyCommand) val stderr = new StringBuilder() @@ -95,7 +89,6 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends logger.info(stderr.toString().trim()) IO(GetmResult(returnCode, stderr.toString().trim())) } - } override def download: IO[DownloadResult] = { // We don't want to log the unmasked signed URL here. On a PAPI backend this log will end up under the user's @@ -105,7 +98,7 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends runGetm map toDownloadResult } - def toDownloadResult(getmResult: GetmResult): DownloadResult = { + def toDownloadResult(getmResult: GetmResult): DownloadResult = getmResult match { case GetmResult(0, stderr) if stderr.isEmpty => DownloadSuccess @@ -133,10 +126,9 @@ case class BulkAccessUrlDownloader(resolvedUrls : List[ResolvedDrsUrl]) extends UnrecognizedRetryableDownloadFailure(ExitCode(rc)) } } - } } -object BulkAccessUrlDownloader{ +object BulkAccessUrlDownloader { type Hashes = Option[Map[String, String]] val ChecksumFailureMessage: Regex = raw""".*AssertionError: Checksum failed!.*""".r diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala index 6c7f27e8a6e..c35a2b1634e 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala @@ -3,10 +3,11 @@ package drs.localizer.downloaders import drs.localizer.ResolvedDrsUrl trait DownloaderFactory { - def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]) : Downloader + def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, - requesterPaysProjectOption: Option[String]): Downloader + requesterPaysProjectOption: Option[String] + ): Downloader } diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala index 8991e79f5fd..74f5bc64621 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala @@ -12,15 +12,16 @@ import scala.sys.process.{Process, ProcessLogger} case class GcsUriDownloader(gcsUrl: String, serviceAccountJson: Option[String], downloadLoc: String, - requesterPaysProjectIdOption: Option[String]) extends Downloader with StrictLogging { + requesterPaysProjectIdOption: Option[String] +) extends Downloader + with StrictLogging { val defaultNumRetries: Int = 5 - val defaultBackoff: CloudNioBackoff = CloudNioSimpleExponentialBackoff( - initialInterval = 1 seconds, maxInterval = 60 seconds, multiplier = 2) + val defaultBackoff: CloudNioBackoff = + CloudNioSimpleExponentialBackoff(initialInterval = 1 seconds, maxInterval = 60 seconds, multiplier = 2) - override def download: IO[DownloadResult] = { + override def download: IO[DownloadResult] = downloadWithRetries(defaultNumRetries, Option(defaultBackoff)) - } def runDownloadCommand: IO[DownloadResult] = { @@ -45,27 +46,30 @@ case class GcsUriDownloader(gcsUrl: String, // run the multiple bash script to download file and log stream sent to stdout and stderr using ProcessLogger val returnCode = copyProcess ! ProcessLogger(logger.underlying.info, logger.underlying.error) - val result = if (returnCode == 0) DownloadSuccess else RecognizedRetryableDownloadFailure(exitCode = ExitCode(returnCode)) + val result = + if (returnCode == 0) DownloadSuccess else RecognizedRetryableDownloadFailure(exitCode = ExitCode(returnCode)) IO.pure(result) } def downloadWithRetries(downloadRetries: Int, backoff: Option[CloudNioBackoff], - downloadAttempt: Int = 0): IO[DownloadResult] = - { + downloadAttempt: Int = 0 + ): IO[DownloadResult] = { - def maybeRetryForDownloadFailure(t: Throwable): IO[DownloadResult] = { + def maybeRetryForDownloadFailure(t: Throwable): IO[DownloadResult] = if (downloadAttempt < downloadRetries) { backoff foreach { b => Thread.sleep(b.backoffMillis) } logger.warn(s"Attempting download retry $downloadAttempt of $downloadRetries for a GCS url", t) - downloadWithRetries(downloadRetries, backoff map { - _.next - }, downloadAttempt + 1) + downloadWithRetries(downloadRetries, + backoff map { + _.next + }, + downloadAttempt + 1 + ) } else { IO.raiseError(new RuntimeException(s"Exhausted $downloadRetries resolution retries to download GCS file", t)) } - } runDownloadCommand.redeemWith( recover = maybeRetryForDownloadFailure, @@ -73,12 +77,13 @@ case class GcsUriDownloader(gcsUrl: String, case s: DownloadSuccess.type => IO.pure(s) case _: RecognizedRetryableDownloadFailure => - downloadWithRetries(downloadRetries, backoff, downloadAttempt+1) + downloadWithRetries(downloadRetries, backoff, downloadAttempt + 1) case _: UnrecognizedRetryableDownloadFailure => - downloadWithRetries(downloadRetries, backoff, downloadAttempt+1) + downloadWithRetries(downloadRetries, backoff, downloadAttempt + 1) case _ => - downloadWithRetries(downloadRetries, backoff, downloadAttempt+1) - }) + downloadWithRetries(downloadRetries, backoff, downloadAttempt + 1) + } + ) } /** @@ -88,7 +93,7 @@ case class GcsUriDownloader(gcsUrl: String, def gcsCopyCommand(flag: String = ""): String = s"gsutil $flag cp $gcsUrl $downloadLoc" - def setServiceAccount(): String = { + def setServiceAccount(): String = saJsonPathOption match { case Some(saJsonPath) => s"""# Set gsutil to use the service account returned from the DRS Resolver @@ -103,9 +108,8 @@ case class GcsUriDownloader(gcsUrl: String, |""".stripMargin case None => "" } - } - def recoverWithRequesterPays(): String = { + def recoverWithRequesterPays(): String = requesterPaysProjectIdOption match { case Some(userProject) => s"""if [ "$$RC_GSUTIL" != "0" ]; then @@ -119,7 +123,6 @@ case class GcsUriDownloader(gcsUrl: String, |""".stripMargin case None => "" } - } // bash to download the GCS file using gsutil s"""set -euo pipefail @@ -145,5 +148,5 @@ case class GcsUriDownloader(gcsUrl: String, } object GcsUriDownloader { - private final val RequesterPaysErrorMsg = "requester pays bucket but no user project" + final private val RequesterPaysErrorMsg = "requester pays bucket but no user project" } diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala index 2ca1bd3d2e3..a72459cdb7b 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala @@ -8,12 +8,11 @@ import org.apache.commons.codec.binary.Base64.encodeBase64String import org.apache.commons.codec.binary.Hex.decodeHex import org.apache.commons.text.StringEscapeUtils - sealed trait GetmChecksum { def getmAlgorithm: String def rawValue: String def value: ErrorOr[String] = rawValue.validNel - def args: ErrorOr[String] = { + def args: ErrorOr[String] = // The value for `--checksum-algorithm` is constrained by the algorithm names in the `sealed` hierarchy of // `GetmChecksum`, but the value for `--checksum` is largely a function of data returned by the DRS server. // Shell escape this to avoid injection. @@ -21,7 +20,6 @@ sealed trait GetmChecksum { val escapedValue = StringEscapeUtils.escapeXSI(v) s"--checksum-algorithm '$getmAlgorithm' --checksum $escapedValue" } - } } case class Md5(override val rawValue: String) extends GetmChecksum { @@ -33,7 +31,8 @@ case class Crc32c(override val rawValue: String) extends GetmChecksum { // The DRS spec says that all hash values should be hex strings, // but getm expects crc32c values to be base64. override def value: ErrorOr[String] = - GetmChecksum.validateHex(rawValue) + GetmChecksum + .validateHex(rawValue) .map(decodeHex) .map(encodeBase64String) @@ -52,7 +51,7 @@ case class Unsupported(override val rawValue: String) extends GetmChecksum { } object GetmChecksum { - def apply(hashes: Hashes, accessUrl: AccessUrl): GetmChecksum = { + def apply(hashes: Hashes, accessUrl: AccessUrl): GetmChecksum = hashes match { case Some(hashes) if hashes.nonEmpty => // `hashes` is keyed by the DRS Resolver names for these hash algorithms, which in turn are the forwarded DRS @@ -61,8 +60,7 @@ object GetmChecksum { // but all of the other algorithm names currently differ between DRS providers and `getm`. if (hashes.contains("md5")) { Md5(hashes("md5")) - } - else if (hashes.contains("crc32c")) { + } else if (hashes.contains("crc32c")) { Crc32c(hashes("crc32c")) } // etags could be anything; only ask `getm` to check s3 etags if this actually looks like an s3 signed url. @@ -81,7 +79,6 @@ object GetmChecksum { } case _ => Null // None or an empty hashes map. } - } def validateHex(s: String): ErrorOr[String] = { val trimmed = s.trim diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/CommandLineParserSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/CommandLineParserSpec.scala index 7be30be8ac0..6658428e650 100644 --- a/cromwell-drs-localizer/src/test/scala/drs/localizer/CommandLineParserSpec.scala +++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/CommandLineParserSpec.scala @@ -63,7 +63,9 @@ class CommandLineParserSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "successfully parse with three arguments and requester pays project" in { - val args = parser.parse(Array(drsObject, containerPath, requesterPaysProject, "-r", requesterPaysProject), CommandLineArguments()).get + val args = parser + .parse(Array(drsObject, containerPath, requesterPaysProject, "-r", requesterPaysProject), CommandLineArguments()) + .get args.drsObject.get shouldBe drsObject args.containerPath.get shouldBe containerPath @@ -74,7 +76,9 @@ class CommandLineParserSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "fail if requester pays argument and flag specify different projects" in { - parser.parse(Array(drsObject, containerPath, requesterPaysProject, "-r", "boom!"), CommandLineArguments()) shouldBe None + parser.parse(Array(drsObject, containerPath, requesterPaysProject, "-r", "boom!"), + CommandLineArguments() + ) shouldBe None } it should "successfully parse args with a manifest file" in { @@ -99,12 +103,18 @@ class CommandLineParserSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "successfully parse an explicit Google access token strategy invocation" in { - val args = parser.parse(Array( - "--access-token-strategy", "google", - drsObject, - containerPath, - "--requester-pays-project", requesterPaysProject - ), CommandLineArguments()).get + val args = parser + .parse(Array( + "--access-token-strategy", + "google", + drsObject, + containerPath, + "--requester-pays-project", + requesterPaysProject + ), + CommandLineArguments() + ) + .get args.drsObject.get shouldBe drsObject args.containerPath.get shouldBe containerPath @@ -115,19 +125,26 @@ class CommandLineParserSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "fail to parse an Azure invocation that specifies requester pays" in { - val args = parser.parse(Array( - "--access-token-strategy", AccessTokenStrategy.Azure, - drsObject, - containerPath, - "--requester-pays-project", requesterPaysProject), CommandLineArguments()) + val args = parser.parse( + Array("--access-token-strategy", + AccessTokenStrategy.Azure, + drsObject, + containerPath, + "--requester-pays-project", + requesterPaysProject + ), + CommandLineArguments() + ) args shouldBe None } it should "successfully parse an Azure invocation" in { - val args = parser.parse(Array( - "--access-token-strategy", AccessTokenStrategy.Azure, - drsObject, containerPath), CommandLineArguments()).get + val args = parser + .parse(Array("--access-token-strategy", AccessTokenStrategy.Azure, drsObject, containerPath), + CommandLineArguments() + ) + .get args.drsObject.get shouldBe drsObject args.containerPath.get shouldBe containerPath @@ -138,10 +155,18 @@ class CommandLineParserSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "successfully parse an Azure invocation with identity" in { - val args = parser.parse(Array( - "--access-token-strategy", AccessTokenStrategy.Azure, - "--identity-client-id", azureIdentityClientId, - drsObject, containerPath), CommandLineArguments()).get + val args = parser + .parse( + Array("--access-token-strategy", + AccessTokenStrategy.Azure, + "--identity-client-id", + azureIdentityClientId, + drsObject, + containerPath + ), + CommandLineArguments() + ) + .get args.drsObject.get shouldBe drsObject args.containerPath.get shouldBe containerPath @@ -152,7 +177,8 @@ class CommandLineParserSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "fail to parse with an unrecognized access token strategy" in { - val args = parser.parse(Array("--access-token-strategy", "nebulous", drsObject, containerPath), CommandLineArguments()) + val args = + parser.parse(Array("--access-token-strategy", "nebulous", drsObject, containerPath), CommandLineArguments()) args shouldBe None } } diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala index 52fa4c99330..dc06ad21a0c 100644 --- a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala +++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala @@ -4,7 +4,7 @@ import cats.data.NonEmptyList import cats.effect.{ExitCode, IO} import cats.syntax.validated._ import drs.localizer.MockDrsPaths.{fakeAccessUrls, fakeDrsUrlWithGcsResolutionOnly, fakeGoogleUrls} -import cloud.nio.impl.drs.{AccessUrl, DrsConfig, DrsCredentials, DrsResolverField, DrsResolverResponse} +import cloud.nio.impl.drs.{AccessUrl, DrsConfig, DrsCredentials, DrsPathResolver, DrsResolverField, DrsResolverResponse} import common.assertion.CromwellTimeoutSpec import common.validation.ErrorOr.ErrorOr import drs.localizer.MockDrsLocalizerDrsPathResolver.{FakeAccessTokenStrategy, FakeHashes} @@ -12,33 +12,40 @@ import drs.localizer.downloaders._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { val fakeDownloadLocation = "/root/foo/foo-123.bam" val fakeRequesterPaysId = "fake-billing-project" - val fakeGoogleInput : IO[List[UnresolvedDrsUrl]] = IO(List( - UnresolvedDrsUrl(fakeDrsUrlWithGcsResolutionOnly, "/path/to/nowhere") - )) + val fakeGoogleInput: IO[List[UnresolvedDrsUrl]] = IO( + List( + UnresolvedDrsUrl(fakeDrsUrlWithGcsResolutionOnly, "/path/to/nowhere") + ) + ) - val fakeAccessInput: IO[List[UnresolvedDrsUrl]] = IO(List( - UnresolvedDrsUrl("https://my-fake-access-url.com", "/path/to/somewhereelse") - )) + val fakeAccessInput: IO[List[UnresolvedDrsUrl]] = IO( + List( + UnresolvedDrsUrl("https://my-fake-access-url.com", "/path/to/somewhereelse") + ) + ) - val fakeBulkGoogleInput: IO[List[UnresolvedDrsUrl]] = IO(List( - UnresolvedDrsUrl("drs://my-fake-google-url.com", "/path/to/nowhere"), - UnresolvedDrsUrl("drs://my-fake-google-url.com2", "/path/to/nowhere2"), - UnresolvedDrsUrl("drs://my-fake-google-url.com3", "/path/to/nowhere3"), - UnresolvedDrsUrl("drs://my-fake-google-url.com4", "/path/to/nowhere4") - )) + val fakeBulkGoogleInput: IO[List[UnresolvedDrsUrl]] = IO( + List( + UnresolvedDrsUrl("drs://my-fake-google-url.com", "/path/to/nowhere"), + UnresolvedDrsUrl("drs://my-fake-google-url.com2", "/path/to/nowhere2"), + UnresolvedDrsUrl("drs://my-fake-google-url.com3", "/path/to/nowhere3"), + UnresolvedDrsUrl("drs://my-fake-google-url.com4", "/path/to/nowhere4") + ) + ) - val fakeBulkAccessInput: IO[List[UnresolvedDrsUrl]] = IO(List( - UnresolvedDrsUrl("drs://my-fake-access-url.com", "/path/to/somewhereelse"), - UnresolvedDrsUrl("drs://my-fake-access-url2.com", "/path/to/somewhereelse2"), - UnresolvedDrsUrl("drs://my-fake-access-url3.com", "/path/to/somewhereelse3"), - UnresolvedDrsUrl("drs://my-fake-access-url4.com", "/path/to/somewhereelse4") - )) + val fakeBulkAccessInput: IO[List[UnresolvedDrsUrl]] = IO( + List( + UnresolvedDrsUrl("drs://my-fake-access-url.com", "/path/to/somewhereelse"), + UnresolvedDrsUrl("drs://my-fake-access-url2.com", "/path/to/somewhereelse2"), + UnresolvedDrsUrl("drs://my-fake-access-url3.com", "/path/to/somewhereelse3"), + UnresolvedDrsUrl("drs://my-fake-access-url4.com", "/path/to/somewhereelse4") + ) + ) behavior of "DrsLocalizerMain" @@ -52,34 +59,43 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat it should "tolerate no URLs being provided" in { val mockDownloadFactory = new DownloaderFactory { - override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): Downloader = { + override def buildGcsUriDownloader(gcsPath: String, + serviceAccountJsonOption: Option[String], + downloadLoc: String, + requesterPaysProjectOption: Option[String] + ): Downloader = // This test path should never ask for the Google downloader throw new RuntimeException("test failure111") - } - override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = { + override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = // This test path should never ask for the Bulk downloader throw new RuntimeException("test failure111") - } } - val mockdrsLocalizer = new MockDrsLocalizerMain(IO(List()), mockDownloadFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId)) + val mockdrsLocalizer = + new MockDrsLocalizerMain(IO(List()), mockDownloadFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId)) val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync() downloaders.length shouldBe 0 } it should "build correct downloader(s) for a single google URL" in { val mockDownloadFactory = new DownloaderFactory { - override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): Downloader = { + override def buildGcsUriDownloader(gcsPath: String, + serviceAccountJsonOption: Option[String], + downloadLoc: String, + requesterPaysProjectOption: Option[String] + ): Downloader = GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption) - } - override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = { + override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = // This test path should never ask for the Bulk downloader throw new RuntimeException("test failure111") - } } - val mockdrsLocalizer = new MockDrsLocalizerMain(IO(List(fakeGoogleUrls.head._1)), mockDownloadFactory,FakeAccessTokenStrategy, Option(fakeRequesterPaysId)) + val mockdrsLocalizer = new MockDrsLocalizerMain(IO(List(fakeGoogleUrls.head._1)), + mockDownloadFactory, + FakeAccessTokenStrategy, + Option(fakeRequesterPaysId) + ) val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync() downloaders.length shouldBe 1 @@ -92,17 +108,23 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat it should "build correct downloader(s) for a single access URL" in { val mockDownloadFactory = new DownloaderFactory { - override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): Downloader = { + override def buildGcsUriDownloader(gcsPath: String, + serviceAccountJsonOption: Option[String], + downloadLoc: String, + requesterPaysProjectOption: Option[String] + ): Downloader = // This test path should never ask for the GCS downloader throw new RuntimeException("test failure") - } - override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = { + override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = BulkAccessUrlDownloader(urlsToDownload) - } } - val mockdrsLocalizer = new MockDrsLocalizerMain(IO(List(fakeAccessUrls.head._1)), mockDownloadFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId)) + val mockdrsLocalizer = new MockDrsLocalizerMain(IO(List(fakeAccessUrls.head._1)), + mockDownloadFactory, + FakeAccessTokenStrategy, + Option(fakeRequesterPaysId) + ) val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync() downloaders.length shouldBe 1 @@ -114,48 +136,64 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat it should "build correct downloader(s) for multiple google URLs" in { val mockDownloadFactory = new DownloaderFactory { - override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): Downloader = { + override def buildGcsUriDownloader(gcsPath: String, + serviceAccountJsonOption: Option[String], + downloadLoc: String, + requesterPaysProjectOption: Option[String] + ): Downloader = GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption) - } - override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = { + override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = // This test path should never ask for the GCS downloader throw new RuntimeException("test failure") - } } - val unresolvedUrls : List[UnresolvedDrsUrl] = fakeGoogleUrls.map(pair => pair._1).toList - val mockdrsLocalizer = new MockDrsLocalizerMain(IO(unresolvedUrls), mockDownloadFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId)) + val unresolvedUrls: List[UnresolvedDrsUrl] = fakeGoogleUrls.map(pair => pair._1).toList + val mockdrsLocalizer = new MockDrsLocalizerMain(IO(unresolvedUrls), + mockDownloadFactory, + FakeAccessTokenStrategy, + Option(fakeRequesterPaysId) + ) val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync() downloaders.length shouldBe unresolvedUrls.length - val countGoogleDownloaders = downloaders.count(downloader => downloader match { - case _: GcsUriDownloader => true - case _ => false - }) + val countGoogleDownloaders = downloaders.count(downloader => + downloader match { + case _: GcsUriDownloader => true + case _ => false + } + ) // We expect one GCS downloader for each GCS uri provided countGoogleDownloaders shouldBe downloaders.length } it should "build a single bulk downloader for multiple access URLs" in { val mockDownloadFactory = new DownloaderFactory { - override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): Downloader = { + override def buildGcsUriDownloader(gcsPath: String, + serviceAccountJsonOption: Option[String], + downloadLoc: String, + requesterPaysProjectOption: Option[String] + ): Downloader = // This test path should never ask for the GCS downloader throw new RuntimeException("test failure") - } - override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = { + override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = BulkAccessUrlDownloader(urlsToDownload) - } } val unresolvedUrls: List[UnresolvedDrsUrl] = fakeAccessUrls.map(pair => pair._1).toList - val mockdrsLocalizer = new MockDrsLocalizerMain(IO(unresolvedUrls), mockDownloadFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId)) + val mockdrsLocalizer = new MockDrsLocalizerMain(IO(unresolvedUrls), + mockDownloadFactory, + FakeAccessTokenStrategy, + Option(fakeRequesterPaysId) + ) val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync() downloaders.length shouldBe 1 - val countBulkDownloaders = downloaders.count(downloader => downloader match { - case _: BulkAccessUrlDownloader => true - case _ => false - }) + val countBulkDownloaders = downloaders.count(downloader => + downloader match { + case _: BulkAccessUrlDownloader => true + case _ => false + } + ) // We expect one total Bulk downloader for all access URIs to share countBulkDownloaders shouldBe 1 val expected = BulkAccessUrlDownloader( @@ -165,23 +203,32 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat } it should "build 1 bulk downloader and 5 google downloaders for a mix of URLs" in { - val unresolvedUrls: List[UnresolvedDrsUrl] = fakeAccessUrls.map(pair => pair._1).toList ++ fakeGoogleUrls.map(pair => pair._1).toList - val mockdrsLocalizer = new MockDrsLocalizerMain(IO(unresolvedUrls), DrsLocalizerMain.defaultDownloaderFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId)) + val unresolvedUrls: List[UnresolvedDrsUrl] = + fakeAccessUrls.map(pair => pair._1).toList ++ fakeGoogleUrls.map(pair => pair._1).toList + val mockdrsLocalizer = new MockDrsLocalizerMain(IO(unresolvedUrls), + DrsLocalizerMain.defaultDownloaderFactory, + FakeAccessTokenStrategy, + Option(fakeRequesterPaysId) + ) val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync() downloaders.length shouldBe 6 - //we expect a single bulk downloader despite 5 access URLs being provided - val countBulkDownloaders = downloaders.count(downloader => downloader match { - case _: BulkAccessUrlDownloader => true - case _ => false - }) + // we expect a single bulk downloader despite 5 access URLs being provided + val countBulkDownloaders = downloaders.count(downloader => + downloader match { + case _: BulkAccessUrlDownloader => true + case _ => false + } + ) // We expect one GCS downloader for each GCS uri provided countBulkDownloaders shouldBe 1 - val countGoogleDownloaders = downloaders.count(downloader => downloader match { - case _: GcsUriDownloader => true - case _ => false - }) + val countGoogleDownloaders = downloaders.count(downloader => + downloader match { + case _: GcsUriDownloader => true + case _ => false + } + ) // We expect one GCS downloader for each GCS uri provided countBulkDownloaders shouldBe 1 countGoogleDownloaders shouldBe 5 @@ -189,24 +236,34 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat it should "accept arguments and run successfully without Requester Pays ID" in { val unresolved = fakeGoogleUrls.head._1 - val mockDrsLocalizer = new MockDrsLocalizerMain(IO(List(unresolved)), DrsLocalizerMain.defaultDownloaderFactory, FakeAccessTokenStrategy, None) + val mockDrsLocalizer = new MockDrsLocalizerMain(IO(List(unresolved)), + DrsLocalizerMain.defaultDownloaderFactory, + FakeAccessTokenStrategy, + None + ) val expected = GcsUriDownloader( gcsUrl = fakeGoogleUrls.get(unresolved).get.drsResponse.gsUri.get, serviceAccountJson = None, downloadLoc = unresolved.downloadDestinationPath, - requesterPaysProjectIdOption = None) + requesterPaysProjectIdOption = None + ) val downloader: Downloader = mockDrsLocalizer.buildDownloaders().unsafeRunSync().head downloader shouldBe expected } it should "run successfully with all 3 arguments" in { val unresolved = fakeGoogleUrls.head._1 - val mockDrsLocalizer = new MockDrsLocalizerMain(IO(List(unresolved)), DrsLocalizerMain.defaultDownloaderFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId)) + val mockDrsLocalizer = new MockDrsLocalizerMain(IO(List(unresolved)), + DrsLocalizerMain.defaultDownloaderFactory, + FakeAccessTokenStrategy, + Option(fakeRequesterPaysId) + ) val expected = GcsUriDownloader( gcsUrl = fakeGoogleUrls.get(unresolved).get.drsResponse.gsUri.get, serviceAccountJson = None, downloadLoc = unresolved.downloadDestinationPath, - requesterPaysProjectIdOption = Option(fakeRequesterPaysId)) + requesterPaysProjectIdOption = Option(fakeRequesterPaysId) + ) val downloader: Downloader = mockDrsLocalizer.buildDownloaders().unsafeRunSync().head downloader shouldBe expected } @@ -214,10 +271,18 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat it should "successfully identify uri types, preferring access" in { val exampleAccessResponse = DrsResolverResponse(accessUrl = Option(AccessUrl("https://something.com", FakeHashes))) val exampleGoogleResponse = DrsResolverResponse(gsUri = Option("gs://something")) - val exampleMixedResponse = DrsResolverResponse(accessUrl = Option(AccessUrl("https://something.com", FakeHashes)), gsUri = Option("gs://something")) - DrsLocalizerMain.toValidatedUriType(exampleAccessResponse.accessUrl, exampleAccessResponse.gsUri) shouldBe URIType.ACCESS - DrsLocalizerMain.toValidatedUriType(exampleGoogleResponse.accessUrl, exampleGoogleResponse.gsUri) shouldBe URIType.GCS - DrsLocalizerMain.toValidatedUriType(exampleMixedResponse.accessUrl, exampleMixedResponse.gsUri) shouldBe URIType.ACCESS + val exampleMixedResponse = DrsResolverResponse(accessUrl = Option(AccessUrl("https://something.com", FakeHashes)), + gsUri = Option("gs://something") + ) + DrsLocalizerMain.toValidatedUriType(exampleAccessResponse.accessUrl, + exampleAccessResponse.gsUri + ) shouldBe URIType.ACCESS + DrsLocalizerMain.toValidatedUriType(exampleGoogleResponse.accessUrl, + exampleGoogleResponse.gsUri + ) shouldBe URIType.GCS + DrsLocalizerMain.toValidatedUriType(exampleMixedResponse.accessUrl, + exampleMixedResponse.gsUri + ) shouldBe URIType.ACCESS } it should "throw an exception if the DRS Resolver response is invalid" in { @@ -246,48 +311,87 @@ object MockDrsPaths { val fakeDrsUrlWithoutAnyResolution = "drs://foo/bar/no-gcs-path" val fakeGoogleUrls: Map[UnresolvedDrsUrl, ResolvedDrsUrl] = Map( - (UnresolvedDrsUrl("drs://abc/foo-123/google/0", "/path/to/google/local0"), ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri0")), "/path/to/google/local0", URIType.GCS)), - (UnresolvedDrsUrl("drs://abc/foo-123/google/1", "/path/to/google/local1"), ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri1")), "/path/to/google/local1", URIType.GCS)), - (UnresolvedDrsUrl("drs://abc/foo-123/google/2", "/path/to/google/local2"), ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri2")), "/path/to/google/local2", URIType.GCS)), - (UnresolvedDrsUrl("drs://abc/foo-123/google/3", "/path/to/google/local3"), ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri3")), "/path/to/google/local3", URIType.GCS)), - (UnresolvedDrsUrl("drs://abc/foo-123/google/4", "/path/to/google/local4"), ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri4")), "/path/to/google/local4", URIType.GCS)) + (UnresolvedDrsUrl("drs://abc/foo-123/google/0", "/path/to/google/local0"), + ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri0")), "/path/to/google/local0", URIType.GCS) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/google/1", "/path/to/google/local1"), + ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri1")), "/path/to/google/local1", URIType.GCS) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/google/2", "/path/to/google/local2"), + ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri2")), "/path/to/google/local2", URIType.GCS) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/google/3", "/path/to/google/local3"), + ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri3")), "/path/to/google/local3", URIType.GCS) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/google/4", "/path/to/google/local4"), + ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri4")), "/path/to/google/local4", URIType.GCS) + ) ) val fakeAccessUrls: Map[UnresolvedDrsUrl, ResolvedDrsUrl] = Map( - (UnresolvedDrsUrl("drs://abc/foo-123/access/0", "/path/to/access/local0"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/0", FakeHashes))), "/path/to/access/local0", URIType.ACCESS)), - (UnresolvedDrsUrl("drs://abc/foo-123/access/1", "/path/to/access/local1"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/1", FakeHashes))), "/path/to/access/local1", URIType.ACCESS)), - (UnresolvedDrsUrl("drs://abc/foo-123/access/2", "/path/to/access/local2"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/2", FakeHashes))), "/path/to/access/local2", URIType.ACCESS)), - (UnresolvedDrsUrl("drs://abc/foo-123/access/3", "/path/to/access/local3"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/3", FakeHashes))), "/path/to/access/local3", URIType.ACCESS)), - (UnresolvedDrsUrl("drs://abc/foo-123/access/4", "/path/to/access/local4"), ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/4", FakeHashes))), "/path/to/access/local4", URIType.ACCESS)) + (UnresolvedDrsUrl("drs://abc/foo-123/access/0", "/path/to/access/local0"), + ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/0", FakeHashes))), + "/path/to/access/local0", + URIType.ACCESS + ) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/access/1", "/path/to/access/local1"), + ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/1", FakeHashes))), + "/path/to/access/local1", + URIType.ACCESS + ) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/access/2", "/path/to/access/local2"), + ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/2", FakeHashes))), + "/path/to/access/local2", + URIType.ACCESS + ) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/access/3", "/path/to/access/local3"), + ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/3", FakeHashes))), + "/path/to/access/local3", + URIType.ACCESS + ) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/access/4", "/path/to/access/local4"), + ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/4", FakeHashes))), + "/path/to/access/local4", + URIType.ACCESS + ) + ) ) } - class MockDrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]], downloaderFactory: DownloaderFactory, drsCredentials: DrsCredentials, requesterPaysProjectIdOption: Option[String] - ) +) extends DrsLocalizerMain(toResolveAndDownload, + downloaderFactory, + FakeAccessTokenStrategy, + requesterPaysProjectIdOption + ) { - extends DrsLocalizerMain(toResolveAndDownload, downloaderFactory, FakeAccessTokenStrategy, requesterPaysProjectIdOption) { - - override def getDrsPathResolver: IO[DrsLocalizerDrsPathResolver] = { + override def getDrsPathResolver: IO[DrsPathResolver] = IO { new MockDrsLocalizerDrsPathResolver(cloud.nio.impl.drs.MockDrsPaths.mockDrsConfig) } - } - override def resolveSingleUrl(resolverObject: DrsLocalizerDrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrl] = { + override def resolveSingleUrl(resolverObject: DrsPathResolver, + drsUrlToResolve: UnresolvedDrsUrl + ): IO[ResolvedDrsUrl] = IO { if (!fakeAccessUrls.contains(drsUrlToResolve) && !fakeGoogleUrls.contains(drsUrlToResolve)) { throw new RuntimeException("Unexpected URI during testing") } - fakeAccessUrls.getOrElse(drsUrlToResolve, fakeGoogleUrls.getOrElse(drsUrlToResolve, ResolvedDrsUrl(DrsResolverResponse(),"/12/3/", URIType.UNKNOWN))) + fakeAccessUrls.getOrElse( + drsUrlToResolve, + fakeGoogleUrls.getOrElse(drsUrlToResolve, ResolvedDrsUrl(DrsResolverResponse(), "/12/3/", URIType.UNKNOWN)) + ) } - } } -class MockDrsLocalizerDrsPathResolver(drsConfig: DrsConfig) extends - DrsLocalizerDrsPathResolver(drsConfig, FakeAccessTokenStrategy) { +class MockDrsLocalizerDrsPathResolver(drsConfig: DrsConfig) + extends DrsPathResolver(drsConfig, FakeAccessTokenStrategy) { override def resolveDrs(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): IO[DrsResolverResponse] = { @@ -298,17 +402,15 @@ class MockDrsLocalizerDrsPathResolver(drsConfig: DrsConfig) extends IO.pure(drsPath) map { case MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly => - drsResolverResponse.copy( - gsUri = Option("gs://abc/foo-123/abc123")) + drsResolverResponse.copy(gsUri = Option("gs://abc/foo-123/abc123")) case MockDrsPaths.fakeDrsUrlWithoutAnyResolution => drsResolverResponse case MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly => - drsResolverResponse.copy( - accessUrl = Option(AccessUrl(url = "http://abc/def/ghi.bam", headers = None))) + drsResolverResponse.copy(accessUrl = Option(AccessUrl(url = "http://abc/def/ghi.bam", headers = None))) case MockDrsPaths.fakeDrsUrlWithAccessUrlAndGcsResolution => - drsResolverResponse.copy( - accessUrl = Option(AccessUrl(url = "http://abc/def/ghi.bam", headers = None)), - gsUri = Option("gs://some/uri")) + drsResolverResponse.copy(accessUrl = Option(AccessUrl(url = "http://abc/def/ghi.bam", headers = None)), + gsUri = Option("gs://some/uri") + ) case e => throw new RuntimeException(s"Unexpected exception in DRS localization test code: $e") } } diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala index 7b96ece8d0a..6eae90aa842 100644 --- a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala +++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala @@ -12,71 +12,75 @@ import org.scalatest.matchers.should.Matchers import java.nio.file.Path class BulkAccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { - val ex1 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url123", None))), "path/to/local/download/dest", URIType.ACCESS) - val ex2 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url1234", None))), "path/to/local/download/dest2", URIType.ACCESS) - val ex3 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url1235", None))), "path/to/local/download/dest3", URIType.ACCESS) - val emptyList : List[ResolvedDrsUrl] = List() + val ex1 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url123", None))), + "path/to/local/download/dest", + URIType.ACCESS + ) + val ex2 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url1234", None))), + "path/to/local/download/dest2", + URIType.ACCESS + ) + val ex3 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url1235", None))), + "path/to/local/download/dest3", + URIType.ACCESS + ) + val emptyList: List[ResolvedDrsUrl] = List() val oneElement: List[ResolvedDrsUrl] = List(ex1) val threeElements: List[ResolvedDrsUrl] = List(ex1, ex2, ex3) it should "correctly parse a collection of Access Urls into a manifest.json" in { - val expected: String = - s"""|[ - | { - | "url" : "https://my.fake/url123", - | "filepath" : "path/to/local/download/dest" - | }, - | { - | "url" : "https://my.fake/url1234", - | "filepath" : "path/to/local/download/dest2" - | }, - | { - | "url" : "https://my.fake/url1235", - | "filepath" : "path/to/local/download/dest3" - | } - |]""".stripMargin - + val expected = + """[{ + | "url": "https://my.fake/url123", + | "filepath": "path/to/local/download/dest" + |}, { + | "url": "https://my.fake/url1234", + | "filepath": "path/to/local/download/dest2" + |}, { + | "url": "https://my.fake/url1235", + | "filepath": "path/to/local/download/dest3" + |}]""".stripMargin val downloader = BulkAccessUrlDownloader(threeElements) val filepath: IO[Path] = downloader.generateJsonManifest(threeElements) val source = scala.io.Source.fromFile(filepath.unsafeRunSync().toString) - val lines = try source.mkString finally source.close() + val lines = + try source.mkString + finally source.close() lines shouldBe expected } it should "properly construct empty JSON array from empty list." in { - val expected: String = - s"""|[ - | - |]""".stripMargin - + val expected: String = "[]" val downloader = BulkAccessUrlDownloader(emptyList) val filepath: IO[Path] = downloader.generateJsonManifest(emptyList) val source = scala.io.Source.fromFile(filepath.unsafeRunSync().toString) - val lines = try source.mkString finally source.close() + val lines = + try source.mkString + finally source.close() lines shouldBe expected } it should "properly construct JSON array from single element list." in { val expected: String = - s"""|[ - | { - | "url" : "https://my.fake/url123", - | "filepath" : "path/to/local/download/dest" - | } - |]""".stripMargin + s"""|[{ + | "url": "https://my.fake/url123", + | "filepath": "path/to/local/download/dest" + |}]""".stripMargin val downloader = BulkAccessUrlDownloader(oneElement) val filepath: IO[Path] = downloader.generateJsonManifest(oneElement) val source = scala.io.Source.fromFile(filepath.unsafeRunSync().toString) - val lines = try source.mkString finally source.close() + val lines = + try source.mkString + finally source.close() lines shouldBe expected } it should "properly construct the invocation command" in { val downloader = BulkAccessUrlDownloader(oneElement) val filepath: Path = downloader.generateJsonManifest(threeElements).unsafeRunSync() - val expected = s"""getm --manifest ${filepath.toString}""" + val expected = s"""timeout 24h getm --manifest ${filepath.toString} -vv""" downloader.generateGetmCommand(filepath) shouldBe expected } @@ -93,15 +97,30 @@ class BulkAccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec w // Unrecognized because of non-zero exit code without an HTTP status. (1, " foobar ", UnrecognizedRetryableDownloadFailure(ExitCode(1))), // Unrecognized because of zero exit status with stderr that does not look like a checksum failure. - (0, """ERROR:getm.cli possibly some words "status_code": 503 words""", UnrecognizedRetryableDownloadFailure(ExitCode(0))), + (0, + """ERROR:getm.cli possibly some words "status_code": 503 words""", + UnrecognizedRetryableDownloadFailure(ExitCode(0)) + ), // Recognized because of non-zero exit status and an HTTP status. - (1, """ERROR:getm.cli possibly some words "status_code": 503 words""", RecognizedRetryableDownloadFailure(ExitCode(1))), + (1, + """ERROR:getm.cli possibly some words "status_code": 503 words""", + RecognizedRetryableDownloadFailure(ExitCode(1)) + ), // Recognized because of non-zero exit status and an HTTP status. - (1, """ERROR:getm.cli possibly some words "status_code": 408 more words""", RecognizedRetryableDownloadFailure(ExitCode(1))), + (1, + """ERROR:getm.cli possibly some words "status_code": 408 more words""", + RecognizedRetryableDownloadFailure(ExitCode(1)) + ), // Recognized and non-retryable because of non-zero exit status and 404 HTTP status. - (1, """ERROR:getm.cli possibly some words "status_code": 404 even more words""", FatalDownloadFailure(ExitCode(1))), + (1, + """ERROR:getm.cli possibly some words "status_code": 404 even more words""", + FatalDownloadFailure(ExitCode(1)) + ), // Unrecognized because of zero exit status and 404 HTTP status. - (0, """ERROR:getm.cli possibly some words "status_code": 404 even more words""", UnrecognizedRetryableDownloadFailure(ExitCode(0))), + (0, + """ERROR:getm.cli possibly some words "status_code": 404 even more words""", + UnrecognizedRetryableDownloadFailure(ExitCode(0)) + ) ) val bulkDownloader = BulkAccessUrlDownloader(null) diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/GetmChecksumSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/GetmChecksumSpec.scala index 69c063ff616..a8ac76fa6c0 100644 --- a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/GetmChecksumSpec.scala +++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/GetmChecksumSpec.scala @@ -17,9 +17,15 @@ class GetmChecksumSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matcher (Option(Map("something weird" -> "012345", "md5" -> "abcdefg")), "https://whatever", Md5("abcdefg")), (Option(Map("something weird" -> "abcdefg", "crc32c" -> "012345")), "https://whatever", Crc32c("012345")), (Option(Map("etag" -> "abcdefg", "crc32c" -> "012345")), "https://whatever", Crc32c("012345")), - (Option(Map("etag" -> "abcdefg", "something weird" -> "012345")), "https://whatever", Unsupported("etag, something weird")), - (Option(Map("etag" -> "abcdefg", "something weird" -> "012345")), "https://whatever.s3.amazonaws.com/foo", AwsEtag("abcdefg")), - (None, "https://whatever.s3.amazonaws.com/foo", Null), + (Option(Map("etag" -> "abcdefg", "something weird" -> "012345")), + "https://whatever", + Unsupported("etag, something weird") + ), + (Option(Map("etag" -> "abcdefg", "something weird" -> "012345")), + "https://whatever.s3.amazonaws.com/foo", + AwsEtag("abcdefg") + ), + (None, "https://whatever.s3.amazonaws.com/foo", Null) ) forAll(results) { (hashes, url, expected) => @@ -31,14 +37,23 @@ class GetmChecksumSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matcher val results = Table( ("description", "algorithm", "expected"), ("md5 hex", Md5("abcdef"), "--checksum-algorithm 'md5' --checksum abcdef".validNel), - ("md5 base64", Md5("cR84lXY1y17c3q7/7riLEA=="), "Invalid checksum value, expected hex but got: cR84lXY1y17c3q7/7riLEA==".invalidNel), - ("md5 gibberish", Md5("what is this???"), "Invalid checksum value, expected hex but got: what is this???".invalidNel), + ("md5 base64", + Md5("cR84lXY1y17c3q7/7riLEA=="), + "Invalid checksum value, expected hex but got: cR84lXY1y17c3q7/7riLEA==".invalidNel + ), + ("md5 gibberish", + Md5("what is this???"), + "Invalid checksum value, expected hex but got: what is this???".invalidNel + ), ("crc32c", Crc32c("012345"), "--checksum-algorithm 'gs_crc32c' --checksum ASNF".validNel), ("crc32c gibberish", Crc32c("????"), "Invalid checksum value, expected hex but got: ????".invalidNel), ("AWS ETag", AwsEtag("012345"), "--checksum-algorithm 's3_etag' --checksum 012345".validNel), // Escape checksum values constructed from unvalidated data returned by DRS servers. - ("Unsupported", Unsupported("Robert'); DROP TABLE Students;\n --\\"), raw"--checksum-algorithm 'null' --checksum Robert\'\)\;\ DROP\ TABLE\ Students\;\ --\\".validNel), - ("Null", Null, "--checksum-algorithm 'null' --checksum null".validNel), + ("Unsupported", + Unsupported("Robert'); DROP TABLE Students;\n --\\"), + raw"--checksum-algorithm 'null' --checksum Robert\'\)\;\ DROP\ TABLE\ Students\;\ --\\".validNel + ), + ("Null", Null, "--checksum-algorithm 'null' --checksum null".validNel) ) forAll(results) { (description, algorithm, expected) => @@ -55,7 +70,7 @@ class GetmChecksumSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matcher (" ", "Invalid checksum value, expected hex but got: ".invalidNel), ("myfavoritestring", "Invalid checksum value, expected hex but got: myfavoritestring".invalidNel), (" AbC123 ", "AbC123".validNel), - ("456", "456".validNel), + ("456", "456".validNel) ) forAll(results) { (testString, expected) => diff --git a/cromwell.example.backends/cromwell.examples.conf b/cromwell.example.backends/cromwell.examples.conf index 4ca250d1202..00f99d2ff5a 100644 --- a/cromwell.example.backends/cromwell.examples.conf +++ b/cromwell.example.backends/cromwell.examples.conf @@ -498,6 +498,11 @@ services { # # count against this limit. # metadata-read-row-number-safety-threshold = 1000000 # + # # Remove any UTF-8 mb4 (4 byte) characters from metadata keys in the list. + # # These characters (namely emojis) will cause metadata writing to fail in database collations + # # that do not support 4 byte UTF-8 characters. + # metadata-keys-to-sanitize-utf8mb4 = ["submittedFiles:workflow", "commandLine"] + # # metadata-write-statistics { # # Not strictly necessary since the 'metadata-write-statistics' section itself is enough for statistics to be recorded. # # However, this can be set to 'false' to disable statistics collection without deleting the section. diff --git a/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala index b2a088e681b..18df2ff5244 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala @@ -20,10 +20,10 @@ import scala.concurrent.ExecutionContext class CromwellClient(val cromwellUrl: URL, val apiVersion: String, - val defaultCredentials: Option[HttpCredentials]=None) - (implicit actorSystem: ActorSystem, materializer: ActorMaterializer) { + val defaultCredentials: Option[HttpCredentials] = None +)(implicit actorSystem: ActorSystem, materializer: ActorMaterializer) { - lazy val defaultAuthorization: Option[Authorization] = defaultCredentials.map { Authorization(_) } + lazy val defaultAuthorization: Option[Authorization] = defaultCredentials.map(Authorization(_)) lazy val defaultHeaders: List[HttpHeader] = defaultAuthorization.toList lazy val engineEndpoint = s"$cromwellUrl/engine/$apiVersion" @@ -33,7 +33,7 @@ class CromwellClient(val cromwellUrl: URL, lazy val submitEndpoint = workflowsEndpoint lazy val batchSubmitEndpoint = s"$submitEndpoint/batch" - def describeEndpoint= s"$womtoolEndpoint/describe" + def describeEndpoint = s"$womtoolEndpoint/describe" def queryEndpoint(args: List[(String, String)]): Uri = { val base = s"$workflowsEndpoint/query" @@ -49,11 +49,20 @@ class CromwellClient(val cromwellUrl: URL, def abortEndpoint(workflowId: WorkflowId): Uri = workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "abort") def statusEndpoint(workflowId: WorkflowId): Uri = workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "status") - def metadataEndpoint(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None): Uri = workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "metadata", args) - def outputsEndpoint(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None): Uri = workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "outputs", args) + def metadataEndpoint(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None): Uri = + workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "metadata", args) + def outputsEndpoint(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None): Uri = + workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "outputs", args) def labelsEndpoint(workflowId: WorkflowId): Uri = workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "labels") - def logsEndpoint(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None): Uri = workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "logs", args) - def diffEndpoint(workflowA: WorkflowId, callA: String, indexA: ShardIndex, workflowB: WorkflowId, callB: String, indexB: ShardIndex): String = { + def logsEndpoint(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None): Uri = + workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "logs", args) + def diffEndpoint(workflowA: WorkflowId, + callA: String, + indexA: ShardIndex, + workflowB: WorkflowId, + callB: String, + indexB: ShardIndex + ): String = { def shardParam(aOrB: String, s: ShardIndex) = s.index.map(i => s"&index$aOrB=$i.toString").getOrElse("") s"$workflowsEndpoint/callcaching/diff?workflowA=$workflowA&callA=$callA&workflowB=$workflowB&callB=$callB${shardParam("A", indexA)}${shardParam("B", indexB)}" } @@ -69,40 +78,48 @@ class CromwellClient(val cromwellUrl: URL, import model.WorkflowDescriptionJsonSupport._ import model.CromwellQueryResultJsonSupport._ - def submit(workflow: WorkflowSubmission) - (implicit ec: ExecutionContext): FailureResponseOrT[SubmittedWorkflow] = { + def submit(workflow: WorkflowSubmission)(implicit ec: ExecutionContext): FailureResponseOrT[SubmittedWorkflow] = { val requestEntity = requestEntityForSubmit(workflow) - makeRequest[CromwellStatus](HttpRequest(HttpMethods.POST, submitEndpoint, List.empty[HttpHeader], requestEntity)) map { status => + makeRequest[CromwellStatus]( + HttpRequest(HttpMethods.POST, submitEndpoint, List.empty[HttpHeader], requestEntity) + ) map { status => SubmittedWorkflow(WorkflowId.fromString(status.id), cromwellUrl, workflow) } } - def describe(workflow: WorkflowDescribeRequest) - (implicit ec: ExecutionContext): FailureResponseOrT[WaasDescription] = { + def describe( + workflow: WorkflowDescribeRequest + )(implicit ec: ExecutionContext): FailureResponseOrT[WaasDescription] = { val requestEntity = requestEntityForDescribe(workflow) makeRequest[WaasDescription](HttpRequest(HttpMethods.POST, describeEndpoint, List.empty[HttpHeader], requestEntity)) } - def submitBatch(workflow: WorkflowBatchSubmission) - (implicit ec: ExecutionContext): FailureResponseOrT[List[SubmittedWorkflow]] = { + def submitBatch( + workflow: WorkflowBatchSubmission + )(implicit ec: ExecutionContext): FailureResponseOrT[List[SubmittedWorkflow]] = { import DefaultJsonProtocol._ val requestEntity = requestEntityForSubmit(workflow) // Make a set of submissions that represent the batch (so we can zip with the results later): - val submissionSet = workflow.inputsBatch.map(inputs => WorkflowSingleSubmission( - workflowSource = workflow.workflowSource, - workflowUrl = workflow.workflowUrl, - workflowRoot = workflow.workflowRoot, - workflowType = workflow.workflowType, - workflowTypeVersion = workflow.workflowTypeVersion, - inputsJson = Option(inputs), - options = workflow.options, - labels = workflow.labels, - zippedImports = workflow.zippedImports)) - - makeRequest[List[CromwellStatus]](HttpRequest(HttpMethods.POST, batchSubmitEndpoint, List.empty[HttpHeader], requestEntity)) map { statuses => + val submissionSet = workflow.inputsBatch.map(inputs => + WorkflowSingleSubmission( + workflowSource = workflow.workflowSource, + workflowUrl = workflow.workflowUrl, + workflowRoot = workflow.workflowRoot, + workflowType = workflow.workflowType, + workflowTypeVersion = workflow.workflowTypeVersion, + inputsJson = Option(inputs), + options = workflow.options, + labels = workflow.labels, + zippedImports = workflow.zippedImports + ) + ) + + makeRequest[List[CromwellStatus]]( + HttpRequest(HttpMethods.POST, batchSubmitEndpoint, List.empty[HttpHeader], requestEntity) + ) map { statuses => val zipped = submissionSet.zip(statuses) zipped map { case (submission, status) => SubmittedWorkflow(WorkflowId.fromString(status.id), cromwellUrl, submission) @@ -110,48 +127,42 @@ class CromwellClient(val cromwellUrl: URL, } } - def abort(workflowId: WorkflowId)(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowStatus] = { + def abort(workflowId: WorkflowId)(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowStatus] = simpleRequest[CromwellStatus](uri = abortEndpoint(workflowId), method = HttpMethods.POST) map WorkflowStatus.apply - } - def status(workflowId: WorkflowId)(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowStatus] = { + def status(workflowId: WorkflowId)(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowStatus] = simpleRequest[CromwellStatus](statusEndpoint(workflowId)) map WorkflowStatus.apply - } def metadata(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None, headers: List[HttpHeader] = defaultHeaders - )(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowMetadata] = { - simpleRequest[String](metadataEndpoint(workflowId, args), headers=headers) map WorkflowMetadata - } + )(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowMetadata] = + simpleRequest[String](metadataEndpoint(workflowId, args), headers = headers) map WorkflowMetadata - def outputs(workflowId: WorkflowId, - args: Option[Map[String, List[String]]] = None)(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowOutputs] = { + def outputs(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None)(implicit + ec: ExecutionContext + ): FailureResponseOrT[WorkflowOutputs] = simpleRequest[WorkflowOutputs](outputsEndpoint(workflowId, args)) - } - def labels(workflowId: WorkflowId, - headers: List[HttpHeader] = defaultHeaders) - (implicit ec: ExecutionContext): FailureResponseOrT[WorkflowLabels] = { + def labels(workflowId: WorkflowId, headers: List[HttpHeader] = defaultHeaders)(implicit + ec: ExecutionContext + ): FailureResponseOrT[WorkflowLabels] = simpleRequest[WorkflowLabels](labelsEndpoint(workflowId), headers = headers) - } - def addLabels(workflowId: WorkflowId, - newLabels: List[Label], - headers: List[HttpHeader] = defaultHeaders) - (implicit ec: ExecutionContext): FailureResponseOrT[WorkflowLabels] = { + def addLabels(workflowId: WorkflowId, newLabels: List[Label], headers: List[HttpHeader] = defaultHeaders)(implicit + ec: ExecutionContext + ): FailureResponseOrT[WorkflowLabels] = { val requestEntity = requestEntityForAddLabels(newLabels) makeRequest[WorkflowLabels](HttpRequest(HttpMethods.PATCH, labelsEndpoint(workflowId), headers, requestEntity)) } - def logs(workflowId: WorkflowId, - args: Option[Map[String, List[String]]] = None)(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowMetadata] = { + def logs(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None)(implicit + ec: ExecutionContext + ): FailureResponseOrT[WorkflowMetadata] = simpleRequest[String](logsEndpoint(workflowId, args)) map WorkflowMetadata - } - def query(workflowId: WorkflowId)(implicit ec: ExecutionContext): FailureResponseOrT[CromwellQueryResults] = { + def query(workflowId: WorkflowId)(implicit ec: ExecutionContext): FailureResponseOrT[CromwellQueryResults] = simpleRequest[CromwellQueryResults](queryEndpoint(List(("id", workflowId.id.toString)))) - } def callCacheDiff(workflowA: WorkflowId, callA: String, @@ -159,27 +170,26 @@ class CromwellClient(val cromwellUrl: URL, workflowB: WorkflowId, callB: String, shardIndexB: ShardIndex - )(implicit ec: ExecutionContext): FailureResponseOrT[CallCacheDiff] = { + )(implicit ec: ExecutionContext): FailureResponseOrT[CallCacheDiff] = simpleRequest[CallCacheDiff](diffEndpoint(workflowA, callA, shardIndexA, workflowB, callB, shardIndexB)) - } - def backends(implicit ec: ExecutionContext): FailureResponseOrT[CromwellBackends] = { + def backends(implicit ec: ExecutionContext): FailureResponseOrT[CromwellBackends] = simpleRequest[CromwellBackends](backendsEndpoint) - } - def version(implicit ec: ExecutionContext): FailureResponseOrT[CromwellVersion] = { + def version(implicit ec: ExecutionContext): FailureResponseOrT[CromwellVersion] = simpleRequest[CromwellVersion](versionEndpoint) - } - private [api] def executeRequest(request: HttpRequest, headers: List[HttpHeader]) = Http().singleRequest(request.withHeaders(headers)) + private[api] def executeRequest(request: HttpRequest, headers: List[HttpHeader]) = + Http().singleRequest(request.withHeaders(headers)) /** * * @tparam A The type of response expected. Must be supported by an implicit unmarshaller from ResponseEntity. */ - private def makeRequest[A](request: HttpRequest, headers: List[HttpHeader] = defaultHeaders) - (implicit um: Unmarshaller[ResponseEntity, A], ec: ExecutionContext): - FailureResponseOrT[A] = { + private def makeRequest[A](request: HttpRequest, headers: List[HttpHeader] = defaultHeaders)(implicit + um: Unmarshaller[ResponseEntity, A], + ec: ExecutionContext + ): FailureResponseOrT[A] = { implicit def cs = IO.contextShift(ec) for { response <- executeRequest(request, headers).asFailureResponseOrT @@ -191,11 +201,9 @@ class CromwellClient(val cromwellUrl: URL, private def simpleRequest[A](uri: Uri, method: HttpMethod = HttpMethods.GET, - headers: List[HttpHeader] = defaultHeaders) - (implicit um: Unmarshaller[ResponseEntity, A], - ec: ExecutionContext): FailureResponseOrT[A] = { + headers: List[HttpHeader] = defaultHeaders + )(implicit um: Unmarshaller[ResponseEntity, A], ec: ExecutionContext): FailureResponseOrT[A] = makeRequest[A](HttpRequest(uri = uri, method = method), headers) - } private val decoders = Map( HttpEncodings.gzip -> Gzip, @@ -203,15 +211,14 @@ class CromwellClient(val cromwellUrl: URL, HttpEncodings.identity -> NoCoding ) - private def decodeResponse(response: HttpResponse): IO[HttpResponse] = { + private def decodeResponse(response: HttpResponse): IO[HttpResponse] = decoders.get(response.encoding) map { decoder => IO(decoder.decodeMessage(response)) } getOrElse IO.raiseError(UnsuccessfulRequestException(s"No decoder for ${response.encoding}", response)) - } } object CromwellClient { - final implicit class EnhancedHttpResponse(val response: HttpResponse) extends AnyVal { + implicit final class EnhancedHttpResponse(val response: HttpResponse) extends AnyVal { def toEntity: IO[Unmarshal[ResponseEntity]] = response match { case HttpResponse(_: StatusCodes.Success, _, entity, _) => IO(Unmarshal(entity)) @@ -233,18 +240,17 @@ object CromwellClient { "workflowInputs" -> workflowSubmission.inputsJson, "workflowOptions" -> workflowSubmission.options, "labels" -> workflowSubmission.labels.map(_.toJson.toString) - ) collect { - case (name, Some(source: String)) => - Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source))) + ) collect { case (name, Some(source: String)) => + Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source))) } val zipBodyParts = Map( "workflowDependencies" -> workflowSubmission.zippedImports - ) collect { - case (name, Some(file)) => Multipart.FormData.BodyPart.fromPath(name, MediaTypes.`application/zip`, file.path) + ) collect { case (name, Some(file)) => + Multipart.FormData.BodyPart.fromPath(name, MediaTypes.`application/zip`, file.path) } - val multipartFormData = Multipart.FormData((sourceBodyParts ++ zipBodyParts).toSeq : _*) + val multipartFormData = Multipart.FormData((sourceBodyParts ++ zipBodyParts).toSeq: _*) multipartFormData.toEntity() } @@ -256,12 +262,11 @@ object CromwellClient { "workflowType" -> describeRequest.workflowType, "workflowTypeVersion" -> describeRequest.workflowTypeVersion, "workflowInputs" -> describeRequest.inputsJson - ) collect { - case (name, Some(source: String)) => - Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source))) + ) collect { case (name, Some(source: String)) => + Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source))) } - val multipartFormData = Multipart.FormData(sourceBodyParts.toSeq : _*) + val multipartFormData = Multipart.FormData(sourceBodyParts.toSeq: _*) multipartFormData.toEntity() } @@ -273,12 +278,16 @@ object CromwellClient { /** * @param args an optional map of HTTP arguments which will be added to the URL */ - private [api] def workflowSpecificGetEndpoint(submitEndpoint: String, workflowId: WorkflowId, endpoint: String, args: Option[Map[String, List[String]]] = None) = { + private[api] def workflowSpecificGetEndpoint(submitEndpoint: String, + workflowId: WorkflowId, + endpoint: String, + args: Option[Map[String, List[String]]] = None + ) = { val url = s"$submitEndpoint/$workflowId/$endpoint" val queryBuilder = Uri.Query.newBuilder - args.getOrElse(Map.empty).foreach({ - case (key, l) => l.foreach(v => queryBuilder.+=(key -> v)) - }) + args.getOrElse(Map.empty).foreach { case (key, l) => + l.foreach(v => queryBuilder.+=(key -> v)) + } val queryResult = queryBuilder.result() Uri(url).withQuery(queryResult) } diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/CallCacheDiff.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/CallCacheDiff.scala index fa4e7fb9187..64cafc2b611 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/CallCacheDiff.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/CallCacheDiff.scala @@ -4,9 +4,17 @@ import ShardIndexFormatter._ import WorkflowIdJsonFormatter._ import spray.json.DefaultJsonProtocol -case class CallCacheDiffCallDescription(executionStatus: String, allowResultReuse: Boolean, callFqn: String, jobIndex: ShardIndex, workflowId: WorkflowId) +case class CallCacheDiffCallDescription(executionStatus: String, + allowResultReuse: Boolean, + callFqn: String, + jobIndex: ShardIndex, + workflowId: WorkflowId +) case class HashDifference(hashKey: String, callA: Option[String], callB: Option[String]) -case class CallCacheDiff(callA: CallCacheDiffCallDescription, callB: CallCacheDiffCallDescription, hashDifferential: List[HashDifference]) +case class CallCacheDiff(callA: CallCacheDiffCallDescription, + callB: CallCacheDiffCallDescription, + hashDifferential: List[HashDifference] +) object CallCacheDiffJsonSupport extends DefaultJsonProtocol { implicit val CallCacheDiffCallDescriptionFormat = jsonFormat5(CallCacheDiffCallDescription) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellQueryResult.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellQueryResult.scala index 4a54cb58469..658ea72489f 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellQueryResult.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellQueryResult.scala @@ -7,7 +7,13 @@ import cromwell.api.model.WorkflowStatusJsonFormatter._ case class CromwellQueryResults(results: Seq[CromwellQueryResult]) -case class CromwellQueryResult(name: Option[String], id: WorkflowId, status: WorkflowStatus, end: Option[OffsetDateTime], start: Option[OffsetDateTime], metadataArchiveStatus: String) +case class CromwellQueryResult(name: Option[String], + id: WorkflowId, + status: WorkflowStatus, + end: Option[OffsetDateTime], + start: Option[OffsetDateTime], + metadataArchiveStatus: String +) object CromwellQueryResultJsonSupport extends DefaultJsonProtocol { implicit val CromwellQueryResultJsonFormat = jsonFormat6(CromwellQueryResult) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/Label.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/Label.scala index fd9d88d2177..0d111626f84 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/Label.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/Label.scala @@ -5,7 +5,7 @@ import scala.language.postfixOps object LabelsJsonFormatter extends DefaultJsonProtocol { implicit object LabelJsonFormat extends RootJsonFormat[List[Label]] { - def write(l: List[Label]) = JsObject(l map { label => label.key -> JsString(label.value)} :_* ) + def write(l: List[Label]) = JsObject(l map { label => label.key -> JsString(label.value) }: _*) def read(value: JsValue) = value.asJsObject.fields map { case (k, JsString(v)) => Label(k, v) case other => throw new UnsupportedOperationException(s"Cannot deserialize $other to a Label") diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/TimeUtil.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/TimeUtil.scala index 1688da92ff7..0db0f1c8f66 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/TimeUtil.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/TimeUtil.scala @@ -4,6 +4,7 @@ import java.time.format.DateTimeFormatter import java.time.{OffsetDateTime, ZoneOffset} object TimeUtil { + /** * Instead of "one of" the valid ISO-8601 formats, standardize on this one: * https://github.com/openjdk/jdk/blob/jdk8-b120/jdk/src/share/classes/java/time/OffsetDateTime.java#L1886 @@ -11,12 +12,15 @@ object TimeUtil { private val Iso8601MillisecondsFormat = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSSXXXXX") implicit class EnhancedOffsetDateTime(val offsetDateTime: OffsetDateTime) extends AnyVal { + /** * Discards the original timezone and shifts the time to UTC, then returns the ISO-8601 formatted string with * exactly three digits of milliseconds. */ - def toUtcMilliString: String = Option(offsetDateTime).map( - _.atZoneSameInstant(ZoneOffset.UTC).format(Iso8601MillisecondsFormat) - ).orNull + def toUtcMilliString: String = Option(offsetDateTime) + .map( + _.atZoneSameInstant(ZoneOffset.UTC).format(Iso8601MillisecondsFormat) + ) + .orNull } } diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WaasDescription.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WaasDescription.scala index e1f679a7b35..f83ca95fb9a 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/WaasDescription.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WaasDescription.scala @@ -22,17 +22,20 @@ final case class WaasDescription(valid: Boolean, importedDescriptorTypes: List[WaasWorkflowDescriptorType], meta: JsObject, parameterMeta: JsObject, - isRunnableWorkflow: Boolean) + isRunnableWorkflow: Boolean +) final case class WaasDescriptionInputDefinition(name: String, valueType: WaasDescriptionWomType, optional: Option[Boolean], default: Option[JsValue], - typeDisplayName: String) + typeDisplayName: String +) final case class WaasDescriptionOutputDefinition(name: String, valueType: WaasDescriptionWomType, - typeDisplayName: String) + typeDisplayName: String +) final case class WaasDescriptionWomType(typeName: String) final case class WaasWorkflowDescriptorType(descriptorType: Option[String], descriptorTypeVersion: Option[String]) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala index 9ab7ea45ec4..1a46e2117f9 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala @@ -4,4 +4,5 @@ final case class WorkflowDescribeRequest(workflowSource: Option[String], workflowUrl: Option[String], workflowType: Option[String], workflowTypeVersion: Option[String], - inputsJson: Option[String]) + inputsJson: Option[String] +) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala index f52495136c3..20c3a558790 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala @@ -30,4 +30,3 @@ object WorkflowIdJsonFormatter extends DefaultJsonProtocol { } } } - diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala index 6da1282d2da..70f9aa1efeb 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala @@ -25,7 +25,7 @@ case object Running extends NonTerminalStatus case object Aborting extends NonTerminalStatus object WorkflowStatus { - def apply(status: String): WorkflowStatus = { + def apply(status: String): WorkflowStatus = status match { case "Submitted" => Submitted case "Running" => Running @@ -35,7 +35,6 @@ object WorkflowStatus { case "Succeeded" => Succeeded case bad => throw new IllegalArgumentException(s"No such status: $bad") } - } def apply(workflowStatus: CromwellStatus): WorkflowStatus = apply(workflowStatus.status) } diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala index 5f59368de13..e22d3e93e2e 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala @@ -22,7 +22,8 @@ final case class WorkflowSingleSubmission(workflowSource: Option[String], inputsJson: Option[String], options: Option[String], labels: Option[List[Label]], - zippedImports: Option[File]) extends WorkflowSubmission + zippedImports: Option[File] +) extends WorkflowSubmission final case class WorkflowBatchSubmission(workflowSource: Option[String], workflowUrl: Option[String], @@ -32,7 +33,8 @@ final case class WorkflowBatchSubmission(workflowSource: Option[String], inputsBatch: List[String], options: Option[String], labels: Option[List[Label]], - zippedImports: Option[File]) extends WorkflowSubmission { + zippedImports: Option[File] +) extends WorkflowSubmission { override val inputsJson: Option[String] = Option(inputsBatch.mkString(start = "[", sep = ",", end = "]")) } diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/package.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/package.scala index 9067ce7157a..ccd75efac1f 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/package.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/package.scala @@ -45,7 +45,7 @@ package object model { } implicit class EnhancedFailureResponseOrHttpResponseT(val responseIoT: FailureResponseOrT[HttpResponse]) - extends AnyVal { + extends AnyVal { def asHttpResponse: Future[HttpResponse] = { val io = responseIoT.value map { case Left(response) => response @@ -55,13 +55,14 @@ package object model { } } - implicit class EnhancedFailureResponseOrT[SuccessType](val responseIoT: FailureResponseOrT[SuccessType]) extends AnyVal { - final def timeout(duration: FiniteDuration) - (implicit timer: Timer[IO], cs: ContextShift[IO]): FailureResponseOrT[SuccessType] = { + implicit class EnhancedFailureResponseOrT[SuccessType](val responseIoT: FailureResponseOrT[SuccessType]) + extends AnyVal { + final def timeout( + duration: FiniteDuration + )(implicit timer: Timer[IO], cs: ContextShift[IO]): FailureResponseOrT[SuccessType] = EitherT(responseIoT.value.timeout(duration)) - } - def asIo(implicit materializer: ActorMaterializer, executionContext: ExecutionContext): IO[SuccessType] = { + def asIo(implicit materializer: ActorMaterializer, executionContext: ExecutionContext): IO[SuccessType] = responseIoT.value flatMap { case Left(response) => implicit def cs = IO.contextShift(executionContext) @@ -72,15 +73,13 @@ package object model { }) case Right(a) => IO.pure(a) } - } /** * Transforms the IO error from one type to another. */ def mapErrorWith(mapper: Throwable => IO[Nothing]): FailureResponseOrT[SuccessType] = { - def handleErrorIo[A](ioIn: IO[A]): IO[A] = { + def handleErrorIo[A](ioIn: IO[A]): IO[A] = ioIn handleErrorWith mapper - } responseIoT.mapK(FunctionK.lift(handleErrorIo)) } diff --git a/cromwellApiClient/src/test/scala/cromwell/api/CromwellClientSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/CromwellClientSpec.scala index 1bfecebc0d5..ab653523801 100644 --- a/cromwellApiClient/src/test/scala/cromwell/api/CromwellClientSpec.scala +++ b/cromwellApiClient/src/test/scala/cromwell/api/CromwellClientSpec.scala @@ -11,7 +11,6 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.flatspec.AsyncFlatSpec import org.scalatest.matchers.should.Matchers - class CromwellClientSpec extends AsyncFlatSpec with BeforeAndAfterAll with Matchers with TableDrivenPropertyChecks { behavior of "CromwellClient" @@ -40,86 +39,81 @@ class CromwellClientSpec extends AsyncFlatSpec with BeforeAndAfterAll with Match private val okRequestEntityTests = Table( ("description", "workflowSubmission", "expectedJsons", "expectedFiles"), - ("submit a wdl", - WorkflowSingleSubmission(Option("wdl"), None, None, None, None, None, None, None, None), - Map("workflowSource" -> "wdl"), - Map() + WorkflowSingleSubmission(Option("wdl"), None, None, None, None, None, None, None, None), + Map("workflowSource" -> "wdl"), + Map() ), - ("batch submit a wdl", - WorkflowBatchSubmission(Option("wdl"), None, None, None, None, List(), None, None, None), - Map("workflowSource" -> "wdl", "workflowInputs" -> "[]"), - Map() + WorkflowBatchSubmission(Option("wdl"), None, None, None, None, List(), None, None, None), + Map("workflowSource" -> "wdl", "workflowInputs" -> "[]"), + Map() ), - ("submit a wdl with data", - WorkflowSingleSubmission( - Option("wdl"), - None, - None, - Option("wfType"), - Option("wfTypeVersion"), - Option("inputsJson"), - Option("optionsJson"), - Option(List(Label("labelKey", "labelValue"))), - Option(tempFile) - ), - Map( - "workflowSource" -> "wdl", - "workflowType" -> "wfType", - "workflowTypeVersion" -> "wfTypeVersion", - "workflowInputs" -> "inputsJson", - "workflowOptions" -> "optionsJson", - "labels" -> """{"labelKey":"labelValue"}""" - ), - Map("workflowDependencies" -> tempFile) + WorkflowSingleSubmission( + Option("wdl"), + None, + None, + Option("wfType"), + Option("wfTypeVersion"), + Option("inputsJson"), + Option("optionsJson"), + Option(List(Label("labelKey", "labelValue"))), + Option(tempFile) + ), + Map( + "workflowSource" -> "wdl", + "workflowType" -> "wfType", + "workflowTypeVersion" -> "wfTypeVersion", + "workflowInputs" -> "inputsJson", + "workflowOptions" -> "optionsJson", + "labels" -> """{"labelKey":"labelValue"}""" + ), + Map("workflowDependencies" -> tempFile) ), - ("submit a wdl using workflow url", - WorkflowSingleSubmission( - None, - Option("https://link-to-url"), - None, - Option("wfType"), - Option("wfTypeVersion"), - Option("inputsJson"), - Option("optionsJson"), - Option(List(Label("labelKey", "labelValue"))), - Option(tempFile) - ), - Map( - "workflowUrl" -> "https://link-to-url", - "workflowType" -> "wfType", - "workflowTypeVersion" -> "wfTypeVersion", - "workflowInputs" -> "inputsJson", - "workflowOptions" -> "optionsJson", - "labels" -> """{"labelKey":"labelValue"}""" - ), - Map("workflowDependencies" -> tempFile) + WorkflowSingleSubmission( + None, + Option("https://link-to-url"), + None, + Option("wfType"), + Option("wfTypeVersion"), + Option("inputsJson"), + Option("optionsJson"), + Option(List(Label("labelKey", "labelValue"))), + Option(tempFile) + ), + Map( + "workflowUrl" -> "https://link-to-url", + "workflowType" -> "wfType", + "workflowTypeVersion" -> "wfTypeVersion", + "workflowInputs" -> "inputsJson", + "workflowOptions" -> "optionsJson", + "labels" -> """{"labelKey":"labelValue"}""" + ), + Map("workflowDependencies" -> tempFile) ), - ("batch submit a wdl with data", - WorkflowBatchSubmission( - Option("wdl"), - None, - None, - Option("wfType"), - Option("wfTypeVersion"), - List("inputsJson1", "inputsJson2"), - Option("optionsJson"), - Option(List(Label("labelKey", "labelValue"))), - Option(tempFile) - ), - Map( - "workflowSource" -> "wdl", - "workflowType" -> "wfType", - "workflowTypeVersion" -> "wfTypeVersion", - "workflowInputs" -> "[inputsJson1,inputsJson2]", - "workflowOptions" -> "optionsJson", - "labels" -> """{"labelKey":"labelValue"}""" - ), - Map("workflowDependencies" -> tempFile) + WorkflowBatchSubmission( + Option("wdl"), + None, + None, + Option("wfType"), + Option("wfTypeVersion"), + List("inputsJson1", "inputsJson2"), + Option("optionsJson"), + Option(List(Label("labelKey", "labelValue"))), + Option(tempFile) + ), + Map( + "workflowSource" -> "wdl", + "workflowType" -> "wfType", + "workflowTypeVersion" -> "wfTypeVersion", + "workflowInputs" -> "[inputsJson1,inputsJson2]", + "workflowOptions" -> "optionsJson", + "labels" -> """{"labelKey":"labelValue"}""" + ), + Map("workflowDependencies" -> tempFile) ) ) @@ -132,24 +126,22 @@ class CromwellClientSpec extends AsyncFlatSpec with BeforeAndAfterAll with Match contentType.mediaType.isMultipart should be(true) val boundary = contentType.mediaType.params("boundary") - val expectedJsonChunks = expectedJsons map { - case (chunkKey, chunkValue) => - s"""|--$boundary - |Content-Type: application/json - |Content-Disposition: form-data; name="$chunkKey" - | - |$chunkValue - |""".stripMargin.replace("\n", "\r\n").trim + val expectedJsonChunks = expectedJsons map { case (chunkKey, chunkValue) => + s"""|--$boundary + |Content-Type: application/json + |Content-Disposition: form-data; name="$chunkKey" + | + |$chunkValue + |""".stripMargin.replace("\n", "\r\n").trim } - val expectedFileChunks = expectedFiles.iterator map { - case (chunkKey, chunkFile) => - s"""|--$boundary - |Content-Type: application/zip - |Content-Disposition: form-data; filename="${chunkFile.name}"; name="$chunkKey" - |""".stripMargin.replace("\n", "\r\n").trim + val expectedFileChunks = expectedFiles.iterator map { case (chunkKey, chunkFile) => + s"""|--$boundary + |Content-Type: application/zip + |Content-Disposition: form-data; filename="${chunkFile.name}"; name="$chunkKey" + |""".stripMargin.replace("\n", "\r\n").trim } - val expectedFileContents = expectedFiles.iterator map { - case (_, chunkFile) => chunkFile.contentAsString + val expectedFileContents = expectedFiles.iterator map { case (_, chunkFile) => + chunkFile.contentAsString } val boundaryEnd = s"--$boundary--" diff --git a/cromwellApiClient/src/test/scala/cromwell/api/CromwellResponseFailedSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/CromwellResponseFailedSpec.scala index 06841bfad71..34e901857b7 100644 --- a/cromwellApiClient/src/test/scala/cromwell/api/CromwellResponseFailedSpec.scala +++ b/cromwellApiClient/src/test/scala/cromwell/api/CromwellResponseFailedSpec.scala @@ -15,15 +15,18 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.duration._ import scala.concurrent.{Await, Future} -class CromwellResponseFailedSpec extends TestKit(ActorSystem("CromwellResponseFailedSpec")) - with AsyncFlatSpecLike with Matchers with BeforeAndAfterAll { +class CromwellResponseFailedSpec + extends TestKit(ActorSystem("CromwellResponseFailedSpec")) + with AsyncFlatSpecLike + with Matchers + with BeforeAndAfterAll { override def afterAll(): Unit = { Await.ready(system.terminate(), 10.seconds.dilated) super.afterAll() } - + implicit val materializer: ActorMaterializer = ActorMaterializer() - + "CromwellAPIClient" should "fail the Future if the HttpResponse is unsuccessful" in { val errorMessage = """|{ @@ -32,14 +35,15 @@ class CromwellResponseFailedSpec extends TestKit(ActorSystem("CromwellResponseFa |} |""".stripMargin.trim val client = new CromwellClient(new URL("http://fakeurl"), "v1") { - override def executeRequest(request: HttpRequest, headers: List[HttpHeader]): Future[HttpResponse] = Future.successful( - new HttpResponse( - StatusCodes.ServiceUnavailable, - List.empty[HttpHeader], - HttpEntity(ContentTypes.`application/json`, errorMessage), - HttpProtocols.`HTTP/1.1` + override def executeRequest(request: HttpRequest, headers: List[HttpHeader]): Future[HttpResponse] = + Future.successful( + new HttpResponse( + StatusCodes.ServiceUnavailable, + List.empty[HttpHeader], + HttpEntity(ContentTypes.`application/json`, errorMessage), + HttpProtocols.`HTTP/1.1` + ) ) - ) } for { diff --git a/cromwellApiClient/src/test/scala/cromwell/api/model/CromwellQueryResultJsonFormatterSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/model/CromwellQueryResultJsonFormatterSpec.scala index 9ac224970bf..4eb0a450aca 100644 --- a/cromwellApiClient/src/test/scala/cromwell/api/model/CromwellQueryResultJsonFormatterSpec.scala +++ b/cromwellApiClient/src/test/scala/cromwell/api/model/CromwellQueryResultJsonFormatterSpec.scala @@ -7,50 +7,51 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import spray.json._ - class CromwellQueryResultJsonFormatterSpec extends AnyFlatSpec with Matchers { behavior of "CromwellQueryResultJsonFormat" - val sampleQueryResult = CromwellQueryResults(results = List( - CromwellQueryResult( - Option("switcheroo"), - WorkflowId.fromString("bee51f36-396d-4e22-8a81-33dedff66bf6"), - Failed, - Option(OffsetDateTime.parse("2017-07-24T14:44:34.010Z")), + val sampleQueryResult = CromwellQueryResults(results = + List( + CromwellQueryResult( + Option("switcheroo"), + WorkflowId.fromString("bee51f36-396d-4e22-8a81-33dedff66bf6"), + Failed, + Option(OffsetDateTime.parse("2017-07-24T14:44:34.010Z")), Option(OffsetDateTime.parse("2017-07-24T14:44:33.227Z")), - "Archived" - ), - CromwellQueryResult( - Option("switcheroo"), - WorkflowId.fromString("0071495e-39eb-478e-bc98-8614b986c91e"), - Succeeded, + "Archived" + ), + CromwellQueryResult( + Option("switcheroo"), + WorkflowId.fromString("0071495e-39eb-478e-bc98-8614b986c91e"), + Succeeded, Option(OffsetDateTime.parse("2017-07-24T15:06:45.940Z")), - Option(OffsetDateTime.parse("2017-07-24T15:04:54.372Z")), - "Unarchived" - ), - )) - - val sampleJson = """|{ - | "results": [ - | { - | "name": "switcheroo", - | "id": "bee51f36-396d-4e22-8a81-33dedff66bf6", - | "status": "Failed", - | "end": "2017-07-24T14:44:34.010Z", - | "start": "2017-07-24T14:44:33.227Z", - | "metadataArchiveStatus": "Archived" - | }, - | { - | "name": "switcheroo", - | "id": "0071495e-39eb-478e-bc98-8614b986c91e", - | "status": "Succeeded", - | "end": "2017-07-24T15:06:45.940Z", - | "start": "2017-07-24T15:04:54.372Z", - | "metadataArchiveStatus": "Unarchived" - | } - | ] - |}""".stripMargin.parseJson.asJsObject + Option(OffsetDateTime.parse("2017-07-24T15:04:54.372Z")), + "Unarchived" + ) + ) + ) + + val sampleJson = """|{ + | "results": [ + | { + | "name": "switcheroo", + | "id": "bee51f36-396d-4e22-8a81-33dedff66bf6", + | "status": "Failed", + | "end": "2017-07-24T14:44:34.010Z", + | "start": "2017-07-24T14:44:33.227Z", + | "metadataArchiveStatus": "Archived" + | }, + | { + | "name": "switcheroo", + | "id": "0071495e-39eb-478e-bc98-8614b986c91e", + | "status": "Succeeded", + | "end": "2017-07-24T15:06:45.940Z", + | "start": "2017-07-24T15:04:54.372Z", + | "metadataArchiveStatus": "Unarchived" + | } + | ] + |}""".stripMargin.parseJson.asJsObject it should "write a query result as a structured JsObject" in { sampleQueryResult.toJson shouldEqual sampleJson diff --git a/cromwellApiClient/src/test/scala/cromwell/api/model/LabelsJsonFormatterSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/model/LabelsJsonFormatterSpec.scala index c1f1e89c08a..ab0c7e08cbc 100644 --- a/cromwellApiClient/src/test/scala/cromwell/api/model/LabelsJsonFormatterSpec.scala +++ b/cromwellApiClient/src/test/scala/cromwell/api/model/LabelsJsonFormatterSpec.scala @@ -4,18 +4,17 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import spray.json._ - class LabelsJsonFormatterSpec extends AnyFlatSpec with Matchers { import cromwell.api.model.LabelsJsonFormatter._ behavior of "WdlValueJsonFormat" val sampleLabels = List(Label("key-1", "value-1"), Label("key-2", "value-2"), Label("key-3", "value-3")) - val sampleJson = """|{ - | "key-1":"value-1", - | "key-2":"value-2", - | "key-3":"value-3" - |}""".stripMargin.parseJson.asJsObject + val sampleJson = """|{ + | "key-1":"value-1", + | "key-2":"value-2", + | "key-3":"value-3" + |}""".stripMargin.parseJson.asJsObject it should "write a Label as a structured JsObject" in { val label = List(Label("test-key", "test-value")) diff --git a/cromwellApiClient/src/test/scala/cromwell/api/model/WaasDescriptionJsonSupportSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/model/WaasDescriptionJsonSupportSpec.scala index 5bad5f8b21f..7e328ee4725 100644 --- a/cromwellApiClient/src/test/scala/cromwell/api/model/WaasDescriptionJsonSupportSpec.scala +++ b/cromwellApiClient/src/test/scala/cromwell/api/model/WaasDescriptionJsonSupportSpec.scala @@ -3,7 +3,6 @@ package cromwell.api.model import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class WaasDescriptionJsonSupportSpec extends AnyFlatSpec with Matchers { it should "deserialize invalid result JSON" in { @@ -25,7 +24,6 @@ class WaasDescriptionJsonSupportSpec extends AnyFlatSpec with Matchers { | "isRunnableWorkflow": false |}""".stripMargin - import cromwell.api.model.WorkflowDescriptionJsonSupport._ import spray.json._ @@ -33,7 +31,11 @@ class WaasDescriptionJsonSupportSpec extends AnyFlatSpec with Matchers { val deserialized = jsonAst.convertTo[WaasDescription] deserialized.valid should be(false) - deserialized.errors should be(List("""Failed to import workflow sub_workflow_aborted_import.wdl.:\nBad import sub_workflow_aborted_import.wdl: Failed to resolve 'sub_workflow_aborted_import.wdl' using resolver: 'http importer (no 'relative-to' origin)' (reason 1 of 1): Relative path""")) + deserialized.errors should be( + List( + """Failed to import workflow sub_workflow_aborted_import.wdl.:\nBad import sub_workflow_aborted_import.wdl: Failed to resolve 'sub_workflow_aborted_import.wdl' using resolver: 'http importer (no 'relative-to' origin)' (reason 1 of 1): Relative path""" + ) + ) deserialized.validWorkflow should be(false) deserialized.name should be("") deserialized.inputs should be(List.empty) diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml index 49d568766b3..cecc4e7b709 100644 --- a/database/migration/src/main/resources/changelog.xml +++ b/database/migration/src/main/resources/changelog.xml @@ -90,6 +90,12 @@ + + diff --git a/database/migration/src/main/resources/changesets/set_table_role.xml b/database/migration/src/main/resources/changesets/set_table_role.xml new file mode 100644 index 00000000000..297cf041d95 --- /dev/null +++ b/database/migration/src/main/resources/changesets/set_table_role.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + SELECT count(1) + FROM pg_roles + where '${engineSharedCromwellDbRole}' != '' and pg_roles.rolname = '${engineSharedCromwellDbRole}'; + + + + ALTER TABLE "CALL_CACHING_AGGREGATION_ENTRY" OWNER TO ${engineSharedCromwellDbRole}; + ALTER TABLE "CALL_CACHING_DETRITUS_ENTRY" OWNER TO ${engineSharedCromwellDbRole}; + ALTER TABLE "CALL_CACHING_ENTRY" OWNER TO ${engineSharedCromwellDbRole}; + ALTER TABLE "CALL_CACHING_HASH_ENTRY" OWNER TO ${engineSharedCromwellDbRole}; + ALTER TABLE "CALL_CACHING_SIMPLETON_ENTRY" OWNER TO ${engineSharedCromwellDbRole}; + ALTER TABLE "DOCKER_HASH_STORE_ENTRY" OWNER TO ${engineSharedCromwellDbRole}; + ALTER TABLE "JOB_KEY_VALUE_ENTRY" OWNER TO ${engineSharedCromwellDbRole}; + ALTER TABLE "JOB_STORE_ENTRY" OWNER TO ${engineSharedCromwellDbRole}; + ALTER TABLE "JOB_STORE_SIMPLETON_ENTRY" OWNER TO ${engineSharedCromwellDbRole}; + ALTER TABLE "SUB_WORKFLOW_STORE_ENTRY" OWNER TO ${engineSharedCromwellDbRole}; + ALTER TABLE "WORKFLOW_STORE_ENTRY" OWNER TO ${engineSharedCromwellDbRole}; + ALTER TABLE "databasechangelog" OWNER TO ${engineSharedCromwellDbRole}; + ALTER TABLE "databasechangeloglock" OWNER TO ${engineSharedCromwellDbRole}; + + + + diff --git a/database/migration/src/main/resources/metadata_changesets/set_table_role.xml b/database/migration/src/main/resources/metadata_changesets/set_table_role.xml index 48a56bb6091..67219feeca3 100644 --- a/database/migration/src/main/resources/metadata_changesets/set_table_role.xml +++ b/database/migration/src/main/resources/metadata_changesets/set_table_role.xml @@ -11,7 +11,7 @@ This changeset will be applied whenever the 'sharedCromwellDbRole' property is set. It runs every time to ensure the role is set correctly after all other changesets. --> - + diff --git a/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala b/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala index cea987fc89a..4f493492d7e 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala @@ -11,15 +11,16 @@ import wom.types.{WomPrimitiveType, WomType} import scala.util.Try -private [migration] object WdlTransformation { +private[migration] object WdlTransformation { def inflate(value: String): Try[String] = Try { Option(value) match { - case Some(v) => IOUtils.toString(new GZIPInputStream(new ByteArrayInputStream(Base64.decodeBase64(v))), Charset.defaultCharset) + case Some(v) => + IOUtils.toString(new GZIPInputStream(new ByteArrayInputStream(Base64.decodeBase64(v))), Charset.defaultCharset) case None => null } - } recover { - case _: IOException => value + } recover { case _: IOException => + value } def coerceStringToWdl(wdlString: String, womType: WomType) = womType match { diff --git a/database/migration/src/main/scala/cromwell/database/migration/custom/BatchedTaskChange.scala b/database/migration/src/main/scala/cromwell/database/migration/custom/BatchedTaskChange.scala index 49db7e4446f..4d5eb16de18 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/custom/BatchedTaskChange.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/custom/BatchedTaskChange.scala @@ -9,6 +9,7 @@ import liquibase.exception.CustomChangeException * Runs a migration as a series of batches. */ trait BatchedTaskChange extends MigrationTaskChange { + /** * Returns sql to retrieve the maximum primary key for the table. * @@ -80,14 +81,16 @@ trait BatchedTaskChange extends MigrationTaskChange { override def migrate(connection: JdbcConnection): Unit = { - logger.info(s"Running migration $migrationName with a read batch size of " + - s"$readBatchSize and a write batch size of $writeBatchSize") + logger.info( + s"Running migration $migrationName with a read batch size of " + + s"$readBatchSize and a write batch size of $writeBatchSize" + ) /* - * Keep count of the size of the batch. - * - * @see writeBatchSize - */ + * Keep count of the size of the batch. + * + * @see writeBatchSize + */ var batchMigrationCounter: Int = 0 val readCount = getReadCount(connection) @@ -101,25 +104,24 @@ trait BatchedTaskChange extends MigrationTaskChange { val paginator = new QueryPaginator(readBatchStatement, readBatchSize, readCount) // Loop over pages - paginator.zipWithIndex foreach { - case (resultBatch, page) => - // Loop over rows in page - new ResultSetIterator(resultBatch) foreach { row => - batchMigrationCounter += migrateBatchRow(row, migrateBatchStatements) - // batchMigrationCounter can actually be bigger than writeBatchSize as wdlValues are processed atomically, - // so this is a best effort - if (batchMigrationCounter >= writeBatchSize) { - migrateBatchStatements.foreach(_.executeBatch()) - connection.commit() - batchMigrationCounter = 0 - } + paginator.zipWithIndex foreach { case (resultBatch, page) => + // Loop over rows in page + new ResultSetIterator(resultBatch) foreach { row => + batchMigrationCounter += migrateBatchRow(row, migrateBatchStatements) + // batchMigrationCounter can actually be bigger than writeBatchSize as wdlValues are processed atomically, + // so this is a best effort + if (batchMigrationCounter >= writeBatchSize) { + migrateBatchStatements.foreach(_.executeBatch()) + connection.commit() + batchMigrationCounter = 0 } + } - resultBatch.close() + resultBatch.close() - val progress = Math.min((page + 1) * 100 / pageCount, 100) - val progressMessage = s"[$migrationName] $progress%" - logger.info(progressMessage) + val progress = Math.min((page + 1) * 100 / pageCount, 100) + val progressMessage = s"[$migrationName] $progress%" + logger.info(progressMessage) } if (batchMigrationCounter != 0) { diff --git a/database/migration/src/main/scala/cromwell/database/migration/custom/MigrationTaskChange.scala b/database/migration/src/main/scala/cromwell/database/migration/custom/MigrationTaskChange.scala index 4fa7935e632..f02d0c46d28 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/custom/MigrationTaskChange.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/custom/MigrationTaskChange.scala @@ -24,7 +24,7 @@ trait MigrationTaskChange extends CustomTaskChange with LazyLogging { */ def migrate(connection: JdbcConnection): Unit - override def execute(database: Database): Unit = { + override def execute(database: Database): Unit = try { val dbConn = database.getConnection.asInstanceOf[JdbcConnection] val autoCommit = dbConn.getAutoCommit @@ -36,7 +36,6 @@ trait MigrationTaskChange extends CustomTaskChange with LazyLogging { case exception: Exception => throw new CustomChangeException(s"Could not apply migration script for $migrationName", exception) } - } override def setUp() = {} diff --git a/database/migration/src/main/scala/cromwell/database/migration/custom/QueryPaginator.scala b/database/migration/src/main/scala/cromwell/database/migration/custom/QueryPaginator.scala index 0e6514043c3..cb85ffdbcac 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/custom/QueryPaginator.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/custom/QueryPaginator.scala @@ -2,13 +2,10 @@ package cromwell.database.migration.custom import java.sql.{PreparedStatement, ResultSet} - -class QueryPaginator(statement: PreparedStatement, - batchSize: Int, - count: Int) extends Iterator[ResultSet] { +class QueryPaginator(statement: PreparedStatement, batchSize: Int, count: Int) extends Iterator[ResultSet] { var cursor = 0 - def next(): ResultSet = { + def next(): ResultSet = { statement.setInt(1, cursor) statement.setInt(2, cursor + batchSize) diff --git a/database/migration/src/main/scala/cromwell/database/migration/failuremetadata/DeduplicateFailureMessageIds.scala b/database/migration/src/main/scala/cromwell/database/migration/failuremetadata/DeduplicateFailureMessageIds.scala index 74ab0769b34..9ffea901284 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/failuremetadata/DeduplicateFailureMessageIds.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/failuremetadata/DeduplicateFailureMessageIds.scala @@ -17,8 +17,10 @@ class DeduplicateFailureMessageIds extends BatchedTaskChange { val callFqnColumn = "CALL_FQN" val jobScatterIndexColumn = "JOB_SCATTER_INDEX" val retryAttemptColumn = "JOB_RETRY_ATTEMPT" - val contentEqualityCheck = List(workflowIdColumn, metadataKeyColumn, callFqnColumn, jobScatterIndexColumn, retryAttemptColumn) - .map(s => s"(t2.$s = t1.$s OR (t2.$s IS NULL AND t1.$s IS NULL))").mkString(" AND ") + val contentEqualityCheck = + List(workflowIdColumn, metadataKeyColumn, callFqnColumn, jobScatterIndexColumn, retryAttemptColumn) + .map(s => s"(t2.$s = t1.$s OR (t2.$s IS NULL AND t1.$s IS NULL))") + .mkString(" AND ") val fixableFailureMessageFilter = "METADATA_KEY LIKE '%failures[%]%:message'" diff --git a/database/migration/src/main/scala/cromwell/database/migration/liquibase/DiffResultFilter.scala b/database/migration/src/main/scala/cromwell/database/migration/liquibase/DiffResultFilter.scala index 2365dcf1e4b..e56fa88bc36 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/liquibase/DiffResultFilter.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/liquibase/DiffResultFilter.scala @@ -1,7 +1,7 @@ package cromwell.database.migration.liquibase import liquibase.database.Database -import liquibase.diff.{DiffResult, Difference, ObjectDifferences} +import liquibase.diff.{Difference, DiffResult, ObjectDifferences} import liquibase.structure.DatabaseObject import liquibase.structure.core._ @@ -11,6 +11,7 @@ import scala.jdk.CollectionConverters._ * Filters liquibase results. */ object DiffResultFilter { + /** * A filter for a database object. */ @@ -42,19 +43,16 @@ object DiffResultFilter { val unexpectedObjects = diffResult.getUnexpectedObjects.asScala val changedObjects = diffResult.getChangedObjects.asScala - val newDiffResult = new DiffResult( - diffResult.getReferenceSnapshot, - diffResult.getComparisonSnapshot, - diffResult.getCompareControl) + val newDiffResult = + new DiffResult(diffResult.getReferenceSnapshot, diffResult.getComparisonSnapshot, diffResult.getCompareControl) missingObjects.filterNot(unchangedFilter(referenceDatabase, _)).foreach(newDiffResult.addMissingObject) unexpectedObjects.filterNot(unchangedFilter(comparisonDatabase, _)).foreach(newDiffResult.addUnexpectedObject) val filteredChangedObjects = changedObjects.filterNot(isSameObject(referenceDatabase, comparisonDatabase, changedFilters)) - for ((obj, difference) <- filteredChangedObjects) { + for ((obj, difference) <- filteredChangedObjects) newDiffResult.addChangedObject(obj, difference) - } newDiffResult } @@ -71,12 +69,14 @@ object DiffResultFilter { * @param objectAndDiff A tuple of the object and the set of differences. * @return True if the object is actually the same. */ - def isSameObject(referenceDatabase: Database, comparisonDatabase: Database, filters: Seq[DiffFilter]) - (objectAndDiff: (DatabaseObject, ObjectDifferences)): Boolean = { + def isSameObject(referenceDatabase: Database, comparisonDatabase: Database, filters: Seq[DiffFilter])( + objectAndDiff: (DatabaseObject, ObjectDifferences) + ): Boolean = { val (obj, objectDifferences) = objectAndDiff val differences = objectDifferences.getDifferences.asScala val filtered = filters.foldLeft(differences)((diffs, diffFilter) => - diffs.filterNot(diffFilter(referenceDatabase, comparisonDatabase, obj, _))) + diffs.filterNot(diffFilter(referenceDatabase, comparisonDatabase, obj, _)) + ) filtered.isEmpty } @@ -89,16 +89,19 @@ object DiffResultFilter { * @param difference The difference reported. * @return True if the object is actually the same with slightly different column widths. */ - def isVarchar255(referenceDatabase: Database, comparisonDatabase: Database, - databaseObject: DatabaseObject, difference: Difference): Boolean = { + def isVarchar255(referenceDatabase: Database, + comparisonDatabase: Database, + databaseObject: DatabaseObject, + difference: Difference + ): Boolean = { val compared = difference.getComparedValue val referenced = difference.getReferenceValue compared.isInstanceOf[DataType] && referenced.isInstanceOf[DataType] && { val comparedDataType = compared.asInstanceOf[DataType] val referencedDataType = referenced.asInstanceOf[DataType] comparedDataType.getTypeName == "VARCHAR" && referencedDataType.getTypeName == "VARCHAR" && - // Our liquibase copypasta defaults VARCHAR to 255. Slick without a value defaults to 254 - (comparedDataType.getColumnSize + referencedDataType.getColumnSize == 255 + 254) + // Our liquibase copypasta defaults VARCHAR to 255. Slick without a value defaults to 254 + (comparedDataType.getColumnSize + referencedDataType.getColumnSize == 255 + 254) } } @@ -114,9 +117,11 @@ object DiffResultFilter { * @param difference The difference reported. * @return True if the object is actually similar based on type. */ - def isTypeSimilar(similarTypes: String*) - (referenceDatabase: Database, comparisonDatabase: Database, - databaseObject: DatabaseObject, difference: Difference): Boolean = { + def isTypeSimilar(similarTypes: String*)(referenceDatabase: Database, + comparisonDatabase: Database, + databaseObject: DatabaseObject, + difference: Difference + ): Boolean = { val compared = difference.getComparedValue val referenced = difference.getReferenceValue compared.isInstanceOf[DataType] && referenced.isInstanceOf[DataType] && { @@ -137,10 +142,12 @@ object DiffResultFilter { * @param difference The difference reported. * @return True if the object is actually similar based on type. */ - def isReordered(referenceDatabase: Database, comparisonDatabase: Database, - databaseObject: DatabaseObject, difference: Difference): Boolean = { + def isReordered(referenceDatabase: Database, + comparisonDatabase: Database, + databaseObject: DatabaseObject, + difference: Difference + ): Boolean = difference.getField == "order" - } /** * Returns true if the object is a change log object. @@ -149,7 +156,7 @@ object DiffResultFilter { * @param databaseObject The database object. * @return True if the object is a change log object. */ - def isChangeLog(database: Database, databaseObject: DatabaseObject): Boolean = { + def isChangeLog(database: Database, databaseObject: DatabaseObject): Boolean = databaseObject match { case table: Table => table.getName.contains("DATABASECHANGELOG") case column: Column => isChangeLog(database, column.getRelation) @@ -157,7 +164,6 @@ object DiffResultFilter { case key: PrimaryKey => isChangeLog(database, key.getTable) case _ => false } - } /** * Returns true if the object is liquibase database object. @@ -166,9 +172,8 @@ object DiffResultFilter { * @param databaseObject The database object. * @return True if the object is a liquibase database object. */ - def isLiquibaseObject(database: Database, databaseObject: DatabaseObject): Boolean = { + def isLiquibaseObject(database: Database, databaseObject: DatabaseObject): Boolean = database.isLiquibaseObject(databaseObject) - } /** * Returns true if the object is a member of the excluded table. @@ -178,23 +183,19 @@ object DiffResultFilter { * @param databaseObject The database object. * @return True if the object is a member of the tables. */ - def isTableObject(tables: Seq[String]) - (database: Database, databaseObject: DatabaseObject): Boolean = { + def isTableObject(tables: Seq[String])(database: Database, databaseObject: DatabaseObject): Boolean = isTableObject(tables, databaseObject) - } - private def isTableObject(tables: Seq[String], databaseObject: DatabaseObject): Boolean = { + private def isTableObject(tables: Seq[String], databaseObject: DatabaseObject): Boolean = tables.exists(table => databaseObject.getName.equalsIgnoreCase(table) || getContainingObjects(databaseObject).exists(isTableObject(tables, _)) ) - } // getContainingObjects is ill-mannered and returns null when really it ought to return an empty array, so wrap // in an `Option` and `getOrElse`. - private def getContainingObjects(databaseObject: DatabaseObject): Array[DatabaseObject] = { + private def getContainingObjects(databaseObject: DatabaseObject): Array[DatabaseObject] = Option(databaseObject.getContainingObjects).getOrElse(Array.empty) - } /** * Adds utility methods to a liquibase diff result. @@ -202,6 +203,7 @@ object DiffResultFilter { * @param diffResult The origin diff result. */ implicit class EnhancedDiffResult(val diffResult: DiffResult) extends AnyVal { + /** * Filters changelogs. * diff --git a/database/migration/src/main/scala/cromwell/database/migration/liquibase/LiquibaseUtils.scala b/database/migration/src/main/scala/cromwell/database/migration/liquibase/LiquibaseUtils.scala index e4823b8b0ac..d8c32f7b8fa 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/liquibase/LiquibaseUtils.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/liquibase/LiquibaseUtils.scala @@ -34,7 +34,7 @@ object LiquibaseUtils { * @param settings The liquibase settings. * @param jdbcConnection A jdbc connection to the database. */ - def updateSchema(settings: LiquibaseSettings)(jdbcConnection: Connection): Unit = { + def updateSchema(settings: LiquibaseSettings)(jdbcConnection: Connection): Unit = mutex.synchronized { val liquibaseConnection = newConnection(jdbcConnection) try { @@ -44,11 +44,9 @@ object LiquibaseUtils { val liquibase = new Liquibase(settings.changeLogResourcePath, new ClassLoaderResourceAccessor(), database) updateSchema(liquibase) - } finally { + } finally closeConnection(liquibaseConnection) - } } - } /** * Wraps a jdbc connection in the database with the appropriate liquibase connection. @@ -58,21 +56,19 @@ object LiquibaseUtils { * @param jdbcConnection The liquibase connection. * @return */ - private def newConnection(jdbcConnection: Connection): DatabaseConnection = { + private def newConnection(jdbcConnection: Connection): DatabaseConnection = jdbcConnection.getMetaData.getDatabaseProductName match { case HsqlDatabaseProperties.PRODUCT_NAME => new HsqlConnection(jdbcConnection) case _ => new JdbcConnection(jdbcConnection) } - } /** * Updates the liquibase database. * * @param liquibase The facade for interacting with liquibase. */ - private def updateSchema(liquibase: Liquibase): Unit = { + private def updateSchema(liquibase: Liquibase): Unit = liquibase.update(DefaultContexts, DefaultLabelExpression) - } /** * Converts a liquibase connection to a liquibase database. @@ -80,9 +76,8 @@ object LiquibaseUtils { * @param liquibaseConnection The liquibase connection. * @return The liquibase database. */ - private def toDatabase(liquibaseConnection: DatabaseConnection): Database = { + private def toDatabase(liquibaseConnection: DatabaseConnection): Database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(liquibaseConnection) - } /** * Compares a reference to a comparison liquibase database. @@ -91,9 +86,8 @@ object LiquibaseUtils { * @param comparisonDatabase The comparison liquibase database. * @return The complete diff results. */ - private def compare(referenceDatabase: Database, comparisonDatabase: Database): DiffResult = { + private def compare(referenceDatabase: Database, comparisonDatabase: Database): DiffResult = DiffGeneratorFactory.getInstance().compare(referenceDatabase, comparisonDatabase, CompareControl.STANDARD) - } /** * Compares a reference to a comparison JDBC connection. @@ -103,7 +97,7 @@ object LiquibaseUtils { * @param block Block of code to run before closing the connections. * @return The complete diff results. */ - def compare[T](referenceJdbc: Connection, comparisonJdbc: Connection)(block: DiffResult => T): T = { + def compare[T](referenceJdbc: Connection, comparisonJdbc: Connection)(block: DiffResult => T): T = mutex.synchronized { withConnection(referenceJdbc) { referenceLiquibase => withConnection(comparisonJdbc) { comparisonLiquibase => @@ -112,7 +106,6 @@ object LiquibaseUtils { } } } - } /** * Provides a connection to a block of code, closing the connection afterwards. @@ -124,11 +117,10 @@ object LiquibaseUtils { */ private def withConnection[T](jdbcConnection: Connection)(block: DatabaseConnection => T): T = { val liquibaseConnection = newConnection(jdbcConnection) - try { + try block(liquibaseConnection) - } finally { + finally closeConnection(liquibaseConnection) - } } /** @@ -136,13 +128,12 @@ object LiquibaseUtils { * * @param connection The liquibase connection. */ - private def closeConnection(connection: DatabaseConnection): Unit = { - try { + private def closeConnection(connection: DatabaseConnection): Unit = + try connection.close() - } finally { + finally { /* ignore */ } - } /** * Returns the changelog for a liquibase setting. @@ -165,11 +156,10 @@ object LiquibaseUtils { * @param settings The liquibase settings. * @return The database change sets. */ - def getChangeSets(settings: LiquibaseSettings): Seq[ChangeSet] = { + def getChangeSets(settings: LiquibaseSettings): Seq[ChangeSet] = mutex.synchronized { getChangeLog(settings).getChangeSets.asScala.toList } - } /** * Returns a schema snapshot. @@ -177,7 +167,7 @@ object LiquibaseUtils { * @param jdbcConnection A jdbc connection to the database. * @return The database change sets. */ - def getSnapshot(jdbcConnection: Connection): DatabaseSnapshot = { + def getSnapshot(jdbcConnection: Connection): DatabaseSnapshot = mutex.synchronized { withConnection(jdbcConnection) { referenceLiquibase => val database = toDatabase(referenceLiquibase) @@ -191,10 +181,8 @@ object LiquibaseUtils { database, new SnapshotControl(database) ) - } finally { + } finally database.setObjectQuotingStrategy(objectQuotingStrategy) - } } } - } } diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/MetadataCustomSql.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/MetadataCustomSql.scala index 13515de2e8c..84734d37b5a 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/MetadataCustomSql.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/MetadataCustomSql.scala @@ -17,15 +17,13 @@ abstract class MetadataCustomSql extends CustomSqlChange { def queries: Array[String] - override def generateStatements(database: Database): Array[SqlStatement] = { - queries map { query => new RawSqlStatement(query) } - } + override def generateStatements(database: Database): Array[SqlStatement] = + queries map { query => new RawSqlStatement(query) } override def setUp(): Unit = () - override def validate(database: Database): ValidationErrors = { + override def validate(database: Database): ValidationErrors = new ValidationErrors() - } override def setFileOpener(resourceAccessor: ResourceAccessor): Unit = () } diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/ExecutionTableMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/ExecutionTableMigration.scala index 2792c00ccf0..991bcacb0f0 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/ExecutionTableMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/ExecutionTableMigration.scala @@ -32,9 +32,8 @@ class ExecutionTableMigration extends MetadataCustomSql { FROM TMP_EXECUTION_MIGRATION e JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID WHERE - e.START_DT IS NOT NULL;""" - , - """INSERT INTO METADATA_JOURNAL ( + e.START_DT IS NOT NULL;""", + """INSERT INTO METADATA_JOURNAL ( WORKFLOW_EXECUTION_UUID, METADATA_KEY, CALL_FQN, @@ -54,9 +53,8 @@ class ExecutionTableMigration extends MetadataCustomSql { 'string', NOW() FROM TMP_EXECUTION_MIGRATION e - JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID;""" - , - s"""INSERT INTO METADATA_JOURNAL ( + JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID;""", + s"""INSERT INTO METADATA_JOURNAL ( WORKFLOW_EXECUTION_UUID, METADATA_KEY, CALL_FQN, @@ -78,9 +76,8 @@ class ExecutionTableMigration extends MetadataCustomSql { FROM TMP_EXECUTION_MIGRATION e JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID WHERE - e.END_DT IS NOT NULL;""" - , - """INSERT INTO METADATA_JOURNAL ( + e.END_DT IS NOT NULL;""", + """INSERT INTO METADATA_JOURNAL ( WORKFLOW_EXECUTION_UUID, METADATA_KEY, CALL_FQN, @@ -100,9 +97,8 @@ class ExecutionTableMigration extends MetadataCustomSql { 'string', NOW() FROM TMP_EXECUTION_MIGRATION e - JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID;""" - , - """INSERT INTO METADATA_JOURNAL ( + JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID;""", + """INSERT INTO METADATA_JOURNAL ( WORKFLOW_EXECUTION_UUID, METADATA_KEY, CALL_FQN, @@ -123,9 +119,8 @@ class ExecutionTableMigration extends MetadataCustomSql { NOW() FROM TMP_EXECUTION_MIGRATION e JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID - WHERE e.RC IS NOT NULL;""" - , - s"""INSERT INTO METADATA_JOURNAL ( + WHERE e.RC IS NOT NULL;""", + s"""INSERT INTO METADATA_JOURNAL ( WORKFLOW_EXECUTION_UUID, METADATA_KEY, CALL_FQN, @@ -145,9 +140,8 @@ class ExecutionTableMigration extends MetadataCustomSql { 'boolean', NOW() FROM TMP_EXECUTION_MIGRATION e - JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID;""" - , - s"""INSERT INTO METADATA_JOURNAL ( + JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID;""", + s"""INSERT INTO METADATA_JOURNAL ( WORKFLOW_EXECUTION_UUID, METADATA_KEY, CALL_FQN, @@ -169,9 +163,8 @@ class ExecutionTableMigration extends MetadataCustomSql { FROM TMP_EXECUTION_MIGRATION e JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID LEFT JOIN RUNTIME_ATTRIBUTES ra ON e.EXECUTION_ID = ra.EXECUTION_ID AND ra.ATTRIBUTE_NAME = 'preemptible' - WHERE ra.ATTRIBUTE_VALUE IS NOT NULL;""" - , - """INSERT INTO METADATA_JOURNAL ( + WHERE ra.ATTRIBUTE_VALUE IS NOT NULL;""", + """INSERT INTO METADATA_JOURNAL ( WORKFLOW_EXECUTION_UUID, METADATA_KEY, CALL_FQN, @@ -187,9 +180,8 @@ class ExecutionTableMigration extends MetadataCustomSql { ATTEMPT, '1900-01-01' FROM TMP_EXECUTION_MIGRATION e - JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID;""" - , - """INSERT INTO METADATA_JOURNAL ( + JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID;""", + """INSERT INTO METADATA_JOURNAL ( WORKFLOW_EXECUTION_UUID, METADATA_KEY, CALL_FQN, @@ -205,9 +197,8 @@ class ExecutionTableMigration extends MetadataCustomSql { ATTEMPT, '1900-01-01' FROM TMP_EXECUTION_MIGRATION e - JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID;""" - , - """INSERT INTO METADATA_JOURNAL ( + JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID;""", + """INSERT INTO METADATA_JOURNAL ( WORKFLOW_EXECUTION_UUID, METADATA_KEY, CALL_FQN, @@ -223,9 +214,8 @@ class ExecutionTableMigration extends MetadataCustomSql { ATTEMPT, '1900-01-01' FROM TMP_EXECUTION_MIGRATION e - JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID;""" - , - """INSERT INTO METADATA_JOURNAL ( + JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID;""", + """INSERT INTO METADATA_JOURNAL ( WORKFLOW_EXECUTION_UUID, METADATA_KEY, CALL_FQN, diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/FailureEventTableMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/FailureEventTableMigration.scala index 60d12342e77..01fa8de0bdf 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/FailureEventTableMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/FailureEventTableMigration.scala @@ -5,7 +5,7 @@ import cromwell.database.migration.metadata.MetadataCustomSql class FailureEventTableMigration extends MetadataCustomSql { import MetadataCustomSql._ - override def queries: Array[String] = { + override def queries: Array[String] = Array( """ |INSERT INTO METADATA_JOURNAL ( @@ -32,31 +32,30 @@ class FailureEventTableMigration extends MetadataCustomSql { | JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID; """.stripMargin, s""" - |INSERT INTO METADATA_JOURNAL ( - | WORKFLOW_EXECUTION_UUID, - | METADATA_KEY, - | CALL_FQN, - | JOB_SCATTER_INDEX, - | JOB_RETRY_ATTEMPT, - | METADATA_VALUE, - | METADATA_VALUE_TYPE, - | METADATA_TIMESTAMP - |) - |SELECT - | WORKFLOW_EXECUTION_UUID, - | CONCAT('failures[', fe.FAILURE_EVENT_ID ,']:timestamp'), - | CALL_FQN, - | IDX, - | ATTEMPT, - | DATE_FORMAT(fe.EVENT_TIMESTAMP, '%Y-%m-%dT%T.%f$Offset'), - | 'string', - | NOW() - |FROM FAILURE_EVENT fe - | LEFT JOIN EXECUTION e ON fe.EXECUTION_ID = e.EXECUTION_ID - | JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID; + |INSERT INTO METADATA_JOURNAL ( + | WORKFLOW_EXECUTION_UUID, + | METADATA_KEY, + | CALL_FQN, + | JOB_SCATTER_INDEX, + | JOB_RETRY_ATTEMPT, + | METADATA_VALUE, + | METADATA_VALUE_TYPE, + | METADATA_TIMESTAMP + |) + |SELECT + | WORKFLOW_EXECUTION_UUID, + | CONCAT('failures[', fe.FAILURE_EVENT_ID ,']:timestamp'), + | CALL_FQN, + | IDX, + | ATTEMPT, + | DATE_FORMAT(fe.EVENT_TIMESTAMP, '%Y-%m-%dT%T.%f$Offset'), + | 'string', + | NOW() + |FROM FAILURE_EVENT fe + | LEFT JOIN EXECUTION e ON fe.EXECUTION_ID = e.EXECUTION_ID + | JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID; """.stripMargin ) - } override def getConfirmationMessage: String = "Failure Table migration complete." } diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/executionevent/ExecutionEventTableDescriptionMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/executionevent/ExecutionEventTableDescriptionMigration.scala index 8ee97fcc90d..4181d369572 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/executionevent/ExecutionEventTableDescriptionMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/executionevent/ExecutionEventTableDescriptionMigration.scala @@ -4,7 +4,7 @@ import cromwell.database.migration.metadata.MetadataCustomSql class ExecutionEventTableDescriptionMigration extends MetadataCustomSql { - override def queries: Array[String] = { + override def queries: Array[String] = Array( """ |INSERT INTO METADATA_JOURNAL ( @@ -31,7 +31,6 @@ class ExecutionEventTableDescriptionMigration extends MetadataCustomSql { | JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID; """.stripMargin ) - } override def getConfirmationMessage: String = "Execution Event Table (Description field) migration complete." } diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/executionevent/ExecutionEventTableEndMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/executionevent/ExecutionEventTableEndMigration.scala index d3d80e74f46..669bfd823bc 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/executionevent/ExecutionEventTableEndMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/executionevent/ExecutionEventTableEndMigration.scala @@ -5,34 +5,33 @@ import MetadataCustomSql._ class ExecutionEventTableEndMigration extends MetadataCustomSql { - override def queries: Array[String] = { + override def queries: Array[String] = Array( s""" - |INSERT INTO METADATA_JOURNAL ( - | WORKFLOW_EXECUTION_UUID, - | METADATA_KEY, - | CALL_FQN, - | JOB_SCATTER_INDEX, - | JOB_RETRY_ATTEMPT, - | METADATA_VALUE, - | METADATA_VALUE_TYPE, - | METADATA_TIMESTAMP - |) - |SELECT - | WORKFLOW_EXECUTION_UUID, - | CONCAT('executionEvents[', ev.EVENT_ID ,']:endTime'), - | CALL_FQN, - | IDX, - | ATTEMPT, - | DATE_FORMAT(ev.END_DT, '%Y-%m-%dT%T.%f$Offset'), - | 'string', - | NOW() - |FROM EXECUTION_EVENT ev - | LEFT JOIN EXECUTION e ON ev.EXECUTION_ID = e.EXECUTION_ID - | JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID; + |INSERT INTO METADATA_JOURNAL ( + | WORKFLOW_EXECUTION_UUID, + | METADATA_KEY, + | CALL_FQN, + | JOB_SCATTER_INDEX, + | JOB_RETRY_ATTEMPT, + | METADATA_VALUE, + | METADATA_VALUE_TYPE, + | METADATA_TIMESTAMP + |) + |SELECT + | WORKFLOW_EXECUTION_UUID, + | CONCAT('executionEvents[', ev.EVENT_ID ,']:endTime'), + | CALL_FQN, + | IDX, + | ATTEMPT, + | DATE_FORMAT(ev.END_DT, '%Y-%m-%dT%T.%f$Offset'), + | 'string', + | NOW() + |FROM EXECUTION_EVENT ev + | LEFT JOIN EXECUTION e ON ev.EXECUTION_ID = e.EXECUTION_ID + | JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID; """.stripMargin ) - } override def getConfirmationMessage: String = "Execution Event Table (End field) migration complete." } diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/executionevent/ExecutionEventTableStartMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/executionevent/ExecutionEventTableStartMigration.scala index e0c9b418460..86168daddf9 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/executionevent/ExecutionEventTableStartMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/executionevent/ExecutionEventTableStartMigration.scala @@ -5,33 +5,31 @@ import MetadataCustomSql._ class ExecutionEventTableStartMigration extends MetadataCustomSql { - override def queries: Array[String] = { - Array( - s""" - |INSERT INTO METADATA_JOURNAL ( - | WORKFLOW_EXECUTION_UUID, - | METADATA_KEY, - | CALL_FQN, - | JOB_SCATTER_INDEX, - | JOB_RETRY_ATTEMPT, - | METADATA_VALUE, - | METADATA_VALUE_TYPE, - | METADATA_TIMESTAMP - |) - |SELECT - | WORKFLOW_EXECUTION_UUID, - | CONCAT('executionEvents[', ev.EVENT_ID ,']:startTime'), - | CALL_FQN, - | IDX, - | ATTEMPT, - | DATE_FORMAT(ev.START_DT, '%Y-%m-%dT%T.%f$Offset'), - | 'string', - | NOW() - |FROM EXECUTION_EVENT ev - | LEFT JOIN EXECUTION e ON ev.EXECUTION_ID = e.EXECUTION_ID - | JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID; + override def queries: Array[String] = + Array(s""" + |INSERT INTO METADATA_JOURNAL ( + | WORKFLOW_EXECUTION_UUID, + | METADATA_KEY, + | CALL_FQN, + | JOB_SCATTER_INDEX, + | JOB_RETRY_ATTEMPT, + | METADATA_VALUE, + | METADATA_VALUE_TYPE, + | METADATA_TIMESTAMP + |) + |SELECT + | WORKFLOW_EXECUTION_UUID, + | CONCAT('executionEvents[', ev.EVENT_ID ,']:startTime'), + | CALL_FQN, + | IDX, + | ATTEMPT, + | DATE_FORMAT(ev.START_DT, '%Y-%m-%dT%T.%f$Offset'), + | 'string', + | NOW() + |FROM EXECUTION_EVENT ev + | LEFT JOIN EXECUTION e ON ev.EXECUTION_ID = e.EXECUTION_ID + | JOIN WORKFLOW_EXECUTION we ON we.WORKFLOW_EXECUTION_ID = e.WORKFLOW_EXECUTION_ID; """.stripMargin) - } override def getConfirmationMessage: String = "Execution Event Table (Start field) migration complete." } diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/CallOutputSymbolTableMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/CallOutputSymbolTableMigration.scala index 822e8e15fbe..c148fc2dce5 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/CallOutputSymbolTableMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/CallOutputSymbolTableMigration.scala @@ -11,23 +11,20 @@ class CallOutputSymbolTableMigration extends SymbolTableMigration { symbolScope: String, symbolIndex: Option[Int], symbolAttempt: Option[Int], - womValue: WomValue): Int = { - + womValue: WomValue + ): Int = (symbolIndex, symbolAttempt) match { case (Some(index), Some(attempt)) => - val metadataStatementForCall = new MetadataStatementForCall(statement, - workflowUuid, - symbolScope, - index, - attempt - ) + val metadataStatementForCall = + new MetadataStatementForCall(statement, workflowUuid, symbolScope, index, attempt) addWdlValue(s"outputs:$symbolName", womValue, metadataStatementForCall) case _ => - logger.warn(s"Found output without index or attempt: [$workflowUuid] $symbolScope - $symbolName:$symbolIndex:$symbolAttempt") + logger.warn( + s"Found output without index or attempt: [$workflowUuid] $symbolScope - $symbolName:$symbolIndex:$symbolAttempt" + ) 0 } - } override def getConfirmationMessage: String = "Call outputs from Symbol Table migration complete." } diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/InputSymbolTableMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/InputSymbolTableMigration.scala index 13021ed9c72..6aed6fb870c 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/InputSymbolTableMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/InputSymbolTableMigration.scala @@ -12,17 +12,13 @@ class InputSymbolTableMigration extends SymbolTableMigration { symbolScope: String, symbolIndex: Option[Int], symbolAttempt: Option[Int], - womValue: WomValue): Int = { - + womValue: WomValue + ): Int = (symbolIndex, symbolAttempt) match { - case (Some(index) , Some(attempt)) => + case (Some(index), Some(attempt)) => // Call scoped - val metadataStatementForCall = new MetadataStatementForCall(statement, - workflowUuid, - symbolScope, - index, - attempt - ) + val metadataStatementForCall = + new MetadataStatementForCall(statement, workflowUuid, symbolScope, index, attempt) addWdlValue(s"inputs:$symbolName", womValue, metadataStatementForCall) case (None, None) if !symbolScope.contains('.') => @@ -31,7 +27,6 @@ class InputSymbolTableMigration extends SymbolTableMigration { case _ => 0 } - } override def getConfirmationMessage: String = "Inputs from Symbol Table migration complete." } diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/MetadataStatement.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/MetadataStatement.scala index da3decae867..eb2d6b89e52 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/MetadataStatement.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/MetadataStatement.scala @@ -42,7 +42,7 @@ class MetadataStatementForWorkflow(preparedStatement: PreparedStatement, workflo val dawn = OffsetDateTime.of(0, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toSystemTimestamp var batchSizeCounter: Int = 0 - private def metadataType(value: Any) = { + private def metadataType(value: Any) = value match { case WomInteger(_) => "int" case WomFloat(_) => "number" @@ -51,16 +51,14 @@ class MetadataStatementForWorkflow(preparedStatement: PreparedStatement, workflo case _: Int | Long => "int" case _: Double | Float => "number" case _: Boolean => "boolean" - case _ =>"string" + case _ => "string" } - } - private def metadataValue(value: Any) = { + private def metadataValue(value: Any) = value match { - case v: WomValue => v.valueString + case v: WomValue => v.valueString case v => v.toString } - } protected def setStatement() = { preparedStatement.setString(MetadataStatement.WorkflowIdIdx, workflowId) @@ -93,12 +91,11 @@ class MetadataStatementForWorkflow(preparedStatement: PreparedStatement, workflo } /** Adds a non-null value to the metadata journal. */ - override def addKeyValue(key: String, value: Any) = { + override def addKeyValue(key: String, value: Any) = if (value != null) { preparedStatement.setTimestamp(MetadataStatement.TimestampIdx, OffsetDateTime.now().toSystemTimestamp) add(key, value, s"Failed to migrate metadata value $value with key $key for workflow $workflowId") } - } override def addEmptyValue(key: String): Unit = { preparedStatement.setTimestamp(MetadataStatement.TimestampIdx, dawn) @@ -106,11 +103,16 @@ class MetadataStatementForWorkflow(preparedStatement: PreparedStatement, workflo } } -class MetadataStatementForCall(preparedStatement: PreparedStatement, workflowId: String, callFqn: String, index: Int, attempt: Int) extends MetadataStatementForWorkflow(preparedStatement, workflowId) { +class MetadataStatementForCall(preparedStatement: PreparedStatement, + workflowId: String, + callFqn: String, + index: Int, + attempt: Int +) extends MetadataStatementForWorkflow(preparedStatement, workflowId) { override def setStatement() = { preparedStatement.setString(MetadataStatement.WorkflowIdIdx, workflowId) preparedStatement.setString(MetadataStatement.CallFqnIdx, callFqn) preparedStatement.setInt(MetadataStatement.CallIndexIdx, index) preparedStatement.setInt(MetadataStatement.CallAttemptIdx, attempt) } -} \ No newline at end of file +} diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala index f3d148ed0c5..4ca834535d7 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala @@ -25,16 +25,16 @@ trait SymbolTableMigration extends BatchedTaskChange { override val readCountQuery = SymbolTableMigration.NbRowsQuery override val readBatchQuery: String = """ - |SELECT - | WORKFLOW_EXECUTION_UUID, - | SYMBOL_NAME, - | SYMBOL_SCOPE, - | SYMBOL_INDEX, - | SYMBOL_ATTEMPT, - | WDL_TYPE, - | WDL_VALUE - | FROM TMP_SYMBOL - | WHERE TMP_SYMBOL_ID >= ? AND TMP_SYMBOL_ID < ?; + |SELECT + | WORKFLOW_EXECUTION_UUID, + | SYMBOL_NAME, + | SYMBOL_SCOPE, + | SYMBOL_INDEX, + | SYMBOL_ATTEMPT, + | WDL_TYPE, + | WDL_VALUE + | FROM TMP_SYMBOL + | WHERE TMP_SYMBOL_ID >= ? AND TMP_SYMBOL_ID < ?; """.stripMargin override val migrateBatchQueries = List(MetadataStatement.InsertSql) @@ -66,7 +66,9 @@ trait SymbolTableMigration extends BatchedTaskChange { case Failure(f) => logger.error( s"""Could not parse symbol of type ${row.getString("WDL_TYPE")} - |for Workflow $workflowUuid - Call $symbolScope:$symbolIndex""".stripMargin, f) + |for Workflow $workflowUuid - Call $symbolScope:$symbolIndex""".stripMargin, + f + ) 0 } } @@ -77,14 +79,18 @@ trait SymbolTableMigration extends BatchedTaskChange { symbolScope: String, symbolIndex: Option[Int], symbolAttempt: Option[Int], - womValue: WomValue): Int + womValue: WomValue + ): Int /** * Add all necessary statements to the batch for the provided WomValue. */ - protected final def addWdlValue(metadataKey: String, womValue: WomValue, metadataStatementForCall: MetadataStatement): Int = { + final protected def addWdlValue(metadataKey: String, + womValue: WomValue, + metadataStatementForCall: MetadataStatement + ): Int = womValue match { - // simplify doesn't handle WdlExpression + // simplify doesn't handle WdlExpression case expr: WdlExpression => metadataStatementForCall.addKeyValue(metadataKey, expr.valueString) 1 @@ -95,5 +101,4 @@ trait SymbolTableMigration extends BatchedTaskChange { } simplified.size } - } } diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/WorkflowOutputSymbolTableMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/WorkflowOutputSymbolTableMigration.scala index 8902739666e..0e7562310c2 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/WorkflowOutputSymbolTableMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/WorkflowOutputSymbolTableMigration.scala @@ -12,7 +12,8 @@ class WorkflowOutputSymbolTableMigration extends SymbolTableMigration { symbolScope: String, symbolIndex: Option[Int], symbolAttempt: Option[Int], - womValue: WomValue): Int = { + womValue: WomValue + ): Int = { val metadataStatementForWorkflow = new MetadataStatementForWorkflow(statement, workflowUuid) addWdlValue(s"outputs:$symbolScope.$symbolName", womValue, metadataStatementForWorkflow) } diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/workflowexecution/WorkflowExecutionTableMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/workflowexecution/WorkflowExecutionTableMigration.scala index 411f596e962..6c343e2c420 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/workflowexecution/WorkflowExecutionTableMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/workflowexecution/WorkflowExecutionTableMigration.scala @@ -5,108 +5,102 @@ import MetadataCustomSql._ class WorkflowExecutionTableMigration extends MetadataCustomSql { - override def queries: Array[String] = { + override def queries: Array[String] = Array( - s""" - |INSERT INTO METADATA_JOURNAL ( - |WORKFLOW_EXECUTION_UUID, - |METADATA_KEY, - |METADATA_VALUE, - |METADATA_VALUE_TYPE, - |METADATA_TIMESTAMP - |) - |SELECT - | WORKFLOW_EXECUTION_UUID, - | 'submission', - | DATE_FORMAT(START_DT, '%Y-%m-%dT%T.%f$Offset'), - | 'string', - | NOW() - |FROM WORKFLOW_EXECUTION - |WHERE START_DT IS NOT NULL;""".stripMargin - , - s""" - |INSERT INTO METADATA_JOURNAL ( - |WORKFLOW_EXECUTION_UUID, - |METADATA_KEY, - |METADATA_VALUE, - |METADATA_VALUE_TYPE, - |METADATA_TIMESTAMP - |) - |SELECT - | WORKFLOW_EXECUTION_UUID, - | 'start', - | DATE_FORMAT(START_DT, '%Y-%m-%dT%T.%f$Offset'), - | 'string', - | NOW() - |FROM WORKFLOW_EXECUTION - |WHERE START_DT IS NOT NULL;""".stripMargin - , - s""" - |INSERT INTO METADATA_JOURNAL ( - |WORKFLOW_EXECUTION_UUID, - |METADATA_KEY, - |METADATA_VALUE, - |METADATA_VALUE_TYPE, - |METADATA_TIMESTAMP - |) - |SELECT - | WORKFLOW_EXECUTION_UUID, - | 'end', - | DATE_FORMAT(END_DT, '%Y-%m-%dT%T.%f$Offset'), - | 'string', - | NOW() - |FROM WORKFLOW_EXECUTION - |WHERE END_DT IS NOT NULL;""".stripMargin - , - s""" - |INSERT INTO METADATA_JOURNAL ( - |WORKFLOW_EXECUTION_UUID, - |METADATA_KEY, - |METADATA_VALUE, - |METADATA_VALUE_TYPE, - |METADATA_TIMESTAMP - |) - |SELECT - | WORKFLOW_EXECUTION_UUID, - | 'status', - | STATUS, - | 'string', - | NOW() - |FROM WORKFLOW_EXECUTION;""".stripMargin - , - """ - |INSERT INTO METADATA_JOURNAL ( - |WORKFLOW_EXECUTION_UUID, - |METADATA_KEY, - |METADATA_VALUE, - |METADATA_VALUE_TYPE, - |METADATA_TIMESTAMP - |) - |SELECT - | WORKFLOW_EXECUTION_UUID, - | 'workflowName', - | WORKFLOW_NAME, - | 'string', - | NOW() - |FROM WORKFLOW_EXECUTION;""".stripMargin - , - """ - |INSERT INTO METADATA_JOURNAL ( - |WORKFLOW_EXECUTION_UUID, - |METADATA_KEY, - |METADATA_VALUE, - |METADATA_VALUE_TYPE, - |METADATA_TIMESTAMP - |) - |SELECT - | WORKFLOW_EXECUTION_UUID, - | 'outputs', - | NULL, - | NULL, - | '1900-01-01 0.000000' - |FROM WORKFLOW_EXECUTION;""".stripMargin + s""" + |INSERT INTO METADATA_JOURNAL ( + |WORKFLOW_EXECUTION_UUID, + |METADATA_KEY, + |METADATA_VALUE, + |METADATA_VALUE_TYPE, + |METADATA_TIMESTAMP + |) + |SELECT + | WORKFLOW_EXECUTION_UUID, + | 'submission', + | DATE_FORMAT(START_DT, '%Y-%m-%dT%T.%f$Offset'), + | 'string', + | NOW() + |FROM WORKFLOW_EXECUTION + |WHERE START_DT IS NOT NULL;""".stripMargin, + s""" + |INSERT INTO METADATA_JOURNAL ( + |WORKFLOW_EXECUTION_UUID, + |METADATA_KEY, + |METADATA_VALUE, + |METADATA_VALUE_TYPE, + |METADATA_TIMESTAMP + |) + |SELECT + | WORKFLOW_EXECUTION_UUID, + | 'start', + | DATE_FORMAT(START_DT, '%Y-%m-%dT%T.%f$Offset'), + | 'string', + | NOW() + |FROM WORKFLOW_EXECUTION + |WHERE START_DT IS NOT NULL;""".stripMargin, + s""" + |INSERT INTO METADATA_JOURNAL ( + |WORKFLOW_EXECUTION_UUID, + |METADATA_KEY, + |METADATA_VALUE, + |METADATA_VALUE_TYPE, + |METADATA_TIMESTAMP + |) + |SELECT + | WORKFLOW_EXECUTION_UUID, + | 'end', + | DATE_FORMAT(END_DT, '%Y-%m-%dT%T.%f$Offset'), + | 'string', + | NOW() + |FROM WORKFLOW_EXECUTION + |WHERE END_DT IS NOT NULL;""".stripMargin, + s""" + |INSERT INTO METADATA_JOURNAL ( + |WORKFLOW_EXECUTION_UUID, + |METADATA_KEY, + |METADATA_VALUE, + |METADATA_VALUE_TYPE, + |METADATA_TIMESTAMP + |) + |SELECT + | WORKFLOW_EXECUTION_UUID, + | 'status', + | STATUS, + | 'string', + | NOW() + |FROM WORKFLOW_EXECUTION;""".stripMargin, + """ + |INSERT INTO METADATA_JOURNAL ( + |WORKFLOW_EXECUTION_UUID, + |METADATA_KEY, + |METADATA_VALUE, + |METADATA_VALUE_TYPE, + |METADATA_TIMESTAMP + |) + |SELECT + | WORKFLOW_EXECUTION_UUID, + | 'workflowName', + | WORKFLOW_NAME, + | 'string', + | NOW() + |FROM WORKFLOW_EXECUTION;""".stripMargin, + """ + |INSERT INTO METADATA_JOURNAL ( + |WORKFLOW_EXECUTION_UUID, + |METADATA_KEY, + |METADATA_VALUE, + |METADATA_VALUE_TYPE, + |METADATA_TIMESTAMP + |) + |SELECT + | WORKFLOW_EXECUTION_UUID, + | 'outputs', + | NULL, + | NULL, + | '1900-01-01 0.000000' + |FROM WORKFLOW_EXECUTION;""".stripMargin ) - } override def getConfirmationMessage: String = "Workflow Execution Table migration complete." } diff --git a/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala index a1857aae21e..13c2ba14e3e 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala @@ -12,7 +12,7 @@ class JobStoreSimpletonMigration extends AbstractRestartMigration { // GOTC (substituting COUNT(*) for the projection): 1 row in set (5.22 sec) private val QueryOutputsForDoneCallsInRunningWorkflows = - """ + """ SELECT js.JOB_STORE_ID, -- 1 s.NAME, -- 2 @@ -51,9 +51,10 @@ class JobStoreSimpletonMigration extends AbstractRestartMigration { def buildJobStoreSimpletonEntries(name: String, womValue: WomValue, womType: WomType) = Option(womValue) match { case None => List(JobStoreSimpletonEntry(name, null, womType.stableName)) - case Some(_) => womValue.simplify(name) map { s => - JobStoreSimpletonEntry(s.simpletonKey, s.simpletonValue.valueString, s.simpletonValue.womType.stableName) - } + case Some(_) => + womValue.simplify(name) map { s => + JobStoreSimpletonEntry(s.simpletonKey, s.simpletonValue.valueString, s.simpletonValue.womType.stableName) + } } while (results.next()) { @@ -73,5 +74,3 @@ class JobStoreSimpletonMigration extends AbstractRestartMigration { } } } - - diff --git a/database/migration/src/main/scala/cromwell/database/migration/restart/table/RenameWorkflowOptionKeysMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/restart/table/RenameWorkflowOptionKeysMigration.scala index c0610ae368b..9019b95f1c0 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/restart/table/RenameWorkflowOptionKeysMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/restart/table/RenameWorkflowOptionKeysMigration.scala @@ -11,7 +11,6 @@ object RenameWorkflowOptionKeysMigration { private val UpdateWorkflowStore = " UPDATE WORKFLOW_STORE SET WORKFLOW_OPTIONS = ? WHERE WORKFLOW_STORE_ID = ? " } - class RenameWorkflowOptionKeysMigration extends AbstractRestartMigration { override protected def description: String = "Workflow option renaming" @@ -27,7 +26,8 @@ class RenameWorkflowOptionKeysMigration extends AbstractRestartMigration { val optionsJson = options.parseJson val newOptionsJson = optionsJson match { case JsObject(fields) => JsObject(fields map renameOptionKeys) - case other => other // There really shouldn't be workflow options of other types, but if there are pass them through. + case other => + other // There really shouldn't be workflow options of other types, but if there are pass them through. } insert.setString(1, newOptionsJson.prettyPrint) diff --git a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/ClearMetadataEntryWorkflowOptions.scala b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/ClearMetadataEntryWorkflowOptions.scala index 2892e9bc830..d07e8be76ef 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/ClearMetadataEntryWorkflowOptions.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/ClearMetadataEntryWorkflowOptions.scala @@ -11,5 +11,6 @@ class ClearMetadataEntryWorkflowOptions extends WorkflowOptionsChange { override val workflowOptionsColumn = "METADATA_VALUE" override val additionalReadBatchFilters = "AND METADATA_KEY = 'submittedFiles:options'" - override def migrateWorkflowOptions(workflowOptions: WorkflowOptions) = workflowOptions.clearEncryptedValues.asPrettyJson + override def migrateWorkflowOptions(workflowOptions: WorkflowOptions) = + workflowOptions.clearEncryptedValues.asPrettyJson } diff --git a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/RenameWorkflowOptionsInMetadata.scala b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/RenameWorkflowOptionsInMetadata.scala index 40dc4533b2d..5f0167b2ac1 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/RenameWorkflowOptionsInMetadata.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/RenameWorkflowOptionsInMetadata.scala @@ -20,12 +20,14 @@ class RenameWorkflowOptionsInMetadata extends BatchedTaskChange { | WHERE $primaryKeyColumn >= ? AND $primaryKeyColumn < ? $additionalReadBatchFilters; |""".stripMargin - override def migrateBatchQueries = List(s"UPDATE $tableName SET $workflowOptionsColumn = ? WHERE $primaryKeyColumn = ?;") + override def migrateBatchQueries = List( + s"UPDATE $tableName SET $workflowOptionsColumn = ? WHERE $primaryKeyColumn = ?;" + ) override def migrateBatchRow(readRow: ResultSet, migrateStatements: List[PreparedStatement]): Int = { val migrateStatement = migrateStatements.head val rowId = readRow.getInt(1) - + val migratedJson = readRow.getString(2).parseJson match { case JsObject(fields) => JsObject(fields map renameOptionKeys) case other => other diff --git a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsChange.scala b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsChange.scala index 1b519fb65fd..aa6e4ed0557 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsChange.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsChange.scala @@ -12,6 +12,7 @@ import scala.util.{Failure, Success} * Edits the workflow options stored in a table. */ trait WorkflowOptionsChange extends BatchedTaskChange { + /** @return name of the table */ def tableName: String @@ -47,7 +48,9 @@ trait WorkflowOptionsChange extends BatchedTaskChange { | WHERE $primaryKeyColumn >= ? AND $primaryKeyColumn < ? $additionalReadBatchFilters; |""".stripMargin - override def migrateBatchQueries = List(s"UPDATE $tableName SET $workflowOptionsColumn = ? WHERE $primaryKeyColumn = ?;") + override def migrateBatchQueries = List( + s"UPDATE $tableName SET $workflowOptionsColumn = ? WHERE $primaryKeyColumn = ?;" + ) override def migrateBatchRow(readRow: ResultSet, migrateStatements: List[PreparedStatement]): Int = { val migrateStatement = migrateStatements.head @@ -61,8 +64,7 @@ trait WorkflowOptionsChange extends BatchedTaskChange { migrateStatement.addBatch() 1 case Failure(exception) => - logger.error( - s"Unable to process $tableName pk $rowId\njson:\n$workflowOptionsJson", exception) + logger.error(s"Unable to process $tableName pk $rowId\njson:\n$workflowOptionsJson", exception) 0 } } diff --git a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsRenaming.scala b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsRenaming.scala index d148a570fdf..aca4a9f6925 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsRenaming.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsRenaming.scala @@ -12,10 +12,9 @@ object WorkflowOptionsRenaming { "call_logs_dir" -> "final_call_logs_dir" ) - def renameOptionKeys(field: JsField): JsField = { + def renameOptionKeys(field: JsField): JsField = field match { case (oldName, value) if RenamedOptionKeys.contains(oldName) => RenamedOptionKeys(oldName) -> value case noop => noop } - } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala index 8e1b98a4a1c..f0968750cba 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala @@ -18,28 +18,38 @@ trait CallCachingSlickDatabase extends CallCachingSqlDatabase { import dataAccess.driver.api._ - override def addCallCaching(joins: Seq[CallCachingJoin], batchSize: Int) - (implicit ec: ExecutionContext): Future[Unit] = { + override def addCallCaching(joins: Seq[CallCachingJoin], batchSize: Int)(implicit + ec: ExecutionContext + ): Future[Unit] = { // Construct parallel lists of parent entries, hashes, simpletons, and detritus from `CallCachingJoin`s. val (entries, hashes, simpletons, detritus, aggregations) = joins.toList.foldMap { j => - (List(j.callCachingEntry), List(j.callCachingHashEntries), List(j.callCachingSimpletonEntries), List(j.callCachingDetritusEntries), List(j.callCachingAggregationEntry.toList)) } + (List(j.callCachingEntry), + List(j.callCachingHashEntries), + List(j.callCachingSimpletonEntries), + List(j.callCachingDetritusEntries), + List(j.callCachingAggregationEntry.toList) + ) + } // Use the supplied `assigner` function to assign parent entry row IDs into the parallel `Seq` of children entities. - def assignEntryIdsToChildren[C](ids: Seq[Long], groupingsOfChildren: Seq[Seq[C]], assigner: (Long, C) => C): Seq[C] = { + def assignEntryIdsToChildren[C](ids: Seq[Long], + groupingsOfChildren: Seq[Seq[C]], + assigner: (Long, C) => C + ): Seq[C] = (ids zip groupingsOfChildren) flatMap { case (id, children) => children.map(assigner(id, _)) } - } // Batch insert entities into the appropriate `Table`. - def batchInsert[E, T <: Table[E]](entries: Seq[E], tableQuery: TableQuery[T]): DBIO[_] = { - DBIO.sequence(entries.grouped(batchSize).map { tableQuery ++= _ }) - } + def batchInsert[E, T <: Table[E]](entries: Seq[E], tableQuery: TableQuery[T]): DBIO[_] = + DBIO.sequence(entries.grouped(batchSize).map(tableQuery ++= _)) // Functions to assign call cache entry IDs into child hash entry, simpleton, and detritus rows. def hashAssigner(id: Long, hash: CallCachingHashEntry) = hash.copy(callCachingEntryId = Option(id)) - def simpletonAssigner(id: Long, simpleton: CallCachingSimpletonEntry) = simpleton.copy(callCachingEntryId = Option(id)) + def simpletonAssigner(id: Long, simpleton: CallCachingSimpletonEntry) = + simpleton.copy(callCachingEntryId = Option(id)) def detritusAssigner(id: Long, detritus: CallCachingDetritusEntry) = detritus.copy(callCachingEntryId = Option(id)) - def aggregationAssigner(id: Long, aggregation: CallCachingAggregationEntry) = aggregation.copy(callCachingEntryId = Option(id)) + def aggregationAssigner(id: Long, aggregation: CallCachingAggregationEntry) = + aggregation.copy(callCachingEntryId = Option(id)) val action = for { entryIds <- dataAccess.callCachingEntryIdsAutoInc ++= entries @@ -70,87 +80,114 @@ trait CallCachingSlickDatabase extends CallCachingSqlDatabase { (0 to 2).toList map { total(_) map { p => PrefixAndLength(p, p.length) } getOrElse doNotMatch } } - override def hasMatchingCallCachingEntriesForBaseAggregation(baseAggregationHash: String, callCachePrefixes: Option[List[String]] = None) - (implicit ec: ExecutionContext): Future[Boolean] = { + override def hasMatchingCallCachingEntriesForBaseAggregation(baseAggregationHash: String, + callCachePrefixes: Option[List[String]] = None + )(implicit ec: ExecutionContext): Future[Boolean] = { val action = callCachePrefixes match { case None => dataAccess.existsCallCachingEntriesForBaseAggregationHash(baseAggregationHash).result case Some(ps) => val one :: two :: three :: _ = prefixesAndLengths(ps) - dataAccess.existsCallCachingEntriesForBaseAggregationHashWithCallCachePrefix( - (baseAggregationHash, - one.prefix, one.length, - two.prefix, two.length, - three.prefix, three.length)).result + dataAccess + .existsCallCachingEntriesForBaseAggregationHashWithCallCachePrefix( + (baseAggregationHash, one.prefix, one.length, two.prefix, two.length, three.prefix, three.length) + ) + .result } runTransaction(action) } - override def findCacheHitForAggregation(baseAggregationHash: String, inputFilesAggregationHash: Option[String], callCachePathPrefixes: Option[List[String]], excludedIds: Set[Long]) - (implicit ec: ExecutionContext): Future[Option[Long]] = { + override def findCacheHitForAggregation(baseAggregationHash: String, + inputFilesAggregationHash: Option[String], + callCachePathPrefixes: Option[List[String]], + excludedIds: Set[Long] + )(implicit ec: ExecutionContext): Future[Option[Long]] = { val action = callCachePathPrefixes match { case None => - dataAccess.callCachingEntriesForAggregatedHashes(baseAggregationHash, inputFilesAggregationHash, excludedIds).result.headOption + dataAccess + .callCachingEntriesForAggregatedHashes(baseAggregationHash, inputFilesAggregationHash, excludedIds) + .result + .headOption case Some(ps) => val one :: two :: three :: _ = prefixesAndLengths(ps) - dataAccess.callCachingEntriesForAggregatedHashesWithPrefixes( - baseAggregationHash, inputFilesAggregationHash, - one.prefix, one.length, - two.prefix, two.length, - three.prefix, three.length, - excludedIds).result.headOption + dataAccess + .callCachingEntriesForAggregatedHashesWithPrefixes(baseAggregationHash, + inputFilesAggregationHash, + one.prefix, + one.length, + two.prefix, + two.length, + three.prefix, + three.length, + excludedIds + ) + .result + .headOption } runTransaction(action) } - override def queryResultsForCacheId(callCachingEntryId: Long) - (implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] = { + override def queryResultsForCacheId( + callCachingEntryId: Long + )(implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] = { val action = for { - callCachingEntryOption <- dataAccess. - callCachingEntriesForId(callCachingEntryId).result.headOption - callCachingSimpletonEntries <- dataAccess. - callCachingSimpletonEntriesForCallCachingEntryId(callCachingEntryId).result - callCachingDetritusEntries <- dataAccess. - callCachingDetritusEntriesForCallCachingEntryId(callCachingEntryId).result + callCachingEntryOption <- dataAccess.callCachingEntriesForId(callCachingEntryId).result.headOption + callCachingSimpletonEntries <- dataAccess + .callCachingSimpletonEntriesForCallCachingEntryId(callCachingEntryId) + .result + callCachingDetritusEntries <- dataAccess + .callCachingDetritusEntriesForCallCachingEntryId(callCachingEntryId) + .result } yield callCachingEntryOption.map( - CallCachingJoin(_, Seq.empty, None, callCachingSimpletonEntries, callCachingDetritusEntries)) + CallCachingJoin(_, Seq.empty, None, callCachingSimpletonEntries, callCachingDetritusEntries) + ) runTransaction(action) } - private def callCacheJoinFromEntryQuery(callCachingEntry: CallCachingEntry) - (implicit ec: ExecutionContext): DBIO[CallCachingJoin] = { + private def callCacheJoinFromEntryQuery( + callCachingEntry: CallCachingEntry + )(implicit ec: ExecutionContext): DBIO[CallCachingJoin] = { val callCachingEntryId = callCachingEntry.callCachingEntryId.get - val callCachingSimpletonEntries: DBIO[Seq[CallCachingSimpletonEntry]] = dataAccess. - callCachingSimpletonEntriesForCallCachingEntryId(callCachingEntryId).result - val callCachingDetritusEntries: DBIO[Seq[CallCachingDetritusEntry]] = dataAccess. - callCachingDetritusEntriesForCallCachingEntryId(callCachingEntryId).result - val callCachingHashEntries: DBIO[Seq[CallCachingHashEntry]] = dataAccess. - callCachingHashEntriesForCallCachingEntryId(callCachingEntryId).result - val callCachingAggregationEntries: DBIO[Option[CallCachingAggregationEntry]] = dataAccess. - callCachingAggregationForCacheEntryId(callCachingEntryId).result.headOption - - (callCachingHashEntries, callCachingAggregationEntries, callCachingSimpletonEntries, callCachingDetritusEntries) mapN { - case (hashes, aggregation, simpletons, detrituses) => - CallCachingJoin(callCachingEntry, hashes, aggregation, simpletons, detrituses) + val callCachingSimpletonEntries: DBIO[Seq[CallCachingSimpletonEntry]] = + dataAccess.callCachingSimpletonEntriesForCallCachingEntryId(callCachingEntryId).result + val callCachingDetritusEntries: DBIO[Seq[CallCachingDetritusEntry]] = + dataAccess.callCachingDetritusEntriesForCallCachingEntryId(callCachingEntryId).result + val callCachingHashEntries: DBIO[Seq[CallCachingHashEntry]] = + dataAccess.callCachingHashEntriesForCallCachingEntryId(callCachingEntryId).result + val callCachingAggregationEntries: DBIO[Option[CallCachingAggregationEntry]] = + dataAccess.callCachingAggregationForCacheEntryId(callCachingEntryId).result.headOption + + (callCachingHashEntries, + callCachingAggregationEntries, + callCachingSimpletonEntries, + callCachingDetritusEntries + ) mapN { case (hashes, aggregation, simpletons, detrituses) => + CallCachingJoin(callCachingEntry, hashes, aggregation, simpletons, detrituses) } } - override def callCacheJoinForCall(workflowExecutionUuid: String, callFqn: String, index: Int) - (implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] = { + override def callCacheJoinForCall(workflowExecutionUuid: String, callFqn: String, index: Int)(implicit + ec: ExecutionContext + ): Future[Option[CallCachingJoin]] = { val action = for { - callCachingEntryOption <- dataAccess. - callCachingEntriesForWorkflowFqnIndex((workflowExecutionUuid, callFqn, index)).result.headOption + callCachingEntryOption <- dataAccess + .callCachingEntriesForWorkflowFqnIndex((workflowExecutionUuid, callFqn, index)) + .result + .headOption callCacheJoin <- callCachingEntryOption - .fold[DBIOAction[Option[CallCachingJoin], NoStream, Effect.All]](DBIO.successful(None))(callCacheJoinFromEntryQuery(_).map(Option.apply)) + .fold[DBIOAction[Option[CallCachingJoin], NoStream, Effect.All]](DBIO.successful(None))( + callCacheJoinFromEntryQuery(_).map(Option.apply) + ) } yield callCacheJoin runTransaction(action) } - override def invalidateCall(callCachingEntryId: Long) - (implicit ec: ExecutionContext): Future[Option[CallCachingEntry]] = { + override def invalidateCall( + callCachingEntryId: Long + )(implicit ec: ExecutionContext): Future[Option[CallCachingEntry]] = { val action = for { _ <- dataAccess.allowResultReuseForCallCachingEntryId(callCachingEntryId).update(false) callCachingEntryOption <- dataAccess.callCachingEntriesForId(callCachingEntryId).result.headOption @@ -159,13 +196,16 @@ trait CallCachingSlickDatabase extends CallCachingSqlDatabase { runTransaction(action) } - override def invalidateCallCacheEntryIdsForWorkflowId(workflowExecutionUuid: String) - (implicit ec: ExecutionContext): Future[Unit] = { + override def invalidateCallCacheEntryIdsForWorkflowId( + workflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Unit] = { val action = dataAccess.allowResultReuseForWorkflowId(workflowExecutionUuid).update(false) runTransaction(action).void } - override def callCacheEntryIdsForWorkflowId(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Seq[Long]] = { + override def callCacheEntryIdsForWorkflowId( + workflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Seq[Long]] = { val action = dataAccess.callCachingEntryIdsForWorkflowId(workflowExecutionUuid).result runTransaction(action) } diff --git a/database/sql/src/main/scala/cromwell/database/slick/DockerHashStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/DockerHashStoreSlickDatabase.scala index 1b70e187083..5109ad17476 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/DockerHashStoreSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/DockerHashStoreSlickDatabase.scala @@ -15,8 +15,9 @@ trait DockerHashStoreSlickDatabase extends DockerHashStoreSqlDatabase { /** * Adds a docker hash entry to the store. */ - override def addDockerHashStoreEntry(dockerHashStoreEntry: DockerHashStoreEntry) - (implicit ec: ExecutionContext): Future[Unit] = { + override def addDockerHashStoreEntry( + dockerHashStoreEntry: DockerHashStoreEntry + )(implicit ec: ExecutionContext): Future[Unit] = { val action = dataAccess.dockerHashStoreEntries += dockerHashStoreEntry runTransaction(action) void } @@ -25,8 +26,9 @@ trait DockerHashStoreSlickDatabase extends DockerHashStoreSqlDatabase { * Retrieves docker hash entries for a workflow. * */ - override def queryDockerHashStoreEntries(workflowExecutionUuid: String) - (implicit ec: ExecutionContext): Future[Seq[DockerHashStoreEntry]] = { + override def queryDockerHashStoreEntries( + workflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Seq[DockerHashStoreEntry]] = { val action = dataAccess.dockerHashStoreEntriesForWorkflowExecutionUuid(workflowExecutionUuid).result runTransaction(action) } @@ -34,7 +36,9 @@ trait DockerHashStoreSlickDatabase extends DockerHashStoreSqlDatabase { /** * Deletes docker hash entries related to a workflow, returning the number of rows affected. */ - override def removeDockerHashStoreEntries(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Int] = { + override def removeDockerHashStoreEntries( + workflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Int] = { val action = dataAccess.dockerHashStoreEntriesForWorkflowExecutionUuid(workflowExecutionUuid).delete runTransaction(action) } diff --git a/database/sql/src/main/scala/cromwell/database/slick/EngineSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/EngineSlickDatabase.scala index bacc10da1ee..fde45e29d6f 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/EngineSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/EngineSlickDatabase.scala @@ -12,7 +12,7 @@ object EngineSlickDatabase { } class EngineSlickDatabase(originalDatabaseConfig: Config) - extends SlickDatabase(originalDatabaseConfig) + extends SlickDatabase(originalDatabaseConfig) with EngineSqlDatabase with WorkflowStoreSlickDatabase with JobKeyValueSlickDatabase diff --git a/database/sql/src/main/scala/cromwell/database/slick/JobKeyValueSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/JobKeyValueSlickDatabase.scala index 4e28cdb33d5..ec416791ad6 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/JobKeyValueSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/JobKeyValueSlickDatabase.scala @@ -16,8 +16,7 @@ trait JobKeyValueSlickDatabase extends JobKeyValueSqlDatabase { runTransaction(action) } - override def addJobKeyValueEntry(jobKeyValueEntry: JobKeyValueEntry) - (implicit ec: ExecutionContext): Future[Unit] = { + override def addJobKeyValueEntry(jobKeyValueEntry: JobKeyValueEntry)(implicit ec: ExecutionContext): Future[Unit] = { val action = if (useSlickUpserts) { for { _ <- dataAccess.jobKeyValueEntryIdsAutoInc.insertOrUpdate(jobKeyValueEntry) @@ -31,24 +30,26 @@ trait JobKeyValueSlickDatabase extends JobKeyValueSqlDatabase { // !!!!!!! updates running in a single transaction. !!!!!!!! // !!!!!!! https://broadworkbench.atlassian.net/browse/BA-6262 !!!!!!!! // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - private def manualUpsertQuery(jobKeyValueEntry: JobKeyValueEntry) - (implicit ec: ExecutionContext) = for { - updateCount <- dataAccess. - storeValuesForJobKeyAndStoreKey(( - jobKeyValueEntry.workflowExecutionUuid, - jobKeyValueEntry.callFullyQualifiedName, - jobKeyValueEntry.jobIndex, - jobKeyValueEntry.jobAttempt, - jobKeyValueEntry.storeKey)). - update(jobKeyValueEntry.storeValue) + private def manualUpsertQuery(jobKeyValueEntry: JobKeyValueEntry)(implicit ec: ExecutionContext) = for { + updateCount <- dataAccess + .storeValuesForJobKeyAndStoreKey( + (jobKeyValueEntry.workflowExecutionUuid, + jobKeyValueEntry.callFullyQualifiedName, + jobKeyValueEntry.jobIndex, + jobKeyValueEntry.jobAttempt, + jobKeyValueEntry.storeKey + ) + ) + .update(jobKeyValueEntry.storeValue) _ <- updateCount match { case 0 => dataAccess.jobKeyValueEntryIdsAutoInc += jobKeyValueEntry case _ => assertUpdateCount("addJobKeyValueEntry", updateCount, 1) } } yield () - def addJobKeyValueEntries(jobKeyValueEntries: Iterable[JobKeyValueEntry]) - (implicit ec: ExecutionContext): Future[Unit] = { + def addJobKeyValueEntries( + jobKeyValueEntries: Iterable[JobKeyValueEntry] + )(implicit ec: ExecutionContext): Future[Unit] = { val action = if (useSlickUpserts) { createBatchUpsert("KeyValueStore", dataAccess.jobKeyValueTableQueryCompiled, jobKeyValueEntries) } else { @@ -57,18 +58,23 @@ trait JobKeyValueSlickDatabase extends JobKeyValueSqlDatabase { runTransaction(action).void } - override def queryJobKeyValueEntries(workflowExecutionUuid: String) - (implicit ec: ExecutionContext): Future[Seq[JobKeyValueEntry]] = { + override def queryJobKeyValueEntries( + workflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Seq[JobKeyValueEntry]] = { val action = dataAccess.jobKeyValueEntriesForWorkflowExecutionUuid(workflowExecutionUuid).result runTransaction(action) } - override def queryStoreValue(workflowExecutionUuid: String, callFqn: String, jobScatterIndex: Int, - jobRetryAttempt: Int, storeKey: String) - (implicit ec: ExecutionContext): Future[Option[String]] = { - val action = dataAccess. - storeValuesForJobKeyAndStoreKey((workflowExecutionUuid, callFqn, jobScatterIndex, jobRetryAttempt, storeKey)). - result.headOption + override def queryStoreValue(workflowExecutionUuid: String, + callFqn: String, + jobScatterIndex: Int, + jobRetryAttempt: Int, + storeKey: String + )(implicit ec: ExecutionContext): Future[Option[String]] = { + val action = dataAccess + .storeValuesForJobKeyAndStoreKey((workflowExecutionUuid, callFqn, jobScatterIndex, jobRetryAttempt, storeKey)) + .result + .headOption runTransaction(action) } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala index 5c13de6dbe3..08c8e637908 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala @@ -11,8 +11,9 @@ trait JobStoreSlickDatabase extends JobStoreSqlDatabase { import dataAccess.driver.api._ - override def addJobStores(jobStoreJoins: Seq[JobStoreJoin], batchSize: Int) - (implicit ec: ExecutionContext): Future[Unit] = { + override def addJobStores(jobStoreJoins: Seq[JobStoreJoin], batchSize: Int)(implicit + ec: ExecutionContext + ): Future[Unit] = { def assignJobStoreIdsToSimpletons(jobStoreIds: Seq[Long]): Seq[JobStoreSimpletonEntry] = { val simpletonsByJobStoreEntry = jobStoreJoins map { _.jobStoreSimpletonEntries } @@ -32,13 +33,17 @@ trait JobStoreSlickDatabase extends JobStoreSqlDatabase { runTransaction(action) } - override def queryJobStores(workflowExecutionUuid: String, callFqn: String, jobScatterIndex: Int, - jobScatterAttempt: Int)(implicit ec: ExecutionContext): - Future[Option[JobStoreJoin]] = { + override def queryJobStores(workflowExecutionUuid: String, + callFqn: String, + jobScatterIndex: Int, + jobScatterAttempt: Int + )(implicit ec: ExecutionContext): Future[Option[JobStoreJoin]] = { val action = for { - jobStoreEntryOption <- dataAccess. - jobStoreEntriesForJobKey((workflowExecutionUuid, callFqn, jobScatterIndex, jobScatterAttempt)).result.headOption + jobStoreEntryOption <- dataAccess + .jobStoreEntriesForJobKey((workflowExecutionUuid, callFqn, jobScatterIndex, jobScatterAttempt)) + .result + .headOption jobStoreSimpletonEntries <- jobStoreEntryOption match { case Some(jobStoreEntry) => dataAccess.jobStoreSimpletonEntriesForJobStoreEntryId(jobStoreEntry.jobStoreEntryId.get).result @@ -49,10 +54,9 @@ trait JobStoreSlickDatabase extends JobStoreSqlDatabase { runTransaction(action) } - override def removeJobStores(workflowExecutionUuids: Seq[String]) - (implicit ec: ExecutionContext): Future[Seq[Int]] = { - val actions = workflowExecutionUuids map { - workflowExecutionUuid => dataAccess.jobStoreEntriesForWorkflowExecutionUuid(workflowExecutionUuid).delete + override def removeJobStores(workflowExecutionUuids: Seq[String])(implicit ec: ExecutionContext): Future[Seq[Int]] = { + val actions = workflowExecutionUuids map { workflowExecutionUuid => + dataAccess.jobStoreEntriesForWorkflowExecutionUuid(workflowExecutionUuid).delete } runTransaction(DBIO.sequence(actions)) } diff --git a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala index eb87f88d101..fbab736ae26 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala @@ -8,8 +8,12 @@ import cromwell.database.slick.tables.MetadataDataAccessComponent import cromwell.database.sql.MetadataSqlDatabase import cromwell.database.sql.SqlConverters._ import cromwell.database.sql.joins.{CallOrWorkflowQuery, CallQuery, MetadataJobQueryValue, WorkflowQuery} -import cromwell.database.sql.tables.{CustomLabelEntry, InformationSchemaEntry, MetadataEntry, WorkflowMetadataSummaryEntry} -import net.ceedubs.ficus.Ficus._ +import cromwell.database.sql.tables.{ + CustomLabelEntry, + InformationSchemaEntry, + MetadataEntry, + WorkflowMetadataSummaryEntry +} import slick.basic.DatabasePublisher import slick.jdbc.{ResultSetConcurrency, ResultSetType} @@ -23,7 +27,8 @@ object MetadataSlickDatabase { } case class SummarizationPartitionedMetadata(nonSummarizableMetadata: Seq[MetadataEntry], - summarizableMetadata: Seq[MetadataEntry]) + summarizableMetadata: Seq[MetadataEntry] + ) def partitionSummarizationMetadata(rawMetadataEntries: Seq[MetadataEntry], startMetadataKey: String, @@ -33,14 +38,24 @@ object MetadataSlickDatabase { submissionMetadataKey: String, parentWorkflowIdKey: String, rootWorkflowIdKey: String, - labelMetadataKey: String): SummarizationPartitionedMetadata = { - - val exactMatchMetadataKeys = Set(startMetadataKey, endMetadataKey, nameMetadataKey, statusMetadataKey, submissionMetadataKey, parentWorkflowIdKey, rootWorkflowIdKey) + labelMetadataKey: String + ): SummarizationPartitionedMetadata = { + + val exactMatchMetadataKeys = Set(startMetadataKey, + endMetadataKey, + nameMetadataKey, + statusMetadataKey, + submissionMetadataKey, + parentWorkflowIdKey, + rootWorkflowIdKey + ) val startsWithMetadataKeys = Set(labelMetadataKey) val (summarizable, nonSummarizable) = rawMetadataEntries partition { entry => entry.callFullyQualifiedName.isEmpty && entry.jobIndex.isEmpty && entry.jobAttempt.isEmpty && - (exactMatchMetadataKeys.contains(entry.metadataKey) || startsWithMetadataKeys.exists(entry.metadataKey.startsWith)) + (exactMatchMetadataKeys.contains(entry.metadataKey) || startsWithMetadataKeys.exists( + entry.metadataKey.startsWith + )) } SummarizationPartitionedMetadata( @@ -51,7 +66,7 @@ object MetadataSlickDatabase { } class MetadataSlickDatabase(originalDatabaseConfig: Config) - extends SlickDatabase(originalDatabaseConfig) + extends SlickDatabase(originalDatabaseConfig) with MetadataSqlDatabase with SummaryStatusSlickDatabase with SummaryQueueSlickDatabase { @@ -60,8 +75,6 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) import dataAccess.driver.api._ import MetadataSlickDatabase._ - lazy val pgLargeObjectWriteRole: Option[String] = originalDatabaseConfig.as[Option[String]]("pgLargeObjectWriteRole") - override def existsMetadataEntries()(implicit ec: ExecutionContext): Future[Boolean] = { val action = dataAccess.metadataEntriesExists.result runTransaction(action) @@ -75,36 +88,39 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) submissionMetadataKey: String, parentWorkflowIdKey: String, rootWorkflowIdKey: String, - labelMetadataKey: String) - (implicit ec: ExecutionContext): Future[Unit] = { + labelMetadataKey: String + )(implicit ec: ExecutionContext): Future[Unit] = { val partitioned = partitionSummarizationMetadata( - rawMetadataEntries = metadataEntries.toSeq, - startMetadataKey, - endMetadataKey, - nameMetadataKey, - statusMetadataKey, - submissionMetadataKey, - parentWorkflowIdKey, - rootWorkflowIdKey, - labelMetadataKey) - - val roleSet = pgLargeObjectWriteRole.map(role => sqlu"""SET ROLE TO "#$role"""") + rawMetadataEntries = metadataEntries.toSeq, + startMetadataKey, + endMetadataKey, + nameMetadataKey, + statusMetadataKey, + submissionMetadataKey, + parentWorkflowIdKey, + rootWorkflowIdKey, + labelMetadataKey + ) // These entries also require a write to the summary queue. - def writeSummarizable(): Future[Unit] = if (partitioned.summarizableMetadata.isEmpty) Future.successful(()) else { + def writeSummarizable(): Future[Unit] = if (partitioned.summarizableMetadata.isEmpty) Future.successful(()) + else { val batchesToWrite = partitioned.summarizableMetadata.grouped(insertBatchSize).toList val insertActions = batchesToWrite.map { batch => val insertMetadata = dataAccess.metadataEntryIdsAutoInc ++= batch insertMetadata.flatMap(ids => writeSummaryQueueEntries(ids)) } - runTransaction(DBIO.sequence(roleSet ++ insertActions)).void + runTransaction(DBIO.sequence(insertActions)).void } // Non-summarizable metadata that only needs to go to the metadata table can be written much more efficiently // than summarizable metadata. - def writeNonSummarizable(): Future[Unit] = if (partitioned.nonSummarizableMetadata.isEmpty) Future.successful(()) else { - val action = DBIO.sequence(roleSet ++ partitioned.nonSummarizableMetadata.grouped(insertBatchSize).map(dataAccess.metadataEntries ++= _)) + def writeNonSummarizable(): Future[Unit] = if (partitioned.nonSummarizableMetadata.isEmpty) Future.successful(()) + else { + val action = DBIO.sequence( + partitioned.nonSummarizableMetadata.grouped(insertBatchSize).map(dataAccess.metadataEntries ++= _) + ) runLobAction(action).void } @@ -119,42 +135,45 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) runTransaction(action) } - override def metadataSummaryEntryExists(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Boolean] = { + override def metadataSummaryEntryExists( + workflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Boolean] = { val action = dataAccess.workflowMetadataSummaryEntryExistsForWorkflowExecutionUuid(workflowExecutionUuid).result runTransaction(action) } - override def queryMetadataEntries(workflowExecutionUuid: String, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { + override def queryMetadataEntries(workflowExecutionUuid: String, timeout: Duration)(implicit + ec: ExecutionContext + ): Future[Seq[MetadataEntry]] = { val action = dataAccess.metadataEntriesForWorkflowExecutionUuid(workflowExecutionUuid).result runTransaction(action, timeout = timeout) } override def streamMetadataEntries(workflowExecutionUuid: String): DatabasePublisher[MetadataEntry] = { - val action = dataAccess.metadataEntriesForWorkflowSortedById(workflowExecutionUuid) + val action = dataAccess + .metadataEntriesForWorkflowSortedById(workflowExecutionUuid) .result .withStatementParameters( rsType = ResultSetType.ForwardOnly, rsConcurrency = ResultSetConcurrency.ReadOnly, // Magic number alert: fetchSize is set to MIN_VALUE for MySQL to stream rather than cache in memory first. // Inspired by: https://github.com/slick/slick/issues/1218 - fetchSize = Integer.MIN_VALUE) + fetchSize = Integer.MIN_VALUE + ) database.stream(action) } - override def countMetadataEntries(workflowExecutionUuid: String, - expandSubWorkflows: Boolean, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Int] = { - val action = dataAccess.countMetadataEntriesForWorkflowExecutionUuid((workflowExecutionUuid, expandSubWorkflows)).result + override def countMetadataEntries(workflowExecutionUuid: String, expandSubWorkflows: Boolean, timeout: Duration)( + implicit ec: ExecutionContext + ): Future[Int] = { + val action = + dataAccess.countMetadataEntriesForWorkflowExecutionUuid((workflowExecutionUuid, expandSubWorkflows)).result runTransaction(action, timeout = timeout) } - override def queryMetadataEntries(workflowExecutionUuid: String, - metadataKey: String, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { + override def queryMetadataEntries(workflowExecutionUuid: String, metadataKey: String, timeout: Duration)(implicit + ec: ExecutionContext + ): Future[Seq[MetadataEntry]] = { val action = dataAccess.metadataEntriesForWorkflowExecutionUuidAndMetadataKey((workflowExecutionUuid, metadataKey)).result runTransaction(action, timeout = timeout) @@ -163,10 +182,14 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) override def countMetadataEntries(workflowExecutionUuid: String, metadataKey: String, expandSubWorkflows: Boolean, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Int] = { + timeout: Duration + )(implicit ec: ExecutionContext): Future[Int] = { val action = - dataAccess.countMetadataEntriesForWorkflowExecutionUuidAndMetadataKey((workflowExecutionUuid, metadataKey, expandSubWorkflows)).result + dataAccess + .countMetadataEntriesForWorkflowExecutionUuidAndMetadataKey( + (workflowExecutionUuid, metadataKey, expandSubWorkflows) + ) + .result runTransaction(action, timeout = timeout) } @@ -174,10 +197,10 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) callFullyQualifiedName: String, jobIndex: Option[Int], jobAttempt: Option[Int], - timeout: Duration) - (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { - val action = dataAccess. - metadataEntriesForJobKey((workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt)).result + timeout: Duration + )(implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { + val action = + dataAccess.metadataEntriesForJobKey((workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt)).result runTransaction(action, timeout = timeout) } @@ -186,10 +209,13 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) jobIndex: Option[Int], jobAttempt: Option[Int], expandSubWorkflows: Boolean, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Int] = { - val action = dataAccess. - countMetadataEntriesForJobKey((workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt, expandSubWorkflows)).result + timeout: Duration + )(implicit ec: ExecutionContext): Future[Int] = { + val action = dataAccess + .countMetadataEntriesForJobKey( + (workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt, expandSubWorkflows) + ) + .result runTransaction(action, timeout = timeout) } @@ -198,10 +224,11 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) callFullyQualifiedName: String, jobIndex: Option[Int], jobAttempt: Option[Int], - timeout: Duration) - (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { - val action = dataAccess.metadataEntriesForJobKeyAndMetadataKey(( - workflowUuid, metadataKey, callFullyQualifiedName, jobIndex, jobAttempt)).result + timeout: Duration + )(implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { + val action = dataAccess + .metadataEntriesForJobKeyAndMetadataKey((workflowUuid, metadataKey, callFullyQualifiedName, jobIndex, jobAttempt)) + .result runTransaction(action, timeout = timeout) } @@ -211,10 +238,13 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) jobIndex: Option[Int], jobAttempt: Option[Int], expandSubWorkflows: Boolean, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Int] = { - val action = dataAccess.countMetadataEntriesForJobKeyAndMetadataKey(( - workflowUuid, metadataKey, callFullyQualifiedName, jobIndex, jobAttempt, expandSubWorkflows)).result + timeout: Duration + )(implicit ec: ExecutionContext): Future[Int] = { + val action = dataAccess + .countMetadataEntriesForJobKeyAndMetadataKey( + (workflowUuid, metadataKey, callFullyQualifiedName, jobIndex, jobAttempt, expandSubWorkflows) + ) + .result runTransaction(action, timeout = timeout) } @@ -222,15 +252,35 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) metadataKeysToFilterFor: List[String], metadataKeysToFilterOut: List[String], metadataJobQueryValue: MetadataJobQueryValue, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { + timeout: Duration + )(implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { val action = metadataJobQueryValue match { case CallQuery(callFqn, jobIndex, jobAttempt) => - dataAccess.metadataEntriesForJobWithKeyConstraints(workflowExecutionUuid, metadataKeysToFilterFor, metadataKeysToFilterOut, callFqn, jobIndex, jobAttempt).result + dataAccess + .metadataEntriesForJobWithKeyConstraints(workflowExecutionUuid, + metadataKeysToFilterFor, + metadataKeysToFilterOut, + callFqn, + jobIndex, + jobAttempt + ) + .result case WorkflowQuery => - dataAccess.metadataEntriesWithKeyConstraints(workflowExecutionUuid, metadataKeysToFilterFor, metadataKeysToFilterOut, requireEmptyJobKey = true).result + dataAccess + .metadataEntriesWithKeyConstraints(workflowExecutionUuid, + metadataKeysToFilterFor, + metadataKeysToFilterOut, + requireEmptyJobKey = true + ) + .result case CallOrWorkflowQuery => - dataAccess.metadataEntriesWithKeyConstraints(workflowExecutionUuid, metadataKeysToFilterFor, metadataKeysToFilterOut, requireEmptyJobKey = false).result + dataAccess + .metadataEntriesWithKeyConstraints(workflowExecutionUuid, + metadataKeysToFilterFor, + metadataKeysToFilterOut, + requireEmptyJobKey = false + ) + .result } runTransaction(action, timeout = timeout) } @@ -240,88 +290,115 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) metadataKeysToFilterOut: List[String], metadataJobQueryValue: MetadataJobQueryValue, expandSubWorkflows: Boolean, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Int] = { + timeout: Duration + )(implicit ec: ExecutionContext): Future[Int] = { val action = metadataJobQueryValue match { case CallQuery(callFqn, jobIndex, jobAttempt) => - dataAccess.countMetadataEntriesForJobWithKeyConstraints(workflowExecutionUuid, metadataKeysToFilterFor, metadataKeysToFilterOut, callFqn, jobIndex, jobAttempt, expandSubWorkflows).result + dataAccess + .countMetadataEntriesForJobWithKeyConstraints(workflowExecutionUuid, + metadataKeysToFilterFor, + metadataKeysToFilterOut, + callFqn, + jobIndex, + jobAttempt, + expandSubWorkflows + ) + .result case WorkflowQuery => - dataAccess.countMetadataEntriesWithKeyConstraints(workflowExecutionUuid, metadataKeysToFilterFor, metadataKeysToFilterOut, requireEmptyJobKey = true, expandSubWorkflows = expandSubWorkflows).result + dataAccess + .countMetadataEntriesWithKeyConstraints(workflowExecutionUuid, + metadataKeysToFilterFor, + metadataKeysToFilterOut, + requireEmptyJobKey = true, + expandSubWorkflows = expandSubWorkflows + ) + .result case CallOrWorkflowQuery => - dataAccess.countMetadataEntriesWithKeyConstraints(workflowExecutionUuid, metadataKeysToFilterFor, metadataKeysToFilterOut, requireEmptyJobKey = false, expandSubWorkflows = expandSubWorkflows).result + dataAccess + .countMetadataEntriesWithKeyConstraints(workflowExecutionUuid, + metadataKeysToFilterFor, + metadataKeysToFilterOut, + requireEmptyJobKey = false, + expandSubWorkflows = expandSubWorkflows + ) + .result } runTransaction(action, timeout = timeout) } - private def updateWorkflowMetadataSummaryEntry(buildUpdatedWorkflowMetadataSummaryEntry: - (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry]) => - WorkflowMetadataSummaryEntry) - (workflowExecutionUuuidAndMetadataEntries: (String, Seq[MetadataEntry])) - (implicit ec: ExecutionContext): DBIO[Unit] = { + private def updateWorkflowMetadataSummaryEntry( + buildUpdatedWorkflowMetadataSummaryEntry: (Option[WorkflowMetadataSummaryEntry], + Seq[MetadataEntry] + ) => WorkflowMetadataSummaryEntry + )( + workflowExecutionUuuidAndMetadataEntries: (String, Seq[MetadataEntry]) + )(implicit ec: ExecutionContext): DBIO[Unit] = { val (workflowExecutionUuid, metadataEntries) = workflowExecutionUuuidAndMetadataEntries for { - // There might not be a preexisting summary for a given UUID, so `headOption` the result - existingWorkflowMetadataSummaryEntry <- dataAccess. - workflowMetadataSummaryEntriesForWorkflowExecutionUuid(workflowExecutionUuid).result.headOption + // There might not be a preexisting summary for a given UUID, so `headOption` the result + existingWorkflowMetadataSummaryEntry <- dataAccess + .workflowMetadataSummaryEntriesForWorkflowExecutionUuid(workflowExecutionUuid) + .result + .headOption updatedWorkflowMetadataSummaryEntry = buildUpdatedWorkflowMetadataSummaryEntry( - existingWorkflowMetadataSummaryEntry, metadataEntries) + existingWorkflowMetadataSummaryEntry, + metadataEntries + ) _ <- upsertWorkflowMetadataSummaryEntry(updatedWorkflowMetadataSummaryEntry) } yield () } private def toCustomLabelEntry(metadataEntry: MetadataEntry): CustomLabelEntry = { - //Extracting the label key from the MetadataEntry key + // Extracting the label key from the MetadataEntry key val labelKey = metadataEntry.metadataKey.split("\\:", 2)(1) val labelValue = metadataEntry.metadataValue.toRawString val customLabelEntry = CustomLabelEntry(labelKey, labelValue, metadataEntry.workflowExecutionUuid) customLabelEntry } - private def upsertCustomLabelEntry(customLabelEntry: CustomLabelEntry) - (implicit ec: ExecutionContext): DBIO[Unit] = { + private def upsertCustomLabelEntry(customLabelEntry: CustomLabelEntry)(implicit ec: ExecutionContext): DBIO[Unit] = if (useSlickUpserts) { for { _ <- dataAccess.customLabelEntryIdsAutoInc.insertOrUpdate(customLabelEntry) } yield () } else { for { - updateCount <- dataAccess. - customLabelEntriesForWorkflowExecutionUuidAndLabelKey( + updateCount <- dataAccess + .customLabelEntriesForWorkflowExecutionUuidAndLabelKey( (customLabelEntry.workflowExecutionUuid, customLabelEntry.customLabelKey) - ).update(customLabelEntry) + ) + .update(customLabelEntry) _ <- updateCount match { case 0 => dataAccess.customLabelEntryIdsAutoInc += customLabelEntry case _ => assertUpdateCount("upsertCustomLabelEntry", updateCount, 1) } } yield () } - } - private def upsertWorkflowMetadataSummaryEntry(workflowMetadataSummaryEntry: WorkflowMetadataSummaryEntry) - (implicit ec: ExecutionContext): DBIO[Unit] = { + private def upsertWorkflowMetadataSummaryEntry( + workflowMetadataSummaryEntry: WorkflowMetadataSummaryEntry + )(implicit ec: ExecutionContext): DBIO[Unit] = if (useSlickUpserts) { for { _ <- dataAccess.workflowMetadataSummaryEntryIdsAutoInc.insertOrUpdate(workflowMetadataSummaryEntry) } yield () } else { for { - updateCount <- dataAccess. - workflowMetadataSummaryEntriesForWorkflowExecutionUuid(workflowMetadataSummaryEntry.workflowExecutionUuid). - update(workflowMetadataSummaryEntry) + updateCount <- dataAccess + .workflowMetadataSummaryEntriesForWorkflowExecutionUuid(workflowMetadataSummaryEntry.workflowExecutionUuid) + .update(workflowMetadataSummaryEntry) _ <- updateCount match { case 0 => dataAccess.workflowMetadataSummaryEntryIdsAutoInc += workflowMetadataSummaryEntry case _ => assertUpdateCount("upsertWorkflowMetadataSummaryEntry", updateCount, 1) } } yield () } - } - override def summarizeIncreasing(labelMetadataKey: String, - limit: Int, - buildUpdatedSummary: - (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry]) - => WorkflowMetadataSummaryEntry) - (implicit ec: ExecutionContext): Future[Long] = { + override def summarizeIncreasing( + labelMetadataKey: String, + limit: Int, + buildUpdatedSummary: (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry]) => WorkflowMetadataSummaryEntry + )(implicit ec: ExecutionContext): Future[Long] = { val action = for { rawMetadataEntries <- dataAccess.metadataEntriesToSummarizeQuery(limit.toLong).result _ <- @@ -337,14 +414,13 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) runTransaction(action) } - override def summarizeDecreasing(summaryNameDecreasing: String, - summaryNameIncreasing: String, - labelMetadataKey: String, - limit: Int, - buildUpdatedSummary: - (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry]) - => WorkflowMetadataSummaryEntry) - (implicit ec: ExecutionContext): Future[(Long, Long)] = { + override def summarizeDecreasing( + summaryNameDecreasing: String, + summaryNameIncreasing: String, + labelMetadataKey: String, + limit: Int, + buildUpdatedSummary: (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry]) => WorkflowMetadataSummaryEntry + )(implicit ec: ExecutionContext): Future[(Long, Long)] = { val action = for { previousExistingMetadataEntryIdOption <- getSummaryStatusEntrySummaryPosition(summaryNameDecreasing) previousInitializedMetadataEntryIdOption <- previousExistingMetadataEntryIdOption match { @@ -372,13 +448,14 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) runTransaction(action) } - private def buildMetadataSummaryFromRawMetadataAndWriteToDb(rawMetadataEntries: Seq[MetadataEntry], - labelMetadataKey: String, - buildUpdatedSummary: - (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry]) => WorkflowMetadataSummaryEntry - )(implicit ec: ExecutionContext): DBIO[Unit] = { + private def buildMetadataSummaryFromRawMetadataAndWriteToDb( + rawMetadataEntries: Seq[MetadataEntry], + labelMetadataKey: String, + buildUpdatedSummary: (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry]) => WorkflowMetadataSummaryEntry + )(implicit ec: ExecutionContext): DBIO[Unit] = { - val (summarizableLabelsMetadata, summarizableRegularMetadata) = rawMetadataEntries.partition(_.metadataKey.contains(labelMetadataKey)) + val (summarizableLabelsMetadata, summarizableRegularMetadata) = + rawMetadataEntries.partition(_.metadataKey.contains(labelMetadataKey)) val groupedSummarizableRegularMetadata = summarizableRegularMetadata.groupBy(_.workflowExecutionUuid) for { @@ -387,25 +464,32 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) } yield () } - override def updateMetadataArchiveStatus(workflowExecutionUuid: String, newArchiveStatus: Option[String]): Future[Int] = { + override def updateMetadataArchiveStatus(workflowExecutionUuid: String, + newArchiveStatus: Option[String] + ): Future[Int] = { val action = dataAccess.metadataArchiveStatusByWorkflowId(workflowExecutionUuid).update(newArchiveStatus) runTransaction(action) } - override def getWorkflowStatus(workflowExecutionUuid: String) - (implicit ec: ExecutionContext): Future[Option[String]] = { + override def getWorkflowStatus( + workflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Option[String]] = { val action = dataAccess.workflowStatusesForWorkflowExecutionUuid(workflowExecutionUuid).result.headOption // The workflow might not exist, so `headOption`. But even if the workflow does exist, the status might be None. // So flatten the Option[Option[String]] to Option[String]. runTransaction(action).map(_.flatten) } - override def getWorkflowLabels(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Map[String, String]] = { + override def getWorkflowLabels( + workflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Map[String, String]] = { val action = dataAccess.labelsForWorkflowExecutionUuid(workflowExecutionUuid).result runTransaction(action).map(_.toMap) } - override def getRootAndSubworkflowLabels(rootWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Map[String, Map[String, String]]] = { + override def getRootAndSubworkflowLabels( + rootWorkflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Map[String, Map[String, String]]] = { val action = dataAccess.labelsForWorkflowAndSubworkflows(rootWorkflowExecutionUuid).result // An empty Map of String workflow IDs to an inner Map of label keys to label values. // The outer Map has a default value so any request for a workflow ID not already present @@ -424,10 +508,10 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) workflowStatuses: Set[String], workflowNames: Set[String], workflowExecutionUuids: Set[String], - labelAndKeyLabelValues: Set[(String,String)], - labelOrKeyLabelValues: Set[(String,String)], - excludeLabelAndValues: Set[(String,String)], - excludeLabelOrValues: Set[(String,String)], + labelAndKeyLabelValues: Set[(String, String)], + labelOrKeyLabelValues: Set[(String, String)], + excludeLabelAndValues: Set[(String, String)], + excludeLabelOrValues: Set[(String, String)], submissionTimestampOption: Option[Timestamp], startTimestampOption: Option[Timestamp], endTimestampOption: Option[Timestamp], @@ -435,12 +519,27 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) includeSubworkflows: Boolean, page: Option[Int], pageSize: Option[Int], - newestFirst: Boolean) - (implicit ec: ExecutionContext): Future[Seq[WorkflowMetadataSummaryEntry]] = { - - val action = dataAccess.queryWorkflowMetadataSummaryEntries(parentIdWorkflowMetadataKey, workflowStatuses, workflowNames, workflowExecutionUuids, - labelAndKeyLabelValues, labelOrKeyLabelValues, excludeLabelAndValues, excludeLabelOrValues, submissionTimestampOption, startTimestampOption, - endTimestampOption, metadataArchiveStatus, includeSubworkflows, page, pageSize, newestFirst) + newestFirst: Boolean + )(implicit ec: ExecutionContext): Future[Seq[WorkflowMetadataSummaryEntry]] = { + + val action = dataAccess.queryWorkflowMetadataSummaryEntries( + parentIdWorkflowMetadataKey, + workflowStatuses, + workflowNames, + workflowExecutionUuids, + labelAndKeyLabelValues, + labelOrKeyLabelValues, + excludeLabelAndValues, + excludeLabelOrValues, + submissionTimestampOption, + startTimestampOption, + endTimestampOption, + metadataArchiveStatus, + includeSubworkflows, + page, + pageSize, + newestFirst + ) runTransaction(action) } @@ -448,79 +547,110 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config) workflowStatuses: Set[String], workflowNames: Set[String], workflowExecutionUuids: Set[String], - labelAndKeyLabelValues: Set[(String,String)], - labelOrKeyLabelValues: Set[(String,String)], - excludeLabelAndValues: Set[(String,String)], - excludeLabelOrValues: Set[(String,String)], + labelAndKeyLabelValues: Set[(String, String)], + labelOrKeyLabelValues: Set[(String, String)], + excludeLabelAndValues: Set[(String, String)], + excludeLabelOrValues: Set[(String, String)], submissionTimestampOption: Option[Timestamp], startTimestampOption: Option[Timestamp], endTimestampOption: Option[Timestamp], metadataArchiveStatus: Set[Option[String]], - includeSubworkflows: Boolean) - (implicit ec: ExecutionContext): Future[Int] = { - val action = dataAccess.countWorkflowMetadataSummaryEntries(parentIdWorkflowMetadataKey, workflowStatuses, workflowNames, workflowExecutionUuids, - labelAndKeyLabelValues, labelOrKeyLabelValues, excludeLabelAndValues, excludeLabelOrValues, submissionTimestampOption, startTimestampOption, - endTimestampOption, metadataArchiveStatus, includeSubworkflows) + includeSubworkflows: Boolean + )(implicit ec: ExecutionContext): Future[Int] = { + val action = dataAccess.countWorkflowMetadataSummaryEntries( + parentIdWorkflowMetadataKey, + workflowStatuses, + workflowNames, + workflowExecutionUuids, + labelAndKeyLabelValues, + labelOrKeyLabelValues, + excludeLabelAndValues, + excludeLabelOrValues, + submissionTimestampOption, + startTimestampOption, + endTimestampOption, + metadataArchiveStatus, + includeSubworkflows + ) runTransaction(action) } - override def deleteAllMetadataForWorkflowAndUpdateArchiveStatus(workflowId: String, newArchiveStatus: Option[String])(implicit ec: ExecutionContext): Future[Int] = { + override def deleteAllMetadataForWorkflowAndUpdateArchiveStatus(workflowId: String, newArchiveStatus: Option[String])( + implicit ec: ExecutionContext + ): Future[Int] = runTransaction { for { numDeleted <- dataAccess.metadataEntriesForWorkflowSortedById(workflowId).delete _ <- dataAccess.metadataArchiveStatusByWorkflowId(workflowId).update(newArchiveStatus) } yield numDeleted } - } - override def getRootWorkflowId(workflowId: String)(implicit ec: ExecutionContext): Future[Option[String]] = { + override def getRootWorkflowId(workflowId: String)(implicit ec: ExecutionContext): Future[Option[String]] = runAction( dataAccess.rootWorkflowId(workflowId).result.headOption ) - } - override def queryWorkflowIdsByArchiveStatusAndEndedOnOrBeforeThresholdTimestamp(archiveStatus: Option[String], thresholdTimestamp: Timestamp, batchSize: Long)(implicit ec: ExecutionContext): Future[Seq[String]] = { + override def queryWorkflowIdsByArchiveStatusAndEndedOnOrBeforeThresholdTimestamp(archiveStatus: Option[String], + thresholdTimestamp: Timestamp, + batchSize: Long + )(implicit ec: ExecutionContext): Future[Seq[String]] = runAction( - dataAccess.workflowIdsByArchiveStatusAndEndedOnOrBeforeThresholdTimestamp((archiveStatus, thresholdTimestamp, batchSize)).result + dataAccess + .workflowIdsByArchiveStatusAndEndedOnOrBeforeThresholdTimestamp((archiveStatus, thresholdTimestamp, batchSize)) + .result ) - } override def getSummaryQueueSize()(implicit ec: ExecutionContext): Future[Int] = runAction( countSummaryQueueEntries() ) - override def getMetadataArchiveStatusAndEndTime(workflowId: String)(implicit ec: ExecutionContext): Future[(Option[String], Option[Timestamp])] = { + override def getMetadataArchiveStatusAndEndTime( + workflowId: String + )(implicit ec: ExecutionContext): Future[(Option[String], Option[Timestamp])] = { val action = dataAccess.metadataArchiveStatusAndEndTimeByWorkflowId(workflowId).result.headOption runTransaction(action).map(_.getOrElse((None, None))) } override def queryWorkflowsToArchiveThatEndedOnOrBeforeThresholdTimestamp(workflowStatuses: List[String], workflowEndTimestampThreshold: Timestamp, - batchSize: Long)(implicit ec: ExecutionContext): Future[Seq[WorkflowMetadataSummaryEntry]] = { + batchSize: Long + )(implicit ec: ExecutionContext): Future[Seq[WorkflowMetadataSummaryEntry]] = runAction( - dataAccess.workflowsToArchiveThatEndedOnOrBeforeThresholdTimestamp(workflowStatuses, workflowEndTimestampThreshold, batchSize).result + dataAccess + .workflowsToArchiveThatEndedOnOrBeforeThresholdTimestamp(workflowStatuses, + workflowEndTimestampThreshold, + batchSize + ) + .result ) - } override def countWorkflowsLeftToArchiveThatEndedOnOrBeforeThresholdTimestamp(workflowStatuses: List[String], - workflowEndTimestampThreshold: Timestamp)(implicit ec: ExecutionContext): Future[Int] = { + workflowEndTimestampThreshold: Timestamp + )(implicit ec: ExecutionContext): Future[Int] = runAction( - dataAccess.countWorkflowsLeftToArchiveThatEndedOnOrBeforeThresholdTimestamp(workflowStatuses, workflowEndTimestampThreshold).result + dataAccess + .countWorkflowsLeftToArchiveThatEndedOnOrBeforeThresholdTimestamp(workflowStatuses, + workflowEndTimestampThreshold + ) + .result ) - } - override def countWorkflowsLeftToDeleteThatEndedOnOrBeforeThresholdTimestamp(workflowEndTimestampThreshold: Timestamp)(implicit ec: ExecutionContext): Future[Int] = { + override def countWorkflowsLeftToDeleteThatEndedOnOrBeforeThresholdTimestamp( + workflowEndTimestampThreshold: Timestamp + )(implicit ec: ExecutionContext): Future[Int] = runAction( dataAccess.countWorkflowsLeftToDeleteThatEndedOnOrBeforeThresholdTimestamp(workflowEndTimestampThreshold).result ) - } - override def getMetadataTableSizeInformation()(implicit ec: ExecutionContext): Future[Option[InformationSchemaEntry]] = { + override def getMetadataTableSizeInformation()(implicit + ec: ExecutionContext + ): Future[Option[InformationSchemaEntry]] = runAction(dataAccess.metadataTableSizeInformation()) - } - override def getFailedJobsMetadataWithWorkflowId(rootWorkflowId: String)(implicit ec: ExecutionContext): Future[Vector[MetadataEntry]] = { + override def getFailedJobsMetadataWithWorkflowId( + rootWorkflowId: String + )(implicit ec: ExecutionContext): Future[Vector[MetadataEntry]] = { val isPostgres = databaseConfig.getValue("db.driver").toString.toLowerCase().contains("postgres") runLobAction(dataAccess.failedJobsMetadataWithWorkflowId(rootWorkflowId, isPostgres)) } diff --git a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala index 459c705480e..bc9b085b1f1 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala @@ -11,11 +11,12 @@ import slick.basic.DatabaseConfig import slick.jdbc.{JdbcCapabilities, JdbcProfile, PostgresProfile, TransactionIsolation} import java.sql.{Connection, PreparedStatement, Statement} -import java.util.concurrent.{ExecutorService, Executors} +import java.util.concurrent.{Executors, ExecutorService} import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext, Future} object SlickDatabase { + /** * Returns either the "url" or "properties.url" */ @@ -23,7 +24,7 @@ object SlickDatabase { lazy val log: Logger = LoggerFactory.getLogger("cromwell.database.slick") - def createSchema(slickDatabase: SlickDatabase): Unit = { + def createSchema(slickDatabase: SlickDatabase): Unit = // NOTE: Slick 3.0.0 schema creation, Clobs, and MySQL don't mix: https://github.com/slick/slick/issues/637 // // Not really an issue, since externally run liquibase is standard way of installing / upgrading MySQL. @@ -43,7 +44,6 @@ object SlickDatabase { import slickDatabase.dataAccess.driver.api._ Await.result(slickDatabase.database.run(slickDatabase.dataAccess.schema.create), Duration.Inf) } - } def getDatabaseConfig(name: String, parentConfig: Config): Config = { val rootDatabaseConfig = parentConfig.getConfig("database") @@ -72,6 +72,19 @@ abstract class SlickDatabase(override val originalDatabaseConfig: Config) extend // NOTE: if you want to refactor database is inner-class type: this.dataAccess.driver.backend.DatabaseFactory val database = slickConfig.db + /* + In some cases we want to write Postgres Large Objects (corresponding to Clob/Blob in Slick) + with a role other than the database user we are authenticated as. This is important + when we want multiple login users to be able to access the records. We can SET ROLE to + a role granted to all of them, and they'll all be able to access the Large Objects. + This SO thread also has a good explanation: + https://dba.stackexchange.com/questions/147607/postgres-large-objects-multiple-users + */ + private lazy val pgLargeObjectWriteRole: Option[String] = + originalDatabaseConfig.as[Option[String]]("pgLargeObjectWriteRole") + private lazy val roleSetCmd = + pgLargeObjectWriteRole.map(role => sqlu"""SET LOCAL ROLE TO "#$role"""") + override lazy val connectionDescription: String = databaseConfig.getString(urlKey) SlickDatabase.log.info(s"Running with database $urlKey = $connectionDescription") @@ -130,7 +143,8 @@ abstract class SlickDatabase(override val originalDatabaseConfig: Config) extend } private val actionExecutionContext: ExecutionContext = ExecutionContext.fromExecutor( - actionThreadPool, database.executor.executionContext.reportFailure + actionThreadPool, + database.executor.executionContext.reportFailure ) protected[this] lazy val insertBatchSize: Int = databaseConfig.getOrElse("insert-batch-size", 2000) @@ -138,23 +152,21 @@ abstract class SlickDatabase(override val originalDatabaseConfig: Config) extend protected[this] lazy val useSlickUpserts: Boolean = dataAccess.driver.capabilities.contains(JdbcCapabilities.insertOrUpdate) - //noinspection SameParameterValue - protected[this] def assertUpdateCount(description: String, updates: Int, expected: Int): DBIO[Unit] = { + // noinspection SameParameterValue + protected[this] def assertUpdateCount(description: String, updates: Int, expected: Int): DBIO[Unit] = if (updates == expected) { DBIO.successful(()) } else { DBIO.failed(new RuntimeException(s"$description expected update count $expected, got $updates")) } - } - override def withConnection[A](block: Connection => A): A = { + override def withConnection[A](block: Connection => A): A = /* TODO: Should this withConnection() method have a (implicit?) timeout parameter, that it passes on to Await.result? If we run completely asynchronously, nest calls to withConnection, and then call flatMap, the outer connection may already be closed before an inner block finishes running. */ Await.result(database.run(SimpleDBIO(context => block(context.connection))), Duration.Inf) - } override def close(): Unit = { actionThreadPool.shutdown() @@ -166,40 +178,51 @@ abstract class SlickDatabase(override val originalDatabaseConfig: Config) extend protected[this] def runTransaction[R](action: DBIO[R], isolationLevel: TransactionIsolation = TransactionIsolation.RepeatableRead, - timeout: Duration = Duration.Inf): Future[R] = { - runActionInternal(action.transactionally.withTransactionIsolation(isolationLevel), timeout = timeout) - } + timeout: Duration = Duration.Inf + ): Future[R] = + runActionInternal(withLobRole(action).transactionally.withTransactionIsolation(isolationLevel), timeout = timeout) + + /* + If we're using Postgres and have been configured to do so, set the desired role on the transaction. + See comments on `roleSetCmd` above for more information. + */ + private def withLobRole[R](action: DBIO[R]): DBIO[R] = + (dataAccess.driver, roleSetCmd) match { + case (PostgresProfile, Some(roleSet)) => roleSet.andThen(action) + case _ => action + } /* Note that this is only appropriate for actions that do not involve Blob * or Clob fields in Postgres, since large object support requires running * transactionally. Use runLobAction instead, which will still run in * auto-commit mode when using other database engines. */ - protected[this] def runAction[R](action: DBIO[R]): Future[R] = { + protected[this] def runAction[R](action: DBIO[R]): Future[R] = runActionInternal(action.withPinnedSession) - } /* Wrapper for queries where Clob/Blob types are used * https://stackoverflow.com/questions/3164072/large-objects-may-not-be-used-in-auto-commit-mode#answer-3164352 */ - protected[this] def runLobAction[R](action: DBIO[R]): Future[R] = { + protected[this] def runLobAction[R](action: DBIO[R]): Future[R] = dataAccess.driver match { case PostgresProfile => runTransaction(action) case _ => runAction(action) } - } - private def runActionInternal[R](action: DBIO[R], timeout: Duration = Duration.Inf): Future[R] = { - //database.run(action) <-- See comment above private val actionThreadPool + private def runActionInternal[R](action: DBIO[R], timeout: Duration = Duration.Inf): Future[R] = + // database.run(action) <-- See comment above private val actionThreadPool Future { - try { + try if (timeout.isFinite) { // https://stackoverflow.com/a/52569275/818054 - Await.result(database.run(action.withStatementParameters(statementInit = _.setQueryTimeout(timeout.toSeconds.toInt))), Duration.Inf) + Await.result( + database.run(action.withStatementParameters(statementInit = _.setQueryTimeout(timeout.toSeconds.toInt))), + Duration.Inf + ) } else { Await.result(database.run(action), Duration.Inf) } - } catch { + catch { case rollbackException: MySQLTransactionRollbackException => debugExitStatusCodeOption match { case Some(status) => @@ -229,17 +252,16 @@ abstract class SlickDatabase(override val originalDatabaseConfig: Config) extend } } }(actionExecutionContext) - } /* - * Upsert the provided values in batch. - * Fails the query if one or more upsert failed. - * Adapted from https://github.com/slick/slick/issues/1781 + * Upsert the provided values in batch. + * Fails the query if one or more upsert failed. + * Adapted from https://github.com/slick/slick/issues/1781 */ protected[this] def createBatchUpsert[T](description: String, compiled: dataAccess.driver.JdbcCompiledInsert, values: Iterable[T] - )(implicit ec: ExecutionContext): DBIO[Unit] = { + )(implicit ec: ExecutionContext): DBIO[Unit] = SimpleDBIO { context => context.session.withPreparedStatement[Array[Int]](compiled.upsert.sql) { st: PreparedStatement => values.foreach { update => @@ -255,10 +277,11 @@ abstract class SlickDatabase(override val originalDatabaseConfig: Config) extend else { val valueList = values.toList val failedRequests = failures.toList.map(valueList(_)) - DBIO.failed(new RuntimeException( - s"$description failed to upsert the following rows: ${failedRequests.mkString(", ")}" - )) + DBIO.failed( + new RuntimeException( + s"$description failed to upsert the following rows: ${failedRequests.mkString(", ")}" + ) + ) } } - } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/SubWorkflowStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SubWorkflowStoreSlickDatabase.scala index 21dd3ece7a0..973f18540fc 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/SubWorkflowStoreSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/SubWorkflowStoreSlickDatabase.scala @@ -12,14 +12,18 @@ trait SubWorkflowStoreSlickDatabase extends SubWorkflowStoreSqlDatabase { import dataAccess.driver.api._ - def addSubWorkflowStoreEntry(rootWorkflowExecutionUuid: String, + def addSubWorkflowStoreEntry(rootWorkflowExecutionUuid: String, parentWorkflowExecutionUuid: String, callFullyQualifiedName: String, jobIndex: Int, jobAttempt: Int, - subWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Unit] = { + subWorkflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Unit] = { val action = for { - workflowStoreEntry <- dataAccess.workflowStoreEntriesForWorkflowExecutionUuid(rootWorkflowExecutionUuid).result.headOption + workflowStoreEntry <- dataAccess + .workflowStoreEntriesForWorkflowExecutionUuid(rootWorkflowExecutionUuid) + .result + .headOption _ <- workflowStoreEntry match { case Some(rootWorkflow) => dataAccess.subWorkflowStoreEntryIdsAutoInc += @@ -31,28 +35,41 @@ trait SubWorkflowStoreSlickDatabase extends SubWorkflowStoreSqlDatabase { jobAttempt, subWorkflowExecutionUuid ) - case None => DBIO.failed(new IllegalArgumentException(s"Could not find root workflow with UUID $rootWorkflowExecutionUuid")) + case None => + DBIO.failed( + new IllegalArgumentException(s"Could not find root workflow with UUID $rootWorkflowExecutionUuid") + ) } } yield () - + runTransaction(action) void } - override def querySubWorkflowStore(parentWorkflowExecutionUuid: String, callFqn: String, jobIndex: Int, jobAttempt: Int) - (implicit ec: ExecutionContext): Future[Option[SubWorkflowStoreEntry]] = { + override def querySubWorkflowStore(parentWorkflowExecutionUuid: String, + callFqn: String, + jobIndex: Int, + jobAttempt: Int + )(implicit ec: ExecutionContext): Future[Option[SubWorkflowStoreEntry]] = { val action = for { - subWorkflowStoreEntryOption <- dataAccess.subWorkflowStoreEntriesForJobKey( - (parentWorkflowExecutionUuid, callFqn, jobIndex, jobAttempt) - ).result.headOption + subWorkflowStoreEntryOption <- dataAccess + .subWorkflowStoreEntriesForJobKey( + (parentWorkflowExecutionUuid, callFqn, jobIndex, jobAttempt) + ) + .result + .headOption } yield subWorkflowStoreEntryOption runTransaction(action) } - override def removeSubWorkflowStoreEntries(rootWorkflowExecutionUuid: String) - (implicit ec: ExecutionContext): Future[Int] = { + override def removeSubWorkflowStoreEntries( + rootWorkflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Int] = { val action = for { - workflowStoreEntry <- dataAccess.workflowStoreEntriesForWorkflowExecutionUuid(rootWorkflowExecutionUuid).result.headOption + workflowStoreEntry <- dataAccess + .workflowStoreEntriesForWorkflowExecutionUuid(rootWorkflowExecutionUuid) + .result + .headOption deleted <- workflowStoreEntry match { case Some(rootWorkflow) => dataAccess.subWorkflowStoreEntriesForRootWorkflowId(rootWorkflow.workflowStoreEntryId.get).delete @@ -60,7 +77,7 @@ trait SubWorkflowStoreSlickDatabase extends SubWorkflowStoreSqlDatabase { DBIO.successful(0) } } yield deleted - + runTransaction(action) } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/SummaryQueueSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SummaryQueueSlickDatabase.scala index af04939b99b..7f6365493d2 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/SummaryQueueSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/SummaryQueueSlickDatabase.scala @@ -7,16 +7,13 @@ trait SummaryQueueSlickDatabase { import dataAccess.driver.api._ - private[slick] def writeSummaryQueueEntries(metadataJournalIds: Seq[Long]) = { + private[slick] def writeSummaryQueueEntries(metadataJournalIds: Seq[Long]) = dataAccess.summaryQueueEntries ++= metadataJournalIds.map(id => SummaryQueueEntry(id)) - } - private[slick] def deleteSummaryQueueEntriesByMetadataJournalIds(metadataJournalIds: Seq[Long]) = { + private[slick] def deleteSummaryQueueEntriesByMetadataJournalIds(metadataJournalIds: Seq[Long]) = dataAccess.summaryQueueEntries.filter(_.metadataJournalId.inSet(metadataJournalIds)).delete - } - private[slick] def countSummaryQueueEntries() = { + private[slick] def countSummaryQueueEntries() = dataAccess.summaryQueueEntries.length.result - } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala index 43a3aff9273..a7fa1a57a62 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala @@ -9,17 +9,15 @@ trait SummaryStatusSlickDatabase { import dataAccess.driver.api._ - private[slick] def getSummaryStatusEntrySummaryPosition(summaryName: String): DBIO[Option[Long]] = { + private[slick] def getSummaryStatusEntrySummaryPosition(summaryName: String): DBIO[Option[Long]] = dataAccess.summaryPositionForSummaryName(summaryName).result.headOption - } - private[slick] def upsertSummaryStatusEntrySummaryPosition(summaryName: String, - summaryPosition: Long) - (implicit ec: ExecutionContext): DBIO[Unit] = { + private[slick] def upsertSummaryStatusEntrySummaryPosition(summaryName: String, summaryPosition: Long)(implicit + ec: ExecutionContext + ): DBIO[Unit] = if (useSlickUpserts) { for { - _ <- dataAccess.summaryStatusEntryIdsAutoInc. - insertOrUpdate(SummaryStatusEntry(summaryName, summaryPosition)) + _ <- dataAccess.summaryStatusEntryIdsAutoInc.insertOrUpdate(SummaryStatusEntry(summaryName, summaryPosition)) } yield () } else { for { @@ -32,5 +30,4 @@ trait SummaryStatusSlickDatabase { } } yield () } - } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala index a39dcbf4195..d7d3e7bb6db 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala @@ -14,8 +14,9 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase { import dataAccess.driver.api._ - override def setStateToState(fromWorkflowState: String, toWorkflowState: String) - (implicit ec: ExecutionContext): Future[Unit] = { + override def setStateToState(fromWorkflowState: String, toWorkflowState: String)(implicit + ec: ExecutionContext + ): Future[Unit] = { val action = dataAccess .workflowStateForWorkflowState(fromWorkflowState) .update(toWorkflowState) @@ -26,8 +27,8 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase { override def deleteOrUpdateWorkflowToState(workflowExecutionUuid: String, workflowStateToDelete1: String, workflowStateToDelete2: String, - workflowStateForUpdate: String) - (implicit ec: ExecutionContext): Future[Option[Boolean]] = { + workflowStateForUpdate: String + )(implicit ec: ExecutionContext): Future[Option[Boolean]] = { val action = for { // First, delete all rows in either of our states to be deleted. deleted <- dataAccess @@ -44,11 +45,14 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase { } } yield (deleted, updated) - runTransaction(action) map { case (deleted, updated) => if (deleted == 0 && updated == 0) None else Option(deleted > 0) } + runTransaction(action) map { case (deleted, updated) => + if (deleted == 0 && updated == 0) None else Option(deleted > 0) + } } - override def addWorkflowStoreEntries(workflowStoreEntries: Iterable[WorkflowStoreEntry]) - (implicit ec: ExecutionContext): Future[Unit] = { + override def addWorkflowStoreEntries( + workflowStoreEntries: Iterable[WorkflowStoreEntry] + )(implicit ec: ExecutionContext): Future[Unit] = { val action = dataAccess.workflowStoreEntryIdsAutoInc ++= workflowStoreEntries runTransaction(action).void } @@ -60,15 +64,14 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase { workflowStateFrom: String, workflowStateTo: String, workflowStateExcluded: String, - excludedGroups: Set[String]) - (implicit ec: ExecutionContext): Future[Seq[WorkflowStoreEntry]] = { + excludedGroups: Set[String] + )(implicit ec: ExecutionContext): Future[Seq[WorkflowStoreEntry]] = { def updateForFetched(cromwellId: String, heartbeatTimestampTo: Timestamp, workflowStateFrom: String, - workflowStateTo: String) - (workflowStoreEntry: WorkflowStoreEntry) - (implicit ec: ExecutionContext): DBIO[Unit] = { + workflowStateTo: String + )(workflowStoreEntry: WorkflowStoreEntry)(implicit ec: ExecutionContext): DBIO[Unit] = { val workflowExecutionUuid = workflowStoreEntry.workflowExecutionUuid val updateState = workflowStoreEntry.workflowState match { case matched if matched == workflowStateFrom => workflowStateTo @@ -88,20 +91,28 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase { } yield () } - def fetchAndUpdateStartableWfs(hogGroup: Option[String]) = { + def fetchAndUpdateStartableWfs(hogGroup: Option[String]) = for { - workflowStoreEntries <- dataAccess.fetchStartableWfsForHogGroup( - (limit.toLong, heartbeatTimestampTimedOut, workflowStateExcluded, hogGroup) - ).result + workflowStoreEntries <- dataAccess + .fetchStartableWfsForHogGroup( + (limit.toLong, heartbeatTimestampTimedOut, workflowStateExcluded, hogGroup) + ) + .result _ <- DBIO.sequence( - workflowStoreEntries map updateForFetched(cromwellId, heartbeatTimestampTo, workflowStateFrom, workflowStateTo) + workflowStoreEntries map updateForFetched(cromwellId, + heartbeatTimestampTo, + workflowStateFrom, + workflowStateTo + ) ) } yield workflowStoreEntries - } val action = for { // find hog group with lowest count of actively running workflows - hogGroupOption <- dataAccess.getHogGroupWithLowestRunningWfs(heartbeatTimestampTimedOut, workflowStateExcluded, excludedGroups).result.headOption + hogGroupOption <- dataAccess + .getHogGroupWithLowestRunningWfs(heartbeatTimestampTimedOut, workflowStateExcluded, excludedGroups) + .result + .headOption workflowStoreEntries <- hogGroupOption match { // if no such hog group was found, all hog groups have workflows that are either actively running or in "OnHold" status case None => DBIO.successful(Seq.empty[WorkflowStoreEntry]) @@ -118,9 +129,9 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase { runTransaction(action, TransactionIsolation.ReadCommitted) } - override def writeWorkflowHeartbeats(workflowExecutionUuids: Seq[String], - heartbeatTimestamp: Timestamp) - (implicit ec: ExecutionContext): Future[Int] = { + override def writeWorkflowHeartbeats(workflowExecutionUuids: Seq[String], heartbeatTimestamp: Timestamp)(implicit + ec: ExecutionContext + ): Future[Int] = { // Return the count of heartbeats written. This could legitimately be less than the size of the `workflowExecutionUuids` // List if any of those workflows completed and their workflow store entries were removed. val action = for { @@ -147,10 +158,9 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase { runTransaction(action) map { _.toMap } } - override def updateWorkflowState(workflowExecutionUuid: String, - fromWorkflowState: String, - toWorkflowState: String) - (implicit ec: ExecutionContext): Future[Int] = { + override def updateWorkflowState(workflowExecutionUuid: String, fromWorkflowState: String, toWorkflowState: String)( + implicit ec: ExecutionContext + ): Future[Int] = { val action = for { updated <- dataAccess .workflowStateForWorkflowExecutionUUidAndWorkflowState((workflowExecutionUuid, fromWorkflowState)) @@ -160,15 +170,14 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase { runTransaction(action) } - override def findWorkflowsWithAbortRequested(cromwellId: String)(implicit ec: ExecutionContext): Future[Iterable[String]] = { + override def findWorkflowsWithAbortRequested(cromwellId: String)(implicit + ec: ExecutionContext + ): Future[Iterable[String]] = runTransaction(dataAccess.findWorkflowsWithAbortRequested(cromwellId).result) - } - override def findWorkflows(cromwellId: String)(implicit ec: ExecutionContext): Future[Iterable[String]] = { + override def findWorkflows(cromwellId: String)(implicit ec: ExecutionContext): Future[Iterable[String]] = runTransaction(dataAccess.findWorkflows(cromwellId).result) - } - override def checkWhetherWorkflowExists(workflowId: String)(implicit ec: ExecutionContext): Future[Boolean] = { + override def checkWhetherWorkflowExists(workflowId: String)(implicit ec: ExecutionContext): Future[Boolean] = runTransaction(dataAccess.checkExists(workflowId).result.map(_.nonEmpty)) - } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingAggregationEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingAggregationEntryComponent.scala index 139558aa7b8..1cbc705e019 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingAggregationEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingAggregationEntryComponent.scala @@ -2,14 +2,14 @@ package cromwell.database.slick.tables import cromwell.database.sql.tables.CallCachingAggregationEntry - trait CallCachingAggregationEntryComponent { this: DriverComponent with CallCachingEntryComponent with CallCachingDetritusEntryComponent => import driver.api._ - class CallCachingAggregationEntries(tag: Tag) extends Table[CallCachingAggregationEntry](tag, "CALL_CACHING_AGGREGATION_ENTRY") { + class CallCachingAggregationEntries(tag: Tag) + extends Table[CallCachingAggregationEntry](tag, "CALL_CACHING_AGGREGATION_ENTRY") { def callCachingAggregationEntryId = column[Long]("CALL_CACHING_AGGREGATION_ENTRY_ID", O.PrimaryKey, O.AutoInc) def baseAggregation = column[String]("BASE_AGGREGATION", O.Length(255)) @@ -21,8 +21,10 @@ trait CallCachingAggregationEntryComponent { override def * = (baseAggregation, inputFilesAggregation, callCachingEntryId.?, callCachingAggregationEntryId.?) <> (CallCachingAggregationEntry.tupled, CallCachingAggregationEntry.unapply) - def fkCallCachingAggregationEntryCallCachingEntryId = foreignKey("FK_CALL_CACHING_AGGREGATION_ENTRY_CALL_CACHING_ENTRY_ID", - callCachingEntryId, callCachingEntries)(_.callCachingEntryId) + def fkCallCachingAggregationEntryCallCachingEntryId = + foreignKey("FK_CALL_CACHING_AGGREGATION_ENTRY_CALL_CACHING_ENTRY_ID", callCachingEntryId, callCachingEntries)( + _.callCachingEntryId + ) def ixCallCachingAggregationEntryBaIfa = index("IX_CALL_CACHING_AGGREGATION_ENTRY_BA_IFA", (baseAggregation, inputFilesAggregation), unique = false) @@ -32,16 +34,16 @@ trait CallCachingAggregationEntryComponent { val callCachingAggregationEntryIdsAutoInc = callCachingAggregationEntries returning callCachingAggregationEntries.map(_.callCachingAggregationEntryId) - - val callCachingAggregationForCacheEntryId = Compiled( - (callCachingEntryId: Rep[Long]) => for { - callCachingAggregationEntry <- callCachingAggregationEntries + + val callCachingAggregationForCacheEntryId = Compiled((callCachingEntryId: Rep[Long]) => + for { + callCachingAggregationEntry <- callCachingAggregationEntries if callCachingAggregationEntry.callCachingEntryId === callCachingEntryId } yield callCachingAggregationEntry ) - val existsCallCachingEntriesForBaseAggregationHash = Compiled( - (baseAggregation: Rep[String]) => (for { + val existsCallCachingEntriesForBaseAggregationHash = Compiled((baseAggregation: Rep[String]) => + (for { callCachingEntry <- callCachingEntries if callCachingEntry.allowResultReuse callCachingAggregationEntry <- callCachingAggregationEntries @@ -52,24 +54,32 @@ trait CallCachingAggregationEntryComponent { val existsCallCachingEntriesForBaseAggregationHashWithCallCachePrefix = Compiled( (baseAggregation: Rep[String], - prefix1: Rep[String], prefix1Length: Rep[Int], - prefix2: Rep[String], prefix2Length: Rep[Int], - prefix3: Rep[String], prefix3Length: Rep[Int] - ) => (for { - callCachingEntry <- callCachingEntries - if callCachingEntry.allowResultReuse - callCachingAggregationEntry <- callCachingAggregationEntries - if callCachingEntry.callCachingEntryId === callCachingAggregationEntry.callCachingEntryId - if callCachingAggregationEntry.baseAggregation === baseAggregation - detritus <- callCachingDetritusEntries - if detritus.callCachingEntryId === callCachingEntry.callCachingEntryId - detritusPath = detritus.detritusValue.map(clobToString) - if (detritusPath.substring(0, prefix1Length) === prefix1) || - (detritusPath.substring(0, prefix2Length) === prefix2) || - (detritusPath.substring(0, prefix3Length) === prefix3)} yield ()).exists + prefix1: Rep[String], + prefix1Length: Rep[Int], + prefix2: Rep[String], + prefix2Length: Rep[Int], + prefix3: Rep[String], + prefix3Length: Rep[Int] + ) => + (for { + callCachingEntry <- callCachingEntries + if callCachingEntry.allowResultReuse + callCachingAggregationEntry <- callCachingAggregationEntries + if callCachingEntry.callCachingEntryId === callCachingAggregationEntry.callCachingEntryId + if callCachingAggregationEntry.baseAggregation === baseAggregation + detritus <- callCachingDetritusEntries + if detritus.callCachingEntryId === callCachingEntry.callCachingEntryId + detritusPath = detritus.detritusValue.map(clobToString) + if (detritusPath.substring(0, prefix1Length) === prefix1) || + (detritusPath.substring(0, prefix2Length) === prefix2) || + (detritusPath.substring(0, prefix3Length) === prefix3) + } yield ()).exists ) - def callCachingEntriesForAggregatedHashes(baseAggregation: Rep[String], inputFilesAggregation: Rep[Option[String]], excludedIds: Set[Long]) = { + def callCachingEntriesForAggregatedHashes(baseAggregation: Rep[String], + inputFilesAggregation: Rep[Option[String]], + excludedIds: Set[Long] + ) = (for { callCachingEntry <- callCachingEntries if callCachingEntry.allowResultReuse && !(callCachingEntry.callCachingEntryId inSet excludedIds) @@ -79,13 +89,17 @@ trait CallCachingAggregationEntryComponent { if (callCachingAggregationEntry.inputFilesAggregation.isEmpty && inputFilesAggregation.isEmpty) || (callCachingAggregationEntry.inputFilesAggregation === inputFilesAggregation) } yield callCachingAggregationEntry.callCachingEntryId).take(1) - } - def callCachingEntriesForAggregatedHashesWithPrefixes(baseAggregation: Rep[String], inputFilesAggregation: Rep[Option[String]], - prefix1: Rep[String], prefix1Length: Rep[Int], - prefix2: Rep[String], prefix2Length: Rep[Int], - prefix3: Rep[String], prefix3Length: Rep[Int], - excludedIds: Set[Long]) = { + def callCachingEntriesForAggregatedHashesWithPrefixes(baseAggregation: Rep[String], + inputFilesAggregation: Rep[Option[String]], + prefix1: Rep[String], + prefix1Length: Rep[Int], + prefix2: Rep[String], + prefix2Length: Rep[Int], + prefix3: Rep[String], + prefix3Length: Rep[Int], + excludedIds: Set[Long] + ) = (for { callCachingEntry <- callCachingEntries if callCachingEntry.allowResultReuse && !(callCachingEntry.callCachingEntryId inSet excludedIds) @@ -104,5 +118,4 @@ trait CallCachingAggregationEntryComponent { (detritusPath.substring(0, prefix2Length) === prefix2) || (detritusPath.substring(0, prefix3Length) === prefix3) } yield callCachingAggregationEntry.callCachingEntryId).take(1) - } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala index b668adb4940..cafaa91d9b0 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala @@ -11,7 +11,7 @@ trait CallCachingDetritusEntryComponent { import driver.api._ class CallCachingDetritusEntries(tag: Tag) - extends Table[CallCachingDetritusEntry](tag, "CALL_CACHING_DETRITUS_ENTRY") { + extends Table[CallCachingDetritusEntry](tag, "CALL_CACHING_DETRITUS_ENTRY") { def callCachingDetritusEntryId = column[Long]("CALL_CACHING_DETRITUS_ENTRY_ID", O.PrimaryKey, O.AutoInc) def detritusKey = column[String]("DETRITUS_KEY", O.Length(255)) @@ -23,9 +23,10 @@ trait CallCachingDetritusEntryComponent { override def * = (detritusKey, detritusValue, callCachingEntryId.?, callCachingDetritusEntryId.?) <> (CallCachingDetritusEntry.tupled, CallCachingDetritusEntry.unapply) - def fkCallCachingDetritusEntryCallCachingEntryId = foreignKey( - "FK_CALL_CACHING_DETRITUS_ENTRY_CALL_CACHING_ENTRY_ID", - callCachingEntryId, callCachingEntries)(_.callCachingEntryId) + def fkCallCachingDetritusEntryCallCachingEntryId = + foreignKey("FK_CALL_CACHING_DETRITUS_ENTRY_CALL_CACHING_ENTRY_ID", callCachingEntryId, callCachingEntries)( + _.callCachingEntryId + ) def ucCallCachingDetritusEntryCceiDk = index("UC_CALL_CACHING_DETRITUS_ENTRY_CCEI_DK", (callCachingEntryId, detritusKey), unique = true) @@ -36,8 +37,8 @@ trait CallCachingDetritusEntryComponent { val callCachingDetritusEntryIdsAutoInc = callCachingDetritusEntries returning callCachingDetritusEntries.map(_.callCachingDetritusEntryId) - val callCachingDetritusEntriesForCallCachingEntryId = Compiled( - (callCachingEntryId: Rep[Long]) => for { + val callCachingDetritusEntriesForCallCachingEntryId = Compiled((callCachingEntryId: Rep[Long]) => + for { callCachingDetritusEntry <- callCachingDetritusEntries if callCachingDetritusEntry.callCachingEntryId === callCachingEntryId } yield callCachingDetritusEntry diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala index db9c5dc1654..b9a26585bec 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala @@ -22,51 +22,60 @@ trait CallCachingEntryComponent { def returnCode = column[Option[Int]]("RETURN_CODE") def allowResultReuse = column[Boolean]("ALLOW_RESULT_REUSE", O.Default(true)) - - override def * = (workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt, returnCode, allowResultReuse, - callCachingEntryId.?) <> (CallCachingEntry.tupled, CallCachingEntry.unapply) + + override def * = (workflowExecutionUuid, + callFullyQualifiedName, + jobIndex, + jobAttempt, + returnCode, + allowResultReuse, + callCachingEntryId.? + ) <> (CallCachingEntry.tupled, CallCachingEntry.unapply) def ucCallCachingEntryWeuCfqnJi = - index("UC_CALL_CACHING_ENTRY_WEU_CFQN_JI", (workflowExecutionUuid, callFullyQualifiedName, jobIndex), - unique = true) + index("UC_CALL_CACHING_ENTRY_WEU_CFQN_JI", + (workflowExecutionUuid, callFullyQualifiedName, jobIndex), + unique = true + ) } protected val callCachingEntries = TableQuery[CallCachingEntries] val callCachingEntryIdsAutoInc = callCachingEntries returning callCachingEntries.map(_.callCachingEntryId) - val callCachingEntriesForId = Compiled( - (callCachingEntryId: Rep[Long]) => for { + val callCachingEntriesForId = Compiled((callCachingEntryId: Rep[Long]) => + for { callCachingEntry <- callCachingEntries if callCachingEntry.callCachingEntryId === callCachingEntryId } yield callCachingEntry ) - val allowResultReuseForCallCachingEntryId = Compiled( - (callCachingEntryId: Rep[Long]) => for { + val allowResultReuseForCallCachingEntryId = Compiled((callCachingEntryId: Rep[Long]) => + for { callCachingEntry <- callCachingEntries if callCachingEntry.callCachingEntryId === callCachingEntryId } yield callCachingEntry.allowResultReuse ) - val callCachingEntriesForWorkflowFqnIndex = Compiled( - (workflowId: Rep[String], callFqn: Rep[String], jobIndex: Rep[Int]) => for { - callCachingEntry <- callCachingEntries - if callCachingEntry.workflowExecutionUuid === workflowId - if callCachingEntry.callFullyQualifiedName === callFqn - if callCachingEntry.jobIndex === jobIndex - } yield callCachingEntry - ) - - val callCachingEntryIdsForWorkflowId = Compiled( - (workflowId: Rep[String]) => for { + val callCachingEntriesForWorkflowFqnIndex = + Compiled((workflowId: Rep[String], callFqn: Rep[String], jobIndex: Rep[Int]) => + for { + callCachingEntry <- callCachingEntries + if callCachingEntry.workflowExecutionUuid === workflowId + if callCachingEntry.callFullyQualifiedName === callFqn + if callCachingEntry.jobIndex === jobIndex + } yield callCachingEntry + ) + + val callCachingEntryIdsForWorkflowId = Compiled((workflowId: Rep[String]) => + for { callCachingEntry <- callCachingEntries if callCachingEntry.workflowExecutionUuid === workflowId } yield callCachingEntry.callCachingEntryId ) - val allowResultReuseForWorkflowId = Compiled( - (workflowId: Rep[String]) => for { + val allowResultReuseForWorkflowId = Compiled((workflowId: Rep[String]) => + for { callCachingEntry <- callCachingEntries if callCachingEntry.workflowExecutionUuid === workflowId } yield callCachingEntry.allowResultReuse diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala index b556b3f22de..b89fe7f5cc8 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala @@ -20,8 +20,10 @@ trait CallCachingHashEntryComponent { override def * = (hashKey, hashValue, callCachingEntryId.?, callCachingHashEntryId.?) <> (CallCachingHashEntry.tupled, CallCachingHashEntry.unapply) - def fkCallCachingHashEntryCallCachingEntryId = foreignKey("FK_CALL_CACHING_HASH_ENTRY_CALL_CACHING_ENTRY_ID", - callCachingEntryId, callCachingEntries)(_.callCachingEntryId) + def fkCallCachingHashEntryCallCachingEntryId = + foreignKey("FK_CALL_CACHING_HASH_ENTRY_CALL_CACHING_ENTRY_ID", callCachingEntryId, callCachingEntries)( + _.callCachingEntryId + ) def ucCallCachingHashEntryCceiHk = index("UC_CALL_CACHING_HASH_ENTRY_CCEI_HK", (callCachingEntryId, hashKey), unique = true) @@ -31,12 +33,12 @@ trait CallCachingHashEntryComponent { val callCachingHashEntryIdsAutoInc = callCachingHashEntries returning callCachingHashEntries.map(_.callCachingHashEntryId) - + /** * Find all hashes for a CALL_CACHING_ENTRY_ID */ - val callCachingHashEntriesForCallCachingEntryId = Compiled( - (callCachingEntryId: Rep[Long]) => for { + val callCachingHashEntriesForCallCachingEntryId = Compiled((callCachingEntryId: Rep[Long]) => + for { callCachingHashEntry <- callCachingHashEntries if callCachingHashEntry.callCachingEntryId === callCachingEntryId } yield callCachingHashEntry diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala index b2e6f02b5b9..8c624798ce4 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala @@ -11,7 +11,7 @@ trait CallCachingSimpletonEntryComponent { import driver.api._ class CallCachingSimpletonEntries(tag: Tag) - extends Table[CallCachingSimpletonEntry](tag, "CALL_CACHING_SIMPLETON_ENTRY") { + extends Table[CallCachingSimpletonEntry](tag, "CALL_CACHING_SIMPLETON_ENTRY") { def callCachingSimpletonEntryId = column[Long]("CALL_CACHING_SIMPLETON_ENTRY_ID", O.PrimaryKey, O.AutoInc) def simpletonKey = column[String]("SIMPLETON_KEY", O.Length(255)) @@ -25,8 +25,10 @@ trait CallCachingSimpletonEntryComponent { override def * = (simpletonKey, simpletonValue, wdlType, callCachingEntryId.?, callCachingSimpletonEntryId.?) <> (CallCachingSimpletonEntry.tupled, CallCachingSimpletonEntry.unapply) - def fkCallCachingSimpletonEntryCallCachingEntryId = foreignKey( - "FK_CALL_CACHING_SIMPLETON_ENTRY_CALL_CACHING_ENTRY_ID", callCachingEntryId, callCachingEntries)(_.callCachingEntryId) + def fkCallCachingSimpletonEntryCallCachingEntryId = + foreignKey("FK_CALL_CACHING_SIMPLETON_ENTRY_CALL_CACHING_ENTRY_ID", callCachingEntryId, callCachingEntries)( + _.callCachingEntryId + ) def ucCallCachingSimpletonEntryCceiSk = index("UC_CALL_CACHING_SIMPLETON_ENTRY_CCEI_SK", (callCachingEntryId, simpletonKey), unique = true) @@ -40,8 +42,8 @@ trait CallCachingSimpletonEntryComponent { /** * Find all result simpletons which match a given CALL_CACHING_ENTRY_ID */ - val callCachingSimpletonEntriesForCallCachingEntryId = Compiled( - (callCachingEntryId: Rep[Long]) => for { + val callCachingSimpletonEntriesForCallCachingEntryId = Compiled((callCachingEntryId: Rep[Long]) => + for { callCachingSimpletonEntry <- callCachingSimpletonEntries if callCachingSimpletonEntry.callCachingEntryId === callCachingEntryId } yield callCachingSimpletonEntry diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CustomLabelEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CustomLabelEntryComponent.scala index 36789edb696..de988ff2bb1 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/CustomLabelEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CustomLabelEntryComponent.scala @@ -11,8 +11,7 @@ trait CustomLabelEntryComponent { import driver.api.TupleMethods._ import driver.api._ - class CustomLabelEntries(tag: Tag) - extends Table[CustomLabelEntry](tag, "CUSTOM_LABEL_ENTRY") { + class CustomLabelEntries(tag: Tag) extends Table[CustomLabelEntry](tag, "CUSTOM_LABEL_ENTRY") { def customLabelEntryId = column[Long]("CUSTOM_LABEL_ENTRY_ID", O.PrimaryKey, O.AutoInc) def customLabelKey = column[String]("CUSTOM_LABEL_KEY", O.Length(255)) @@ -31,45 +30,52 @@ trait CustomLabelEntryComponent { ) def fkCustomLabelEntryWorkflowExecutionUuid = foreignKey("FK_CUSTOM_LABEL_ENTRY_WORKFLOW_EXECUTION_UUID", - workflowExecutionUuid, workflowMetadataSummaryEntries)(_.workflowExecutionUuid, onDelete = Cascade) + workflowExecutionUuid, + workflowMetadataSummaryEntries + )(_.workflowExecutionUuid, onDelete = Cascade) - def ucCustomLabelEntryClkWeu = index("UC_CUSTOM_LABEL_ENTRY_CLK_WEU", - (customLabelKey, workflowExecutionUuid), unique = true) + def ucCustomLabelEntryClkWeu = + index("UC_CUSTOM_LABEL_ENTRY_CLK_WEU", (customLabelKey, workflowExecutionUuid), unique = true) - def ixCustomLabelEntryClkClv = index("IX_CUSTOM_LABEL_ENTRY_CLK_CLV", (customLabelKey, customLabelValue), unique = false) -} + def ixCustomLabelEntryClkClv = + index("IX_CUSTOM_LABEL_ENTRY_CLK_CLV", (customLabelKey, customLabelValue), unique = false) + } val customLabelEntries = TableQuery[CustomLabelEntries] val customLabelEntryIdsAutoInc = customLabelEntries returning customLabelEntries.map(_.customLabelEntryId) - val customLabelEntriesForWorkflowExecutionUuidAndLabelKey = Compiled( - (workflowExecutionUuid: Rep[String], labelKey: Rep[String]) => for { - customLabelEntry <- customLabelEntries - if customLabelEntry.workflowExecutionUuid === workflowExecutionUuid && - customLabelEntry.customLabelKey === labelKey - } yield customLabelEntry.forUpdate) + val customLabelEntriesForWorkflowExecutionUuidAndLabelKey = + Compiled((workflowExecutionUuid: Rep[String], labelKey: Rep[String]) => + for { + customLabelEntry <- customLabelEntries + if customLabelEntry.workflowExecutionUuid === workflowExecutionUuid && + customLabelEntry.customLabelKey === labelKey + } yield customLabelEntry.forUpdate + ) def existsWorkflowIdLabelKeyAndValue(workflowId: Rep[String], labelKey: Rep[String], - labelValue: Rep[String]): Rep[Boolean] = { - customLabelEntries.filter(customLabelEntry => - customLabelEntry.workflowExecutionUuid === workflowId && - customLabelEntry.customLabelKey === labelKey && - customLabelEntry.customLabelValue === labelValue - ).exists - } - - val labelsForWorkflowExecutionUuid = Compiled( - (workflowExecutionUuid: Rep[String]) => for { + labelValue: Rep[String] + ): Rep[Boolean] = + customLabelEntries + .filter(customLabelEntry => + customLabelEntry.workflowExecutionUuid === workflowId && + customLabelEntry.customLabelKey === labelKey && + customLabelEntry.customLabelValue === labelValue + ) + .exists + + val labelsForWorkflowExecutionUuid = Compiled((workflowExecutionUuid: Rep[String]) => + for { customLabelEntry <- customLabelEntries if customLabelEntry.workflowExecutionUuid === workflowExecutionUuid } yield (customLabelEntry.customLabelKey, customLabelEntry.customLabelValue) ) - val labelsForWorkflowAndSubworkflows = Compiled( - (workflowExecutionUuid: Rep[String]) => for { + val labelsForWorkflowAndSubworkflows = Compiled((workflowExecutionUuid: Rep[String]) => + for { summary <- workflowMetadataSummaryEntries if summary.rootWorkflowExecutionUuid === workflowExecutionUuid || summary.workflowExecutionUuid === workflowExecutionUuid customLabelEntry <- customLabelEntries diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/DockerHashStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/DockerHashStoreEntryComponent.scala index 5929b2e5a6a..be8bda6863b 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/DockerHashStoreEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/DockerHashStoreEntryComponent.scala @@ -19,20 +19,27 @@ trait DockerHashStoreEntryComponent { def dockerSize = column[Option[Long]]("DOCKER_SIZE", O.Default(None)) - override def * = (workflowExecutionUuid, dockerTag, dockerHash, dockerSize, dockerHashStoreEntryId.?) <> (DockerHashStoreEntry.tupled, DockerHashStoreEntry.unapply) - - def ucDockerHashStoreEntryWeuDt = index("UC_DOCKER_HASH_STORE_ENTRY_WEU_DT", (workflowExecutionUuid, dockerTag), unique = true) + override def * = (workflowExecutionUuid, + dockerTag, + dockerHash, + dockerSize, + dockerHashStoreEntryId.? + ) <> (DockerHashStoreEntry.tupled, DockerHashStoreEntry.unapply) + + def ucDockerHashStoreEntryWeuDt = + index("UC_DOCKER_HASH_STORE_ENTRY_WEU_DT", (workflowExecutionUuid, dockerTag), unique = true) } val dockerHashStoreEntries = TableQuery[DockerHashStoreEntries] - val dockerHashStoreEntryIdsAutoInc = dockerHashStoreEntries returning dockerHashStoreEntries.map(_.dockerHashStoreEntryId) + val dockerHashStoreEntryIdsAutoInc = + dockerHashStoreEntries returning dockerHashStoreEntries.map(_.dockerHashStoreEntryId) /** * Useful for finding the docker hash store for a given workflow execution UUID */ - val dockerHashStoreEntriesForWorkflowExecutionUuid = Compiled( - (workflowExecutionUuid: Rep[String]) => for { + val dockerHashStoreEntriesForWorkflowExecutionUuid = Compiled((workflowExecutionUuid: Rep[String]) => + for { dockerHashStoreEntry <- dockerHashStoreEntries if dockerHashStoreEntry.workflowExecutionUuid === workflowExecutionUuid } yield dockerHashStoreEntry diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala index 7822e9896ff..fb5160ba69d 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala @@ -53,16 +53,15 @@ trait DriverComponent { /** Adds quotes around the string if required by the DBMS. */ def quoted(string: String) = if (shouldQuote) s""""$string"""" else string - val clobToString: Rep[SerialClob] => Rep[String] = { + val clobToString: Rep[SerialClob] => Rep[String] = this.driver match { - /* + /* Workaround https://jira.mariadb.org/browse/CONJ-717 Bypass Slick `asColumnOf[String]` calling the JDBC `{fn convert(Column, VARCHAR)}`. Instead directly call `concat(Column)` supported by both the MariaDB driver and the MySQL driver. - */ + */ case MySQLProfile => SimpleFunction.unary("concat") case _ => _.asColumnOf[String] } - } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/EngineDataAccessComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/EngineDataAccessComponent.scala index 6b1bda3d12d..cd2c16379ab 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/EngineDataAccessComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/EngineDataAccessComponent.scala @@ -2,18 +2,19 @@ package cromwell.database.slick.tables import slick.jdbc.JdbcProfile -class EngineDataAccessComponent(val driver: JdbcProfile) extends DataAccessComponent - with CallCachingDetritusEntryComponent - with CallCachingEntryComponent - with CallCachingHashEntryComponent - with CallCachingAggregationEntryComponent - with CallCachingSimpletonEntryComponent - with DockerHashStoreEntryComponent - with JobKeyValueEntryComponent - with JobStoreEntryComponent - with JobStoreSimpletonEntryComponent - with SubWorkflowStoreEntryComponent - with WorkflowStoreEntryComponent { +class EngineDataAccessComponent(val driver: JdbcProfile) + extends DataAccessComponent + with CallCachingDetritusEntryComponent + with CallCachingEntryComponent + with CallCachingHashEntryComponent + with CallCachingAggregationEntryComponent + with CallCachingSimpletonEntryComponent + with DockerHashStoreEntryComponent + with JobKeyValueEntryComponent + with JobStoreEntryComponent + with JobStoreSimpletonEntryComponent + with SubWorkflowStoreEntryComponent + with WorkflowStoreEntryComponent { import driver.api._ diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/JobKeyValueEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/JobKeyValueEntryComponent.scala index ee75cda27c0..cd730a5c154 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/JobKeyValueEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/JobKeyValueEntryComponent.scala @@ -23,11 +23,20 @@ trait JobKeyValueEntryComponent { def storeValue = column[String]("STORE_VALUE", O.Length(255)) - override def * = (workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt, storeKey, storeValue, - jobKeyValueEntryId.?) <> (JobKeyValueEntry.tupled, JobKeyValueEntry.unapply) - - def ucJobKeyValueEntryWeuCfqnJiJaSk = index("UC_JOB_KEY_VALUE_ENTRY_WEU_CFQN_JI_JA_SK", - (workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt, storeKey), unique = true) + override def * = (workflowExecutionUuid, + callFullyQualifiedName, + jobIndex, + jobAttempt, + storeKey, + storeValue, + jobKeyValueEntryId.? + ) <> (JobKeyValueEntry.tupled, JobKeyValueEntry.unapply) + + def ucJobKeyValueEntryWeuCfqnJiJaSk = index( + "UC_JOB_KEY_VALUE_ENTRY_WEU_CFQN_JI_JA_SK", + (workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt, storeKey), + unique = true + ) } protected val jobKeyValueEntries = TableQuery[JobKeyValueEntries] @@ -37,21 +46,27 @@ trait JobKeyValueEntryComponent { val jobKeyValueEntriesExists = Compiled(jobKeyValueEntries.take(1).exists) - val jobKeyValueEntriesForWorkflowExecutionUuid = Compiled((workflowExecutionUuid: Rep[String]) => for { + val jobKeyValueEntriesForWorkflowExecutionUuid = Compiled((workflowExecutionUuid: Rep[String]) => + for { jobKeyValueEntry <- jobKeyValueEntries if jobKeyValueEntry.workflowExecutionUuid === workflowExecutionUuid } yield jobKeyValueEntry ) val storeValuesForJobKeyAndStoreKey = Compiled( - (workflowExecutionUuid: Rep[String], callFullyQualifiedName: Rep[String], jobIndex: Rep[Int], jobAttempt: Rep[Int], - storeKey: Rep[String]) => for { - jobKeyValueEntry <- jobKeyValueEntries - if jobKeyValueEntry.workflowExecutionUuid === workflowExecutionUuid - if jobKeyValueEntry.callFullyQualifiedName === callFullyQualifiedName - if jobKeyValueEntry.jobIndex === jobIndex - if jobKeyValueEntry.jobAttempt === jobAttempt - if jobKeyValueEntry.storeKey === storeKey - } yield jobKeyValueEntry.storeValue + (workflowExecutionUuid: Rep[String], + callFullyQualifiedName: Rep[String], + jobIndex: Rep[Int], + jobAttempt: Rep[Int], + storeKey: Rep[String] + ) => + for { + jobKeyValueEntry <- jobKeyValueEntries + if jobKeyValueEntry.workflowExecutionUuid === workflowExecutionUuid + if jobKeyValueEntry.callFullyQualifiedName === callFullyQualifiedName + if jobKeyValueEntry.jobIndex === jobIndex + if jobKeyValueEntry.jobAttempt === jobAttempt + if jobKeyValueEntry.storeKey === storeKey + } yield jobKeyValueEntry.storeValue ) } diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreEntryComponent.scala index 04c01ead0cb..acc9c6f12d3 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreEntryComponent.scala @@ -30,11 +30,21 @@ trait JobStoreEntryComponent { def retryableFailure = column[Option[Boolean]]("RETRYABLE_FAILURE") - override def * = (workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt, jobSuccessful, returnCode, - exceptionMessage, retryableFailure, jobStoreEntryId.?) <> (JobStoreEntry.tupled, JobStoreEntry.unapply) + override def * = (workflowExecutionUuid, + callFullyQualifiedName, + jobIndex, + jobAttempt, + jobSuccessful, + returnCode, + exceptionMessage, + retryableFailure, + jobStoreEntryId.? + ) <> (JobStoreEntry.tupled, JobStoreEntry.unapply) def ucJobStoreEntryWeuCfqnJiJa = index("UC_JOB_STORE_ENTRY_WEU_CFQN_JI_JA", - (workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt), unique = true) + (workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt), + unique = true + ) def ixJobStoreEntryWeu = index("IX_JOB_STORE_ENTRY_WEU", workflowExecutionUuid, unique = false) } @@ -46,8 +56,8 @@ trait JobStoreEntryComponent { /** * Useful for finding all job stores for a given workflow execution UUID (e.g. so you can delete them! Bwahaha) */ - val jobStoreEntriesForWorkflowExecutionUuid = Compiled( - (workflowExecutionUuid: Rep[String]) => for { + val jobStoreEntriesForWorkflowExecutionUuid = Compiled((workflowExecutionUuid: Rep[String]) => + for { jobStoreEntry <- jobStoreEntries if jobStoreEntry.workflowExecutionUuid === workflowExecutionUuid } yield jobStoreEntry @@ -57,8 +67,11 @@ trait JobStoreEntryComponent { * Useful for finding the unique job store for a given job key */ val jobStoreEntriesForJobKey = Compiled( - (workflowExecutionUuid: Rep[String], callFullyQualifiedName: Rep[String], jobIndex: Rep[Int], - jobAttempt: Rep[Int]) => + (workflowExecutionUuid: Rep[String], + callFullyQualifiedName: Rep[String], + jobIndex: Rep[Int], + jobAttempt: Rep[Int] + ) => for { jobStoreEntry <- jobStoreEntries if jobStoreEntry.workflowExecutionUuid === workflowExecutionUuid && diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala index 285ccc93b82..da14cede6e8 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala @@ -26,7 +26,9 @@ trait JobStoreSimpletonEntryComponent { (JobStoreSimpletonEntry.tupled, JobStoreSimpletonEntry.unapply) def fkJobStoreSimpletonEntryJobStoreEntryId = foreignKey("FK_JOB_STORE_SIMPLETON_ENTRY_JOB_STORE_ENTRY_ID", - jobStoreEntryId, jobStoreEntries)(_.jobStoreEntryId, onDelete = Cascade) + jobStoreEntryId, + jobStoreEntries + )(_.jobStoreEntryId, onDelete = Cascade) def ucJobStoreSimpletonEntryJseiSk = index("UC_JOB_STORE_SIMPLETON_ENTRY_JSEI_SK", (jobStoreEntryId, simpletonKey), unique = true) @@ -40,8 +42,8 @@ trait JobStoreSimpletonEntryComponent { /** * Find all result simpletons which match a given JOB_STORE_ENTRY_ID */ - val jobStoreSimpletonEntriesForJobStoreEntryId = Compiled( - (jobStoreEntryId: Rep[Long]) => for { + val jobStoreSimpletonEntriesForJobStoreEntryId = Compiled((jobStoreEntryId: Rep[Long]) => + for { jobStoreSimpletonEntry <- jobStoreSimpletonEntries if jobStoreSimpletonEntry.jobStoreEntryId === jobStoreEntryId } yield jobStoreSimpletonEntry ) diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataDataAccessComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataDataAccessComponent.scala index f3acdd6736d..a8eb1e2c22e 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataDataAccessComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataDataAccessComponent.scala @@ -2,17 +2,18 @@ package cromwell.database.slick.tables import slick.jdbc.JdbcProfile -class MetadataDataAccessComponent(val driver: JdbcProfile) extends DataAccessComponent - with CustomLabelEntryComponent - with MetadataEntryComponent - with SummaryStatusEntryComponent - with SummaryQueueEntryComponent - with WorkflowMetadataSummaryEntryComponent { +class MetadataDataAccessComponent(val driver: JdbcProfile) + extends DataAccessComponent + with CustomLabelEntryComponent + with MetadataEntryComponent + with SummaryStatusEntryComponent + with SummaryQueueEntryComponent + with WorkflowMetadataSummaryEntryComponent { import driver.api._ override lazy val schema: driver.SchemaDescription = - customLabelEntries.schema ++ + customLabelEntries.schema ++ metadataEntries.schema ++ summaryStatusEntries.schema ++ workflowMetadataSummaryEntries.schema ++ @@ -20,12 +21,11 @@ class MetadataDataAccessComponent(val driver: JdbcProfile) extends DataAccessCom // Looks like here is the most appropriate place for this val since it doesn't fit neither in // SummaryQueueEntryComponent nor in MetadataEntryComponent - val metadataEntriesToSummarizeQuery = { - Compiled( - (limit: ConstColumn[Long]) => (for { + val metadataEntriesToSummarizeQuery = + Compiled((limit: ConstColumn[Long]) => + (for { summaryEntry <- summaryQueueEntries.take(limit) metadataEntry <- metadataEntries if metadataEntry.metadataEntryId === summaryEntry.metadataJournalId } yield metadataEntry).sortBy(_.metadataEntryId) ) - } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala index 1c1225c195d..3ba54ee2760 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala @@ -27,7 +27,8 @@ trait MetadataEntryComponent { def metadataEntryId = column[Long]("METADATA_JOURNAL_ID", O.PrimaryKey, O.AutoInc) - def workflowExecutionUuid = column[String]("WORKFLOW_EXECUTION_UUID", O.Length(255)) // TODO: rename column via liquibase + def workflowExecutionUuid = + column[String]("WORKFLOW_EXECUTION_UUID", O.Length(255)) // TODO: rename column via liquibase def callFullyQualifiedName = column[Option[String]]("CALL_FQN", O.Length(255)) // TODO: rename column via liquibase @@ -43,8 +44,16 @@ trait MetadataEntryComponent { def metadataTimestamp = column[Timestamp]("METADATA_TIMESTAMP") - override def * = (workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt, metadataKey, metadataValue, - metadataValueType, metadataTimestamp, metadataEntryId.?) <> (MetadataEntry.tupled, MetadataEntry.unapply) + override def * = (workflowExecutionUuid, + callFullyQualifiedName, + jobIndex, + jobAttempt, + metadataKey, + metadataValue, + metadataValueType, + metadataTimestamp, + metadataEntryId.? + ) <> (MetadataEntry.tupled, MetadataEntry.unapply) // TODO: rename index via liquibase def ixMetadataEntryWeu = index("METADATA_WORKFLOW_IDX", workflowExecutionUuid, unique = false) @@ -56,151 +65,173 @@ trait MetadataEntryComponent { val metadataEntriesExists = Compiled(metadataEntries.take(1).exists) - val metadataEntriesForWorkflowExecutionUuid = Compiled( - (workflowExecutionUuid: Rep[String]) => (for { + val metadataEntriesForWorkflowExecutionUuid = Compiled((workflowExecutionUuid: Rep[String]) => + (for { metadataEntry <- metadataEntries if metadataEntry.workflowExecutionUuid === workflowExecutionUuid } yield metadataEntry).sortBy(_.metadataTimestamp) ) - val metadataEntriesForWorkflowSortedById = Compiled( - (workflowExecutionUuid: Rep[String]) => (for { + val metadataEntriesForWorkflowSortedById = Compiled((workflowExecutionUuid: Rep[String]) => + (for { metadataEntry <- metadataEntries if metadataEntry.workflowExecutionUuid === workflowExecutionUuid } yield metadataEntry).sortBy(_.metadataEntryId) ) - val countMetadataEntriesForWorkflowExecutionUuid = Compiled( - (rootWorkflowId: Rep[String], expandSubWorkflows: Rep[Boolean]) => { - val targetWorkflowIds = for { - summary <- workflowMetadataSummaryEntries - // Uses `IX_WORKFLOW_METADATA_SUMMARY_ENTRY_RWEU`, `UC_WORKFLOW_METADATA_SUMMARY_ENTRY_WEU` - if summary.workflowExecutionUuid === rootWorkflowId || ((summary.rootWorkflowExecutionUuid === rootWorkflowId) && expandSubWorkflows) - } yield summary.workflowExecutionUuid - - for { - metadata <- metadataEntries - if metadata.workflowExecutionUuid in targetWorkflowIds // Uses `METADATA_WORKFLOW_IDX` - } yield metadata - }.size - ) - - val metadataEntryExistsForWorkflowExecutionUuid = Compiled( - (workflowExecutionUuid: Rep[String]) => (for { + val countMetadataEntriesForWorkflowExecutionUuid = + Compiled((rootWorkflowId: Rep[String], expandSubWorkflows: Rep[Boolean]) => + { + val targetWorkflowIds = for { + summary <- workflowMetadataSummaryEntries + // Uses `IX_WORKFLOW_METADATA_SUMMARY_ENTRY_RWEU`, `UC_WORKFLOW_METADATA_SUMMARY_ENTRY_WEU` + if summary.workflowExecutionUuid === rootWorkflowId || ((summary.rootWorkflowExecutionUuid === rootWorkflowId) && expandSubWorkflows) + } yield summary.workflowExecutionUuid + + for { + metadata <- metadataEntries + if metadata.workflowExecutionUuid in targetWorkflowIds // Uses `METADATA_WORKFLOW_IDX` + } yield metadata + }.size + ) + + val metadataEntryExistsForWorkflowExecutionUuid = Compiled((workflowExecutionUuid: Rep[String]) => + (for { metadataEntry <- metadataEntries if metadataEntry.workflowExecutionUuid === workflowExecutionUuid } yield metadataEntry).exists ) - def metadataEntryExistsForWorkflowExecutionUuid(workflowId: Rep[String], key: Rep[String]): Rep[Boolean] = { - metadataEntries.filter( metadataEntry => - metadataEntry.workflowExecutionUuid === workflowId && - metadataEntry.metadataKey === key && - metadataEntry.metadataValue.isDefined - ).exists - } - - val metadataEntriesForWorkflowExecutionUuidAndMetadataKey = Compiled( - (workflowExecutionUuid: Rep[String], metadataKey: Rep[String]) => (for { - metadataEntry <- metadataEntries - if metadataEntry.workflowExecutionUuid === workflowExecutionUuid - if metadataEntry.metadataKey === metadataKey - if metadataEntry.callFullyQualifiedName.isEmpty - if metadataEntry.jobIndex.isEmpty - if metadataEntry.jobAttempt.isEmpty - } yield metadataEntry).sortBy(_.metadataTimestamp) - ) + def metadataEntryExistsForWorkflowExecutionUuid(workflowId: Rep[String], key: Rep[String]): Rep[Boolean] = + metadataEntries + .filter(metadataEntry => + metadataEntry.workflowExecutionUuid === workflowId && + metadataEntry.metadataKey === key && + metadataEntry.metadataValue.isDefined + ) + .exists - val countMetadataEntriesForWorkflowExecutionUuidAndMetadataKey = Compiled( - (rootWorkflowId: Rep[String], metadataKey: Rep[String], expandSubWorkflows: Rep[Boolean]) => { - val targetWorkflowIds = for { - summary <- workflowMetadataSummaryEntries - // Uses `IX_WORKFLOW_METADATA_SUMMARY_ENTRY_RWEU`, `UC_WORKFLOW_METADATA_SUMMARY_ENTRY_WEU` - if summary.workflowExecutionUuid === rootWorkflowId || ((summary.rootWorkflowExecutionUuid === rootWorkflowId) && expandSubWorkflows) - } yield summary.workflowExecutionUuid - - for { - metadata <- metadataEntries - if metadata.workflowExecutionUuid in targetWorkflowIds // Uses `METADATA_WORKFLOW_IDX` - if metadata.metadataKey === metadataKey - if metadata.callFullyQualifiedName.isEmpty - if metadata.jobIndex.isEmpty - if metadata.jobAttempt.isEmpty - } yield metadata - }.size - ) + val metadataEntriesForWorkflowExecutionUuidAndMetadataKey = + Compiled((workflowExecutionUuid: Rep[String], metadataKey: Rep[String]) => + (for { + metadataEntry <- metadataEntries + if metadataEntry.workflowExecutionUuid === workflowExecutionUuid + if metadataEntry.metadataKey === metadataKey + if metadataEntry.callFullyQualifiedName.isEmpty + if metadataEntry.jobIndex.isEmpty + if metadataEntry.jobAttempt.isEmpty + } yield metadataEntry).sortBy(_.metadataTimestamp) + ) + + val countMetadataEntriesForWorkflowExecutionUuidAndMetadataKey = + Compiled((rootWorkflowId: Rep[String], metadataKey: Rep[String], expandSubWorkflows: Rep[Boolean]) => + { + val targetWorkflowIds = for { + summary <- workflowMetadataSummaryEntries + // Uses `IX_WORKFLOW_METADATA_SUMMARY_ENTRY_RWEU`, `UC_WORKFLOW_METADATA_SUMMARY_ENTRY_WEU` + if summary.workflowExecutionUuid === rootWorkflowId || ((summary.rootWorkflowExecutionUuid === rootWorkflowId) && expandSubWorkflows) + } yield summary.workflowExecutionUuid + + for { + metadata <- metadataEntries + if metadata.workflowExecutionUuid in targetWorkflowIds // Uses `METADATA_WORKFLOW_IDX` + if metadata.metadataKey === metadataKey + if metadata.callFullyQualifiedName.isEmpty + if metadata.jobIndex.isEmpty + if metadata.jobAttempt.isEmpty + } yield metadata + }.size + ) val metadataEntriesForJobKey = Compiled( - (workflowExecutionUuid: Rep[String], callFullyQualifiedName: Rep[String], jobIndex: Rep[Option[Int]], - jobAttempt: Rep[Option[Int]]) => (for { - metadataEntry <- metadataEntries - if metadataEntry.workflowExecutionUuid === workflowExecutionUuid - if metadataEntry.callFullyQualifiedName === callFullyQualifiedName - if hasSameIndex(metadataEntry, jobIndex) - if hasSameAttempt(metadataEntry, jobAttempt) - } yield metadataEntry).sortBy(_.metadataTimestamp) + (workflowExecutionUuid: Rep[String], + callFullyQualifiedName: Rep[String], + jobIndex: Rep[Option[Int]], + jobAttempt: Rep[Option[Int]] + ) => + (for { + metadataEntry <- metadataEntries + if metadataEntry.workflowExecutionUuid === workflowExecutionUuid + if metadataEntry.callFullyQualifiedName === callFullyQualifiedName + if hasSameIndex(metadataEntry, jobIndex) + if hasSameAttempt(metadataEntry, jobAttempt) + } yield metadataEntry).sortBy(_.metadataTimestamp) ) val countMetadataEntriesForJobKey = Compiled( - (rootWorkflowId: Rep[String], callFullyQualifiedName: Rep[String], jobIndex: Rep[Option[Int]], - jobAttempt: Rep[Option[Int]], expandSubWorkflows: Rep[Boolean]) => { - val targetWorkflowIds = for { - summary <- workflowMetadataSummaryEntries - // Uses `IX_WORKFLOW_METADATA_SUMMARY_ENTRY_RWEU`, `UC_WORKFLOW_METADATA_SUMMARY_ENTRY_WEU` - if summary.workflowExecutionUuid === rootWorkflowId || ((summary.rootWorkflowExecutionUuid === rootWorkflowId) && expandSubWorkflows) - } yield summary.workflowExecutionUuid - - for { - metadata <- metadataEntries - if metadata.workflowExecutionUuid in targetWorkflowIds // Uses `METADATA_WORKFLOW_IDX` - if metadata.callFullyQualifiedName === callFullyQualifiedName - if hasSameIndex(metadata, jobIndex) - if hasSameAttempt(metadata, jobAttempt) - } yield metadata - }.size + (rootWorkflowId: Rep[String], + callFullyQualifiedName: Rep[String], + jobIndex: Rep[Option[Int]], + jobAttempt: Rep[Option[Int]], + expandSubWorkflows: Rep[Boolean] + ) => + { + val targetWorkflowIds = for { + summary <- workflowMetadataSummaryEntries + // Uses `IX_WORKFLOW_METADATA_SUMMARY_ENTRY_RWEU`, `UC_WORKFLOW_METADATA_SUMMARY_ENTRY_WEU` + if summary.workflowExecutionUuid === rootWorkflowId || ((summary.rootWorkflowExecutionUuid === rootWorkflowId) && expandSubWorkflows) + } yield summary.workflowExecutionUuid + + for { + metadata <- metadataEntries + if metadata.workflowExecutionUuid in targetWorkflowIds // Uses `METADATA_WORKFLOW_IDX` + if metadata.callFullyQualifiedName === callFullyQualifiedName + if hasSameIndex(metadata, jobIndex) + if hasSameAttempt(metadata, jobAttempt) + } yield metadata + }.size ) val metadataEntriesForJobKeyAndMetadataKey = Compiled( - (workflowExecutionUuid: Rep[String], metadataKey: Rep[String], callFullyQualifiedName: Rep[String], - jobIndex: Rep[Option[Int]], jobAttempt: Rep[Option[Int]]) => (for { - metadataEntry <- metadataEntries - if metadataEntry.workflowExecutionUuid === workflowExecutionUuid - if metadataEntry.metadataKey === metadataKey - if metadataEntry.callFullyQualifiedName === callFullyQualifiedName - if hasSameIndex(metadataEntry, jobIndex) - if hasSameAttempt(metadataEntry, jobAttempt) - } yield metadataEntry).sortBy(_.metadataTimestamp) + (workflowExecutionUuid: Rep[String], + metadataKey: Rep[String], + callFullyQualifiedName: Rep[String], + jobIndex: Rep[Option[Int]], + jobAttempt: Rep[Option[Int]] + ) => + (for { + metadataEntry <- metadataEntries + if metadataEntry.workflowExecutionUuid === workflowExecutionUuid + if metadataEntry.metadataKey === metadataKey + if metadataEntry.callFullyQualifiedName === callFullyQualifiedName + if hasSameIndex(metadataEntry, jobIndex) + if hasSameAttempt(metadataEntry, jobAttempt) + } yield metadataEntry).sortBy(_.metadataTimestamp) ) val countMetadataEntriesForJobKeyAndMetadataKey = Compiled( - (rootWorkflowId: Rep[String], metadataKey: Rep[String], callFullyQualifiedName: Rep[String], - jobIndex: Rep[Option[Int]], jobAttempt: Rep[Option[Int]], expandSubWorkflows: Rep[Boolean]) => { - val targetWorkflowIds = for { - summary <- workflowMetadataSummaryEntries - // Uses `IX_WORKFLOW_METADATA_SUMMARY_ENTRY_RWEU`, `UC_WORKFLOW_METADATA_SUMMARY_ENTRY_WEU` - if summary.workflowExecutionUuid === rootWorkflowId || ((summary.rootWorkflowExecutionUuid === rootWorkflowId) && expandSubWorkflows) - } yield summary.workflowExecutionUuid - - for { - metadata <- metadataEntries - if metadata.workflowExecutionUuid in targetWorkflowIds // Uses `METADATA_WORKFLOW_IDX` - if metadata.metadataKey === metadataKey - if metadata.callFullyQualifiedName === callFullyQualifiedName - if hasSameIndex(metadata, jobIndex) - if hasSameAttempt(metadata, jobAttempt) - } yield metadata - }.size + (rootWorkflowId: Rep[String], + metadataKey: Rep[String], + callFullyQualifiedName: Rep[String], + jobIndex: Rep[Option[Int]], + jobAttempt: Rep[Option[Int]], + expandSubWorkflows: Rep[Boolean] + ) => + { + val targetWorkflowIds = for { + summary <- workflowMetadataSummaryEntries + // Uses `IX_WORKFLOW_METADATA_SUMMARY_ENTRY_RWEU`, `UC_WORKFLOW_METADATA_SUMMARY_ENTRY_WEU` + if summary.workflowExecutionUuid === rootWorkflowId || ((summary.rootWorkflowExecutionUuid === rootWorkflowId) && expandSubWorkflows) + } yield summary.workflowExecutionUuid + + for { + metadata <- metadataEntries + if metadata.workflowExecutionUuid in targetWorkflowIds // Uses `METADATA_WORKFLOW_IDX` + if metadata.metadataKey === metadataKey + if metadata.callFullyQualifiedName === callFullyQualifiedName + if hasSameIndex(metadata, jobIndex) + if hasSameAttempt(metadata, jobAttempt) + } yield metadata + }.size ) - val metadataEntriesForIdRange = Compiled( - (minMetadataEntryId: Rep[Long], maxMetadataEntryId: Rep[Long]) => { - for { - metadataEntry <- metadataEntries - if metadataEntry.metadataEntryId >= minMetadataEntryId - if metadataEntry.metadataEntryId <= maxMetadataEntryId - } yield metadataEntry - } - ) + val metadataEntriesForIdRange = Compiled { (minMetadataEntryId: Rep[Long], maxMetadataEntryId: Rep[Long]) => + for { + metadataEntry <- metadataEntries + if metadataEntry.metadataEntryId >= minMetadataEntryId + if metadataEntry.metadataEntryId <= maxMetadataEntryId + } yield metadataEntry + } /** * Returns metadata entries that are "like" metadataKeys for the specified workflow. @@ -210,14 +241,14 @@ trait MetadataEntryComponent { def metadataEntriesWithKeyConstraints(workflowExecutionUuid: String, metadataKeysToFilterFor: List[String], metadataKeysToFilterOut: List[String], - requireEmptyJobKey: Boolean) = { + requireEmptyJobKey: Boolean + ) = (for { metadataEntry <- metadataEntries if metadataEntry.workflowExecutionUuid === workflowExecutionUuid if metadataEntryHasMetadataKeysLike(metadataEntry, metadataKeysToFilterFor, metadataKeysToFilterOut) if metadataEntryHasEmptyJobKey(metadataEntry, requireEmptyJobKey) } yield metadataEntry).sortBy(_.metadataTimestamp) - } /** * Counts metadata entries that are "like" metadataKeys for the specified workflow. @@ -228,7 +259,8 @@ trait MetadataEntryComponent { metadataKeysToFilterFor: List[String], metadataKeysToFilterOut: List[String], requireEmptyJobKey: Boolean, - expandSubWorkflows: Boolean) = { + expandSubWorkflows: Boolean + ) = { val targetWorkflowIds = for { summary <- workflowMetadataSummaryEntries @@ -253,7 +285,8 @@ trait MetadataEntryComponent { metadataKeysToFilterOut: List[String], callFqn: String, jobIndex: Option[Int], - jobAttempt: Option[Int]) = { + jobAttempt: Option[Int] + ) = (for { metadataEntry <- metadataEntries if metadataEntry.workflowExecutionUuid === workflowExecutionUuid @@ -265,7 +298,6 @@ trait MetadataEntryComponent { // regardless of the attempt if (metadataEntry.jobAttempt === jobAttempt) || jobAttempt.isEmpty } yield metadataEntry).sortBy(_.metadataTimestamp) - } /** * Counts metadata entries that are "like" metadataKeys for the specified call. @@ -277,7 +309,8 @@ trait MetadataEntryComponent { callFqn: String, jobIndex: Option[Int], jobAttempt: Option[Int], - expandSubWorkflows: Boolean) = { + expandSubWorkflows: Boolean + ) = { val targetWorkflowIds = for { summary <- workflowMetadataSummaryEntries @@ -301,39 +334,43 @@ trait MetadataEntryComponent { def metadataTableSizeInformation() = { val query = sql""" - |SELECT DATA_LENGTH, INDEX_LENGTH, DATA_FREE - |FROM information_schema.tables - |WHERE TABLE_NAME = 'METADATA_ENTRY' + |SELECT DATA_LENGTH, INDEX_LENGTH, DATA_FREE + |FROM information_schema.tables + |WHERE TABLE_NAME = 'METADATA_ENTRY' """.stripMargin - query.as[InformationSchemaEntry](rconv = GetResult { r => - InformationSchemaEntry(r.<<, r.<<, r.<<) - }).headOption + query + .as[InformationSchemaEntry](rconv = GetResult { r => + InformationSchemaEntry(r.<<, r.<<, r.<<) + }) + .headOption } def failedJobsMetadataWithWorkflowId(rootWorkflowId: String, isPostgres: Boolean) = { - val getMetadataEntryResult = GetResult(r => { - MetadataEntry(r.<<, r.<<, r.<<, r.<<, r.<<, r.nextClobOption().map(clob => new SerialClob(clob)), r.<<, r.<<, r.<<) - }) - - def dbIdentifierWrapper(identifier: String, isPostgres: Boolean) = { - if(isPostgres) s"${'"'}$identifier${'"'}" else identifier - } - - def dbMetadataValueColCheckName(isPostgres: Boolean): String = { - if(isPostgres) "obj.data" else "METADATA_VALUE" + val getMetadataEntryResult = GetResult { r => + MetadataEntry(r.<<, + r.<<, + r.<<, + r.<<, + r.<<, + r.nextClobOption().map(clob => new SerialClob(clob)), + r.<<, + r.<<, + r.<< + ) } - def attemptAndIndexSelectStatement(callFqn: String, scatterIndex: String, retryAttempt: String, variablePrefix: String): String = { - s"SELECT ${callFqn}, MAX(COALESCE(${scatterIndex}, 0)) as ${variablePrefix}Scatter, MAX(COALESCE(${retryAttempt}, 0)) AS ${variablePrefix}Retry" - } + def dbIdentifierWrapper(identifier: String, isPostgres: Boolean) = + if (isPostgres) s"${'"'}$identifier${'"'}" else identifier - def pgObjectInnerJoinStatement(isPostgres: Boolean, metadataValColName: String): String = { - if(isPostgres) s"INNER JOIN pg_largeobject obj ON me.${metadataValColName} = cast(obj.loid as text)" else "" - } + def evaluateMetadataValue(isPostgres: Boolean, colName: String): String = + if (isPostgres) s"convert_from(lo_get(${colName}::oid), 'UTF8')" else colName - def failedTaskGroupByClause(metadataValue: String, callFqn: String): String = { - return s"GROUP BY ${callFqn}, ${metadataValue}" - } + def attemptAndIndexSelectStatement(callFqn: String, + scatterIndex: String, + retryAttempt: String, + variablePrefix: String + ): String = + s"SELECT ${callFqn}, MAX(COALESCE(${scatterIndex}, 0)) as ${variablePrefix}Scatter, MAX(COALESCE(${retryAttempt}, 0)) AS ${variablePrefix}Retry" val workflowUuid = dbIdentifierWrapper("WORKFLOW_EXECUTION_UUID", isPostgres) val callFqn = dbIdentifierWrapper("CALL_FQN", isPostgres) @@ -347,7 +384,8 @@ trait MetadataEntryComponent { val metadataValue = dbIdentifierWrapper("METADATA_VALUE", isPostgres) val metadataEntry = dbIdentifierWrapper("METADATA_ENTRY", isPostgres) val wmse = dbIdentifierWrapper("WORKFLOW_METADATA_SUMMARY_ENTRY", isPostgres) - val resultSetColumnNames = s"me.${workflowUuid}, me.${callFqn}, me.${scatterIndex}, me.${retryAttempt}, me.${metadataKey}, me.${metadataValue}, me.${metadataValueType}, me.${metadataTimestamp}, me.${metadataJournalId}" + val resultSetColumnNames = + s"me.${workflowUuid}, me.${callFqn}, me.${scatterIndex}, me.${retryAttempt}, me.${metadataKey}, me.${metadataValue}, me.${metadataValueType}, me.${metadataTimestamp}, me.${metadataJournalId}" val query = sql""" @@ -358,11 +396,12 @@ trait MetadataEntryComponent { FROM #${metadataEntry} me INNER JOIN #${wmse} wmse ON wmse.#${workflowUuid} = me.#${workflowUuid} - #${pgObjectInnerJoinStatement(isPostgres, metadataValue)} WHERE (wmse.#${rootUuid} = $rootWorkflowId OR wmse.#${workflowUuid} = $rootWorkflowId) - AND (me.#${metadataKey} in ('executionStatus', 'backendStatus') AND #${dbMetadataValueColCheckName(isPostgres)} = 'Failed') - #${failedTaskGroupByClause(dbMetadataValueColCheckName(isPostgres), callFqn)} - HAVING #${dbMetadataValueColCheckName(isPostgres)} = 'Failed' + AND (me.#${metadataKey} in ('executionStatus', 'backendStatus') AND #${evaluateMetadataValue(isPostgres, + metadataValue + )} = 'Failed') + GROUP BY #${callFqn}, #${metadataValue} + HAVING #${evaluateMetadataValue(isPostgres, metadataValue)} = 'Failed' ) AS failedCalls ON me.#${callFqn} = failedCalls.#${callFqn} INNER JOIN ( @@ -405,7 +444,8 @@ trait MetadataEntryComponent { private[this] def metadataEntryHasMetadataKeysLike(metadataEntry: MetadataEntries, metadataKeysToFilterFor: List[String], - metadataKeysToFilterOut: List[String]): Rep[Boolean] = { + metadataKeysToFilterOut: List[String] + ): Rep[Boolean] = { def containsKey(key: String): Rep[Boolean] = metadataEntry.metadataKey like key @@ -423,19 +463,17 @@ trait MetadataEntryComponent { } } - private[this] def hasSameIndex(metadataEntry: MetadataEntries, jobIndex: Rep[Option[Int]]) = { + private[this] def hasSameIndex(metadataEntry: MetadataEntries, jobIndex: Rep[Option[Int]]) = (metadataEntry.jobIndex.isEmpty && jobIndex.isEmpty) || (metadataEntry.jobIndex === jobIndex) - } - private[this] def hasSameAttempt(metadataEntry: MetadataEntries, jobAttempt: Rep[Option[Int]]) = { + private[this] def hasSameAttempt(metadataEntry: MetadataEntries, jobAttempt: Rep[Option[Int]]) = (metadataEntry.jobAttempt.isEmpty && jobAttempt.isEmpty) || (metadataEntry.jobAttempt === jobAttempt) - } private[this] def metadataEntryHasEmptyJobKey(metadataEntry: MetadataEntries, - requireEmptyJobKey: Rep[Boolean]): Rep[Boolean] = { + requireEmptyJobKey: Rep[Boolean] + ): Rep[Boolean] = !requireEmptyJobKey || (metadataEntry.callFullyQualifiedName.isEmpty && metadataEntry.jobIndex.isEmpty && metadataEntry.jobAttempt.isEmpty) - } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/SubWorkflowStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/SubWorkflowStoreEntryComponent.scala index 11fa2191cb2..77e39b43816 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/SubWorkflowStoreEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/SubWorkflowStoreEntryComponent.scala @@ -13,7 +13,7 @@ trait SubWorkflowStoreEntryComponent { def subWorkflowStoreEntryId = column[Long]("SUB_WORKFLOW_STORE_ENTRY_ID", O.PrimaryKey, O.AutoInc) def rootWorkflowId = column[Long]("ROOT_WORKFLOW_ID") - + def parentWorkflowExecutionUuid = column[String]("PARENT_WORKFLOW_EXECUTION_UUID", O.Length(255)) def callFullyQualifiedName = column[String]("CALL_FULLY_QUALIFIED_NAME", O.Length(255)) @@ -24,23 +24,37 @@ trait SubWorkflowStoreEntryComponent { def subWorkflowExecutionUuid = column[String]("SUB_WORKFLOW_EXECUTION_UUID", O.Length(255)) - override def * = (rootWorkflowId.?, parentWorkflowExecutionUuid, callFullyQualifiedName, callIndex, callAttempt, subWorkflowExecutionUuid, subWorkflowStoreEntryId.?) <> (SubWorkflowStoreEntry.tupled, SubWorkflowStoreEntry.unapply) - - def ucSubWorkflowStoreEntryPweuCfqnCiCa = index("UC_SUB_WORKFLOW_STORE_ENTRY_PWEU_CFQN_CI_CA", - (parentWorkflowExecutionUuid, callFullyQualifiedName, callIndex, callAttempt), unique = true) + override def * = (rootWorkflowId.?, + parentWorkflowExecutionUuid, + callFullyQualifiedName, + callIndex, + callAttempt, + subWorkflowExecutionUuid, + subWorkflowStoreEntryId.? + ) <> (SubWorkflowStoreEntry.tupled, SubWorkflowStoreEntry.unapply) + + def ucSubWorkflowStoreEntryPweuCfqnCiCa = index( + "UC_SUB_WORKFLOW_STORE_ENTRY_PWEU_CFQN_CI_CA", + (parentWorkflowExecutionUuid, callFullyQualifiedName, callIndex, callAttempt), + unique = true + ) def fkSubWorkflowStoreEntryRootWorkflowId = foreignKey("FK_SUB_WORKFLOW_STORE_ENTRY_ROOT_WORKFLOW_ID", - rootWorkflowId, workflowStoreEntries)(_.workflowStoreEntryId, onDelete = Cascade) + rootWorkflowId, + workflowStoreEntries + )(_.workflowStoreEntryId, onDelete = Cascade) - def ixSubWorkflowStoreEntryPweu = index("IX_SUB_WORKFLOW_STORE_ENTRY_PWEU", parentWorkflowExecutionUuid, unique = false) + def ixSubWorkflowStoreEntryPweu = + index("IX_SUB_WORKFLOW_STORE_ENTRY_PWEU", parentWorkflowExecutionUuid, unique = false) } protected val subWorkflowStoreEntries = TableQuery[SubWorkflowStoreEntries] - val subWorkflowStoreEntryIdsAutoInc = subWorkflowStoreEntries returning subWorkflowStoreEntries.map(_.subWorkflowStoreEntryId) + val subWorkflowStoreEntryIdsAutoInc = + subWorkflowStoreEntries returning subWorkflowStoreEntries.map(_.subWorkflowStoreEntryId) - val subWorkflowStoreEntriesForRootWorkflowId = Compiled( - (rootWorkflowId: Rep[Long]) => for { + val subWorkflowStoreEntriesForRootWorkflowId = Compiled((rootWorkflowId: Rep[Long]) => + for { subWorkflowStoreEntry <- subWorkflowStoreEntries if subWorkflowStoreEntry.rootWorkflowId === rootWorkflowId } yield subWorkflowStoreEntry @@ -50,8 +64,11 @@ trait SubWorkflowStoreEntryComponent { * Useful for finding the unique sub workflow entry for a given job key */ val subWorkflowStoreEntriesForJobKey = Compiled( - (parentWorkflowExecutionUuid: Rep[String], callFullyQualifiedName: Rep[String], jobIndex: Rep[Int], - jobAttempt: Rep[Int]) => + (parentWorkflowExecutionUuid: Rep[String], + callFullyQualifiedName: Rep[String], + jobIndex: Rep[Int], + jobAttempt: Rep[Int] + ) => for { subWorkflowStoreEntry <- subWorkflowStoreEntries if subWorkflowStoreEntry.parentWorkflowExecutionUuid === parentWorkflowExecutionUuid && diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/SummaryStatusEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/SummaryStatusEntryComponent.scala index 807cb0802f7..72d1ece4bb0 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/SummaryStatusEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/SummaryStatusEntryComponent.scala @@ -24,8 +24,8 @@ trait SummaryStatusEntryComponent { val summaryStatusEntryIdsAutoInc = summaryStatusEntries returning summaryStatusEntries.map(_.summaryStatusEntryId) - val summaryPositionForSummaryName = Compiled( - (summaryName: Rep[String]) => for { + val summaryPositionForSummaryName = Compiled((summaryName: Rep[String]) => + for { summaryStatusEntry <- summaryStatusEntries if summaryStatusEntry.summaryName === summaryName } yield summaryStatusEntry.summaryPosition diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala index c3020790be3..82489d4ae28 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala @@ -17,7 +17,7 @@ trait WorkflowMetadataSummaryEntryComponent { import driver.api._ class WorkflowMetadataSummaryEntries(tag: Tag) - extends Table[WorkflowMetadataSummaryEntry](tag, "WORKFLOW_METADATA_SUMMARY_ENTRY") { + extends Table[WorkflowMetadataSummaryEntry](tag, "WORKFLOW_METADATA_SUMMARY_ENTRY") { def workflowMetadataSummaryEntryId = column[Long]("WORKFLOW_METADATA_SUMMARY_ENTRY_ID", O.PrimaryKey, O.AutoInc) def workflowExecutionUuid = column[String]("WORKFLOW_EXECUTION_UUID", O.Length(100)) @@ -38,10 +38,19 @@ trait WorkflowMetadataSummaryEntryComponent { def metadataArchiveStatus: Rep[Option[String]] = column[Option[String]]("METADATA_ARCHIVE_STATUS", O.Length(30)) - def baseProjection = (workflowExecutionUuid, workflowName, workflowStatus, startTimestamp, endTimestamp, - submissionTimestamp, parentWorkflowExecutionUuid, rootWorkflowExecutionUuid, metadataArchiveStatus) + def baseProjection = (workflowExecutionUuid, + workflowName, + workflowStatus, + startTimestamp, + endTimestamp, + submissionTimestamp, + parentWorkflowExecutionUuid, + rootWorkflowExecutionUuid, + metadataArchiveStatus + ) - override def * = baseProjection ~ workflowMetadataSummaryEntryId.? <> (WorkflowMetadataSummaryEntry.tupled, WorkflowMetadataSummaryEntry.unapply) + override def * = + baseProjection ~ workflowMetadataSummaryEntryId.? <> (WorkflowMetadataSummaryEntry.tupled, WorkflowMetadataSummaryEntry.unapply) def forUpdate = baseProjection.shaped <> ( tuple => WorkflowMetadataSummaryEntry.tupled(tuple :+ None), @@ -71,112 +80,123 @@ trait WorkflowMetadataSummaryEntryComponent { val workflowMetadataSummaryEntryIdsAutoInc = workflowMetadataSummaryEntries returning workflowMetadataSummaryEntries.map(_.workflowMetadataSummaryEntryId) - val workflowIdsByArchiveStatusAndEndedOnOrBeforeThresholdTimestamp = Compiled( - (metadataArchiveStatus: Rep[Option[String]], workflowEndTimestampThreshold: Rep[Timestamp], batchSize: ConstColumn[Long]) => { + val workflowIdsByArchiveStatusAndEndedOnOrBeforeThresholdTimestamp = Compiled { + (metadataArchiveStatus: Rep[Option[String]], + workflowEndTimestampThreshold: Rep[Timestamp], + batchSize: ConstColumn[Long] + ) => (for { summary <- workflowMetadataSummaryEntries if summary.metadataArchiveStatus === metadataArchiveStatus if summary.endTimestamp <= workflowEndTimestampThreshold } yield summary.workflowExecutionUuid).take(batchSize) - }) + } - val workflowMetadataSummaryEntriesForWorkflowExecutionUuid = Compiled( - (workflowExecutionUuid: Rep[String]) => for { + val workflowMetadataSummaryEntriesForWorkflowExecutionUuid = Compiled((workflowExecutionUuid: Rep[String]) => + for { workflowMetadataSummaryEntry <- workflowMetadataSummaryEntries if workflowMetadataSummaryEntry.workflowExecutionUuid === workflowExecutionUuid - } yield workflowMetadataSummaryEntry.forUpdate) + } yield workflowMetadataSummaryEntry.forUpdate + ) - val workflowMetadataSummaryEntryExistsForWorkflowExecutionUuid = Compiled( - (workflowExecutionUuid: Rep[String]) => (for { + val workflowMetadataSummaryEntryExistsForWorkflowExecutionUuid = Compiled((workflowExecutionUuid: Rep[String]) => + (for { summaryEntry <- workflowMetadataSummaryEntries if summaryEntry.workflowExecutionUuid === workflowExecutionUuid } yield summaryEntry).exists ) - val workflowStatusesForWorkflowExecutionUuid = Compiled( - (workflowExecutionUuid: Rep[String]) => for { + val workflowStatusesForWorkflowExecutionUuid = Compiled((workflowExecutionUuid: Rep[String]) => + for { workflowMetadataSummaryEntry <- workflowMetadataSummaryEntries if workflowMetadataSummaryEntry.workflowExecutionUuid === workflowExecutionUuid } yield workflowMetadataSummaryEntry.workflowStatus ) - val rootWorkflowId = Compiled( - (workflowId: Rep[String]) => for { + val rootWorkflowId = Compiled((workflowId: Rep[String]) => + for { summary <- workflowMetadataSummaryEntries if summary.workflowExecutionUuid === workflowId - } yield { - summary.rootWorkflowExecutionUuid.getOrElse(summary.workflowExecutionUuid) - } + } yield summary.rootWorkflowExecutionUuid.getOrElse(summary.workflowExecutionUuid) ) - val metadataArchiveStatusByWorkflowId = Compiled( - (workflowExecutionUuid: Rep[String]) => for { + val metadataArchiveStatusByWorkflowId = Compiled((workflowExecutionUuid: Rep[String]) => + for { workflowMetadataSummaryEntry <- workflowMetadataSummaryEntries if workflowMetadataSummaryEntry.workflowExecutionUuid === workflowExecutionUuid - } yield workflowMetadataSummaryEntry.metadataArchiveStatus) + } yield workflowMetadataSummaryEntry.metadataArchiveStatus + ) - val metadataArchiveStatusAndEndTimeByWorkflowId = Compiled( - (workflowExecutionUuid: Rep[String]) => for { + val metadataArchiveStatusAndEndTimeByWorkflowId = Compiled((workflowExecutionUuid: Rep[String]) => + for { workflowMetadataSummaryEntry <- workflowMetadataSummaryEntries if workflowMetadataSummaryEntry.workflowExecutionUuid === workflowExecutionUuid - } yield (workflowMetadataSummaryEntry.metadataArchiveStatus, workflowMetadataSummaryEntry.endTimestamp)) + } yield (workflowMetadataSummaryEntry.metadataArchiveStatus, workflowMetadataSummaryEntry.endTimestamp) + ) private def fetchAllWorkflowsToArchiveThatEndedOnOrBeforeThresholdTimestamp(workflowStatuses: List[String], - workflowEndTimestampThreshold: Timestamp): Query[WorkflowMetadataSummaryEntries, WorkflowMetadataSummaryEntry, Seq] = { + workflowEndTimestampThreshold: Timestamp + ): Query[WorkflowMetadataSummaryEntries, WorkflowMetadataSummaryEntry, Seq] = for { summaryEntry <- workflowMetadataSummaryEntries if summaryEntry.workflowStatus.inSet(workflowStatuses) if summaryEntry.metadataArchiveStatus.isEmpty // get Unarchived workflows only if summaryEntry.endTimestamp <= workflowEndTimestampThreshold } yield summaryEntry - } - private def fetchAllWorkflowsToDeleteThatEndedOnOrBeforeThresholdTimestamp(workflowEndTimestampThreshold: Timestamp): Query[WorkflowMetadataSummaryEntries, WorkflowMetadataSummaryEntry, Seq] = { + private def fetchAllWorkflowsToDeleteThatEndedOnOrBeforeThresholdTimestamp( + workflowEndTimestampThreshold: Timestamp + ): Query[WorkflowMetadataSummaryEntries, WorkflowMetadataSummaryEntry, Seq] = for { summaryEntry <- workflowMetadataSummaryEntries if summaryEntry.metadataArchiveStatus === Option("Archived") // get archived but not deleted workflows only if summaryEntry.endTimestamp <= workflowEndTimestampThreshold } yield summaryEntry - } def workflowsToArchiveThatEndedOnOrBeforeThresholdTimestamp(workflowStatuses: List[String], workflowEndTimestampThreshold: Timestamp, - batchSize: Long): Query[WorkflowMetadataSummaryEntries, WorkflowMetadataSummaryEntry, Seq] = { + batchSize: Long + ): Query[WorkflowMetadataSummaryEntries, WorkflowMetadataSummaryEntry, Seq] = fetchAllWorkflowsToArchiveThatEndedOnOrBeforeThresholdTimestamp( workflowStatuses, workflowEndTimestampThreshold ).sortBy(_.endTimestamp).take(batchSize) - } def countWorkflowsLeftToArchiveThatEndedOnOrBeforeThresholdTimestamp(workflowStatuses: List[String], - workflowEndTimestampThreshold: Timestamp): Rep[Int] = { + workflowEndTimestampThreshold: Timestamp + ): Rep[Int] = fetchAllWorkflowsToArchiveThatEndedOnOrBeforeThresholdTimestamp( workflowStatuses, workflowEndTimestampThreshold ).length - } - def countWorkflowsLeftToDeleteThatEndedOnOrBeforeThresholdTimestamp(workflowEndTimestampThreshold: Timestamp): Rep[Int] = { + def countWorkflowsLeftToDeleteThatEndedOnOrBeforeThresholdTimestamp( + workflowEndTimestampThreshold: Timestamp + ): Rep[Int] = fetchAllWorkflowsToDeleteThatEndedOnOrBeforeThresholdTimestamp( workflowEndTimestampThreshold ).length - } - def concat(a: SQLActionBuilder, b: SQLActionBuilder): SQLActionBuilder = { - SQLActionBuilder(a.queryParts ++ b.queryParts, (p: Unit, pp: PositionedParameters) => { - a.unitPConv.apply(p, pp) - b.unitPConv.apply(p, pp) - }) - } + def concat(a: SQLActionBuilder, b: SQLActionBuilder): SQLActionBuilder = + SQLActionBuilder(a.queryParts ++ b.queryParts, + (p: Unit, pp: PositionedParameters) => { + a.unitPConv.apply(p, pp) + b.unitPConv.apply(p, pp) + } + ) - def concatNel(nel: NonEmptyList[SQLActionBuilder]): SQLActionBuilder = nel.tail.foldLeft(nel.head) { (acc, next) => concat(acc, next) } + def concatNel(nel: NonEmptyList[SQLActionBuilder]): SQLActionBuilder = nel.tail.foldLeft(nel.head) { (acc, next) => + concat(acc, next) + } - def and(list: NonEmptyList[SQLActionBuilder]): SQLActionBuilder = if (list.size == 1) list.head else { + def and(list: NonEmptyList[SQLActionBuilder]): SQLActionBuilder = if (list.size == 1) list.head + else { val fullList = data.NonEmptyList.of(sql"(") ++ list.init.flatMap(x => List(x, sql" AND ")) :+ list.last :+ sql")" concatNel(fullList) } - def or(list: NonEmptyList[SQLActionBuilder]): SQLActionBuilder = if (list.size == 1) list.head else { + def or(list: NonEmptyList[SQLActionBuilder]): SQLActionBuilder = if (list.size == 1) list.head + else { val fullList = data.NonEmptyList.of(sql"(") ++ list.init.flatMap(x => List(x, sql" OR ")) :+ list.last :+ sql")" concatNel(fullList) } @@ -192,15 +212,16 @@ trait WorkflowMetadataSummaryEntryComponent { workflowStatuses: Set[String], workflowNames: Set[String], workflowExecutionUuids: Set[String], - labelAndKeyLabelValues: Set[(String,String)], - labelOrKeyLabelValues: Set[(String,String)], - excludeLabelAndValues: Set[(String,String)], - excludeLabelOrValues: Set[(String,String)], + labelAndKeyLabelValues: Set[(String, String)], + labelOrKeyLabelValues: Set[(String, String)], + excludeLabelAndValues: Set[(String, String)], + excludeLabelOrValues: Set[(String, String)], submissionTimestampOption: Option[Timestamp], startTimestampOption: Option[Timestamp], endTimestampOption: Option[Timestamp], metadataArchiveStatus: Set[Option[String]], - includeSubworkflows: Boolean): SQLActionBuilder = { + includeSubworkflows: Boolean + ): SQLActionBuilder = { val customLabelEntryTable = quoted("CUSTOM_LABEL_ENTRY") val workflowMetadataSummaryEntryTable = quoted("WORKFLOW_METADATA_SUMMARY_ENTRY") @@ -212,8 +233,8 @@ trait WorkflowMetadataSummaryEntryComponent { val summaryTableAlias = quoted("summaryTable") val labelsOrTableAlias = quoted("labelsOrMixin") - val labelsAndTableAliases = labelAndKeyLabelValues.zipWithIndex.map { - case (labelPair, i) => quoted(s"labelAndTable$i") -> labelPair + val labelsAndTableAliases = labelAndKeyLabelValues.zipWithIndex.map { case (labelPair, i) => + quoted(s"labelAndTable$i") -> labelPair }.toMap val selectColumns = List( @@ -226,7 +247,7 @@ trait WorkflowMetadataSummaryEntryComponent { "PARENT_WORKFLOW_EXECUTION_UUID", "ROOT_WORKFLOW_EXECUTION_UUID", "METADATA_ARCHIVE_STATUS", - "WORKFLOW_METADATA_SUMMARY_ENTRY_ID", + "WORKFLOW_METADATA_SUMMARY_ENTRY_ID" ) .map(quoted) .mkString(s"$summaryTableAlias.", ", ", "") @@ -241,11 +262,10 @@ trait WorkflowMetadataSummaryEntryComponent { } val labelOrJoin = if (labelOrKeyLabelValues.nonEmpty) { - Option( - sql"""| JOIN #$customLabelEntryTable #$labelsOrTableAlias - | ON #$summaryTableAlias.#$workflowExecutionUuidColumn - | = #$labelsOrTableAlias.#$workflowExecutionUuidColumn - |""".stripMargin) + Option(sql"""| JOIN #$customLabelEntryTable #$labelsOrTableAlias + | ON #$summaryTableAlias.#$workflowExecutionUuidColumn + | = #$labelsOrTableAlias.#$workflowExecutionUuidColumn + |""".stripMargin) } else None val labelAndJoins = labelsAndTableAliases.toList.map { case (labelAndTableAlias, _) => @@ -255,9 +275,8 @@ trait WorkflowMetadataSummaryEntryComponent { |""".stripMargin } - val from = concatNel(NonEmptyList.of( - sql"""|FROM #$workflowMetadataSummaryEntryTable #$summaryTableAlias - |""".stripMargin) ++ labelOrJoin.toList ++ labelAndJoins) + val from = concatNel(NonEmptyList.of(sql"""|FROM #$workflowMetadataSummaryEntryTable #$summaryTableAlias + |""".stripMargin) ++ labelOrJoin.toList ++ labelAndJoins) def makeSetConstraint(column: String, elements: Set[String]) = { val list = elements.toList.map(element => sql"""#$summaryTableAlias.#${quoted(column)} = $element""") @@ -272,9 +291,8 @@ trait WorkflowMetadataSummaryEntryComponent { NonEmptyList.fromList(list).map(or).toList } - def makeTimeConstraint(column: String, comparison: String, elementOption: Option[Timestamp]) = { + def makeTimeConstraint(column: String, comparison: String, elementOption: Option[Timestamp]) = elementOption.map(element => sql"""#$summaryTableAlias.#${quoted(column)} #$comparison $element""").toList - } val statusConstraint = makeSetConstraint("WORKFLOW_STATUS", workflowStatuses) val nameConstraint = makeSetConstraint("WORKFLOW_NAME", workflowNames) @@ -287,23 +305,30 @@ trait WorkflowMetadataSummaryEntryComponent { val metadataArchiveStatusConstraint = makeSetConstraintWithNulls("METADATA_ARCHIVE_STATUS", metadataArchiveStatus) // *ALL* of the labelAnd list of KV pairs must exist: - val labelsAndConstraint = NonEmptyList.fromList(labelsAndTableAliases.toList.map { - case (labelsAndTableAlias, (labelKey, labelValue)) => - and(NonEmptyList.of( - sql"""#$labelsAndTableAlias.#$customLabelKeyColumn = $labelKey""", - sql"""#$labelsAndTableAlias.#$customLabelValueColumn = $labelValue""", - )) - }).map(and).toList + val labelsAndConstraint = NonEmptyList + .fromList(labelsAndTableAliases.toList.map { case (labelsAndTableAlias, (labelKey, labelValue)) => + and( + NonEmptyList.of( + sql"""#$labelsAndTableAlias.#$customLabelKeyColumn = $labelKey""", + sql"""#$labelsAndTableAlias.#$customLabelValueColumn = $labelValue""" + ) + ) + }) + .map(and) + .toList // At least one of the labelOr list of KV pairs must exist: - val labelOrConstraint = NonEmptyList.fromList(labelOrKeyLabelValues.toList.map { - case (labelKey, labelValue) => - and(NonEmptyList.of( - sql"""#$labelsOrTableAlias.#$customLabelKeyColumn = $labelKey""", - sql"""#$labelsOrTableAlias.#$customLabelValueColumn = $labelValue""", - )) - }).map(or).toList - + val labelOrConstraint = NonEmptyList + .fromList(labelOrKeyLabelValues.toList.map { case (labelKey, labelValue) => + and( + NonEmptyList.of( + sql"""#$labelsOrTableAlias.#$customLabelKeyColumn = $labelKey""", + sql"""#$labelsOrTableAlias.#$customLabelValueColumn = $labelValue""" + ) + ) + }) + .map(or) + .toList var mixinTableCounter = 0 @@ -322,18 +347,26 @@ trait WorkflowMetadataSummaryEntryComponent { } // *ALL* of the excludeLabelOr list of KV pairs must *NOT* exist: - val excludeLabelsOrConstraint = NonEmptyList.fromList(excludeLabelOrValues.toList map { - case (labelKey, labelValue) => not(labelExists(labelKey, labelValue)) - }).map(and).toList + val excludeLabelsOrConstraint = NonEmptyList + .fromList(excludeLabelOrValues.toList map { case (labelKey, labelValue) => + not(labelExists(labelKey, labelValue)) + }) + .map(and) + .toList // At least one of the excludeLabelAnd list of KV pairs must *NOT* exist: - val excludeLabelsAndConstraint = NonEmptyList.fromList(excludeLabelAndValues.toList.map { - case (labelKey, labelValue) => not(labelExists(labelKey, labelValue)) - }).map(or).toList - - val includeSubworkflowsConstraint = if (includeSubworkflows) List.empty else { - List(sql"""#$summaryTableAlias.#$parentWorkflowExecutionUuidColumn IS NULL""".stripMargin) - } + val excludeLabelsAndConstraint = NonEmptyList + .fromList(excludeLabelAndValues.toList.map { case (labelKey, labelValue) => + not(labelExists(labelKey, labelValue)) + }) + .map(or) + .toList + + val includeSubworkflowsConstraint = + if (includeSubworkflows) List.empty + else { + List(sql"""#$summaryTableAlias.#$parentWorkflowExecutionUuidColumn IS NULL""".stripMargin) + } val constraintList = statusConstraint ++ @@ -359,19 +392,20 @@ trait WorkflowMetadataSummaryEntryComponent { concatNel((NonEmptyList.of(select) :+ from) ++ where) } - def countWorkflowMetadataSummaryEntries(parentIdWorkflowMetadataKey: String, - workflowStatuses: Set[String], workflowNames: Set[String], + workflowStatuses: Set[String], + workflowNames: Set[String], workflowExecutionUuids: Set[String], - labelAndKeyLabelValues: Set[(String,String)], - labelOrKeyLabelValues: Set[(String,String)], - excludeLabelAndValues: Set[(String,String)], - excludeLabelOrValues: Set[(String,String)], + labelAndKeyLabelValues: Set[(String, String)], + labelOrKeyLabelValues: Set[(String, String)], + excludeLabelAndValues: Set[(String, String)], + excludeLabelOrValues: Set[(String, String)], submissionTimestampOption: Option[Timestamp], startTimestampOption: Option[Timestamp], endTimestampOption: Option[Timestamp], metadataArchiveStatus: Set[Option[String]], - includeSubworkflows: Boolean) = { + includeSubworkflows: Boolean + ) = buildQueryAction( selectOrCount = Count, parentIdWorkflowMetadataKey, @@ -388,18 +422,18 @@ trait WorkflowMetadataSummaryEntryComponent { metadataArchiveStatus, includeSubworkflows = includeSubworkflows ).as[Int].head - } /** * Query workflow execution using the filter criteria encapsulated by the `WorkflowExecutionQueryParameters`. */ def queryWorkflowMetadataSummaryEntries(parentIdWorkflowMetadataKey: String, - workflowStatuses: Set[String], workflowNames: Set[String], + workflowStatuses: Set[String], + workflowNames: Set[String], workflowExecutionUuids: Set[String], - labelAndKeyLabelValues: Set[(String,String)], - labelOrKeyLabelValues: Set[(String,String)], - excludeLabelAndValues: Set[(String,String)], - excludeLabelOrValues: Set[(String,String)], + labelAndKeyLabelValues: Set[(String, String)], + labelOrKeyLabelValues: Set[(String, String)], + excludeLabelAndValues: Set[(String, String)], + excludeLabelOrValues: Set[(String, String)], submissionTimestampOption: Option[Timestamp], startTimestampOption: Option[Timestamp], endTimestampOption: Option[Timestamp], @@ -407,7 +441,8 @@ trait WorkflowMetadataSummaryEntryComponent { includeSubworkflows: Boolean, page: Option[Int], pageSize: Option[Int], - newestFirst: Boolean) = { + newestFirst: Boolean + ) = { val mainQuery = buildQueryAction( selectOrCount = Select, parentIdWorkflowMetadataKey, @@ -433,13 +468,13 @@ trait WorkflowMetadataSummaryEntryComponent { // `true` for queries, newest workflows are the most relevant // `false` for archiving, going oldest-to-newest - val orderByAddendum = if (newestFirst) - sql"""| ORDER BY #${quoted("WORKFLOW_METADATA_SUMMARY_ENTRY_ID")} DESC - |""".stripMargin - else - sql"""| ORDER BY #${quoted("WORKFLOW_METADATA_SUMMARY_ENTRY_ID")} ASC - |""".stripMargin - + val orderByAddendum = + if (newestFirst) + sql"""| ORDER BY #${quoted("WORKFLOW_METADATA_SUMMARY_ENTRY_ID")} DESC + |""".stripMargin + else + sql"""| ORDER BY #${quoted("WORKFLOW_METADATA_SUMMARY_ENTRY_ID")} ASC + |""".stripMargin // NB you can preview the prepared statement created here by using, for example: println(result.statements.head) diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala index 847fa862405..874fcbd0c0c 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala @@ -43,8 +43,23 @@ trait WorkflowStoreEntryComponent { def hogGroup = column[Option[String]]("HOG_GROUP", O.Length(100)) - override def * = (workflowExecutionUuid, workflowDefinition, workflowUrl, workflowRoot, workflowType, workflowTypeVersion, workflowInputs, workflowOptions, workflowState, - submissionTime, importsZip, customLabels, cromwellId, heartbeatTimestamp, hogGroup, workflowStoreEntryId.?) <> ((WorkflowStoreEntry.apply _).tupled, WorkflowStoreEntry.unapply) + override def * = (workflowExecutionUuid, + workflowDefinition, + workflowUrl, + workflowRoot, + workflowType, + workflowTypeVersion, + workflowInputs, + workflowOptions, + workflowState, + submissionTime, + importsZip, + customLabels, + cromwellId, + heartbeatTimestamp, + hogGroup, + workflowStoreEntryId.? + ) <> ((WorkflowStoreEntry.apply _).tupled, WorkflowStoreEntry.unapply) def ucWorkflowStoreEntryWeu = index("UC_WORKFLOW_STORE_ENTRY_WEU", workflowExecutionUuid, unique = true) @@ -58,15 +73,15 @@ trait WorkflowStoreEntryComponent { /** * Useful for finding the workflow store for a given workflow execution UUID */ - val workflowStoreEntriesForWorkflowExecutionUuid = Compiled( - (workflowExecutionUuid: Rep[String]) => for { + val workflowStoreEntriesForWorkflowExecutionUuid = Compiled((workflowExecutionUuid: Rep[String]) => + for { workflowStoreEntry <- workflowStoreEntries if workflowStoreEntry.workflowExecutionUuid === workflowExecutionUuid } yield workflowStoreEntry ) - val heartbeatForWorkflowStoreEntry = Compiled( - (workflowExecutionUuid: Rep[String]) => for { + val heartbeatForWorkflowStoreEntry = Compiled((workflowExecutionUuid: Rep[String]) => + for { workflowStoreEntry <- workflowStoreEntries if workflowStoreEntry.workflowExecutionUuid === workflowExecutionUuid } yield workflowStoreEntry.heartbeatTimestamp @@ -77,7 +92,8 @@ trait WorkflowStoreEntryComponent { */ def getHogGroupWithLowestRunningWfs(heartbeatTimestampTimedOut: Timestamp, excludeWorkflowState: String, - excludedGroups: Set[String]): Query[Rep[Option[String]], Option[String], Seq] = { + excludedGroups: Set[String] + ): Query[Rep[Option[String]], Option[String], Seq] = { val startableWorkflows = for { row <- workflowStoreEntries /* @@ -86,7 +102,7 @@ trait WorkflowStoreEntryComponent { 2) Workflows with old heartbeats, presumably abandoned by a defunct Cromwell. 3) Workflows not in "OnHold" state 4) Workflows that don't belong to hog groups in excludedGroups - */ + */ if (row.heartbeatTimestamp.isEmpty || row.heartbeatTimestamp < heartbeatTimestampTimedOut) && (row.workflowState =!= excludeWorkflowState) && !(row.hogGroup inSet excludedGroups) @@ -109,7 +125,7 @@ trait WorkflowStoreEntryComponent { This looks for: 1) Workflows not in "OnHold" state 2) Workflows that don't belong to hog groups in excludedGroups - */ + */ if row.workflowState =!= excludeWorkflowState && !(row.hogGroup inSet excludedGroups) } yield row @@ -129,27 +145,31 @@ trait WorkflowStoreEntryComponent { Seq ] = for { (hog_group, workflows_ct) <- totalWorkflowsByHogGroup - (startable_hog_group, startable_workflows_ct, oldest_submission_time) <- numOfStartableWfsByHogGroup if hog_group === startable_hog_group + (startable_hog_group, startable_workflows_ct, oldest_submission_time) <- numOfStartableWfsByHogGroup + if hog_group === startable_hog_group } yield (hog_group, workflows_ct - startable_workflows_ct, oldest_submission_time) // sort the above calculated result set first by the count of actively running workflows, then by hog group with // oldest submission timestamp and then sort it alphabetically by hog group name. Then take the first row of // the result and return the hog group name. - wfsRunningPerHogGroup.sortBy { - case (hogGroupName, running_wf_ct, oldest_submission_time) => (running_wf_ct.asc, oldest_submission_time, hogGroupName) - }.take(1).map(_._1) + wfsRunningPerHogGroup + .sortBy { case (hogGroupName, running_wf_ct, oldest_submission_time) => + (running_wf_ct.asc, oldest_submission_time, hogGroupName) + } + .take(1) + .map(_._1) } /** * Returns up to "limit" startable workflows, sorted by submission time, that belong to * given hog group and are not in "OnHold" status. */ - val fetchStartableWfsForHogGroup = Compiled( + val fetchStartableWfsForHogGroup = Compiled { (limit: ConstColumn[Long], heartbeatTimestampTimedOut: ConstColumn[Timestamp], excludeWorkflowState: Rep[String], - hogGroup: Rep[Option[String]]) => { - + hogGroup: Rep[Option[String]] + ) => val workflowsToStart = for { row <- workflowStoreEntries /* @@ -158,7 +178,7 @@ trait WorkflowStoreEntryComponent { 2) Workflows with old heartbeats, presumably abandoned by a defunct Cromwell. 3) Workflows not in "OnHold" state 4) Workflows that belong to included hog group - */ + */ if (row.heartbeatTimestamp.isEmpty || row.heartbeatTimestamp < heartbeatTimestampTimedOut) && (row.workflowState =!= excludeWorkflowState) && (row.hogGroup === hogGroup) @@ -169,8 +189,7 @@ trait WorkflowStoreEntryComponent { do an update subsequent to this select in the same transaction that we know will impact those readers. */ workflowsToStart.forUpdate.sortBy(_.submissionTime.asc).take(limit) - } - ) + } /** * Useful for counting workflows in a given state. @@ -184,8 +203,8 @@ trait WorkflowStoreEntryComponent { /** * Useful for updating the relevant fields of a workflow store entry when a workflow is picked up for processing. */ - val workflowStoreFieldsForPickup = Compiled( - (workflowExecutionUuid: Rep[String]) => for { + val workflowStoreFieldsForPickup = Compiled((workflowExecutionUuid: Rep[String]) => + for { row <- workflowStoreEntries if row.workflowExecutionUuid === workflowExecutionUuid } yield (row.workflowState, row.cromwellId, row.heartbeatTimestamp) @@ -194,8 +213,8 @@ trait WorkflowStoreEntryComponent { /** * Useful for clearing out cromwellId and heartbeatTimestamp on an orderly Cromwell shutdown. */ - val releaseWorkflowStoreEntries = Compiled( - (cromwellId: Rep[String]) => for { + val releaseWorkflowStoreEntries = Compiled((cromwellId: Rep[String]) => + for { row <- workflowStoreEntries if row.cromwellId === cromwellId } yield (row.cromwellId, row.heartbeatTimestamp) @@ -204,8 +223,8 @@ trait WorkflowStoreEntryComponent { /** * Useful for updating state for all entries matching a given state */ - val workflowStateForWorkflowState = Compiled( - (workflowState: Rep[String]) => for { + val workflowStateForWorkflowState = Compiled((workflowState: Rep[String]) => + for { workflowStoreEntry <- workflowStoreEntries if workflowStoreEntry.workflowState === workflowState } yield workflowStoreEntry.workflowState @@ -214,8 +233,8 @@ trait WorkflowStoreEntryComponent { /** * Useful for updating a given workflow to a new state */ - val workflowStateForWorkflowExecutionUUid = Compiled( - (workflowId: Rep[String]) => for { + val workflowStateForWorkflowExecutionUUid = Compiled((workflowId: Rep[String]) => + for { workflowStoreEntry <- workflowStoreEntries if workflowStoreEntry.workflowExecutionUuid === workflowId } yield workflowStoreEntry.workflowState @@ -224,53 +243,48 @@ trait WorkflowStoreEntryComponent { /** * Useful for updating a given workflow to a 'Submitted' state when it's currently 'On Hold' */ - val workflowStateForWorkflowExecutionUUidAndWorkflowState = Compiled( - (workflowId: Rep[String], workflowState: Rep[String]) => { + val workflowStateForWorkflowExecutionUUidAndWorkflowState = Compiled { + (workflowId: Rep[String], workflowState: Rep[String]) => for { workflowStoreEntry <- workflowStoreEntries if workflowStoreEntry.workflowExecutionUuid === workflowId if workflowStoreEntry.workflowState === workflowState } yield workflowStoreEntry.workflowState - } - ) + } /** * Useful for deleting a given workflow to a 'Submitted' state when it's currently 'On Hold' or 'Submitted' */ - val workflowStoreEntryForWorkflowExecutionUUidAndWorkflowStates = Compiled( - (workflowId: Rep[String], - workflowStateOr1: Rep[String], - workflowStateOr2: Rep[String] - ) => { + val workflowStoreEntryForWorkflowExecutionUUidAndWorkflowStates = Compiled { + (workflowId: Rep[String], workflowStateOr1: Rep[String], workflowStateOr2: Rep[String]) => for { workflowStoreEntry <- workflowStoreEntries if workflowStoreEntry.workflowExecutionUuid === workflowId if workflowStoreEntry.workflowState === workflowStateOr1 || workflowStoreEntry.workflowState === workflowStateOr2 } yield workflowStoreEntry - } - ) + } // Find workflows running on a given Cromwell instance with abort requested: - val findWorkflowsWithAbortRequested = Compiled( - (cromwellId: Rep[String]) => for { + val findWorkflowsWithAbortRequested = Compiled((cromwellId: Rep[String]) => + for { workflowStoreEntry <- workflowStoreEntries if workflowStoreEntry.workflowState === "Aborting" && workflowStoreEntry.cromwellId === cromwellId } yield workflowStoreEntry.workflowExecutionUuid ) // Find workflows running on a given Cromwell instance: - val findWorkflows = Compiled( - (cromwellId: Rep[String]) => for { + val findWorkflows = Compiled((cromwellId: Rep[String]) => + for { workflowStoreEntry <- workflowStoreEntries if workflowStoreEntry.cromwellId === cromwellId } yield workflowStoreEntry.workflowExecutionUuid ) - val checkExists = Compiled( - (workflowId: Rep[String]) => (for { + val checkExists = Compiled((workflowId: Rep[String]) => + for { workflowStoreEntry <- workflowStoreEntries if workflowStoreEntry.workflowExecutionUuid === workflowId - } yield 1) + } yield 1 ) } diff --git a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala index 25380dc41b5..ef51b15b2ff 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala @@ -8,23 +8,27 @@ import scala.concurrent.{ExecutionContext, Future} trait CallCachingSqlDatabase { def addCallCaching(joins: Seq[CallCachingJoin], batchSize: Int)(implicit ec: ExecutionContext): Future[Unit] - def hasMatchingCallCachingEntriesForBaseAggregation(baseAggregationHash: String, callCachePathPrefixes: Option[List[String]]) - (implicit ec: ExecutionContext): Future[Boolean] + def hasMatchingCallCachingEntriesForBaseAggregation(baseAggregationHash: String, + callCachePathPrefixes: Option[List[String]] + )(implicit ec: ExecutionContext): Future[Boolean] - def findCacheHitForAggregation(baseAggregationHash: String, inputFilesAggregationHash: Option[String], callCachePathPrefixes: Option[List[String]], excludedIds: Set[Long]) - (implicit ec: ExecutionContext): Future[Option[Long]] + def findCacheHitForAggregation(baseAggregationHash: String, + inputFilesAggregationHash: Option[String], + callCachePathPrefixes: Option[List[String]], + excludedIds: Set[Long] + )(implicit ec: ExecutionContext): Future[Option[Long]] - def queryResultsForCacheId(callCachingEntryId: Long) - (implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] - - def callCacheJoinForCall(workflowExecutionUuid: String, callFqn: String, index: Int) - (implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] + def queryResultsForCacheId(callCachingEntryId: Long)(implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] - def invalidateCall(callCachingEntryId: Long) - (implicit ec: ExecutionContext): Future[Option[CallCachingEntry]] + def callCacheJoinForCall(workflowExecutionUuid: String, callFqn: String, index: Int)(implicit + ec: ExecutionContext + ): Future[Option[CallCachingJoin]] - def invalidateCallCacheEntryIdsForWorkflowId(workflowExecutionUuid: String) - (implicit ec: ExecutionContext): Future[Unit] + def invalidateCall(callCachingEntryId: Long)(implicit ec: ExecutionContext): Future[Option[CallCachingEntry]] + + def invalidateCallCacheEntryIdsForWorkflowId(workflowExecutionUuid: String)(implicit + ec: ExecutionContext + ): Future[Unit] def callCacheEntryIdsForWorkflowId(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Seq[Long]] } diff --git a/database/sql/src/main/scala/cromwell/database/sql/DockerHashStoreSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/DockerHashStoreSqlDatabase.scala index 8a719185c98..11c09154364 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/DockerHashStoreSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/DockerHashStoreSqlDatabase.scala @@ -11,15 +11,15 @@ trait DockerHashStoreSqlDatabase { * Adds a docker hash entry to the store. * */ - def addDockerHashStoreEntry(dockerHashStoreEntry: DockerHashStoreEntry) - (implicit ec: ExecutionContext): Future[Unit] + def addDockerHashStoreEntry(dockerHashStoreEntry: DockerHashStoreEntry)(implicit ec: ExecutionContext): Future[Unit] /** * Retrieves docker hash entries for a workflow. * */ - def queryDockerHashStoreEntries(workflowExecutionUuid: String) - (implicit ec: ExecutionContext): Future[Seq[DockerHashStoreEntry]] + def queryDockerHashStoreEntries(workflowExecutionUuid: String)(implicit + ec: ExecutionContext + ): Future[Seq[DockerHashStoreEntry]] /** * Deletes docker hash entries related to a workflow, returning the number of rows affected. diff --git a/database/sql/src/main/scala/cromwell/database/sql/EngineSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/EngineSqlDatabase.scala index 1062e18ed25..3e5e16f881c 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/EngineSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/EngineSqlDatabase.scala @@ -1,9 +1,10 @@ package cromwell.database.sql -trait EngineSqlDatabase extends SqlDatabase - with JobKeyValueSqlDatabase - with CallCachingSqlDatabase - with JobStoreSqlDatabase - with WorkflowStoreSqlDatabase - with SubWorkflowStoreSqlDatabase - with DockerHashStoreSqlDatabase +trait EngineSqlDatabase + extends SqlDatabase + with JobKeyValueSqlDatabase + with CallCachingSqlDatabase + with JobStoreSqlDatabase + with WorkflowStoreSqlDatabase + with SubWorkflowStoreSqlDatabase + with DockerHashStoreSqlDatabase diff --git a/database/sql/src/main/scala/cromwell/database/sql/JobKeyValueSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/JobKeyValueSqlDatabase.scala index 372c4907df0..efda8010eb0 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/JobKeyValueSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/JobKeyValueSqlDatabase.scala @@ -9,16 +9,18 @@ trait JobKeyValueSqlDatabase { def existsJobKeyValueEntries()(implicit ec: ExecutionContext): Future[Boolean] - def addJobKeyValueEntry(jobKeyValueEntry: JobKeyValueEntry) - (implicit ec: ExecutionContext): Future[Unit] + def addJobKeyValueEntry(jobKeyValueEntry: JobKeyValueEntry)(implicit ec: ExecutionContext): Future[Unit] - def addJobKeyValueEntries(jobKeyValueEntries: Iterable[JobKeyValueEntry]) - (implicit ec: ExecutionContext): Future[Unit] + def addJobKeyValueEntries(jobKeyValueEntries: Iterable[JobKeyValueEntry])(implicit ec: ExecutionContext): Future[Unit] - def queryJobKeyValueEntries(workflowExecutionUuid: String) - (implicit ec: ExecutionContext): Future[Seq[JobKeyValueEntry]] + def queryJobKeyValueEntries(workflowExecutionUuid: String)(implicit + ec: ExecutionContext + ): Future[Seq[JobKeyValueEntry]] - def queryStoreValue(workflowExecutionUuid: String, callFqn: String, jobScatterIndex: Int, - jobRetryAttempt: Int, storeKey: String) - (implicit ec: ExecutionContext): Future[Option[String]] + def queryStoreValue(workflowExecutionUuid: String, + callFqn: String, + jobScatterIndex: Int, + jobRetryAttempt: Int, + storeKey: String + )(implicit ec: ExecutionContext): Future[Option[String]] } diff --git a/database/sql/src/main/scala/cromwell/database/sql/JobStoreSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/JobStoreSqlDatabase.scala index a5497ae49c2..769eac0a3b2 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/JobStoreSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/JobStoreSqlDatabase.scala @@ -9,8 +9,9 @@ trait JobStoreSqlDatabase { def addJobStores(jobStoreJoins: Seq[JobStoreJoin], batchSize: Int)(implicit ec: ExecutionContext): Future[Unit] - def queryJobStores(workflowExecutionUuid: String, callFqn: String, jobScatterIndex: Int, jobScatterAttempt: Int) - (implicit ec: ExecutionContext): Future[Option[JobStoreJoin]] + def queryJobStores(workflowExecutionUuid: String, callFqn: String, jobScatterIndex: Int, jobScatterAttempt: Int)( + implicit ec: ExecutionContext + ): Future[Option[JobStoreJoin]] def removeJobStores(workflowExecutionUuids: Seq[String])(implicit ec: ExecutionContext): Future[Seq[Int]] } diff --git a/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala index 9139c819999..5746af4f501 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala @@ -34,56 +34,55 @@ trait MetadataSqlDatabase extends SqlDatabase { submissionMetadataKey: String, parentWorkflowIdKey: String, rootWorkflowIdKey: String, - labelMetadataKey: String)(implicit ec: ExecutionContext): Future[Unit] + labelMetadataKey: String + )(implicit ec: ExecutionContext): Future[Unit] def metadataEntryExists(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Boolean] def metadataSummaryEntryExists(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Boolean] - def queryMetadataEntries(workflowExecutionUuid: String, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] + def queryMetadataEntries(workflowExecutionUuid: String, timeout: Duration)(implicit + ec: ExecutionContext + ): Future[Seq[MetadataEntry]] def streamMetadataEntries(workflowExecutionUuid: String): DatabasePublisher[MetadataEntry] - def countMetadataEntries(workflowExecutionUuid: String, - expandSubWorkflows: Boolean, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Int] + def countMetadataEntries(workflowExecutionUuid: String, expandSubWorkflows: Boolean, timeout: Duration)(implicit + ec: ExecutionContext + ): Future[Int] - def queryMetadataEntries(workflowExecutionUuid: String, - metadataKey: String, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] + def queryMetadataEntries(workflowExecutionUuid: String, metadataKey: String, timeout: Duration)(implicit + ec: ExecutionContext + ): Future[Seq[MetadataEntry]] def countMetadataEntries(workflowExecutionUuid: String, metadataKey: String, expandSubWorkflows: Boolean, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Int] + timeout: Duration + )(implicit ec: ExecutionContext): Future[Int] def queryMetadataEntries(workflowExecutionUuid: String, callFullyQualifiedName: String, jobIndex: Option[Int], jobAttempt: Option[Int], - timeout: Duration) - (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] + timeout: Duration + )(implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] def countMetadataEntries(workflowExecutionUuid: String, callFullyQualifiedName: String, jobIndex: Option[Int], jobAttempt: Option[Int], expandSubWorkflows: Boolean, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Int] + timeout: Duration + )(implicit ec: ExecutionContext): Future[Int] def queryMetadataEntries(workflowUuid: String, metadataKey: String, callFullyQualifiedName: String, jobIndex: Option[Int], jobAttempt: Option[Int], - timeout: Duration) - (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] + timeout: Duration + )(implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] def countMetadataEntries(workflowUuid: String, metadataKey: String, @@ -91,23 +90,23 @@ trait MetadataSqlDatabase extends SqlDatabase { jobIndex: Option[Int], jobAttempt: Option[Int], expandSubWorkflows: Boolean, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Int] + timeout: Duration + )(implicit ec: ExecutionContext): Future[Int] def queryMetadataEntryWithKeyConstraints(workflowExecutionUuid: String, metadataKeysToFilterFor: List[String], metadataKeysToFilterAgainst: List[String], metadataJobQueryValue: MetadataJobQueryValue, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] + timeout: Duration + )(implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] def countMetadataEntryWithKeyConstraints(workflowExecutionUuid: String, metadataKeysToFilterFor: List[String], metadataKeysToFilterAgainst: List[String], metadataJobQueryValue: MetadataJobQueryValue, expandSubWorkflows: Boolean, - timeout: Duration) - (implicit ec: ExecutionContext): Future[Int] + timeout: Duration + )(implicit ec: ExecutionContext): Future[Int] /** * Retrieves next summarizable block of metadata satisfying the specified criteria. @@ -115,12 +114,11 @@ trait MetadataSqlDatabase extends SqlDatabase { * @param buildUpdatedSummary Takes in the optional existing summary and the metadata, returns the new summary. * @return A `Future` with the number of rows summarized by the invocation, and the number of rows still to summarize. */ - def summarizeIncreasing(labelMetadataKey: String, - limit: Int, - buildUpdatedSummary: - (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry]) - => WorkflowMetadataSummaryEntry) - (implicit ec: ExecutionContext): Future[Long] + def summarizeIncreasing( + labelMetadataKey: String, + limit: Int, + buildUpdatedSummary: (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry]) => WorkflowMetadataSummaryEntry + )(implicit ec: ExecutionContext): Future[Long] /** * Retrieves a window of summarizable metadata satisfying the specified criteria. @@ -128,14 +126,13 @@ trait MetadataSqlDatabase extends SqlDatabase { * @param buildUpdatedSummary Takes in the optional existing summary and the metadata, returns the new summary. * @return A `Future` with the number of rows summarized by this invocation, and the number of rows still to summarize. */ - def summarizeDecreasing(summaryNameDecreasing: String, - summaryNameIncreasing: String, - labelMetadataKey: String, - limit: Int, - buildUpdatedSummary: - (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry]) - => WorkflowMetadataSummaryEntry) - (implicit ec: ExecutionContext): Future[(Long, Long)] + def summarizeDecreasing( + summaryNameDecreasing: String, + summaryNameIncreasing: String, + labelMetadataKey: String, + limit: Int, + buildUpdatedSummary: (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry]) => WorkflowMetadataSummaryEntry + )(implicit ec: ExecutionContext): Future[(Long, Long)] def updateMetadataArchiveStatus(workflowExecutionUuid: String, newArchiveStatus: Option[String]): Future[Int] @@ -143,16 +140,18 @@ trait MetadataSqlDatabase extends SqlDatabase { def getWorkflowLabels(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Map[String, String]] - def getRootAndSubworkflowLabels(rootWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Map[String, Map[String, String]]] + def getRootAndSubworkflowLabels(rootWorkflowExecutionUuid: String)(implicit + ec: ExecutionContext + ): Future[Map[String, Map[String, String]]] def queryWorkflowSummaries(parentIdWorkflowMetadataKey: String, workflowStatuses: Set[String], workflowNames: Set[String], workflowExecutionUuids: Set[String], - labelAndKeyLabelValues: Set[(String,String)], - labelOrKeyLabelValues: Set[(String,String)], - excludeLabelAndValues: Set[(String,String)], - excludeLabelOrValues: Set[(String,String)], + labelAndKeyLabelValues: Set[(String, String)], + labelOrKeyLabelValues: Set[(String, String)], + excludeLabelAndValues: Set[(String, String)], + excludeLabelOrValues: Set[(String, String)], submissionTimestamp: Option[Timestamp], startTimestampOption: Option[Timestamp], endTimestampOption: Option[Timestamp], @@ -160,43 +159,57 @@ trait MetadataSqlDatabase extends SqlDatabase { includeSubworkflows: Boolean, page: Option[Int], pageSize: Option[Int], - newestFirst: Boolean) - (implicit ec: ExecutionContext): Future[Iterable[WorkflowMetadataSummaryEntry]] + newestFirst: Boolean + )(implicit ec: ExecutionContext): Future[Iterable[WorkflowMetadataSummaryEntry]] def countWorkflowSummaries(parentIdWorkflowMetadataKey: String, - workflowStatuses: Set[String], workflowNames: Set[String], + workflowStatuses: Set[String], + workflowNames: Set[String], workflowExecutionUuids: Set[String], - labelAndKeyLabelValues: Set[(String,String)], - labelOrKeyLabelValues: Set[(String,String)], - excludeLabelAndValues: Set[(String,String)], - excludeLabelOrValues: Set[(String,String)], + labelAndKeyLabelValues: Set[(String, String)], + labelOrKeyLabelValues: Set[(String, String)], + excludeLabelAndValues: Set[(String, String)], + excludeLabelOrValues: Set[(String, String)], submissionTimestamp: Option[Timestamp], startTimestampOption: Option[Timestamp], endTimestampOption: Option[Timestamp], metadataArchiveStatus: Set[Option[String]], - includeSubworkflows: Boolean) - (implicit ec: ExecutionContext): Future[Int] + includeSubworkflows: Boolean + )(implicit ec: ExecutionContext): Future[Int] - def deleteAllMetadataForWorkflowAndUpdateArchiveStatus(rootWorkflowId: String, newArchiveStatus: Option[String])(implicit ec: ExecutionContext): Future[Int] + def deleteAllMetadataForWorkflowAndUpdateArchiveStatus(rootWorkflowId: String, newArchiveStatus: Option[String])( + implicit ec: ExecutionContext + ): Future[Int] def getRootWorkflowId(workflowId: String)(implicit ec: ExecutionContext): Future[Option[String]] - def queryWorkflowIdsByArchiveStatusAndEndedOnOrBeforeThresholdTimestamp(archiveStatus: Option[String], thresholdTimestamp: Timestamp, batchSizeOpt: Long)(implicit ec: ExecutionContext): Future[Seq[String]] + def queryWorkflowIdsByArchiveStatusAndEndedOnOrBeforeThresholdTimestamp(archiveStatus: Option[String], + thresholdTimestamp: Timestamp, + batchSizeOpt: Long + )(implicit ec: ExecutionContext): Future[Seq[String]] def getSummaryQueueSize()(implicit ec: ExecutionContext): Future[Int] - def getMetadataArchiveStatusAndEndTime(workflowId: String)(implicit ec: ExecutionContext): Future[(Option[String], Option[Timestamp])] + def getMetadataArchiveStatusAndEndTime(workflowId: String)(implicit + ec: ExecutionContext + ): Future[(Option[String], Option[Timestamp])] def queryWorkflowsToArchiveThatEndedOnOrBeforeThresholdTimestamp(workflowStatuses: List[String], workflowEndTimestampThreshold: Timestamp, - batchSize: Long)(implicit ec: ExecutionContext): Future[Seq[WorkflowMetadataSummaryEntry]] + batchSize: Long + )(implicit ec: ExecutionContext): Future[Seq[WorkflowMetadataSummaryEntry]] def countWorkflowsLeftToArchiveThatEndedOnOrBeforeThresholdTimestamp(workflowStatuses: List[String], - workflowEndTimestampThreshold: Timestamp)(implicit ec: ExecutionContext): Future[Int] + workflowEndTimestampThreshold: Timestamp + )(implicit ec: ExecutionContext): Future[Int] - def countWorkflowsLeftToDeleteThatEndedOnOrBeforeThresholdTimestamp(workflowEndTimestampThreshold: Timestamp)(implicit ec: ExecutionContext): Future[Int] + def countWorkflowsLeftToDeleteThatEndedOnOrBeforeThresholdTimestamp(workflowEndTimestampThreshold: Timestamp)(implicit + ec: ExecutionContext + ): Future[Int] def getMetadataTableSizeInformation()(implicit ec: ExecutionContext): Future[Option[InformationSchemaEntry]] - def getFailedJobsMetadataWithWorkflowId(rootWorkflowId: String)(implicit ec: ExecutionContext): Future[Vector[MetadataEntry]] + def getFailedJobsMetadataWithWorkflowId(rootWorkflowId: String)(implicit + ec: ExecutionContext + ): Future[Vector[MetadataEntry]] } diff --git a/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala index 842dcd775d4..64a4f2dd2a3 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala @@ -16,6 +16,7 @@ trait SqlDatabase extends AutoCloseable { } object SqlDatabase { + /** * Modifies config.getString("url") to return a unique schema, if the original url contains the text * "\${uniqueSchema}". diff --git a/database/sql/src/main/scala/cromwell/database/sql/SubWorkflowStoreSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/SubWorkflowStoreSqlDatabase.scala index 10707dc9031..e6430730497 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/SubWorkflowStoreSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/SubWorkflowStoreSqlDatabase.scala @@ -12,10 +12,12 @@ trait SubWorkflowStoreSqlDatabase { callFullyQualifiedName: String, jobIndex: Int, jobAttempt: Int, - subWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Unit] + subWorkflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Unit] - def querySubWorkflowStore(parentWorkflowExecutionUuid: String, callFqn: String, jobIndex: Int, jobAttempt: Int) - (implicit ec: ExecutionContext): Future[Option[SubWorkflowStoreEntry]] + def querySubWorkflowStore(parentWorkflowExecutionUuid: String, callFqn: String, jobIndex: Int, jobAttempt: Int)( + implicit ec: ExecutionContext + ): Future[Option[SubWorkflowStoreEntry]] def removeSubWorkflowStoreEntries(parentWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Int] } diff --git a/database/sql/src/main/scala/cromwell/database/sql/WorkflowStoreSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/WorkflowStoreSqlDatabase.scala index 82dc51c875b..66dc97be3da 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/WorkflowStoreSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/WorkflowStoreSqlDatabase.scala @@ -29,8 +29,7 @@ ____ __ ____ ______ .______ __ ___ _______ __ ______ /** * Set all running workflows to aborting state. */ - def setStateToState(fromWorkflowState: String, toWorkflowState: String) - (implicit ec: ExecutionContext): Future[Unit] + def setStateToState(fromWorkflowState: String, toWorkflowState: String)(implicit ec: ExecutionContext): Future[Unit] /** * Set the workflow Id from one state to another. @@ -45,14 +44,15 @@ ____ __ ____ ______ .______ __ ___ _______ __ ______ def deleteOrUpdateWorkflowToState(workflowExecutionUuid: String, workflowStateToDelete1: String, workflowStateToDelete2: String, - workflowStateForUpdate: String) - (implicit ec: ExecutionContext): Future[Option[Boolean]] + workflowStateForUpdate: String + )(implicit ec: ExecutionContext): Future[Option[Boolean]] /** * Adds the requested WorkflowSourceFiles to the store. */ - def addWorkflowStoreEntries(workflowStoreEntries: Iterable[WorkflowStoreEntry]) - (implicit ec: ExecutionContext): Future[Unit] + def addWorkflowStoreEntries(workflowStoreEntries: Iterable[WorkflowStoreEntry])(implicit + ec: ExecutionContext + ): Future[Unit] /** * Retrieves a limited number of workflows which have not already been pulled into the engine and updates their @@ -75,12 +75,12 @@ ____ __ ____ ______ .______ __ ___ _______ __ ______ workflowStateFrom: String, workflowStateTo: String, workflowStateExcluded: String, - excludedGroups: Set[String]) - (implicit ec: ExecutionContext): Future[Seq[WorkflowStoreEntry]] + excludedGroups: Set[String] + )(implicit ec: ExecutionContext): Future[Seq[WorkflowStoreEntry]] - def writeWorkflowHeartbeats(workflowExecutionUuids: Seq[String], - heartbeatTimestamp: Timestamp) - (implicit ec: ExecutionContext): Future[Int] + def writeWorkflowHeartbeats(workflowExecutionUuids: Seq[String], heartbeatTimestamp: Timestamp)(implicit + ec: ExecutionContext + ): Future[Int] /** * Clears out cromwellId and heartbeatTimestamp for all workflow store entries currently assigned @@ -98,8 +98,9 @@ ____ __ ____ ______ .______ __ ___ _______ __ ______ /** * Returns the number of rows updated from one state to another. */ - def updateWorkflowState(workflowExecutionUuid: String, fromWorkflowState: String, toWorkflowState: String) - (implicit ec: ExecutionContext): Future[Int] + def updateWorkflowState(workflowExecutionUuid: String, fromWorkflowState: String, toWorkflowState: String)(implicit + ec: ExecutionContext + ): Future[Int] def findWorkflowsWithAbortRequested(cromwellId: String)(implicit ec: ExecutionContext): Future[Iterable[String]] diff --git a/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingDiffJoin.scala b/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingDiffJoin.scala index 2b56e9f88d5..d45e7a72cf3 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingDiffJoin.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingDiffJoin.scala @@ -2,4 +2,7 @@ package cromwell.database.sql.joins import cromwell.database.sql.tables.CallCachingEntry -case class CallCachingDiffJoin(cacheEntryA: CallCachingEntry, cacheEntryB: CallCachingEntry, diff: Seq[(Option[(String, String)], Option[(String, String)])]) +case class CallCachingDiffJoin(cacheEntryA: CallCachingEntry, + cacheEntryB: CallCachingEntry, + diff: Seq[(Option[(String, String)], Option[(String, String)])] +) diff --git a/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingJoin.scala b/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingJoin.scala index 07313980bc1..a9d32ee9115 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingJoin.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingJoin.scala @@ -2,8 +2,7 @@ package cromwell.database.sql.joins import cromwell.database.sql.tables._ -case class CallCachingJoin -( +case class CallCachingJoin( callCachingEntry: CallCachingEntry, callCachingHashEntries: Seq[CallCachingHashEntry], callCachingAggregationEntry: Option[CallCachingAggregationEntry], diff --git a/database/sql/src/main/scala/cromwell/database/sql/joins/JobStoreJoin.scala b/database/sql/src/main/scala/cromwell/database/sql/joins/JobStoreJoin.scala index 3a2c19a0b81..d47ffd8f615 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/joins/JobStoreJoin.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/joins/JobStoreJoin.scala @@ -2,8 +2,7 @@ package cromwell.database.sql.joins import cromwell.database.sql.tables.{JobStoreEntry, JobStoreSimpletonEntry} -case class JobStoreJoin -( +case class JobStoreJoin( jobStoreEntry: JobStoreEntry, jobStoreSimpletonEntries: Seq[JobStoreSimpletonEntry] ) diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingAggregationEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingAggregationEntry.scala index 088c07bf3c7..c1f87bd1b02 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingAggregationEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingAggregationEntry.scala @@ -1,7 +1,6 @@ package cromwell.database.sql.tables -case class CallCachingAggregationEntry -( +case class CallCachingAggregationEntry( baseAggregation: String, inputFilesAggregation: Option[String], callCachingEntryId: Option[Long] = None, diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala index 36afadd4c8e..3462ad527cc 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala @@ -2,8 +2,7 @@ package cromwell.database.sql.tables import javax.sql.rowset.serial.SerialClob -case class CallCachingDetritusEntry -( +case class CallCachingDetritusEntry( detritusKey: String, detritusValue: Option[SerialClob], callCachingEntryId: Option[Long] = None, diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingEntry.scala index 24263b0cb80..ba748261e82 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingEntry.scala @@ -1,7 +1,6 @@ package cromwell.database.sql.tables -case class CallCachingEntry -( +case class CallCachingEntry( workflowExecutionUuid: String, callFullyQualifiedName: String, jobIndex: Int, diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingHashEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingHashEntry.scala index 5f2aff9fb14..580c5e5bb3f 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingHashEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingHashEntry.scala @@ -1,7 +1,6 @@ package cromwell.database.sql.tables -case class CallCachingHashEntry -( +case class CallCachingHashEntry( hashKey: String, hashValue: String, callCachingEntryId: Option[Long] = None, diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala index c4e6628ee81..36a39f7fd5f 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala @@ -2,8 +2,7 @@ package cromwell.database.sql.tables import javax.sql.rowset.serial.SerialClob -case class CallCachingSimpletonEntry -( +case class CallCachingSimpletonEntry( simpletonKey: String, simpletonValue: Option[SerialClob], wdlType: String, diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CustomLabelEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CustomLabelEntry.scala index 030a5e0bb25..19c959be929 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/CustomLabelEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CustomLabelEntry.scala @@ -1,7 +1,6 @@ package cromwell.database.sql.tables -case class CustomLabelEntry -( +case class CustomLabelEntry( customLabelKey: String, customLabelValue: String, workflowExecutionUuid: String, diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/DockerHashStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/DockerHashStoreEntry.scala index 542558324f5..05fc6c4e7db 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/DockerHashStoreEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/DockerHashStoreEntry.scala @@ -1,7 +1,6 @@ package cromwell.database.sql.tables -case class DockerHashStoreEntry -( +case class DockerHashStoreEntry( workflowExecutionUuid: String, dockerTag: String, dockerHash: String, diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/InformationSchemaEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/InformationSchemaEntry.scala index 0cf05e0b5b5..b29bba61205 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/InformationSchemaEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/InformationSchemaEntry.scala @@ -1,7 +1,6 @@ package cromwell.database.sql.tables -final case class InformationSchemaEntry -( +final case class InformationSchemaEntry( dataLength: Long, indexLength: Long, dataFree: Long diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/JobKeyValueEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/JobKeyValueEntry.scala index 5f91bee2091..8b768ff66de 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/JobKeyValueEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/JobKeyValueEntry.scala @@ -1,7 +1,6 @@ package cromwell.database.sql.tables -case class JobKeyValueEntry -( +case class JobKeyValueEntry( workflowExecutionUuid: String, callFullyQualifiedName: String, jobIndex: Int, diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreEntry.scala index 6f59a0c738c..a7be44561c4 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreEntry.scala @@ -2,8 +2,7 @@ package cromwell.database.sql.tables import javax.sql.rowset.serial.SerialClob -case class JobStoreEntry -( +case class JobStoreEntry( workflowExecutionUuid: String, callFullyQualifiedName: String, jobIndex: Int, diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala index 4a6c19864dd..b0c3463625f 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala @@ -2,8 +2,7 @@ package cromwell.database.sql.tables import javax.sql.rowset.serial.SerialClob -case class JobStoreSimpletonEntry -( +case class JobStoreSimpletonEntry( simpletonKey: String, simpletonValue: Option[SerialClob], wdlType: String, diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/MetadataEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/MetadataEntry.scala index c273c3e47e3..97c4cc9acf2 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/MetadataEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/MetadataEntry.scala @@ -4,8 +4,7 @@ import java.sql.Timestamp import javax.sql.rowset.serial.SerialClob -case class MetadataEntry -( +case class MetadataEntry( workflowExecutionUuid: String, callFullyQualifiedName: Option[String], jobIndex: Option[Int], diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/SubWorkflowStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/SubWorkflowStoreEntry.scala index 4cf89381ad0..6d081845a84 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/SubWorkflowStoreEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/SubWorkflowStoreEntry.scala @@ -1,7 +1,6 @@ package cromwell.database.sql.tables -case class SubWorkflowStoreEntry -( +case class SubWorkflowStoreEntry( rootWorkflowId: Option[Long], parentWorkflowExecutionUuid: String, callFullyQualifiedName: String, diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/SummaryStatusEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/SummaryStatusEntry.scala index 6315295d478..8dfdeea1626 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/SummaryStatusEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/SummaryStatusEntry.scala @@ -1,7 +1,6 @@ package cromwell.database.sql.tables -case class SummaryStatusEntry -( +case class SummaryStatusEntry( summaryName: String, summaryPosition: Long, summaryStatusEntryId: Option[Int] = None diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowMetadataSummaryEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowMetadataSummaryEntry.scala index 19e553630e6..bd98d398d75 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowMetadataSummaryEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowMetadataSummaryEntry.scala @@ -2,8 +2,7 @@ package cromwell.database.sql.tables import java.sql.Timestamp -case class WorkflowMetadataSummaryEntry -( +case class WorkflowMetadataSummaryEntry( workflowExecutionUuid: String, workflowName: Option[String], workflowStatus: Option[String], diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala index efb594444d4..6017bb5d539 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala @@ -4,8 +4,7 @@ import java.sql.Timestamp import javax.sql.rowset.serial.{SerialBlob, SerialClob} -case class WorkflowStoreEntry -( +case class WorkflowStoreEntry( workflowExecutionUuid: String, workflowDefinition: Option[SerialClob], workflowUrl: Option[String], diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerClientHelper.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerClientHelper.scala index 706cbfef3d8..e034319033a 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/DockerClientHelper.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerClientHelper.scala @@ -7,10 +7,10 @@ import cromwell.docker.DockerInfoActor.DockerInfoResponse import scala.concurrent.duration.FiniteDuration trait DockerClientHelper extends RobustClientHelper { this: Actor with ActorLogging => - + protected def dockerHashingActor: ActorRef - - private [docker] def dockerResponseReceive: Receive = { + + private[docker] def dockerResponseReceive: Receive = { case dockerResponse: DockerInfoResponse if hasTimeout(dockerResponse.request) => cancelTimeout(dockerResponse.request) receive.apply(dockerResponse) @@ -19,9 +19,10 @@ trait DockerClientHelper extends RobustClientHelper { this: Actor with ActorLogg receive.apply(context -> dockerResponse) } - def sendDockerCommand(dockerHashRequest: DockerInfoRequest, timeout: FiniteDuration = RobustClientHelper.DefaultRequestLostTimeout) = { + def sendDockerCommand(dockerHashRequest: DockerInfoRequest, + timeout: FiniteDuration = RobustClientHelper.DefaultRequestLostTimeout + ) = robustSend(dockerHashRequest, dockerHashingActor, timeout) - } def dockerReceive = robustReceive orElse dockerResponseReceive } diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerHashResult.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerHashResult.scala index e3120516a9a..771db6d6576 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/DockerHashResult.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerHashResult.scala @@ -5,13 +5,13 @@ import scala.util.{Failure, Success, Try} object DockerHashResult { // See https://docs.docker.com/registry/spec/api/#/content-digests val DigestRegex = """([a-zA-Z0-9_+.-]+):([a-zA-Z0-9]+)""".r - - def fromString(str: String): Try[DockerHashResult] = { + + def fromString(str: String): Try[DockerHashResult] = str match { case DigestRegex(alg, hash) => Success(DockerHashResult(alg, hash)) - case _ => Failure(new IllegalArgumentException(s"Hash value $str does not have the expected 'algorithm:hash' syntax")) + case _ => + Failure(new IllegalArgumentException(s"Hash value $str does not have the expected 'algorithm:hash' syntax")) } - } } case class DockerHashResult(hashAlgorithm: String, hashValue: String) { diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala index a798f351f17..7c20760b644 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala @@ -16,25 +16,33 @@ sealed trait DockerImageIdentifier { lazy val name = repository map { r => s"$r/$image" } getOrElse image // The name of the image with a repository prefix if a repository was specified, or with a default repository prefix of // "library" if no repository was specified. - lazy val nameWithDefaultRepository = { + lazy val nameWithDefaultRepository = // In ACR, the repository is part of the registry domain instead of the path // e.g. `terrabatchdev.azurecr.io` if (host.exists(_.contains(AzureContainerRegistry.domain))) image else repository.getOrElse("library") + s"/$image" - } lazy val hostAsString = host map { h => s"$h/" } getOrElse "" // The full name of this image, including a repository prefix only if a repository was explicitly specified. lazy val fullName = s"$hostAsString$name:$reference" } -case class DockerImageIdentifierWithoutHash(host: Option[String], repository: Option[String], image: String, reference: String) extends DockerImageIdentifier { +case class DockerImageIdentifierWithoutHash(host: Option[String], + repository: Option[String], + image: String, + reference: String +) extends DockerImageIdentifier { def withHash(hash: DockerHashResult) = DockerImageIdentifierWithHash(host, repository, image, reference, hash) override def swapReference(newReference: String) = this.copy(reference = newReference) } -case class DockerImageIdentifierWithHash(host: Option[String], repository: Option[String], image: String, reference: String, hash: DockerHashResult) extends DockerImageIdentifier { +case class DockerImageIdentifierWithHash(host: Option[String], + repository: Option[String], + image: String, + reference: String, + hash: DockerHashResult +) extends DockerImageIdentifier { override lazy val fullName: String = s"$hostAsString$name@${hash.algorithmAndHash}" override def swapReference(newReference: String) = this.copy(reference = newReference) } @@ -48,7 +56,7 @@ object DockerImageIdentifier { ( # Begin capturing group for name [a-z0-9]+(?:[._-][a-z0-9]+)* # API v2 name component regex - see https://docs.docker.com/registry/spec/api/#/overview - (?::[0-9]+)? # Optional port + (?::[0-9]{4,5}|:443)? # Optional port (expect 4 or 5 digits OR :443) (?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)* # Optional additional name components separated by / ) # End capturing group for name @@ -68,12 +76,11 @@ object DockerImageIdentifier { )? """.trim.r - def fromString(dockerString: String): Try[DockerImageIdentifier] = { + def fromString(dockerString: String): Try[DockerImageIdentifier] = dockerString.trim match { case DockerStringRegex(name, tag, hash) => buildId(name, Option(tag), Option(hash)) case _ => Failure(new IllegalArgumentException(s"Docker image $dockerString has an invalid syntax.")) } - } private def isRegistryHostName(str: String) = str.contains('.') || str.startsWith("localhost") @@ -97,10 +104,17 @@ object DockerImageIdentifier { } (tag, hash) match { - case (None, None) => Success(DockerImageIdentifierWithoutHash(dockerHost, dockerRepo, dockerImage, DefaultDockerTag)) + case (None, None) => + Success(DockerImageIdentifierWithoutHash(dockerHost, dockerRepo, dockerImage, DefaultDockerTag)) case (Some(t), None) => Success(DockerImageIdentifierWithoutHash(dockerHost, dockerRepo, dockerImage, t)) - case (None, Some(h)) => DockerHashResult.fromString(h) map { hash => DockerImageIdentifierWithHash(dockerHost, dockerRepo, dockerImage, h, hash) } - case (Some(t), Some(h)) => DockerHashResult.fromString(h) map { hash => DockerImageIdentifierWithHash(dockerHost, dockerRepo, dockerImage, s"$t@$h", hash) } + case (None, Some(h)) => + DockerHashResult.fromString(h) map { hash => + DockerImageIdentifierWithHash(dockerHost, dockerRepo, dockerImage, h, hash) + } + case (Some(t), Some(h)) => + DockerHashResult.fromString(h) map { hash => + DockerImageIdentifierWithHash(dockerHost, dockerRepo, dockerImage, s"$t@$h", hash) + } } } } diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala index 3ebb8d98f39..3b3c0e5d7c5 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala @@ -30,16 +30,21 @@ import scala.concurrent.ExecutionContextExecutor import scala.concurrent.duration._ final class DockerInfoActor( - dockerRegistryFlows: Seq[DockerRegistry], - queueBufferSize: Int, - cacheEntryTTL: FiniteDuration, - cacheSize: Long - ) extends Actor with ActorLogging { + dockerRegistryFlows: Seq[DockerRegistry], + queueBufferSize: Int, + cacheEntryTTL: FiniteDuration, + cacheSize: Long +) extends Actor + with ActorLogging { implicit val system: ActorSystem = context.system implicit val ec: ExecutionContextExecutor = context.dispatcher implicit val cs: ContextShift[IO] = IO.contextShift(ec) - val retryPolicy: RetryPolicy[IO] = RetryPolicy[IO](RetryPolicy.exponentialBackoff(DockerConfiguration.instance.maxTimeBetweenRetries, DockerConfiguration.instance.maxRetries)) + val retryPolicy: RetryPolicy[IO] = RetryPolicy[IO]( + RetryPolicy.exponentialBackoff(DockerConfiguration.instance.maxTimeBetweenRetries, + DockerConfiguration.instance.maxRetries + ) + ) /* Use the guava CacheBuilder class that implements a thread safe map with built in cache features. * https://google.github.io/guava/releases/20.0/api/docs/com/google/common/cache/CacheBuilder.html @@ -47,29 +52,29 @@ final class DockerInfoActor( * - Added to the cache by the streamSink, running on a thread from the stream thread pool * - Accessed by this actor on its receive method thread, to check if an element is in the cache and use it * - Automatically removed from the cache by the cache itself - * + * * + Concurrency level is the number of expected threads to modify the cache. * Set to "2" because the stream will add elements, and the cache itself remove them. - * This value has not a critical impact: + * This value has not a critical impact: * https://google.github.io/guava/releases/20.0/api/docs/com/google/common/cache/CacheBuilder.html#concurrencyLevel-int- - * - * + expireAfterWrite sets the time after which cache entries must expire. + * + * + expireAfterWrite sets the time after which cache entries must expire. * We use expireAfterWrite (as opposed to expireAfterAccess because we want to the entry to expire - * even if it's accessed. The goal here is to force the actor to ask again for the hash after a certain + * even if it's accessed. The goal here is to force the actor to ask again for the hash after a certain * amount of time to guarantee its relative accuracy. - * - * + maximumSize sets the maximum amount of entries the cache can contain. + * + * + maximumSize sets the maximum amount of entries the cache can contain. * If/when this size is reached, least used entries will be expired */ - private val cache = CacheBuilder.newBuilder() + private val cache = CacheBuilder + .newBuilder() .concurrencyLevel(2) .expireAfterWrite(cacheEntryTTL._1, cacheEntryTTL._2) .maximumSize(cacheSize) .build[DockerImageIdentifier, DockerInformation]() - private def checkCache(dockerHashRequest: DockerInfoRequest) = { + private def checkCache(dockerHashRequest: DockerInfoRequest) = Option(cache.getIfPresent(dockerHashRequest.dockerImageID)) - } override def receive: Receive = receiveBehavior(Map.empty) @@ -83,26 +88,24 @@ final class DockerInfoActor( } case ShutdownCommand => // Shutdown all streams by sending None to the queue - registries - .values.toList + registries.values.toList .parTraverse[IO, Unit](_.enqueue1(None)) .unsafeRunSync() context stop self } - def sendToStream(registries: Map[DockerRegistry, StreamQueue], context: DockerInfoActor.DockerInfoContext): Unit = { + def sendToStream(registries: Map[DockerRegistry, StreamQueue], context: DockerInfoActor.DockerInfoContext): Unit = registries collectFirst { case (registry, queue) if registry.accepts(context.dockerImageID) => queue } match { case Some(queue) => enqueue(context, queue) case None => context.replyTo ! DockerHashUnknownRegistry(context.request) } - } def enqueue(dockerInfoContext: DockerInfoContext, queue: StreamQueue): Unit = { val enqueueIO = queue.offer1(Option(dockerInfoContext)).runAsync { - case Right(true) => IO.unit// Good ! + case Right(true) => IO.unit // Good ! case _ => backpressure(dockerInfoContext) } @@ -148,13 +151,13 @@ final class DockerInfoActor( // If the registry imposes throttling, debounce the stream to ensure the throttling rate is respected val throttledSource = registry.config.throttle.map(_.delay).map(source.metered[IO]).getOrElse(source) - val stream = clientStream.flatMap({ client => + val stream = clientStream.flatMap { client => throttledSource // Run requests in parallel - allow nbThreads max concurrent requests - order doesn't matter - .parEvalMapUnordered(registry.config.nbThreads)({ request => registry.run(request)(client) }) + .parEvalMapUnordered(registry.config.nbThreads)(request => registry.run(request)(client)) // Send to the sink for finalization of the result .through(streamSink) - }) + } // Start the stream now asynchronously. It will keep running until we terminate the queue by sending None to it stream.compile.drain.unsafeRunAsyncAndForget() @@ -166,7 +169,7 @@ final class DockerInfoActor( override def preStart(): Unit = { // Force initialization of the header constants to make sure they're valid locally(DockerRegistryV2Abstract) - + val registries = dockerRegistryFlows.toList .parTraverse(startAndRegisterStream) @@ -194,12 +197,14 @@ object DockerInfoActor { } case class DockerInformation(dockerHash: DockerHashResult, dockerCompressedSize: Option[DockerSize]) - case class DockerInfoSuccessResponse(dockerInformation: DockerInformation, request: DockerInfoRequest) extends DockerInfoResponse + case class DockerInfoSuccessResponse(dockerInformation: DockerInformation, request: DockerInfoRequest) + extends DockerInfoResponse sealed trait DockerHashFailureResponse extends DockerInfoResponse { def reason: String } - case class DockerInfoFailedResponse(failure: Throwable, request: DockerInfoRequest) extends DockerHashFailureResponse { + case class DockerInfoFailedResponse(failure: Throwable, request: DockerInfoRequest) + extends DockerHashFailureResponse { override val reason = s"Failed to get docker hash for ${request.dockerImageID.fullName} ${failure.getMessage}" } case class DockerHashUnknownRegistry(request: DockerInfoRequest) extends DockerHashFailureResponse { @@ -224,9 +229,10 @@ object DockerInfoActor { def props(dockerRegistryFlows: Seq[DockerRegistry], queueBufferSize: Int = 100, cacheEntryTTL: FiniteDuration, - cacheSize: Long): Props = { - Props(new DockerInfoActor(dockerRegistryFlows, queueBufferSize, cacheEntryTTL, cacheSize)).withDispatcher(Dispatcher.IoDispatcher) - } + cacheSize: Long + ): Props = + Props(new DockerInfoActor(dockerRegistryFlows, queueBufferSize, cacheEntryTTL, cacheSize)) + .withDispatcher(Dispatcher.IoDispatcher) def remoteRegistriesFromConfig(config: Config): List[DockerRegistry] = { import cats.syntax.traverse._ @@ -237,8 +243,8 @@ object DockerInfoActor { ("dockerhub", { c: DockerRegistryConfig => new DockerHubRegistry(c) }), ("google", { c: DockerRegistryConfig => new GoogleRegistry(c) }), ("quay", { c: DockerRegistryConfig => new QuayRegistry(c) }) - ).traverse[ErrorOr, DockerRegistry]({ - case (configPath, constructor) => DockerRegistryConfig.fromConfig(config.as[Config](configPath)).map(constructor) - }).unsafe("Docker registry configuration") + ).traverse[ErrorOr, DockerRegistry] { case (configPath, constructor) => + DockerRegistryConfig.fromConfig(config.as[Config](configPath)).map(constructor) + }.unsafe("Docker registry configuration") } } diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerRegistry.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerRegistry.scala index d35e6e0c91b..fac486d114d 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/DockerRegistry.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerRegistry.scala @@ -8,7 +8,9 @@ import org.http4s.client.Client * Interface used by the docker hash actor to build a flow and validate whether or not it can accept an image. */ trait DockerRegistry { - def run(dockerInfoContext: DockerInfoContext)(implicit client: Client[IO]): IO[(DockerInfoResponse, DockerInfoContext)] + def run(dockerInfoContext: DockerInfoContext)(implicit + client: Client[IO] + ): IO[(DockerInfoResponse, DockerInfoContext)] def accepts(dockerImageIdentifier: DockerImageIdentifier): Boolean def config: DockerRegistryConfig } diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerRegistryConfig.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerRegistryConfig.scala index a6e15846e5e..327d9e8038c 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/DockerRegistryConfig.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerRegistryConfig.scala @@ -19,9 +19,9 @@ object DockerRegistryConfig { lazy val default = DockerRegistryConfig(None, 5) def fromConfig(config: Config): ErrorOr[DockerRegistryConfig] = { - val throttle = validate { config.getAs[Throttle]("throttle") } - val threads = validate { config.as[Int]("num-threads") } - + val throttle = validate(config.getAs[Throttle]("throttle")) + val threads = validate(config.as[Int]("num-threads")) + (throttle, threads) mapN DockerRegistryConfig.apply } } diff --git a/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliClient.scala b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliClient.scala index e0e2a383d92..4f2e2b7e891 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliClient.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliClient.scala @@ -11,13 +11,14 @@ import scala.util.Try * https://docs.docker.com/engine/api/v1.27/ */ trait DockerCliClient { + /** * Looks up a docker hash. * * @param dockerCliKey The docker hash to lookup. * @return The hash if found, None if not found, and Failure if an error occurs. */ - def lookupHash(dockerCliKey: DockerCliKey): Try[Option[String]] = { + def lookupHash(dockerCliKey: DockerCliKey): Try[Option[String]] = /* The stdout contains the tab separated repository/tag/digest for __all__ local images. Would be great to just get a single hash using the key... unfortunately @@ -26,16 +27,14 @@ trait DockerCliClient { forRun("docker", "images", "--digests", "--format", """{{printf "%s\t%s\t%s" .Repository .Tag .Digest}}""") { _.flatMap(parseHashLine).find(_.key == dockerCliKey).map(_.digest) } - } /** * Pulls a docker image. * @param dockerCliKey The docker hash to lookup. * @return Failure if an error occurs. */ - def pull(dockerCliKey: DockerCliKey): Try[Unit] = { - forRun("docker", "pull", dockerCliKey.fullName) { const(()) } - } + def pull(dockerCliKey: DockerCliKey): Try[Unit] = + forRun("docker", "pull", dockerCliKey.fullName)(const(())) /** * Tries to run the command, then feeds the stdout to `f`. If the exit code is non-zero, returns a `Failure` with @@ -46,20 +45,18 @@ trait DockerCliClient { * @tparam A Return type. * @return An attempt to run A. */ - private def forRun[A](cmd: String*)(f: Seq[String] => A): Try[A] = { + private def forRun[A](cmd: String*)(f: Seq[String] => A): Try[A] = Try { val dockerCliResult = run(cmd) if (dockerCliResult.exitCode == 0) { f(dockerCliResult.stdout) } else { - throw new RuntimeException( - s"""|Error running: ${cmd.mkString(" ")} - |Exit code: ${dockerCliResult.exitCode} - |${dockerCliResult.stderr.mkString("\n")} - |""".stripMargin) + throw new RuntimeException(s"""|Error running: ${cmd.mkString(" ")} + |Exit code: ${dockerCliResult.exitCode} + |${dockerCliResult.stderr.mkString("\n")} + |""".stripMargin) } } - } /** * Run a command and return the result. Overridable for testing. diff --git a/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliFlow.scala index f6bf6e96d4c..513d27bc001 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliFlow.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliFlow.scala @@ -28,20 +28,22 @@ class DockerCliFlow(implicit ec: ExecutionContext) extends DockerRegistry { override def run(dockerInfoContext: DockerInfoContext)(implicit client: Client[IO]) = { implicit val timer = IO.timer(ec) - DockerCliFlow.lookupHashOrTimeout(firstLookupTimeout)(dockerInfoContext) - .flatMap({ + DockerCliFlow + .lookupHashOrTimeout(firstLookupTimeout)(dockerInfoContext) + .flatMap { // If the image isn't there, pull it and try again case (_: DockerInfoNotFound, _) => DockerCliFlow.pull(dockerInfoContext) DockerCliFlow.lookupHashOrTimeout(firstLookupTimeout)(dockerInfoContext) case other => IO.pure(other) - }) + } } override def config = DockerRegistryConfig.default } object DockerCliFlow { + /** * Lookup the hash for the image referenced in the context. * @@ -53,10 +55,11 @@ object DockerCliFlow { DockerInfoActor.logger.debug("Looking up hash of {}", dockerCliKey.fullName) val result = DockerCliClient.lookupHash(dockerCliKey) match { case Success(None) => DockerInfoNotFound(context.request) - case Success(Some(hash)) => DockerHashResult.fromString(hash) match { - case Success(r) => DockerInfoSuccessResponse(DockerInformation(r, None), context.request) - case Failure(t) => DockerInfoFailedResponse(t, context.request) - } + case Success(Some(hash)) => + DockerHashResult.fromString(hash) match { + case Success(r) => DockerInfoSuccessResponse(DockerInformation(r, None), context.request) + case Failure(t) => DockerInfoFailedResponse(t, context.request) + } case Failure(throwable) => DockerInfoFailedResponse(throwable, context.request) } // give the compiler a hint on the debug() override we're trying to use. @@ -72,22 +75,22 @@ object DockerCliFlow { * @param context The image to lookup. * @return The docker hash response plus the context of our flow. */ - private def lookupHashOrTimeout(timeout: FiniteDuration) - (context: DockerInfoContext) - (implicit cs: ContextShift[IO], timer: Timer[IO]): IO[(DockerInfoResponse, DockerInfoContext)] = { - IO(lookupHash(context)).timeout(timeout) - .handleErrorWith({ - case _: TimeoutException => IO.pure { - val dockerCliKey = cliKeyFromImageId(context) - val exception = new TimeoutException( - s"""|Timeout while looking up hash of ${dockerCliKey.fullName}. - |Ensure that docker is running correctly. - |""".stripMargin) - DockerInfoFailedResponse(exception, context.request) -> context - } + private def lookupHashOrTimeout(timeout: FiniteDuration)( + context: DockerInfoContext + )(implicit cs: ContextShift[IO], timer: Timer[IO]): IO[(DockerInfoResponse, DockerInfoContext)] = + IO(lookupHash(context)) + .timeout(timeout) + .handleErrorWith { + case _: TimeoutException => + IO.pure { + val dockerCliKey = cliKeyFromImageId(context) + val exception = new TimeoutException(s"""|Timeout while looking up hash of ${dockerCliKey.fullName}. + |Ensure that docker is running correctly. + |""".stripMargin) + DockerInfoFailedResponse(exception, context.request) -> context + } case other => IO.pure(DockerInfoFailedResponse(other, context.request) -> context) - }) - } + } /** * Pull the docker image referenced in context. diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerManifest.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerManifest.scala index a7990d3cb09..8f9a2743d20 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerManifest.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerManifest.scala @@ -3,7 +3,7 @@ package cromwell.docker.registryv2 // From https://docs.docker.com/registry/spec/manifest-v2-2/ sealed trait DockerManifestResponse -case class DockerManifest(layers: Array[DockerLayer]) extends DockerManifestResponse{ +case class DockerManifest(layers: Array[DockerLayer]) extends DockerManifestResponse { lazy val compressedSize: Long = layers.map(_.size).sum } case class DockerLayer(size: Long) diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2Abstract.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2Abstract.scala index bb25cb4bc3d..234b762429d 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2Abstract.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2Abstract.scala @@ -34,16 +34,21 @@ object DockerRegistryV2Abstract { val OCIIndexV1MediaType = "application/vnd.oci.image.index.v1+json" // If one of those fails it means someone changed one of the strings above to an invalid one. - val DockerManifestV2MediaRange = MediaRange.parse(DockerManifestV2MediaType) + val DockerManifestV2MediaRange = MediaRange + .parse(DockerManifestV2MediaType) .unsafe("Cannot parse invalid manifest v2 content type. Please report this error.") - val DockerManifestListV2MediaRange = MediaRange.parse(DockerManifestListV2MediaType) + val DockerManifestListV2MediaRange = MediaRange + .parse(DockerManifestListV2MediaType) .unsafe("Cannot parse invalid manifest list v2 content type. Please report this error.") - val AcceptDockerManifestV2Header = Accept.parse(DockerManifestV2MediaType) + val AcceptDockerManifestV2Header = Accept + .parse(DockerManifestV2MediaType) .unsafe("Cannot parse invalid manifest v2 Accept header. Please report this error.") - val OCIIndexV1MediaRange = MediaRange.parse(OCIIndexV1MediaType) + val OCIIndexV1MediaRange = MediaRange + .parse(OCIIndexV1MediaType) .unsafe("Cannot parse invalid OCI index v1 content type. Please report this error.") - val AcceptOCIIndexV1Header = Accept.parse(OCIIndexV1MediaType) + val AcceptOCIIndexV1Header = Accept + .parse(OCIIndexV1MediaType) .unsafe("Cannot parse invalid OCI index v1 Accept header. Please report this error.") implicit val entityManifestDecoder = jsonEntityDecoder[DockerManifest](DockerManifestV2MediaRange) @@ -55,19 +60,21 @@ object DockerRegistryV2Abstract { * This is necessary because the docker registry API responds with an "application/vnd.docker.distribution.manifest.v2+json" * and not the traditional "application/json". Adapted from CirceInstances.jsonOf */ - private def jsonEntityDecoder[A](mediaRange: MediaRange)(implicit decoder: Decoder[A]): EntityDecoder[IO, A] = EntityDecoder.decodeBy[IO, A](mediaRange) { message => - CirceInstances.builder.build.jsonDecoderByteBuffer[IO] - .decode(message, strict = false) - .flatMap({ json => - decoder.decodeJson(json) - .fold( - failure => - DecodeResult.failureT( - InvalidMessageBodyFailure(s"Could not decode JSON: $json", Some(failure))), - DecodeResult.successT(_) - ) - }) - } + private def jsonEntityDecoder[A](mediaRange: MediaRange)(implicit decoder: Decoder[A]): EntityDecoder[IO, A] = + EntityDecoder.decodeBy[IO, A](mediaRange) { message => + CirceInstances.builder.build + .jsonDecoderByteBuffer[IO] + .decode(message, strict = false) + .flatMap { json => + decoder + .decodeJson(json) + .fold( + failure => + DecodeResult.failureT(InvalidMessageBodyFailure(s"Could not decode JSON: $json", Some(failure))), + DecodeResult.successT(_) + ) + } + } // Placeholder exceptions that can be carried through IO before being converted to a DockerInfoFailedResponse private class Unauthorized(message: String) extends Exception(message) @@ -88,7 +95,9 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi /** * This is the main function. Given a docker context and an http client, retrieve information about the docker image. */ - def run(dockerInfoContext: DockerInfoContext)(implicit client: Client[IO]): IO[(DockerInfoResponse, DockerInfoContext)] = { + def run( + dockerInfoContext: DockerInfoContext + )(implicit client: Client[IO]): IO[(DockerInfoResponse, DockerInfoContext)] = { val dockerResponse = for { token <- getToken(dockerInfoContext) dockerSuccessResponse <- getDockerResponse(token, dockerInfoContext) @@ -97,19 +106,20 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi // Always map failures to a DockerHashFailedResponse instead of letting the IO fail, this is important so that the stream // that is calling this function will not fail dockerResponse.attempt - .map({ + .map { case Left(_: Unauthorized) => DockerInfoUnauthorized(dockerInfoContext.request) case Left(_: NotFound) => DockerInfoNotFound(dockerInfoContext.request) case Left(failure) => DockerInfoFailedResponse(failure, dockerInfoContext.request) case Right(value) => value - }) + } .map(_ -> dockerInfoContext) } // Execute a request. No retries because they're expected to already be handled by the client - protected def executeRequest[A](request: IO[Request[IO]], handler: Response[IO] => IO[A])(implicit client: Client[IO]): IO[A] = { + protected def executeRequest[A](request: IO[Request[IO]], handler: Response[IO] => IO[A])(implicit + client: Client[IO] + ): IO[A] = request.flatMap(client.run(_).use[IO, A](handler)) - } /** * Obtain an authorization token for the subsequent manifest request. Return IO.pure(None) if no token is needed @@ -129,11 +139,16 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi * @param client http client * @return docker info response */ - protected def getDockerResponse(token: Option[String], dockerInfoContext: DockerInfoContext)(implicit client: Client[IO]): IO[DockerInfoSuccessResponse] = { + protected def getDockerResponse(token: Option[String], dockerInfoContext: DockerInfoContext)(implicit + client: Client[IO] + ): IO[DockerInfoSuccessResponse] = { val requestDockerManifest = manifestRequest(token, dockerInfoContext.dockerImageID, AcceptDockerManifestV2Header) lazy val requestOCIManifest = manifestRequest(token, dockerInfoContext.dockerImageID, AcceptOCIIndexV1Header) def tryOCIManifest(err: Throwable) = { - logger.info(s"Manifest request failed for docker manifest V2, falling back to OCI manifest. Image: ${dockerInfoContext.dockerImageID}", err) + logger.info( + s"Manifest request failed for docker manifest V2, falling back to OCI manifest. Image: ${dockerInfoContext.dockerImageID}", + err + ) executeRequest(requestOCIManifest, handleManifestResponse(dockerInfoContext, token)) } // Try to execute a request using the Docker Manifest format, and if that fails, try using the newer OCI manifest format @@ -145,7 +160,8 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi * Returns true if this flow is able to process this docker image, * false otherwise */ - def accepts(dockerImageIdentifier: DockerImageIdentifier) = dockerImageIdentifier.host.contains(registryHostName(dockerImageIdentifier)) + def accepts(dockerImageIdentifier: DockerImageIdentifier) = + dockerImageIdentifier.host.contains(registryHostName(dockerImageIdentifier)) /* Methods that must to be implemented by a subclass */ @@ -201,26 +217,30 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi */ private def tokenResponseHandler(response: Response[IO]): IO[String] = response match { case Status.Successful(r) => r.as[DockerAccessToken].map(_.token) - case r => r.as[String] - .flatMap(b => IO.raiseError(new Exception(s"Request failed with status ${r.status.code} and body $b"))) + case r => + r.as[String] + .flatMap(b => IO.raiseError(new Exception(s"Request failed with status ${r.status.code} and body $b"))) } /** * Builds the manifest URI to be queried based on a DockerImageID */ - private def buildManifestUri(dockerImageID: DockerImageIdentifier): Uri = { + private def buildManifestUri(dockerImageID: DockerImageIdentifier): Uri = Uri.apply( scheme = Option(Scheme.https), authority = Option(Authority(host = Uri.RegName(registryHostName(dockerImageID)))), path = s"/v2/${dockerImageID.nameWithDefaultRepository}/manifests/${dockerImageID.reference}" ) - } /** * Request to get the manifest, using the auth token if provided */ - private def manifestRequest(token: Option[String], imageId: DockerImageIdentifier, manifestHeader: Accept): IO[Request[IO]] = { - val authorizationHeader: Option[Authorization] = token.map(t => Authorization(Credentials.Token(AuthScheme.Bearer, t))) + private def manifestRequest(token: Option[String], + imageId: DockerImageIdentifier, + manifestHeader: Accept + ): IO[Request[IO]] = { + val authorizationHeader: Option[Authorization] = + token.map(t => Authorization(Credentials.Token(AuthScheme.Bearer, t))) val request = Method.GET( buildManifestUri(imageId), List( @@ -231,19 +251,26 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi request } - private def handleManifestResponse(dockerInfoContext: DockerInfoContext, token: Option[String])(response: Response[IO])(implicit client: Client[IO]): IO[DockerInfoSuccessResponse] = { + private def handleManifestResponse(dockerInfoContext: DockerInfoContext, token: Option[String])( + response: Response[IO] + )(implicit client: Client[IO]): IO[DockerInfoSuccessResponse] = { // Getting the manifest content is not strictly necessary but just a bonus to get the size. If it fails, log the error and return None - def handleManifestAttempt(attempt: Either[Throwable, Option[DockerManifest]]): Option[DockerManifest] = attempt match { - case Left(failure) => - logger.warn(s"Could not get manifest for ${dockerInfoContext.dockerImageID.fullName}", failure) - None - case Right(manifest) => manifest - } + def handleManifestAttempt(attempt: Either[Throwable, Option[DockerManifest]]): Option[DockerManifest] = + attempt match { + case Left(failure) => + logger.warn(s"Could not get manifest for ${dockerInfoContext.dockerImageID.fullName}", failure) + None + case Right(manifest) => manifest + } for { hashResult <- getDigestFromResponse(response) - maybeManifest <- parseManifest(dockerInfoContext.dockerImageID, token)(response).attempt.map(handleManifestAttempt) - } yield DockerInfoSuccessResponse(DockerInformation(hashResult, maybeManifest.map(_.compressedSize).map(DockerSize.apply)), dockerInfoContext.request) + maybeManifest <- parseManifest(dockerInfoContext.dockerImageID, token)(response).attempt + .map(handleManifestAttempt) + } yield DockerInfoSuccessResponse( + DockerInformation(hashResult, maybeManifest.map(_.compressedSize).map(DockerSize.apply)), + dockerInfoContext.request + ) } /** @@ -259,19 +286,22 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi * between platforms. * If that assumption turns out to be incorrect, a smarter decision may need to be made to choose the manifest to lookup. */ - private def parseManifest(dockerImageIdentifier: DockerImageIdentifier, token: Option[String])(response: Response[IO])(implicit client: Client[IO]): IO[Option[DockerManifest]] = response match { + private def parseManifest(dockerImageIdentifier: DockerImageIdentifier, token: Option[String])( + response: Response[IO] + )(implicit client: Client[IO]): IO[Option[DockerManifest]] = response match { case Status.Successful(r) if r.headers.exists(_.value.equalsIgnoreCase(DockerManifestV2MediaType)) => r.as[DockerManifest].map(Option.apply) case Status.Successful(r) if r.headers.exists(_.value.equalsIgnoreCase(DockerManifestListV2MediaType)) => - r.as[DockerManifestList].flatMap({ dockerManifestList => + r.as[DockerManifestList].flatMap { dockerManifestList => obtainManifestFromList(dockerManifestList, dockerImageIdentifier, token) - }) + } case _ => IO.pure(None) } private def obtainManifestFromList(dockerManifestList: DockerManifestList, dockerImageIdentifier: DockerImageIdentifier, - token: Option[String])(implicit client: Client[IO]): IO[Option[DockerManifest]] = { + token: Option[String] + )(implicit client: Client[IO]): IO[Option[DockerManifest]] = dockerManifestList.manifests.headOption .map(_.digest) .map(dockerImageIdentifier.swapReference) match { @@ -279,23 +309,24 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi val request = manifestRequest(token, identifierWithNewHash, AcceptDockerManifestV2Header) executeRequest(request, parseManifest(dockerImageIdentifier, token)) case None => - logger.error(s"The manifest list for ${dockerImageIdentifier.fullName} was empty. Cannot proceed to obtain the size of image") + logger.error( + s"The manifest list for ${dockerImageIdentifier.fullName} was empty. Cannot proceed to obtain the size of image" + ) IO.pure(None) } - } private def getDigestFromResponse(response: Response[IO]): IO[DockerHashResult] = response match { case Status.Successful(r) => extractDigestFromHeaders(r.headers) - case Status.Unauthorized(r) => r.as[String].flatMap(body => IO.raiseError(new Unauthorized(r.status.toString + " " + body))) + case Status.Unauthorized(r) => + r.as[String].flatMap(body => IO.raiseError(new Unauthorized(r.status.toString + " " + body))) case Status.NotFound(r) => r.as[String].flatMap(body => IO.raiseError(new NotFound(r.status.toString + " " + body))) - case failed => failed.as[String].flatMap(body => IO.raiseError(new UnknownError(failed.status.toString + " " + body)) - ) + case failed => + failed.as[String].flatMap(body => IO.raiseError(new UnknownError(failed.status.toString + " " + body))) } - private def extractDigestFromHeaders(headers: Headers) = { + private def extractDigestFromHeaders(headers: Headers) = headers.find(a => a.toRaw.name.equals(DigestHeaderName)) match { case Some(digest) => IO.fromEither(DockerHashResult.fromString(digest.value).toEither) case None => IO.raiseError(new Exception(s"Manifest response did not have a digest header")) } - } } diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/azure/AzureContainerRegistry.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/azure/AzureContainerRegistry.scala index 46dfd116bc6..ff251839ba9 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/azure/AzureContainerRegistry.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/azure/AzureContainerRegistry.scala @@ -15,7 +15,6 @@ import org.http4s.client.Client import io.circe.generic.auto._ import org.http4s._ - class AzureContainerRegistry(config: DockerRegistryConfig) extends DockerRegistryV2Abstract(config) with LazyLogging { /** @@ -25,7 +24,7 @@ class AzureContainerRegistry(config: DockerRegistryConfig) extends DockerRegistr dockerImageIdentifier.host.getOrElse("") override def accepts(dockerImageIdentifier: DockerImageIdentifier): Boolean = - dockerImageIdentifier.hostAsString.contains(domain) + dockerImageIdentifier.hostAsString.contains(domain) override protected def authorizationServerHostName(dockerImageIdentifier: DockerImageIdentifier): String = dockerImageIdentifier.host.getOrElse("") @@ -35,13 +34,12 @@ class AzureContainerRegistry(config: DockerRegistryConfig) extends DockerRegistr */ override def serviceName: Option[String] = throw new Exception("ACR service name is host of user-defined registry, must derive from `DockerImageIdentifier`") - + /** * Builds the list of headers for the token request */ - override protected def buildTokenRequestHeaders(dockerInfoContext: DockerInfoContext): List[Header] = { + override protected def buildTokenRequestHeaders(dockerInfoContext: DockerInfoContext): List[Header] = List(contentTypeHeader) - } private val contentTypeHeader: Header = { import org.http4s.headers.`Content-Type` @@ -49,7 +47,7 @@ class AzureContainerRegistry(config: DockerRegistryConfig) extends DockerRegistr `Content-Type`(MediaType.application.`x-www-form-urlencoded`) } - + private def getRefreshToken(authServerHostname: String, defaultAccessToken: String): IO[Request[IO]] = { import org.http4s.Uri.{Authority, Scheme} import org.http4s.client.dsl.io._ @@ -69,16 +67,16 @@ class AzureContainerRegistry(config: DockerRegistryConfig) extends DockerRegistr "grant_type" -> "access_token" ), uri, - List(contentTypeHeader): _* + List(contentTypeHeader): _* ) } /* Unlike other repositories, Azure reserves `GET /oauth2/token` for Basic Authentication [0] In order to use Oauth we must `POST /oauth2/token` [1] - + [0] https://github.com/Azure/acr/blob/main/docs/Token-BasicAuth.md#using-the-token-api - [1] https://github.com/Azure/acr/blob/main/docs/AAD-OAuth.md#calling-post-oauth2token-to-get-an-acr-access-token + [1] https://github.com/Azure/acr/blob/main/docs/AAD-OAuth.md#calling-post-oauth2token-to-get-an-acr-access-token */ private def getDockerAccessToken(hostname: String, repository: String, refreshToken: String): IO[Request[IO]] = { import org.http4s.Uri.{Authority, Scheme} @@ -102,14 +100,17 @@ class AzureContainerRegistry(config: DockerRegistryConfig) extends DockerRegistr "grant_type" -> "refresh_token" ), uri, - List(contentTypeHeader): _* + List(contentTypeHeader): _* ) } - override protected def getToken(dockerInfoContext: DockerInfoContext)(implicit client: Client[IO]): IO[Option[String]] = { + override protected def getToken( + dockerInfoContext: DockerInfoContext + )(implicit client: Client[IO]): IO[Option[String]] = { val hostname = authorizationServerHostName(dockerInfoContext.dockerImageID) - val maybeAadAccessToken: ErrorOr[String] = AzureCredentials.getAccessToken(None) // AAD token suitable for get-refresh-token request - val repository = dockerInfoContext.dockerImageID.image // ACR uses what we think of image name, as the repository + val maybeAadAccessToken: ErrorOr[String] = + AzureCredentials.getAccessToken(None) // AAD token suitable for get-refresh-token request + val repository = dockerInfoContext.dockerImageID.image // ACR uses what we think of image name, as the repository // Top-level flow: AAD access token -> refresh token -> ACR access token maybeAadAccessToken match { @@ -131,19 +132,21 @@ class AzureContainerRegistry(config: DockerRegistryConfig) extends DockerRegistr private def parseRefreshToken(response: Response[IO]): IO[String] = response match { case Status.Successful(r) => r.as[AcrRefreshToken].map(_.refresh_token) case r => - r.as[String].flatMap(b => IO.raiseError(new Exception(s"Request failed with status ${r.status.code} and body $b"))) + r.as[String] + .flatMap(b => IO.raiseError(new Exception(s"Request failed with status ${r.status.code} and body $b"))) } private def parseAccessToken(response: Response[IO]): IO[String] = response match { case Status.Successful(r) => r.as[AcrAccessToken].map(_.access_token) case r => - r.as[String].flatMap(b => IO.raiseError(new Exception(s"Request failed with status ${r.status.code} and body $b"))) + r.as[String] + .flatMap(b => IO.raiseError(new Exception(s"Request failed with status ${r.status.code} and body $b"))) } } object AzureContainerRegistry { - + def domain: String = "azurecr.io" - + } diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/dockerhub/DockerHubRegistry.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/dockerhub/DockerHubRegistry.scala index 33fb2dabfa9..c523fe71169 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/dockerhub/DockerHubRegistry.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/dockerhub/DockerHubRegistry.scala @@ -17,12 +17,10 @@ class DockerHubRegistry(config: DockerRegistryConfig) extends DockerRegistryV2Ab /** * Builds the list of headers for the token request */ - def buildTokenRequestHeaders(dockerInfoContext: DockerInfoContext) = { - dockerInfoContext.credentials collect { - case DockerCredentials(token) => - Authorization(org.http4s.BasicCredentials(token)) + def buildTokenRequestHeaders(dockerInfoContext: DockerInfoContext) = + dockerInfoContext.credentials collect { case DockerCredentials(token) => + Authorization(org.http4s.BasicCredentials(token)) } - } override def accepts(dockerImageIdentifier: DockerImageIdentifier): Boolean = dockerImageIdentifier.host |> isValidDockerHubHost diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/google/GoogleRegistry.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/google/GoogleRegistry.scala index 4db74317e07..81bcb4dd4ec 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/google/GoogleRegistry.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/google/GoogleRegistry.scala @@ -11,12 +11,17 @@ import scala.concurrent.duration._ class GoogleRegistry(config: DockerRegistryConfig) extends DockerRegistryV2Abstract(config) { private val AccessTokenAcceptableTTL = 1.minute - - def googleRegion(dockerImageIdentifier: DockerImageIdentifier): String = dockerImageIdentifier.host.flatMap(_.split("/").headOption).getOrElse("") - override def registryHostName(dockerImageIdentifier: DockerImageIdentifier): String = googleRegion(dockerImageIdentifier) - override def authorizationServerHostName(dockerImageIdentifier: DockerImageIdentifier): String = googleRegion(dockerImageIdentifier) - + def googleRegion(dockerImageIdentifier: DockerImageIdentifier): String = + dockerImageIdentifier.host.flatMap(_.split("/").headOption).getOrElse("") + + override def registryHostName(dockerImageIdentifier: DockerImageIdentifier): String = googleRegion( + dockerImageIdentifier + ) + override def authorizationServerHostName(dockerImageIdentifier: DockerImageIdentifier): String = googleRegion( + dockerImageIdentifier + ) + override protected def buildTokenRequestUri(dockerImageID: DockerImageIdentifier): Uri = { val uri = super.buildTokenRequestUri(dockerImageID) uri.withPath(s"/v2${uri.path}") @@ -25,17 +30,15 @@ class GoogleRegistry(config: DockerRegistryConfig) extends DockerRegistryV2Abstr /** * Builds the list of headers for the token request */ - def buildTokenRequestHeaders(dockerInfoContext: DockerInfoContext): List[Authorization] = { - dockerInfoContext.credentials collect { - case credentials: OAuth2Credentials => Authorization(org.http4s.Credentials.Token(AuthScheme.Bearer, freshAccessToken(credentials))) + def buildTokenRequestHeaders(dockerInfoContext: DockerInfoContext): List[Authorization] = + dockerInfoContext.credentials collect { case credentials: OAuth2Credentials => + Authorization(org.http4s.Credentials.Token(AuthScheme.Bearer, freshAccessToken(credentials))) } - } - + private def freshAccessToken(credential: OAuth2Credentials) = { - def accessTokenTTLIsAcceptable(accessToken: AccessToken) = { + def accessTokenTTLIsAcceptable(accessToken: AccessToken) = (accessToken.getExpirationTime.getTime - System.currentTimeMillis()).millis.gteq(AccessTokenAcceptableTTL) - } - + Option(credential.getAccessToken) match { case Some(accessToken) if accessTokenTTLIsAcceptable(accessToken) => accessToken.getTokenValue case _ => @@ -44,8 +47,9 @@ class GoogleRegistry(config: DockerRegistryConfig) extends DockerRegistryV2Abstr } } - override def accepts(dockerImageIdentifier: DockerImageIdentifier): Boolean = { + override def accepts(dockerImageIdentifier: DockerImageIdentifier): Boolean = // Supports both GCR (Google Container Registry) and GAR (Google Artifact Registry). - dockerImageIdentifier.hostAsString.contains("gcr.io") || dockerImageIdentifier.hostAsString.contains("-docker.pkg.dev") - } + dockerImageIdentifier.hostAsString.contains("gcr.io") || dockerImageIdentifier.hostAsString.contains( + "-docker.pkg.dev" + ) } diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/quay/QuayRegistry.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/quay/QuayRegistry.scala index d739eba2651..00ca2836636 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/quay/QuayRegistry.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/quay/QuayRegistry.scala @@ -9,12 +9,13 @@ import org.http4s.client.Client class QuayRegistry(config: DockerRegistryConfig) extends DockerRegistryV2Abstract(config) { override protected def registryHostName(dockerImageIdentifier: DockerImageIdentifier): String = "quay.io" - // Not used for now because we bypass the token part as quay doesn't require one for public images + // Not used for now because we bypass the token part as quay doesn't require one for public images override protected def authorizationServerHostName(dockerImageIdentifier: DockerImageIdentifier): String = "quay.io" // Not used for now, same reason as above override protected def buildTokenRequestHeaders(dockerInfoContext: DockerInfoContext): List[Header] = List.empty - override protected def getToken(dockerInfoContext: DockerInfoContext)(implicit client: Client[IO]): IO[Option[String]] = { + override protected def getToken(dockerInfoContext: DockerInfoContext)(implicit + client: Client[IO] + ): IO[Option[String]] = IO.pure(None) - } } diff --git a/dockerHashing/src/test/scala/cromwell/docker/DockerHashMocks.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerHashMocks.scala index 4c664912aeb..d39cd538d17 100644 --- a/dockerHashing/src/test/scala/cromwell/docker/DockerHashMocks.scala +++ b/dockerHashing/src/test/scala/cromwell/docker/DockerHashMocks.scala @@ -11,26 +11,27 @@ class DockerRegistryMock(responses: MockHashResponse*) extends DockerRegistry { // Counts the number of elements going through this "flow" private var _count: Int = 0 - private def nextResponse(context: DockerInfoContext): (DockerInfoResponse, DockerInfoContext) = - responsesLeft.headOption match { - case Some(mockResponse) => - _count += 1 - if (mockResponse.nb > 1) - responsesLeft.update(0, mockResponse.copy(nb = mockResponse.nb - 1)) - else - responsesLeft.remove(0) - (mockResponse.hashResponse, context) - // When we hit the end, loop - case None => - responsesLeft = responses.toBuffer - nextResponse(context) - } - + private def nextResponse(context: DockerInfoContext): (DockerInfoResponse, DockerInfoContext) = + responsesLeft.headOption match { + case Some(mockResponse) => + _count += 1 + if (mockResponse.nb > 1) + responsesLeft.update(0, mockResponse.copy(nb = mockResponse.nb - 1)) + else + responsesLeft.remove(0) + (mockResponse.hashResponse, context) + // When we hit the end, loop + case None => + responsesLeft = responses.toBuffer + nextResponse(context) + } + def count() = _count override def accepts(dockerImageIdentifier: DockerImageIdentifier): Boolean = true - override def run(dockerInfoContext: DockerInfoContext)(implicit client: Client[IO]) = IO.pure(nextResponse(dockerInfoContext)) + override def run(dockerInfoContext: DockerInfoContext)(implicit client: Client[IO]) = + IO.pure(nextResponse(dockerInfoContext)) override def config = DockerRegistryConfig.default } diff --git a/dockerHashing/src/test/scala/cromwell/docker/DockerImageIdentifierSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerImageIdentifierSpec.scala index 41353934fc6..30c8cb6a010 100644 --- a/dockerHashing/src/test/scala/cromwell/docker/DockerImageIdentifierSpec.scala +++ b/dockerHashing/src/test/scala/cromwell/docker/DockerImageIdentifierSpec.scala @@ -5,30 +5,51 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks -class DockerImageIdentifierSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with TableDrivenPropertyChecks { +class DockerImageIdentifierSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with TableDrivenPropertyChecks { behavior of "DockerImageID" it should "parse valid docker images" in { val valid = Table( - ("sourceString", "host", "repo", "image", "reference"), + ("sourceString", "host", "repo", "image", "reference"), // Without tags -> latest - ("ubuntu", None, None, "ubuntu", "latest"), - ("broad/cromwell", None, Option("broad"), "cromwell", "latest"), - ("index.docker.io/ubuntu", Option("index.docker.io"), None, "ubuntu", "latest"), - ("broad/cromwell/submarine", None, Option("broad/cromwell"), "submarine", "latest"), - ("gcr.io/google/slim", Option("gcr.io"), Option("google"), "slim", "latest"), - ("us-central1-docker.pkg.dev/google/slim", Option("us-central1-docker.pkg.dev"), Option("google"), "slim", "latest"), - ("terrabatchdev.azurecr.io/postgres", Option("terrabatchdev.azurecr.io"), None, "postgres", "latest"), + ("ubuntu", None, None, "ubuntu", "latest"), + ("broad/cromwell", None, Option("broad"), "cromwell", "latest"), + ("index.docker.io/ubuntu", Option("index.docker.io"), None, "ubuntu", "latest"), + ("broad/cromwell/submarine", None, Option("broad/cromwell"), "submarine", "latest"), + ("gcr.io/google/slim", Option("gcr.io"), Option("google"), "slim", "latest"), + ("us-central1-docker.pkg.dev/google/slim", + Option("us-central1-docker.pkg.dev"), + Option("google"), + "slim", + "latest" + ), + ("terrabatchdev.azurecr.io/postgres", Option("terrabatchdev.azurecr.io"), None, "postgres", "latest"), // With tags - ("ubuntu:latest", None, None, "ubuntu", "latest"), - ("ubuntu:1235-SNAP", None, None, "ubuntu", "1235-SNAP"), - ("ubuntu:V3.8-5_1", None, None, "ubuntu", "V3.8-5_1"), - ("index.docker.io:9999/ubuntu:170904", Option("index.docker.io:9999"), None, "ubuntu", "170904"), - ("localhost:5000/capture/transwf:170904", Option("localhost:5000"), Option("capture"), "transwf", "170904"), - ("quay.io/biocontainers/platypus-variant:0.8.1.1--htslib1.5_0", Option("quay.io"), Option("biocontainers"), "platypus-variant", "0.8.1.1--htslib1.5_0"), + ("ubuntu:latest", None, None, "ubuntu", "latest"), + ("ubuntu:1235-SNAP", None, None, "ubuntu", "1235-SNAP"), + ("ubuntu:V3.8-5_1", None, None, "ubuntu", "V3.8-5_1"), + ("index.docker.io:9999/ubuntu:170904", Option("index.docker.io:9999"), None, "ubuntu", "170904"), + ("localhost:5000/capture/transwf:170904", Option("localhost:5000"), Option("capture"), "transwf", "170904"), + ("quay.io/biocontainers/platypus-variant:0.8.1.1--htslib1.5_0", + Option("quay.io"), + Option("biocontainers"), + "platypus-variant", + "0.8.1.1--htslib1.5_0" + ), ("terrabatchdev.azurecr.io/postgres:latest", Option("terrabatchdev.azurecr.io"), None, "postgres", "latest"), + ("python:3", None, None, "python", "3"), + ("localhost:443/ubuntu", Option("localhost:443"), None, "ubuntu", "latest"), // Very long tags with trailing spaces cause problems for the re engine - ("someuser/someimage:supercalifragilisticexpialidociouseventhoughthesoundofitissomethingquiteatrociousifyousayitloudenoughyoullalwayssoundprecocious ", None, Some("someuser"), "someimage", "supercalifragilisticexpialidociouseventhoughthesoundofitissomethingquiteatrociousifyousayitloudenoughyoullalwayssoundprecocious") + ("someuser/someimage:supercalifragilisticexpialidociouseventhoughthesoundofitissomethingquiteatrociousifyousayitloudenoughyoullalwayssoundprecocious ", + None, + Some("someuser"), + "someimage", + "supercalifragilisticexpialidociouseventhoughthesoundofitissomethingquiteatrociousifyousayitloudenoughyoullalwayssoundprecocious" + ) ) forAll(valid) { (dockerString, host, repo, image, reference) => diff --git a/dockerHashing/src/test/scala/cromwell/docker/DockerInfoActorSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerInfoActorSpec.scala index 72baec70825..ed48762c2cb 100644 --- a/dockerHashing/src/test/scala/cromwell/docker/DockerInfoActorSpec.scala +++ b/dockerHashing/src/test/scala/cromwell/docker/DockerInfoActorSpec.scala @@ -25,44 +25,40 @@ class DockerInfoActorSpec extends DockerRegistrySpec with AnyFlatSpecLike with M it should "retrieve a public docker hash" taggedAs IntegrationTest in { dockerActor ! makeRequest("ubuntu:latest") - - expectMsgPF(5 second) { - case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => - alg shouldBe "sha256" - hash should not be empty + + expectMsgPF(5 second) { case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => + alg shouldBe "sha256" + hash should not be empty } } it should "retrieve a public docker hash on gcr" taggedAs IntegrationTest in { dockerActor ! makeRequest("gcr.io/google-containers/alpine-with-bash:1.0") - expectMsgPF(5 second) { - case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => - alg shouldBe "sha256" - hash should not be empty + expectMsgPF(5 second) { case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => + alg shouldBe "sha256" + hash should not be empty } } it should "retrieve a public docker hash on gar" taggedAs IntegrationTest in { dockerActor ! makeRequest("us-central1-docker.pkg.dev/broad-dsde-cromwell-dev/bt-335/ubuntu:bt-335") - expectMsgPF(5 second) { - case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => - alg shouldBe "sha256" - hash should not be empty + expectMsgPF(5 second) { case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => + alg shouldBe "sha256" + hash should not be empty } } it should "retrieve a private docker hash on acr" taggedAs IntegrationTest in { dockerActor ! makeRequest("terrabatchdev.azurecr.io/postgres:latest") - expectMsgPF(15 second) { - case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => - alg shouldBe "sha256" - hash should not be empty + expectMsgPF(15 second) { case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => + alg shouldBe "sha256" + hash should not be empty } } - + it should "send image not found message back if the image does not exist" taggedAs IntegrationTest in { val notFound = makeRequest("ubuntu:nonexistingtag") dockerActor ! notFound @@ -76,38 +72,38 @@ class DockerInfoActorSpec extends DockerRegistrySpec with AnyFlatSpecLike with M expectMsgClass(5 seconds, classOf[DockerInfoUnauthorized]) } - + it should "send an unrecognized host message if no flow can process the docker string" taggedAs IntegrationTest in { val unauthorized = makeRequest("unknown.io/image:v1") dockerActor ! unauthorized expectMsgClass(5 seconds, classOf[DockerHashUnknownRegistry]) } - + it should "cache results" in { - + val image1 = dockerImage("ubuntu:latest") val request = DockerInfoRequest(image1) - + val hashSuccess = DockerHashResult("sha256", "hashvalue") val responseSuccess = DockerInfoSuccessResponse(DockerInformation(hashSuccess, None), request) val mockResponseSuccess = MockHashResponse(responseSuccess, 1) - + val responseFailure = DockerInfoFailedResponse(new Exception("Docker hash failed - part of test flow"), request) val mockResponseFailure = MockHashResponse(responseFailure, 1) - + // Send back success, failure, success, failure, ... val mockHttpFlow = new DockerRegistryMock(mockResponseSuccess, mockResponseFailure) val dockerActorWithCache = system.actorOf( props = DockerInfoActor.props(Seq(mockHttpFlow), 1000, 3 seconds, 10), - name = "dockerActorWithCache", + name = "dockerActorWithCache" ) - + dockerActorWithCache ! request expectMsg(DockerInfoSuccessResponse(DockerInformation(hashSuccess, None), request)) // Necessary to give some time to the cache to be updated - as it's decoupled from sending back the response Thread.sleep(1000) - + dockerActorWithCache ! request // Without caching, the second request would have yielded a Failure since the mock flow alternates between a success and a failure // Getting a success here means the request didn't make it to the stream @@ -122,11 +118,10 @@ class DockerInfoActorSpec extends DockerRegistrySpec with AnyFlatSpecLike with M mockHttpFlow.count() shouldBe 2 } - it should "not deadlock" taggedAs IntegrationTest in { lazy val dockerActorScale = system.actorOf( props = DockerInfoActor.props(registryFlows, 1000, 20.minutes, 0), - name = "dockerActorScale", + name = "dockerActorScale" ) 0 until 400 foreach { _ => dockerActorScale ! makeRequest("gcr.io/google-containers/alpine-with-bash:1.0") diff --git a/dockerHashing/src/test/scala/cromwell/docker/DockerRegistrySpec.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerRegistrySpec.scala index ab35c216b51..5742f44bc2c 100644 --- a/dockerHashing/src/test/scala/cromwell/docker/DockerRegistrySpec.scala +++ b/dockerHashing/src/test/scala/cromwell/docker/DockerRegistrySpec.scala @@ -18,14 +18,13 @@ abstract class DockerRegistrySpec extends TestKitSuite with ImplicitSender { // Disable cache by setting a cache size of 0 - A separate test tests the cache lazy val dockerActor: ActorRef = system.actorOf( props = DockerInfoActor.props(registryFlows, 1000, 20.minutes, 0), - name = "dockerActor", + name = "dockerActor" ) def dockerImage(string: String): DockerImageIdentifier = DockerImageIdentifier.fromString(string).get - def makeRequest(string: String): DockerInfoRequest = { + def makeRequest(string: String): DockerInfoRequest = DockerInfoRequest(dockerImage(string)) - } override protected def afterAll(): Unit = { system.stop(dockerActor) diff --git a/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliClientSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliClientSpec.scala index 6553979f9ff..14b9e4f3ba3 100644 --- a/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliClientSpec.scala +++ b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliClientSpec.scala @@ -7,18 +7,24 @@ import org.scalatest.prop.TableDrivenPropertyChecks import scala.util.{Failure, Success} -class DockerCliClientSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with Matchers with TableDrivenPropertyChecks { +class DockerCliClientSpec + extends AnyFlatSpecLike + with CromwellTimeoutSpec + with Matchers + with TableDrivenPropertyChecks { behavior of "DockerCliClient" private val lookupSuccessStdout = Seq( "\t\t", "fauxbuntu\tlatest\tsha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - "fauxbuntu\tmytag\tsha256:00001111222233334444555566667777888899990000aaaabbbbccccddddeeee") + "fauxbuntu\tmytag\tsha256:00001111222233334444555566667777888899990000aaaabbbbccccddddeeee" + ) private val lookupSuccessHashes = Table( ("dockerCliKey", "hashValue"), (DockerCliKey("fauxbuntu", "latest"), "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), - (DockerCliKey("fauxbuntu", "mytag"), "00001111222233334444555566667777888899990000aaaabbbbccccddddeeee")) + (DockerCliKey("fauxbuntu", "mytag"), "00001111222233334444555566667777888899990000aaaabbbbccccddddeeee") + ) forAll(lookupSuccessHashes) { (dockerCliKey, hashValue) => it should s"successfully lookup simulated hash for ${dockerCliKey.fullName}" in { @@ -47,7 +53,8 @@ class DockerCliClientSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with """|Error running: docker images --digests --format {{printf "%s\t%s\t%s" .Repository .Tag .Digest}} |Exit code: 1 |Error response from daemon: Bad response from Docker engine - |""".stripMargin) + |""".stripMargin + ) } } diff --git a/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliSpec.scala index 3a8d88cdb01..7565195f1d2 100644 --- a/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliSpec.scala +++ b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliSpec.scala @@ -1,7 +1,7 @@ package cromwell.docker.local import cromwell.core.Tags.IntegrationTest -import cromwell.docker.DockerInfoActor.{DockerInfoNotFound, DockerInfoSuccessResponse, DockerInformation} +import cromwell.docker.DockerInfoActor.{DockerInfoNotFound, DockerInformation, DockerInfoSuccessResponse} import cromwell.docker.{DockerHashResult, DockerRegistry, DockerRegistrySpec} import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers @@ -16,30 +16,27 @@ class DockerCliSpec extends DockerRegistrySpec with AnyFlatSpecLike with Matcher it should "retrieve a public docker hash" taggedAs IntegrationTest in { dockerActor ! makeRequest("ubuntu:latest") - expectMsgPF(30.seconds) { - case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => - alg shouldBe "sha256" - hash should not be empty + expectMsgPF(30.seconds) { case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => + alg shouldBe "sha256" + hash should not be empty } } it should "retrieve a public docker hash on gcr" taggedAs IntegrationTest in { dockerActor ! makeRequest("gcr.io/google-containers/alpine-with-bash:1.0") - expectMsgPF(30.seconds) { - case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => - alg shouldBe "sha256" - hash should not be empty + expectMsgPF(30.seconds) { case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => + alg shouldBe "sha256" + hash should not be empty } } it should "retrieve a public docker hash on gar" taggedAs IntegrationTest in { dockerActor ! makeRequest("us-central1-docker.pkg.dev/broad-dsde-cromwell-dev/bt-335/ubuntu:bt-335") - expectMsgPF(30.seconds) { - case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => - alg shouldBe "sha256" - hash should not be empty + expectMsgPF(30.seconds) { case DockerInfoSuccessResponse(DockerInformation(DockerHashResult(alg, hash), _), _) => + alg shouldBe "sha256" + hash should not be empty } } diff --git a/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliTimeoutSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliTimeoutSpec.scala index 90b3a2fa0ca..445c4378dd1 100644 --- a/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliTimeoutSpec.scala +++ b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliTimeoutSpec.scala @@ -19,24 +19,20 @@ class DockerCliTimeoutSpec extends DockerRegistrySpec with AnyFlatSpecLike with it should "timeout retrieving a public docker hash" taggedAs IntegrationTest in { dockerActor ! makeRequest("ubuntu:latest") - expectMsgPF(5.seconds) { - case DockerInfoFailedResponse(exception: TimeoutException, _) => - exception.getMessage should be( - """|Timeout while looking up hash of ubuntu:latest. - |Ensure that docker is running correctly. - |""".stripMargin) + expectMsgPF(5.seconds) { case DockerInfoFailedResponse(exception: TimeoutException, _) => + exception.getMessage should be("""|Timeout while looking up hash of ubuntu:latest. + |Ensure that docker is running correctly. + |""".stripMargin) } } it should "timeout retrieving a public docker hash on gcr" taggedAs IntegrationTest in { dockerActor ! makeRequest("gcr.io/google-containers/alpine-with-bash:1.0") - expectMsgPF(5.seconds) { - case DockerInfoFailedResponse(exception: TimeoutException, _) => - exception.getMessage should be( - """|Timeout while looking up hash of gcr.io/google-containers/alpine-with-bash:1.0. - |Ensure that docker is running correctly. - |""".stripMargin) + expectMsgPF(5.seconds) { case DockerInfoFailedResponse(exception: TimeoutException, _) => + exception.getMessage should be("""|Timeout while looking up hash of gcr.io/google-containers/alpine-with-bash:1.0. + |Ensure that docker is running correctly. + |""".stripMargin) } } @@ -44,12 +40,12 @@ class DockerCliTimeoutSpec extends DockerRegistrySpec with AnyFlatSpecLike with it should "timeout retrieving a public docker hash on gar" taggedAs IntegrationTest in { dockerActor ! makeRequest("us-central1-docker.pkg.dev/broad-dsde-cromwell-dev/bt-335/ubuntu:bt-335") - expectMsgPF(5.seconds) { - case DockerInfoFailedResponse(exception: TimeoutException, _) => - exception.getMessage should be( - """|Timeout while looking up hash of us-central1-docker.pkg.dev/broad-dsde-cromwell-dev/bt-335/ubuntu:bt-335. - |Ensure that docker is running correctly. - |""".stripMargin) + expectMsgPF(5.seconds) { case DockerInfoFailedResponse(exception: TimeoutException, _) => + exception.getMessage should be( + """|Timeout while looking up hash of us-central1-docker.pkg.dev/broad-dsde-cromwell-dev/bt-335/ubuntu:bt-335. + |Ensure that docker is running correctly. + |""".stripMargin + ) } } @@ -58,12 +54,10 @@ class DockerCliTimeoutSpec extends DockerRegistrySpec with AnyFlatSpecLike with val notFound = makeRequest("ubuntu:nonexistingtag") dockerActor ! notFound - expectMsgPF(5.seconds) { - case DockerInfoFailedResponse(exception: TimeoutException, _) => - exception.getMessage should be( - """|Timeout while looking up hash of ubuntu:nonexistingtag. - |Ensure that docker is running correctly. - |""".stripMargin) + expectMsgPF(5.seconds) { case DockerInfoFailedResponse(exception: TimeoutException, _) => + exception.getMessage should be("""|Timeout while looking up hash of ubuntu:nonexistingtag. + |Ensure that docker is running correctly. + |""".stripMargin) } } @@ -71,12 +65,10 @@ class DockerCliTimeoutSpec extends DockerRegistrySpec with AnyFlatSpecLike with val unauthorized = makeRequest("tjeandet/sinatra:v1") dockerActor ! unauthorized - expectMsgPF(5.seconds) { - case DockerInfoFailedResponse(exception: TimeoutException, _) => - exception.getMessage should be( - """|Timeout while looking up hash of tjeandet/sinatra:v1. - |Ensure that docker is running correctly. - |""".stripMargin) + expectMsgPF(5.seconds) { case DockerInfoFailedResponse(exception: TimeoutException, _) => + exception.getMessage should be("""|Timeout while looking up hash of tjeandet/sinatra:v1. + |Ensure that docker is running correctly. + |""".stripMargin) } } @@ -84,12 +76,10 @@ class DockerCliTimeoutSpec extends DockerRegistrySpec with AnyFlatSpecLike with val unauthorized = makeRequest("unknown.io/image:v1") dockerActor ! unauthorized - expectMsgPF(5.seconds) { - case DockerInfoFailedResponse(exception: TimeoutException, _) => - exception.getMessage should be( - """|Timeout while looking up hash of unknown.io/library/image:v1. - |Ensure that docker is running correctly. - |""".stripMargin) + expectMsgPF(5.seconds) { case DockerInfoFailedResponse(exception: TimeoutException, _) => + exception.getMessage should be("""|Timeout while looking up hash of unknown.io/library/image:v1. + |Ensure that docker is running correctly. + |""".stripMargin) } } } diff --git a/dockerHashing/src/test/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractSpec.scala index e99383ac131..f084a85b035 100644 --- a/dockerHashing/src/test/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractSpec.scala +++ b/dockerHashing/src/test/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractSpec.scala @@ -23,15 +23,18 @@ class DockerRegistryV2AbstractSpec extends AnyFlatSpec with CromwellTimeoutSpec val mediaType = MediaType.parse(DockerRegistryV2Abstract.DockerManifestV2MediaType).toOption.get val contentType: Header = `Content-Type`(mediaType) - val mockClient = Client({ _: Request[IO] => + val mockClient = Client { _: Request[IO] => // This response will have an empty body, so we need to be explicit about the typing: - Resource.pure[IO, Response[IO]](Response(headers = Headers.of(contentType))) : Resource[IO, Response[IO]] - }) + Resource.pure[IO, Response[IO]](Response(headers = Headers.of(contentType))): Resource[IO, Response[IO]] + } val dockerImageIdentifier = DockerImageIdentifier.fromString("ubuntu").get val dockerInfoRequest = DockerInfoRequest(dockerImageIdentifier) val context = DockerInfoContext(dockerInfoRequest, null) val result = registry.run(context)(mockClient).unsafeRunSync() - result.asInstanceOf[(DockerInfoFailedResponse, DockerInfoContext)]._1.reason shouldBe "Failed to get docker hash for ubuntu:latest Malformed message body: Invalid JSON: empty body" + result + .asInstanceOf[(DockerInfoFailedResponse, DockerInfoContext)] + ._1 + .reason shouldBe "Failed to get docker hash for ubuntu:latest Malformed message body: Invalid JSON: empty body" } } diff --git a/docs/Configuring.md b/docs/Configuring.md index aa810d19a0c..5dc2a43d2ca 100644 --- a/docs/Configuring.md +++ b/docs/Configuring.md @@ -134,18 +134,18 @@ system.max-concurrent-workflows = 5000 **New Workflow Poll Rate** -Cromwell will look for new workflows to start on a regular interval, configured as a number of seconds. You can change the polling rate from the default `2` seconds by editing the value: +Cromwell will look for new workflows to start on a regular interval, configured as a number of seconds. You can change the polling rate from the default `20` seconds by editing the value: ```hocon -system.new-workflow-poll-rate = 2 +system.new-workflow-poll-rate = 20 ``` **Max Workflow Launch Count** -On every poll, Cromwell will take at limited number of new submissions, provided there are new workflows to launch and the `system.max-concurrent-workflows` number has not been reached. While the default is to launch up to `50` workflows, you can override this by setting: +On every poll, Cromwell will take at limited number of new submissions, provided there are new workflows to launch and the `system.max-concurrent-workflows` number has not been reached. While the default is to launch up to `1` workflow, you can override this by setting: ```hocon -system.max-workflow-launch-count = 50 +system.max-workflow-launch-count = 1 ``` ***Abort configuration*** @@ -324,6 +324,34 @@ database { } ``` +If you want multiple database users to be able to read Cromwell's data from a Postgresql database, you'll need to create a +role that all relevant users have access to, and adjust Cromwell to use this role. This is because each Large Object is owned +by, and only readable by, the role that wrote it. + +First, pass these options when executing Cromwell. They will ensure that Cromwell's database tables are +owned by the role, not the initial login user. + * `-DengineSharedCromwellDbRole=your_role` to control the role that owns the engine tables + * `-DsharedCromwellDbRole=your_role` to control the role that owns the metadata tables + +Next, use the config key `pgLargeObjectWriteRole` to set the role that should own all large objects, as shown below. +This config will have no effect if you aren't using Postgresql. The configured login user can be any user that is +granted the shared role. + +```hocon +database { + profile = "slick.jdbc.PostgresProfile$" + pgLargeObjectWriteRole = "your_role" + db { + driver = "org.postgresql.Driver" + url = "jdbc:postgresql://localhost:5432/cromwell" + user = "user" + password = "pass" + port = 5432 + connectionTimeout = 5000 + } +} +``` + **Using Cromwell with file-based database (No server required)** SQLite is currently not supported. However, HSQLDB does support running with a persistence file. diff --git a/docs/RuntimeAttributes.md b/docs/RuntimeAttributes.md index 3924c1de3e4..60216946a37 100644 --- a/docs/RuntimeAttributes.md +++ b/docs/RuntimeAttributes.md @@ -429,6 +429,7 @@ runtime { Note that when this options is specified, make sure the requested CPU platform is [available](https://cloud.google.com/compute/docs/regions-zones/#available) in the `zones` you selected. The following CPU platforms are currently supported by the Google Cloud backend: +- `Intel Ice Lake` - `Intel Cascade Lake` - `Intel Skylake` - `Intel Broadwell` diff --git a/docs/Scaling.md b/docs/Scaling.md index beca560212e..6ee7524446c 100644 --- a/docs/Scaling.md +++ b/docs/Scaling.md @@ -38,7 +38,7 @@ system { # Cromwell will launch up to N submitted workflows at a time, regardless of how many open workflow slots exist # Set this to 0 for a non-runner. - max-workflow-launch-count = 50 + max-workflow-launch-count = 1 # The maximum number of workflows to run concurrently. # Set this to 0 for a non-runner. diff --git a/docs/developers/Building.md b/docs/developers/Building.md index 5c5cc21eaf7..08ffe7a3ddd 100644 --- a/docs/developers/Building.md +++ b/docs/developers/Building.md @@ -8,13 +8,6 @@ features or fixes, the following are required to build Cromwell from source: * [AdoptOpenJDK 11 HotSpot](https://adoptopenjdk.net/) * [Git](https://git-scm.com/) -You can also use the [development image](https://github.com/broadinstitute/cromwell/tree/develop/scripts/docker-develop), and build a development container to work inside: - -```bash -$ docker build -t cromwell-dev . -$ docker run -it cromwell-dev bash -``` - First start by cloning the Cromwell repository from GitHub: ```bash @@ -35,15 +28,15 @@ Finally build the Cromwell jar: $ sbt assembly ``` -NOTE: This command will run for a long time the first time. -NOTE: Compiling will not succeed on directories encrypted with ecryptfs (ubuntu encrypted home dirs for example), due to long file paths. - `sbt assembly` will build the runnable Cromwell JAR in `server/target/scala-2.13/` with a name like `cromwell-.jar`. It will also build a runnable Womtool JAR in `womtool/target/scala-2.13/` with a name like `womtool-.jar`. -To build a [Docker](https://www.docker.com/) image, run: +## Docker -```bash -$ sbt server/docker -``` +The following Docker build configurations are supported. Most users will want Snapshot, resulting in an image like `broadinstitute/cromwell:-SNAP`. -This will build and tag a Docker image with a name like `broadinstitute/cromwell:-SNAP`. +| Command | Build Type | Debug Tools | Description | +|------------------------------------------------|------------|-------------|--------------------------------------| +| `sbt server/docker` | Snapshot | No | Most common local build | +| `sbt -Dproject.isDebug=true server/docker` | Debug | Yes | Local build with debugging/profiling | +| `sbt -Dproject.isSnapshot=false server/docker` | Standard | No | Reserved for CI: commit on `develop` | +| `sbt -Dproject.isRelease=true server/docker` | Release | No | Reserved for CI: numbered release | diff --git a/docs/developers/Centaur.md b/docs/developers/Centaur.md index 35d2e350e7d..bbbb011ba32 100644 --- a/docs/developers/Centaur.md +++ b/docs/developers/Centaur.md @@ -133,7 +133,6 @@ our tests into groups depending on which configuration files they require. Below | PAPI Upgrade | `PapiUpgradeTestCaseSpec` | `papiUpgradeTestCases` | | PAPI Upgrade
New Workflows | `CentaurTestSuite` | `papiUpgradeNewWorkflowsTestCases` | | Azure Blob | `CentaurTestSuite ` | `azureBlobTestCases` | -| WDL Upgrade | `WdlUpgradeTestCaseSpec` | `standardTestCases`**** | | (other) | `CentaurTestSuite` | `standardTestCases` | @@ -143,9 +142,7 @@ or `papi_v2beta_centaur_application.conf` \*\* Cromwell Config overrides ([47 link](https://github.com/broadinstitute/cromwell/blob/47/src/ci/bin/test.inc.sh#L213-L221)) \*\*\* Test Directory overrides - ([47 link](https://github.com/broadinstitute/cromwell/blob/47/src/ci/bin/test.inc.sh#L440-L449)) -\*\*\*\* Test Directory only tests tagged with `wdl_upgrade` - ([47 link](https://github.com/broadinstitute/cromwell/blob/47/centaur/src/main/resources/standardTestCases/write_lines.test#L3)) + ([47 link](https://github.com/broadinstitute/cromwell/blob/47/src/ci/bin/test.inc.sh#L440-L449)) - Engine Upgrade: Retrieves the [Cromwell Version](https://github.com/broadinstitute/cromwell/blob/47/project/Version.scala#L8) then retrieves the previous jar/docker-image from DockerHub. Centaur starts with the prior version, then restarts with the compiled source code. diff --git a/docs/filesystems/Filesystems.md b/docs/filesystems/Filesystems.md index 8ca03825c6e..ecde17f89a9 100644 --- a/docs/filesystems/Filesystems.md +++ b/docs/filesystems/Filesystems.md @@ -19,7 +19,7 @@ filesystems { class = "cromwell.filesystems.drs.DrsFileSystemConfig" config { resolver { - url = "https://martha-url-here or https://drshub-url-here" + url = https://drshub-url-here" # The number of times to retry failures connecting or HTTP 429 or HTTP 5XX responses, default 3. num-retries = 3 # How long to wait between retrying HTTP 429 or HTTP 5XX responses, default 10 seconds. diff --git a/docs/set_copyright.py b/docs/set_copyright.py new file mode 100644 index 00000000000..cb0cb6f0718 --- /dev/null +++ b/docs/set_copyright.py @@ -0,0 +1,3 @@ +from datetime import datetime +def on_config(config, **kwargs): + config.copyright = f"Copyright © {datetime.now().year} Broad Institute" diff --git a/docs/tutorials/Batch101.md b/docs/tutorials/Batch101.md index 4c31d910ab1..d62a4e9de1a 100644 --- a/docs/tutorials/Batch101.md +++ b/docs/tutorials/Batch101.md @@ -122,7 +122,7 @@ backend { providers { batch { - actor-factory = "cromwell.backend.google.pipelines.batch.GcpBatchBackendLifecycleActorFactory" + actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory" config { # Google project project = "my-cromwell-workflows" diff --git a/docs/tutorials/HPCSlurmWithLocalScratch.md b/docs/tutorials/HPCSlurmWithLocalScratch.md index 85bed57e531..fb5b95200d9 100644 --- a/docs/tutorials/HPCSlurmWithLocalScratch.md +++ b/docs/tutorials/HPCSlurmWithLocalScratch.md @@ -3,6 +3,8 @@ # Installing the Cromwell To Use Local Scratch Device #### These instructions are a community contribution +### In the process of being updated as of 2024-02 + In this section we will install the Cromwell Workflow Management system and configure it, so it can use the local scratch device on the compute nodes. (these installations are done in a ```centos 8``` enviornment) diff --git a/engine/src/main/scala/cromwell/Simpletons.scala b/engine/src/main/scala/cromwell/Simpletons.scala index 259e57a62e8..6ef40548438 100644 --- a/engine/src/main/scala/cromwell/Simpletons.scala +++ b/engine/src/main/scala/cromwell/Simpletons.scala @@ -12,13 +12,11 @@ import scala.util.Try * to `WdlSingleFile` instances. */ object Simpletons { - def toSimpleton(entry: CallCachingSimpletonEntry): WomValueSimpleton = { + def toSimpleton(entry: CallCachingSimpletonEntry): WomValueSimpleton = toSimpleton(entry.wdlType, entry.simpletonKey, entry.simpletonValue.toRawString) - } - def toSimpleton(entry: JobStoreSimpletonEntry): WomValueSimpleton = { + def toSimpleton(entry: JobStoreSimpletonEntry): WomValueSimpleton = toSimpleton(entry.wdlType, entry.simpletonKey, entry.simpletonValue.toRawString) - } private def toSimpleton(womType: String, simpletonKey: String, simpletonValue: String): WomValueSimpleton = { val womValue: String => Try[WomValue] = womType match { diff --git a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala index c648261a7f4..332664f1d56 100644 --- a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala +++ b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala @@ -19,17 +19,18 @@ object EngineFilesystems { .filter(_ => config.as[Boolean]("engine.filesystems.local.enabled")) .to(collection.immutable.SortedMap) - private val pathBuilderFactories: SortedMap[String, PathBuilderFactory] = { + private val pathBuilderFactories: SortedMap[String, PathBuilderFactory] = // Unordered maps are a classical source of randomness injection into a system ( - CromwellFileSystems.instance.factoriesFromConfig(config.as[Config]("engine")) + CromwellFileSystems.instance + .factoriesFromConfig(config.as[Config]("engine")) .unsafe("Failed to instantiate engine filesystem") ++ defaultFileSystemFactory ).to(collection.immutable.SortedMap) - } def configuredPathBuilderFactories: List[PathBuilderFactory] = pathBuilderFactories.values.toList - def pathBuildersForWorkflow(workflowOptions: WorkflowOptions, factories: List[PathBuilderFactory])(implicit as: ActorSystem): Future[List[PathBuilder]] = { + def pathBuildersForWorkflow(workflowOptions: WorkflowOptions, factories: List[PathBuilderFactory])(implicit + as: ActorSystem + ): Future[List[PathBuilder]] = PathBuilderFactory.instantiatePathBuilders(factories, workflowOptions) - } } diff --git a/engine/src/main/scala/cromwell/engine/EngineIoFunctions.scala b/engine/src/main/scala/cromwell/engine/EngineIoFunctions.scala index 6dcc487b8d6..d752c3b4d42 100644 --- a/engine/src/main/scala/cromwell/engine/EngineIoFunctions.scala +++ b/engine/src/main/scala/cromwell/engine/EngineIoFunctions.scala @@ -8,16 +8,23 @@ import better.files.File._ import scala.concurrent.{ExecutionContext, Future} -class EngineIoFunctions(val pathBuilders: List[PathBuilder], override val asyncIo: AsyncIo, override val ec: ExecutionContext) extends ReadLikeFunctions with WorkflowCorePathFunctions { - override def glob(pattern: String): Future[Seq[String]] = throw new UnsupportedOperationException(s"glob(path, pattern) not implemented yet") +class EngineIoFunctions(val pathBuilders: List[PathBuilder], + override val asyncIo: AsyncIo, + override val ec: ExecutionContext +) extends ReadLikeFunctions + with WorkflowCorePathFunctions { + override def glob(pattern: String): Future[Seq[String]] = throw new UnsupportedOperationException( + s"glob(path, pattern) not implemented yet" + ) // TODO: This is not suited for multi backend / multi filesystem use. Keep local for now to not break local CWL conf tests override def writeFile(path: String, content: String): Future[WomSingleFile] = Future.successful { val cromwellPath = buildPath(path) - val string = if (cromwellPath.isAbsolute) - cromwellPath.write(content).pathAsString - else - (newTemporaryDirectory() / path).write(content).pathAsString + val string = + if (cromwellPath.isAbsolute) + cromwellPath.write(content).pathAsString + else + (newTemporaryDirectory() / path).write(content).pathAsString WomSingleFile(string) } @@ -27,7 +34,9 @@ class EngineIoFunctions(val pathBuilders: List[PathBuilder], override val asyncI override def listAllFilesUnderDirectory(dirPath: String): Nothing = throw new UnsupportedOperationException(s"listAllFilesUnderDirectory not implemented yet") - override def listDirectory(path: String)(visited: Vector[String]) = throw new UnsupportedOperationException(s"listDirectory not implemented yet") + override def listDirectory(path: String)(visited: Vector[String]) = throw new UnsupportedOperationException( + s"listDirectory not implemented yet" + ) override def isDirectory(path: String) = Future.successful(buildPath(path).isDirectory) diff --git a/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala b/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala index ab062853a2b..99c93ba330b 100644 --- a/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala +++ b/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala @@ -13,7 +13,8 @@ case class EngineWorkflowDescriptor(topLevelCallable: Callable, failureMode: WorkflowFailureMode, pathBuilders: List[PathBuilder], callCachingMode: CallCachingMode, - parentWorkflow: Option[EngineWorkflowDescriptor] = None) { + parentWorkflow: Option[EngineWorkflowDescriptor] = None +) { val rootWorkflow: EngineWorkflowDescriptor = parentWorkflow match { case Some(parent) => parent.rootWorkflow diff --git a/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala b/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala index 410a6261b5c..74d652b741d 100644 --- a/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala +++ b/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala @@ -8,10 +8,11 @@ import scala.util.{Failure, Success, Try} case class BackendConfigurationEntry(name: String, lifecycleActorFactoryClass: String, config: Config) { def asBackendLifecycleActorFactory: Try[BackendLifecycleActorFactory] = Try { - Class.forName(lifecycleActorFactoryClass) - .getConstructor(classOf[String], classOf[BackendConfigurationDescriptor]) - .newInstance(name, asBackendConfigurationDescriptor) - .asInstanceOf[BackendLifecycleActorFactory] + Class + .forName(lifecycleActorFactoryClass) + .getConstructor(classOf[String], classOf[BackendConfigurationDescriptor]) + .newInstance(name, asBackendConfigurationDescriptor) + .asInstanceOf[BackendLifecycleActorFactory] } def asBackendConfigurationDescriptor = BackendConfigurationDescriptor(config, ConfigFactory.load) @@ -21,7 +22,8 @@ object BackendConfiguration { private val BackendConfig = ConfigFactory.load.getConfig("backend") private val DefaultBackendName = BackendConfig.getString("default") private val BackendProviders = BackendConfig.getConfig("providers") - private val BackendNames: Set[String] = BackendProviders.entrySet().asScala.map(_.getKey.split("\\.").toSeq.head).toSet + private val BackendNames: Set[String] = + BackendProviders.entrySet().asScala.map(_.getKey.split("\\.").toSeq.head).toSet val AllBackendEntries: List[BackendConfigurationEntry] = BackendNames.toList map { backendName => val entry = BackendProviders.getConfig(backendName) @@ -33,14 +35,17 @@ object BackendConfiguration { } val DefaultBackendEntry: BackendConfigurationEntry = AllBackendEntries.find(_.name == DefaultBackendName) getOrElse { - throw new IllegalArgumentException(s"Could not find specified default backend name '$DefaultBackendName' " + - s"in '${BackendNames.mkString("', '")}'.") + throw new IllegalArgumentException( + s"Could not find specified default backend name '$DefaultBackendName' " + + s"in '${BackendNames.mkString("', '")}'." + ) } - def backendConfigurationDescriptor(backendName: String): Try[BackendConfigurationDescriptor] = { - AllBackendEntries.collect({case entry if entry.name.equalsIgnoreCase(backendName) => entry.asBackendConfigurationDescriptor}).headOption match { + def backendConfigurationDescriptor(backendName: String): Try[BackendConfigurationDescriptor] = + AllBackendEntries.collect { + case entry if entry.name.equalsIgnoreCase(backendName) => entry.asBackendConfigurationDescriptor + }.headOption match { case Some(descriptor) => Success(descriptor) case None => Failure(new Exception(s"invalid backend: $backendName")) } - } } diff --git a/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala b/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala index 24eb59d2d95..09acd74476a 100644 --- a/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala +++ b/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala @@ -11,11 +11,11 @@ import cromwell.backend.BackendLifecycleActorFactory case class CromwellBackends(backendEntries: List[BackendConfigurationEntry]) { // Raise the exception here if some backend factories failed to instantiate - val backendLifecycleActorFactories = TryUtil.sequenceMap(backendEntries.map(e => e.name -> e.asBackendLifecycleActorFactory).toMap).get + val backendLifecycleActorFactories = + TryUtil.sequenceMap(backendEntries.map(e => e.name -> e.asBackendLifecycleActorFactory).toMap).get - def backendLifecycleActorFactoryByName(backendName: String): ErrorOr[BackendLifecycleActorFactory] = { + def backendLifecycleActorFactoryByName(backendName: String): ErrorOr[BackendLifecycleActorFactory] = backendLifecycleActorFactories.get(backendName).toValidNel(s"Backend $backendName was not found") - } def isValidBackendName(name: String): Boolean = backendLifecycleActorFactories.contains(name) } @@ -24,21 +24,17 @@ object CromwellBackends { var instance: Option[CromwellBackends] = None - def isValidBackendName(name: String): Boolean = evaluateIfInitialized(_.isValidBackendName(name)) - def backendLifecycleFactoryActorByName(backendName: String): ErrorOr[BackendLifecycleActorFactory] = { + def backendLifecycleFactoryActorByName(backendName: String): ErrorOr[BackendLifecycleActorFactory] = evaluateIfInitialized(_.backendLifecycleActorFactoryByName(backendName)) - } - private def evaluateIfInitialized[A](func: CromwellBackends => A): A = { + private def evaluateIfInitialized[A](func: CromwellBackends => A): A = instance match { case Some(cromwellBackend) => func(cromwellBackend) case None => throw new Exception("Cannot use CromwellBackend until initBackends is called") } - } - def initBackends(backendEntries: List[BackendConfigurationEntry]): Unit = { + def initBackends(backendEntries: List[BackendConfigurationEntry]): Unit = instance = Option(CromwellBackends(backendEntries)) - } } diff --git a/engine/src/main/scala/cromwell/engine/engine.scala b/engine/src/main/scala/cromwell/engine/engine.scala index 1c282aab703..e5971622beb 100644 --- a/engine/src/main/scala/cromwell/engine/engine.scala +++ b/engine/src/main/scala/cromwell/engine/engine.scala @@ -13,7 +13,9 @@ final case class CallAttempt(fqn: FullyQualifiedName, attempt: Int) object WorkflowFailureMode { def tryParse(mode: String): Try[WorkflowFailureMode] = { val modes = Seq(ContinueWhilePossible, NoNewCalls) - modes find { _.toString.equalsIgnoreCase(mode) } map { Success(_) } getOrElse Failure(new Exception(s"Invalid workflow failure mode: $mode")) + modes find { _.toString.equalsIgnoreCase(mode) } map { Success(_) } getOrElse Failure( + new Exception(s"Invalid workflow failure mode: $mode") + ) } } sealed trait WorkflowFailureMode { diff --git a/engine/src/main/scala/cromwell/engine/instrumentation/HttpInstrumentation.scala b/engine/src/main/scala/cromwell/engine/instrumentation/HttpInstrumentation.scala index 2c909e1f776..aa65d6506be 100644 --- a/engine/src/main/scala/cromwell/engine/instrumentation/HttpInstrumentation.scala +++ b/engine/src/main/scala/cromwell/engine/instrumentation/HttpInstrumentation.scala @@ -18,29 +18,31 @@ object HttpInstrumentation { } trait HttpInstrumentation extends CromwellInstrumentation { - - private def makeRequestPath(httpRequest: HttpRequest, httpResponse: HttpResponse): InstrumentationPath = NonEmptyList.of( - // Returns the path of the URI only, without query parameters (e.g: api/engine/workflows/metadata) - httpRequest.uri.path.toString().stripPrefix("/") - // Replace UUIDs with [id] to keep paths same regardless of the workflow - .replaceAll(HttpInstrumentation.UUIDRegex, "[id]"), - // Name of the method (e.g: GET) - httpRequest.method.value, - // Status code of the Response (e.g: 200) - httpResponse.status.intValue.toString - ) - private def sendTimingApi(statsDPath: InstrumentationPath, timing: FiniteDuration) = { + private def makeRequestPath(httpRequest: HttpRequest, httpResponse: HttpResponse): InstrumentationPath = + NonEmptyList.of( + // Returns the path of the URI only, without query parameters (e.g: api/engine/workflows/metadata) + httpRequest.uri.path + .toString() + .stripPrefix("/") + // Replace UUIDs with [id] to keep paths same regardless of the workflow + .replaceAll(HttpInstrumentation.UUIDRegex, "[id]"), + // Name of the method (e.g: GET) + httpRequest.method.value, + // Status code of the Response (e.g: 200) + httpResponse.status.intValue.toString + ) + + private def sendTimingApi(statsDPath: InstrumentationPath, timing: FiniteDuration) = sendTiming(statsDPath, timing, ApiPrefix) - } def instrumentRequest: Directive0 = extractRequest flatMap { request => val timeStamp = System.currentTimeMillis mapResponse { response => /* - * Send a metric corresponding to the request response time. - * Note: The current StatsD implementation always pairs a timing metric with a counter metric - * So no need to explicitly send one + * Send a metric corresponding to the request response time. + * Note: The current StatsD implementation always pairs a timing metric with a counter metric + * So no need to explicitly send one */ sendTimingApi(makeRequestPath(request, response), (System.currentTimeMillis - timeStamp).millis) response diff --git a/engine/src/main/scala/cromwell/engine/instrumentation/IoInstrumentation.scala b/engine/src/main/scala/cromwell/engine/instrumentation/IoInstrumentation.scala index 15d799d222b..ba0c4591c91 100644 --- a/engine/src/main/scala/cromwell/engine/instrumentation/IoInstrumentation.scala +++ b/engine/src/main/scala/cromwell/engine/instrumentation/IoInstrumentation.scala @@ -27,6 +27,7 @@ private object IoInstrumentationImplicits { * Augments IoResult to provide instrumentation conversion methods */ implicit class InstrumentedIoResult(val ioResult: IoResult) extends AnyVal { + /** * Returns the instrumentation path of this IoResult */ @@ -34,6 +35,7 @@ private object IoInstrumentationImplicits { case (_: IoSuccess[_], ioCommandContext) => ioCommandContext.request.successPath case (f: IoFailAck[_], ioCommandContext) => ioCommandContext.request.failedPath(f.failure) } + /** * Returns the instrumentation path of this IoResult */ @@ -44,19 +46,22 @@ private object IoInstrumentationImplicits { * Augments IoCommand to provide instrumentation conversion methods */ implicit class InstrumentedIoCommand(val ioCommand: IoCommand[_]) extends AnyVal { + /** * Returns the instrumentation path of this IoCommand */ def toPath: InstrumentationPath = { val path = ioCommand match { - case copy: IoCopyCommand => (copy.source, copy.destination) match { - case (_: GcsPath, _) | (_, _: GcsPath) => GcsPath - case _ => LocalPath - } - case singleFileCommand: SingleFileIoCommand[_] => singleFileCommand.file match { - case _: GcsPath => GcsPath - case _ => LocalPath - } + case copy: IoCopyCommand => + (copy.source, copy.destination) match { + case (_: GcsPath, _) | (_, _: GcsPath) => GcsPath + case _ => LocalPath + } + case singleFileCommand: SingleFileIoCommand[_] => + singleFileCommand.file match { + case _: GcsPath => GcsPath + case _ => LocalPath + } case _ => UnknownFileSystemPath } @@ -71,16 +76,14 @@ private object IoInstrumentationImplicits { /** * Returns a failed instrumentation path for this IoCommand provided a throwable */ - def failedPath(failure: Throwable): InstrumentationPath = { + def failedPath(failure: Throwable): InstrumentationPath = ioCommand.toPath.concatNel(FailureKey).withStatusCodeFailure(GoogleUtil.extractStatusCode(failure)) - } /** * Returns a retried instrumentation path for this IoCommand provided a throwable */ - def retriedPath(failure: Throwable): InstrumentationPath = { + def retriedPath(failure: Throwable): InstrumentationPath = ioCommand.toPath.concatNel(RetryKey).withStatusCodeFailure(GoogleUtil.extractStatusCode(failure)) - } } } @@ -100,7 +103,9 @@ trait IoInstrumentation extends CromwellInstrumentationActor { this: Actor => */ final def instrumentIoResult(ioResult: IoResult): Unit = { incrementIo(ioResult.toCounterPath) - sendTiming(ioResult.toDurationPath, (OffsetDateTime.now.toEpochSecond - ioResult._2.creationTime.toEpochSecond).seconds) + sendTiming(ioResult.toDurationPath, + (OffsetDateTime.now.toEpochSecond - ioResult._2.creationTime.toEpochSecond).seconds + ) } final def incrementBackpressure(): Unit = incrementIo(backpressure) @@ -108,5 +113,7 @@ trait IoInstrumentation extends CromwellInstrumentationActor { this: Actor => /** * Increment an IoCommand to the proper bucket depending on the request type. */ - final def incrementIoRetry(ioCommand: IoCommand[_], failure: Throwable): Unit = incrementIo(ioCommand.retriedPath(failure)) + final def incrementIoRetry(ioCommand: IoCommand[_], failure: Throwable): Unit = incrementIo( + ioCommand.retriedPath(failure) + ) } diff --git a/engine/src/main/scala/cromwell/engine/instrumentation/JobInstrumentation.scala b/engine/src/main/scala/cromwell/engine/instrumentation/JobInstrumentation.scala index 91613a7c2cb..49e479e7b3d 100644 --- a/engine/src/main/scala/cromwell/engine/instrumentation/JobInstrumentation.scala +++ b/engine/src/main/scala/cromwell/engine/instrumentation/JobInstrumentation.scala @@ -35,25 +35,21 @@ trait JobInstrumentation extends CromwellInstrumentationActor { this: Actor => /** * Generic method to add a workflow related timing metric value */ - def setTimingJob(statsDPath: InstrumentationPath, duration: FiniteDuration): Unit = { + def setTimingJob(statsDPath: InstrumentationPath, duration: FiniteDuration): Unit = sendTiming(statsDPath, duration, JobPrefix) - } /** * Generic method to update a job related gauge metric value */ - def sendGaugeJob(statsDPath: InstrumentationPath, value: Long): Unit = { + def sendGaugeJob(statsDPath: InstrumentationPath, value: Long): Unit = sendGauge(statsDPath, value, JobPrefix) - } /** * Add a timing value for the run time of a job in a given state */ - def setJobTimePerState(response: BackendJobExecutionResponse, duration: FiniteDuration): Unit = { + def setJobTimePerState(response: BackendJobExecutionResponse, duration: FiniteDuration): Unit = setTimingJob(backendJobExecutionResponsePaths(response), duration) - } - - def recordExecutionStepTiming(state: String, duration: FiniteDuration): Unit = { + + def recordExecutionStepTiming(state: String, duration: FiniteDuration): Unit = sendTiming(jobTimingKey.concatNel("state").concatNel(state), duration, JobPrefix) - } } diff --git a/engine/src/main/scala/cromwell/engine/instrumentation/WorkflowInstrumentation.scala b/engine/src/main/scala/cromwell/engine/instrumentation/WorkflowInstrumentation.scala index ac9d28a9eb6..a9272bcf7d3 100644 --- a/engine/src/main/scala/cromwell/engine/instrumentation/WorkflowInstrumentation.scala +++ b/engine/src/main/scala/cromwell/engine/instrumentation/WorkflowInstrumentation.scala @@ -13,8 +13,9 @@ import scala.concurrent.duration.FiniteDuration import scala.language.postfixOps object WorkflowInstrumentation { - private val WorkflowStatePaths: Map[WorkflowState, InstrumentationPath] = WorkflowState.WorkflowStateValues map { state => - state -> NonEmptyList.of(state.toString) + private val WorkflowStatePaths: Map[WorkflowState, InstrumentationPath] = WorkflowState.WorkflowStateValues map { + state => + state -> NonEmptyList.of(state.toString) } toMap // Use "Queued" instead of "Submitted" as it seems to reflect better the actual state @@ -28,43 +29,39 @@ object WorkflowInstrumentation { * Provides helper methods for workflow instrumentation */ trait WorkflowInstrumentation extends CromwellInstrumentationActor { this: Actor => - private def workflowStatePath(workflowState: WorkflowState): InstrumentationPath = WorkflowInstrumentation.WorkflowStatePaths(workflowState) + private def workflowStatePath(workflowState: WorkflowState): InstrumentationPath = + WorkflowInstrumentation.WorkflowStatePaths(workflowState) /** * Generic method to increment a workflow related counter metric value */ - def incrementWorkflow(statsDPath: InstrumentationPath): Unit = { + def incrementWorkflow(statsDPath: InstrumentationPath): Unit = increment(statsDPath, WorkflowPrefix) - } /** * Generic method to add a workflow related timing metric value */ - def setTimingWorkflow(statsDPath: InstrumentationPath, duration: FiniteDuration): Unit = { + def setTimingWorkflow(statsDPath: InstrumentationPath, duration: FiniteDuration): Unit = sendTiming(statsDPath, duration, WorkflowPrefix) - } /** * Generic method to update a workflow related gauge metric value */ - def sendGaugeWorkflow(statsDPath: InstrumentationPath, value: Long): Unit = { + def sendGaugeWorkflow(statsDPath: InstrumentationPath, value: Long): Unit = sendGauge(statsDPath, value, WorkflowPrefix) - } /** * Counts every time a workflow enters a given state */ - def incrementWorkflowState(workflowState: WorkflowState): Unit = { + def incrementWorkflowState(workflowState: WorkflowState): Unit = incrementWorkflow(workflowStatePath(workflowState)) - } /** * Add a timing value for the run time of a workflow in a given state */ - //* TODO: enforce a terminal state ? - def setWorkflowTimePerState(workflowState: WorkflowState, duration: FiniteDuration): Unit = { + // * TODO: enforce a terminal state ? + def setWorkflowTimePerState(workflowState: WorkflowState, duration: FiniteDuration): Unit = setTimingWorkflow(workflowStatePath(workflowState), duration) - } /** * Set the current number of submitted workflows (queued but not running) diff --git a/engine/src/main/scala/cromwell/engine/io/IoActor.scala b/engine/src/main/scala/cromwell/engine/io/IoActor.scala index b4b3a0b191f..5d69ba844a0 100644 --- a/engine/src/main/scala/cromwell/engine/io/IoActor.scala +++ b/engine/src/main/scala/cromwell/engine/io/IoActor.scala @@ -27,7 +27,6 @@ import java.time.temporal.ChronoUnit import scala.concurrent.ExecutionContext import scala.concurrent.duration._ - /** * Actor that performs IO operations asynchronously using akka streams * @@ -35,26 +34,28 @@ import scala.concurrent.duration._ * @param materializer actor materializer to run the stream * @param serviceRegistryActor actorRef for the serviceRegistryActor */ -final class IoActor(ioConfig: IoConfig, - override val serviceRegistryActor: ActorRef, - applicationName: String)(implicit val materializer: ActorMaterializer) - extends Actor with ActorLogging with StreamActorHelper[IoCommandContext[_]] with IoInstrumentation with Timers { +final class IoActor(ioConfig: IoConfig, override val serviceRegistryActor: ActorRef, applicationName: String)(implicit + val materializer: ActorMaterializer +) extends Actor + with ActorLogging + with StreamActorHelper[IoCommandContext[_]] + with IoInstrumentation + with Timers { implicit val ec: ExecutionContext = context.dispatcher implicit val system: ActorSystem = context.system // IntelliJ disapproves of mutable state in Actors, but this should be safe as long as access occurs only in // the `receive` method. Alternatively IntelliJ does suggest a `become` workaround we might try in the future. - //noinspection ActorMutableStateInspection + // noinspection ActorMutableStateInspection private var backpressureExpiration: Option[OffsetDateTime] = None /** * Method for instrumentation to be executed when a IoCommand failed and is being retried. * Can be passed to flows so they can invoke it when necessary. */ - private def onRetry(commandContext: IoCommandContext[_])(throwable: Throwable): Unit = { + private def onRetry(commandContext: IoCommandContext[_])(throwable: Throwable): Unit = incrementIoRetry(commandContext.request, throwable) - } override def preStart(): Unit = { // On start up, let the controller know that the load is normal @@ -62,25 +63,25 @@ final class IoActor(ioConfig: IoConfig, super.preStart() } - private [io] lazy val defaultFlow = + private[io] lazy val defaultFlow = new NioFlow( parallelism = ioConfig.nio.parallelism, onRetryCallback = onRetry, onBackpressure = onBackpressure, numberOfAttempts = ioConfig.numberOfAttempts, - commandBackpressureStaleness = ioConfig.commandBackpressureStaleness) - .flow + commandBackpressureStaleness = ioConfig.commandBackpressureStaleness + ).flow .withAttributes(ActorAttributes.dispatcher(Dispatcher.IoDispatcher)) - private [io] lazy val gcsBatchFlow = + private[io] lazy val gcsBatchFlow = new ParallelGcsBatchFlow( config = ioConfig.gcsBatch, scheduler = context.system.scheduler, onRetry = onRetry, onBackpressure = onBackpressure, applicationName = applicationName, - commandBackpressureStaleness = ioConfig.commandBackpressureStaleness) - .flow + commandBackpressureStaleness = ioConfig.commandBackpressureStaleness + ).flow .withAttributes(ActorAttributes.dispatcher(Dispatcher.IoDispatcher)) private val source = Source.queue[IoCommandContext[_]](ioConfig.queueSize, OverflowStrategy.dropNew) @@ -91,10 +92,14 @@ final class IoActor(ioConfig: IoConfig, val input = builder.add(Flow[IoCommandContext[_]]) // Partitions requests between gcs batch, and single nio requests - val batchPartitioner = builder.add(Partition[IoCommandContext[_]](2, { - case _: GcsBatchCommandContext[_, _] => 0 - case _ => 1 - })) + val batchPartitioner = builder.add( + Partition[IoCommandContext[_]](2, + { + case _: GcsBatchCommandContext[_, _] => 0 + case _ => 1 + } + ) + ) // Sub flow for batched gcs requests val batches = batchPartitioner.out(0) collect { case batch: GcsBatchCommandContext[_, _] => batch } @@ -112,8 +117,8 @@ final class IoActor(ioConfig: IoConfig, val batchFlowPorts = builder.add(gcsBatchFlow) input ~> batchPartitioner - defaults.outlet ~> defaultFlowPorts ~> merger - batches.outlet ~> batchFlowPorts ~> merger + defaults.outlet ~> defaultFlowPorts ~> merger + batches.outlet ~> batchFlowPorts ~> merger FlowShape[IoCommandContext[_], IoResult](input.in, merger.out) } @@ -136,6 +141,7 @@ final class IoActor(ioConfig: IoConfig, override def onBackpressure(scale: Option[Double] = None): Unit = { incrementBackpressure() + log.warning("IoActor notifying HighLoad") serviceRegistryActor ! LoadMetric("IO", HighLoad) val uncappedDelay = scale.getOrElse(1.0d) * LoadConfig.IoNormalWindowMinimum @@ -148,11 +154,11 @@ final class IoActor(ioConfig: IoConfig, /* GCS Batch command with context */ case (clientContext: Any, gcsBatchCommand: GcsBatchIoCommand[_, _]) => val replyTo = sender() - val commandContext = GcsBatchCommandContext( - request = gcsBatchCommand, - maxAttemptsNumber = ioConfig.numberOfAttempts, - replyTo = replyTo, - clientContext = Option(clientContext)) + val commandContext = GcsBatchCommandContext(request = gcsBatchCommand, + maxAttemptsNumber = ioConfig.numberOfAttempts, + replyTo = replyTo, + clientContext = Option(clientContext) + ) sendToStream(commandContext) /* GCS Batch command without context */ @@ -201,7 +207,10 @@ final class IoActor(ioConfig: IoConfig, } val newExpiration = OffsetDateTime.now().until(proposedExpiry, ChronoUnit.MILLIS) - timers.startSingleTimer(BackPressureTimerResetKey, BackPressureTimerResetAction, FiniteDuration(newExpiration, MILLISECONDS)) + timers.startSingleTimer(BackPressureTimerResetKey, + BackPressureTimerResetAction, + FiniteDuration(newExpiration, MILLISECONDS) + ) backpressureExpiration = Option(proposedExpiry) case _ => // Ignore proposed expiries that would be before the current expiry @@ -219,7 +228,8 @@ trait IoCommandContext[T] extends StreamContext { def request: IoCommand[T] def replyTo: ActorRef def fail(failure: Throwable): IoResult = (request.fail(failure), this) - def failReadForbidden(failure: Throwable, forbiddenPath: String): IoResult = (request.failReadForbidden(failure, forbiddenPath), this) + def failReadForbidden(failure: Throwable, forbiddenPath: String): IoResult = + (request.failReadForbidden(failure, forbiddenPath), this) def success(value: T): IoResult = (request.success(value), this) } @@ -233,7 +243,10 @@ object IoActor { /** Result type of an IoFlow, contains the original command context and the final IoAck response. */ type IoResult = (IoAck[_], IoCommandContext[_]) - case class DefaultCommandContext[T](request: IoCommand[T], replyTo: ActorRef, override val clientContext: Option[Any] = None) extends IoCommandContext[T] + case class DefaultCommandContext[T](request: IoCommand[T], + replyTo: ActorRef, + override val clientContext: Option[Any] = None + ) extends IoCommandContext[T] case object BackPressureTimerResetKey @@ -241,13 +254,10 @@ object IoActor { case class BackPressure(duration: FiniteDuration) extends ControlMessage - def props(ioConfig: IoConfig, - serviceRegistryActor: ActorRef, - applicationName: String, - ) - (implicit materializer: ActorMaterializer): Props = { + def props(ioConfig: IoConfig, serviceRegistryActor: ActorRef, applicationName: String)(implicit + materializer: ActorMaterializer + ): Props = Props(new IoActor(ioConfig, serviceRegistryActor, applicationName)).withDispatcher(IoDispatcher) - } case class IoConfig(queueSize: Int, numberOfAttempts: Int, @@ -257,7 +267,8 @@ object IoActor { ioNormalWindowMaximum: FiniteDuration, nio: NioFlowConfig, gcsBatch: GcsBatchFlowConfig, - throttle: Option[Throttle]) + throttle: Option[Throttle] + ) implicit val ioConfigReader: ValueReader[IoConfig] = (config: Config, _: String) => { diff --git a/engine/src/main/scala/cromwell/engine/io/IoActorProxy.scala b/engine/src/main/scala/cromwell/engine/io/IoActorProxy.scala index 5491440a979..c8ed10ca86b 100644 --- a/engine/src/main/scala/cromwell/engine/io/IoActorProxy.scala +++ b/engine/src/main/scala/cromwell/engine/io/IoActorProxy.scala @@ -14,7 +14,8 @@ object IoActorProxy { } class IoActorProxy(ioActor: ActorRef) extends Actor with ActorLogging with GracefulShutdownHelper { - private val cache = CacheBuilder.newBuilder() + private val cache = CacheBuilder + .newBuilder() .build[IoCommandWithPromise[_], IoResult]() private val ioPromiseProxyActor: ActorRef = context.actorOf(IoPromiseProxyActor.props(ioActor), "IoPromiseProxyActor") @@ -31,7 +32,7 @@ class IoActorProxy(ioActor: ActorRef) extends Actor with ActorLogging with Grace case ioCommand: IoCommand[_] => ioActor forward ioCommand case withContext: (Any, IoCommand[_]) @unchecked => ioActor forward withContext - case ShutdownCommand => + case ShutdownCommand => context stop ioPromiseProxyActor waitForActorsAndShutdown(NonEmptyList.one(ioActor)) } diff --git a/engine/src/main/scala/cromwell/engine/io/IoAttempts.scala b/engine/src/main/scala/cromwell/engine/io/IoAttempts.scala index d43e003ecb1..a51112029e9 100644 --- a/engine/src/main/scala/cromwell/engine/io/IoAttempts.scala +++ b/engine/src/main/scala/cromwell/engine/io/IoAttempts.scala @@ -7,18 +7,18 @@ import org.apache.commons.lang3.exception.ExceptionUtils object IoAttempts { object EnhancedCromwellIoException { - def apply[S](state: S, cause: Throwable)(implicit showState: Show[S]): EnhancedCromwellIoException = { + def apply[S](state: S, cause: Throwable)(implicit showState: Show[S]): EnhancedCromwellIoException = EnhancedCromwellIoException(s"[${showState.show(state)}] - ${ExceptionUtils.getMessage(cause)}", cause) - } } - - case class EnhancedCromwellIoException(message: String, cause: Throwable) - extends Throwable(message, cause, true, false) with CromwellFatalExceptionMarker - + + case class EnhancedCromwellIoException(message: String, cause: Throwable) + extends Throwable(message, cause, true, false) + with CromwellFatalExceptionMarker + implicit val showState = new Show[IoAttempts] { override def show(t: IoAttempts) = s"Attempted ${t.attempts} time(s)" } - + implicit val stateToThrowable = new StatefulIoError[IoAttempts] { override def toThrowable(state: IoAttempts, throwable: Throwable) = { state.throwables.foreach(throwable.addSuppressed) @@ -26,9 +26,8 @@ object IoAttempts { } } - val updateState: (Throwable, IoAttempts) => IoAttempts = (throwable, state) => { + val updateState: (Throwable, IoAttempts) => IoAttempts = (throwable, state) => state.copy(attempts = state.attempts + 1, throwables = state.throwables :+ throwable) - } } case class IoAttempts(attempts: Int, throwables: List[Throwable] = List.empty) diff --git a/engine/src/main/scala/cromwell/engine/io/IoCommandStalenessBackpressuring.scala b/engine/src/main/scala/cromwell/engine/io/IoCommandStalenessBackpressuring.scala index e8ceda43656..47591516d7b 100644 --- a/engine/src/main/scala/cromwell/engine/io/IoCommandStalenessBackpressuring.scala +++ b/engine/src/main/scala/cromwell/engine/io/IoCommandStalenessBackpressuring.scala @@ -19,17 +19,18 @@ trait IoCommandStalenessBackpressuring extends StrictLogging { val seconds = millis / 1000.0 logger.info("I/O command {} seconds stale, applying I/O subsystem backpressure with scale {}", - f"$seconds%,.3f", f"$scale%.2f") + f"$seconds%,.3f", + f"$scale%.2f" + ) onBackpressure(Option(scale)) } /** Invokes `onBackpressure` if `ioCommand` is older than the staleness limit returned by `maxStaleness`. */ - def backpressureIfStale(ioCommand: IoCommand[_], onBackpressure: Option[Double] => Unit): Unit = { + def backpressureIfStale(ioCommand: IoCommand[_], onBackpressure: Option[Double] => Unit): Unit = if (ioCommand.creation.isBefore(commandStalenessThreshold)) { logAndBackpressure(ioCommand, onBackpressure) } - } /** Invokes `onBackpressure` if at least one IoCommandContext in `contexts` is older than the * staleness limit returned by `maxStaleness`. */ diff --git a/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala b/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala index d2aa427a567..1a75bdd4c15 100644 --- a/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala +++ b/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala @@ -15,17 +15,19 @@ object RetryableRequestSupport { * The default count is `5` and may be customized with `system.io.number-of-attempts`. */ def isRetryable(failure: Throwable): Boolean = failure match { - case gcs: StorageException => gcs.isRetryable || + case gcs: StorageException => + gcs.isRetryable || isRetryable(gcs.getCause) || AdditionalRetryableHttpCodes.contains(gcs.getCode) || - Option(gcs.getMessage).exists(msg => - AdditionalRetryableErrorMessages.contains(msg.toLowerCase)) + Option(gcs.getMessage).exists(msg => AdditionalRetryableErrorMessages.contains(msg.toLowerCase)) case _: SSLException => true case _: BatchFailedException => true case _: ChecksumFailedException => true case _: SocketException => true case _: SocketTimeoutException => true - case ioE: IOException if Option(ioE.getMessage).exists(_.contains("Error getting access token for service account")) => true + case ioE: IOException + if Option(ioE.getMessage).exists(_.contains("Error getting access token for service account")) => + true case ioE: IOException => isGcs500(ioE) || isGcs503(ioE) || isGcs504(ioE) case other => // Infinitely retryable is a subset of retryable @@ -68,24 +70,21 @@ object RetryableRequestSupport { isGcsRateLimitException(failure) } - def isGcs500(failure: Throwable): Boolean = { + def isGcs500(failure: Throwable): Boolean = Option(failure.getMessage).exists(msg => msg.contains("Could not read from gs") && - msg.contains("500 Internal Server Error") + msg.contains("500 Internal Server Error") ) - } - def isGcs503(failure: Throwable): Boolean = { + def isGcs503(failure: Throwable): Boolean = Option(failure.getMessage).exists(msg => msg.contains("Could not read from gs") && - msg.contains("503 Service Unavailable") + msg.contains("503 Service Unavailable") ) - } - def isGcs504(failure: Throwable): Boolean = { + def isGcs504(failure: Throwable): Boolean = Option(failure.getMessage).exists(msg => msg.contains("Could not read from gs") && - msg.contains("504 Gateway Timeout") + msg.contains("504 Gateway Timeout") ) - } } diff --git a/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchCommandContext.scala b/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchCommandContext.scala index d41b0539dc6..27f6afe4299 100644 --- a/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchCommandContext.scala +++ b/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchCommandContext.scala @@ -34,7 +34,7 @@ object GcsBatchCommandContext { .setInitialIntervalMillis(1.second.toMillis.toInt) .setMultiplier(4) .setMaxIntervalMillis(30.seconds.toMillis.toInt) - .setRandomizationFactor(0.2D) + .setRandomizationFactor(0.2d) .setMaxElapsedTimeMillis(30.minutes.toMillis.toInt) .build() ) @@ -48,14 +48,14 @@ final case class GcsBatchCommandContext[T, U](request: GcsBatchIoCommand[T, U], backoff: Backoff = GcsBatchCommandContext.defaultBackoff, currentAttempt: Int = 1, promise: Promise[BatchResponse] = Promise[BatchResponse]() - ) - extends IoCommandContext[T] - with StrictLogging { +) extends IoCommandContext[T] + with StrictLogging { /** * None if no retry should be attempted, Some(timeToWaitBeforeNextAttempt) otherwise */ - lazy val retryIn: Option[FiniteDuration] = if (currentAttempt >= maxAttemptsNumber) None else Option(backoff.backoffMillis milliseconds) + lazy val retryIn: Option[FiniteDuration] = + if (currentAttempt >= maxAttemptsNumber) None else Option(backoff.backoffMillis milliseconds) /** * Json batch call back for a batched request @@ -63,19 +63,20 @@ final case class GcsBatchCommandContext[T, U](request: GcsBatchIoCommand[T, U], lazy val callback: JsonBatchCallback[U] = new JsonBatchCallback[U]() { // These callbacks are only called once, therefore it's imperative that they set the promise value before exiting. // This tryCallbackOrFail ensures that if the callback function itself errors, we get _some_ result back on the future. - def onSuccess(response: U, httpHeaders: HttpHeaders): Unit = tryCallbackOrFail("onSuccessCallback", onSuccessCallback(response, httpHeaders)) - def onFailure(googleJsonError: GoogleJsonError, httpHeaders: HttpHeaders): Unit = tryCallbackOrFail(callbackName = "onFailureCallback", onFailureCallback(googleJsonError, httpHeaders)) + def onSuccess(response: U, httpHeaders: HttpHeaders): Unit = + tryCallbackOrFail("onSuccessCallback", onSuccessCallback(response, httpHeaders)) + def onFailure(googleJsonError: GoogleJsonError, httpHeaders: HttpHeaders): Unit = + tryCallbackOrFail(callbackName = "onFailureCallback", onFailureCallback(googleJsonError, httpHeaders)) } def tryCallbackOrFail(callbackName: String, callback: () => Unit): Unit = { Try { callback.apply() - }.recover { - case t => - // Ideally we would catch and handle the cases which might lead us here before they actually get this far: - logger.error(s"Programmer Error: Error processing IO response in $callbackName", t) - promise.tryFailure(new Exception(s"Error processing IO response in $callbackName: ${t.getMessage}")) - () + }.recover { case t => + // Ideally we would catch and handle the cases which might lead us here before they actually get this far: + logger.error(s"Programmer Error: Error processing IO response in $callbackName", t) + promise.tryFailure(new Exception(s"Error processing IO response in $callbackName: ${t.getMessage}")) + () } () } @@ -83,16 +84,14 @@ final case class GcsBatchCommandContext[T, U](request: GcsBatchIoCommand[T, U], /** * Increment backoff time and attempt count */ - lazy val next: GcsBatchCommandContext[T, U] = { + lazy val next: GcsBatchCommandContext[T, U] = this.copy(backoff = backoff.next, currentAttempt = currentAttempt + 1, promise = Promise[BatchResponse]()) - } /** * Only increment backoff. To be used for failures that should be retried infinitely */ - lazy val nextTransient: GcsBatchCommandContext[T, U] = { + lazy val nextTransient: GcsBatchCommandContext[T, U] = this.copy(backoff = backoff.next, promise = Promise[BatchResponse]()) - } /** * Queue the request for batching @@ -106,7 +105,9 @@ final case class GcsBatchCommandContext[T, U](request: GcsBatchIoCommand[T, U], * On success callback. Transform the request response to a stream-ready response that can complete the promise */ private def onSuccessCallback(response: U, httpHeaders: HttpHeaders)(): Unit = { - request.logIOMsgOverLimit(s"GcsBatchCommandContext.onSuccessCallback '${response.toPrettyElidedString(limit = 1000)}'") + request.logIOMsgOverLimit( + s"GcsBatchCommandContext.onSuccessCallback '${response.toPrettyElidedString(limit = 1000)}'" + ) handleSuccessOrNextRequest(request.onSuccess(response, httpHeaders)) } @@ -131,7 +132,9 @@ final case class GcsBatchCommandContext[T, U](request: GcsBatchIoCommand[T, U], * On failure callback. Fail the promise with a StorageException */ private def onFailureCallback(googleJsonError: GoogleJsonError, httpHeaders: HttpHeaders)(): Unit = { - request.logIOMsgOverLimit(s"GcsBatchCommandContext.onFailureCallback '${googleJsonError.toPrettyElidedString(limit = 1000)}'") + request.logIOMsgOverLimit( + s"GcsBatchCommandContext.onFailureCallback '${googleJsonError.toPrettyElidedString(limit = 1000)}'" + ) if (isProjectNotProvidedError(googleJsonError)) { // Returning an Either.Right here means that the operation is not complete and that we need to do another request handleSuccessOrNextRequest(Right(request.withUserProject).validNel) diff --git a/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchFlow.scala b/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchFlow.scala index 0007b77a42f..972a52530de 100644 --- a/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchFlow.scala +++ b/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchFlow.scala @@ -35,7 +35,8 @@ object GcsBatchFlow { */ case class BatchFailedException(failure: Throwable) extends IOException(failure) - private val ReadForbiddenPattern = ".*does not have storage\\.objects\\.(?:get|list|copy) access to ([^/]+).*".r.pattern + private val ReadForbiddenPattern = + ".*does not have storage\\.objects\\.(?:get|list|copy) access to ([^/]+).*".r.pattern /* Returns `Some(bucket)` if the specified argument represents a forbidden attempt to read from `bucket`. */ private[gcs] def getReadForbiddenBucket(errorMsg: String): Option[String] = { @@ -60,8 +61,9 @@ class GcsBatchFlow(batchSize: Int, onRetry: IoCommandContext[_] => Throwable => Unit, onBackpressure: Option[Double] => Unit, applicationName: String, - backpressureStaleness: FiniteDuration) - (implicit ec: ExecutionContext) extends IoCommandStalenessBackpressuring { + backpressureStaleness: FiniteDuration +)(implicit ec: ExecutionContext) + extends IoCommandStalenessBackpressuring { // Does not carry any authentication, assumes all underlying requests are properly authenticated private val httpRequestInitializer = new HttpRequestInitializer { @@ -113,9 +115,9 @@ class GcsBatchFlow(batchSize: Int, val flow: Graph[ FlowShape[ GcsBatchCommandContext[_, _], - (IoAck[_], IoCommandContext[_]), + (IoAck[_], IoCommandContext[_]) ], - NotUsed, + NotUsed ] = GraphDSL.create() { implicit builder => import GraphDSL.Implicits._ @@ -129,21 +131,22 @@ class GcsBatchFlow(batchSize: Int, val batchProcessor = builder.add( Flow[GcsBatchCommandContext[_, _]] // Group commands together in batches so they can be processed as such - .groupedWithin(batchSize, batchTimespan) + .groupedWithin(batchSize, batchTimespan) // execute the batch and outputs each sub-response individually, as a Future - .mapConcat[Future[GcsBatchResponse[_]]](executeBatch) + .mapConcat[Future[GcsBatchResponse[_]]](executeBatch) // Wait for each Future to complete - .mapAsyncUnordered[GcsBatchResponse[_]](batchSize) { identity } + .mapAsyncUnordered[GcsBatchResponse[_]](batchSize)(identity) ) // Partitions the responses: Terminal responses exit the flow, others go back to the sourceMerger val responseHandler = builder.add(responseHandlerFlow) // Buffer commands to be retried to avoid backpressuring too rapidly - val nextRequestBuffer = builder.add(Flow[GcsBatchCommandContext[_, _]].buffer(batchSize, OverflowStrategy.backpressure)) + val nextRequestBuffer = + builder.add(Flow[GcsBatchCommandContext[_, _]].buffer(batchSize, OverflowStrategy.backpressure)) source ~> sourceMerger ~> batchProcessor ~> responseHandler.in - sourceMerger.preferred <~ nextRequestBuffer <~ responseHandler.out1 + sourceMerger.preferred <~ nextRequestBuffer <~ responseHandler.out1 FlowShape[GcsBatchCommandContext[_, _], IoResult](source.in, responseHandler.out0) } @@ -156,10 +159,14 @@ class GcsBatchFlow(batchSize: Int, private lazy val responseHandlerFlow = GraphDSL.create() { implicit builder => import GraphDSL.Implicits._ - val source = builder.add(Partition[GcsBatchResponse[_]](2, { - case _: GcsBatchTerminal[_] => 0 - case _ => 1 - })) + val source = builder.add( + Partition[GcsBatchResponse[_]](2, + { + case _: GcsBatchTerminal[_] => 0 + case _ => 1 + } + ) + ) // Terminal responses: output of this flow val terminals = source.out(0) collect { case terminal: GcsBatchTerminal[_] => terminal.ioResult } @@ -170,7 +177,10 @@ class GcsBatchFlow(batchSize: Int, case nextRequest: GcsBatchNextRequest[_] => nextRequest.context } - new FanOutShape2[GcsBatchResponse[_], IoResult, GcsBatchCommandContext[_, _]](source.in, terminals.outlet, nextRequest.outlet) + new FanOutShape2[GcsBatchResponse[_], IoResult, GcsBatchCommandContext[_, _]](source.in, + terminals.outlet, + nextRequest.outlet + ) } private def executeBatch(contexts: Seq[GcsBatchCommandContext[_, _]]): List[Future[GcsBatchResponse[_]]] = { @@ -193,12 +203,16 @@ class GcsBatchFlow(batchSize: Int, // Otherwise fail with the original exception Try(batchRequest.execute()) match { case Failure(failure: IOException) => - logger.info(s"Failed to execute GCS Batch request. Failed request belonged to batch of size ${batchRequest.size()} containing commands: " + - s"${batchCommandNamesList.mkString("\n")}.\n${failure.toPrettyElidedString(limit = 1000)}") + logger.info( + s"Failed to execute GCS Batch request. Failed request belonged to batch of size ${batchRequest.size()} containing commands: " + + s"${batchCommandNamesList.mkString("\n")}.\n${failure.toPrettyElidedString(limit = 1000)}" + ) failAllPromisesWith(BatchFailedException(failure)) case Failure(failure) => - logger.info(s"Failed to execute GCS Batch request. Failed request belonged to batch of size ${batchRequest.size()} containing commands: " + - s"${batchCommandNamesList.mkString("\n")}.\n${failure.toPrettyElidedString(limit = 1000)}") + logger.info( + s"Failed to execute GCS Batch request. Failed request belonged to batch of size ${batchRequest.size()} containing commands: " + + s"${batchCommandNamesList.mkString("\n")}.\n${failure.toPrettyElidedString(limit = 1000)}" + ) failAllPromisesWith(failure) case _ => } @@ -219,7 +233,9 @@ class GcsBatchFlow(batchSize: Int, * Otherwise create a GcsBatchTerminal response with the IoFailure * In both cases, returns a successful Future to avoid failing the stream or dropping elements */ - private def recoverCommand(context: GcsBatchCommandContext[_, _]): PartialFunction[Throwable, Future[GcsBatchResponse[_]]] = { + private def recoverCommand( + context: GcsBatchCommandContext[_, _] + ): PartialFunction[Throwable, Future[GcsBatchResponse[_]]] = { // If the failure is retryable - recover with a GcsBatchRetry so it can be retried in the next batch case failure if isRetryable(failure) => context.retryIn match { @@ -246,22 +262,22 @@ class GcsBatchFlow(batchSize: Int, /** * Fail a command context with a failure. */ - private def fail(context: GcsBatchCommandContext[_, _], failure: Throwable) = { + private def fail(context: GcsBatchCommandContext[_, _], failure: Throwable) = Future.successful( GcsBatchTerminal( context.fail(EnhancedCromwellIoException(IoAttempts(context.currentAttempt), failure)) ) ) - } /** * Fail a command context with a forbidden failure. */ - private def failReadForbidden(context: GcsBatchCommandContext[_, _], failure: Throwable, forbiddenPath: String) = { + private def failReadForbidden(context: GcsBatchCommandContext[_, _], failure: Throwable, forbiddenPath: String) = Future.successful( GcsBatchTerminal( - context.failReadForbidden(EnhancedCromwellIoException(IoAttempts(context.currentAttempt), failure), forbiddenPath) + context.failReadForbidden(EnhancedCromwellIoException(IoAttempts(context.currentAttempt), failure), + forbiddenPath + ) ) ) - } } diff --git a/engine/src/main/scala/cromwell/engine/io/gcs/GcsResponse.scala b/engine/src/main/scala/cromwell/engine/io/gcs/GcsResponse.scala index 937e2b66063..b8a4cb4a873 100644 --- a/engine/src/main/scala/cromwell/engine/io/gcs/GcsResponse.scala +++ b/engine/src/main/scala/cromwell/engine/io/gcs/GcsResponse.scala @@ -6,7 +6,8 @@ import cromwell.engine.io.IoActor._ * ADT used only inside the batch stream * @tparam T final type of the result of the Command */ -private [gcs] sealed trait GcsBatchResponse[T] -private [gcs] case class GcsBatchTerminal[T](ioResult: IoResult) extends GcsBatchResponse[T] -private [gcs] case class GcsBatchRetry[T](context: GcsBatchCommandContext[T, _], failure: Throwable) extends GcsBatchResponse[T] -private [gcs] case class GcsBatchNextRequest[T](context: GcsBatchCommandContext[T, _]) extends GcsBatchResponse[T] +sealed private[gcs] trait GcsBatchResponse[T] +private[gcs] case class GcsBatchTerminal[T](ioResult: IoResult) extends GcsBatchResponse[T] +private[gcs] case class GcsBatchRetry[T](context: GcsBatchCommandContext[T, _], failure: Throwable) + extends GcsBatchResponse[T] +private[gcs] case class GcsBatchNextRequest[T](context: GcsBatchCommandContext[T, _]) extends GcsBatchResponse[T] diff --git a/engine/src/main/scala/cromwell/engine/io/gcs/ParallelGcsBatchFlow.scala b/engine/src/main/scala/cromwell/engine/io/gcs/ParallelGcsBatchFlow.scala index 3ef3548bdec..9fc5c5fc7ea 100644 --- a/engine/src/main/scala/cromwell/engine/io/gcs/ParallelGcsBatchFlow.scala +++ b/engine/src/main/scala/cromwell/engine/io/gcs/ParallelGcsBatchFlow.scala @@ -18,10 +18,10 @@ class ParallelGcsBatchFlow(config: GcsBatchFlowConfig, onRetry: IoCommandContext[_] => Throwable => Unit, onBackpressure: Option[Double] => Unit, applicationName: String, - commandBackpressureStaleness: FiniteDuration) - (implicit ec: ExecutionContext) { + commandBackpressureStaleness: FiniteDuration +)(implicit ec: ExecutionContext) { - //noinspection TypeAnnotation + // noinspection TypeAnnotation val flow = GraphDSL.create() { implicit builder => import GraphDSL.Implicits._ val balancer = builder.add(Balance[GcsBatchCommandContext[_, _]](config.parallelism, waitForAllDownstreams = false)) @@ -35,7 +35,8 @@ class ParallelGcsBatchFlow(config: GcsBatchFlowConfig, onRetry = onRetry, onBackpressure = onBackpressure, applicationName = applicationName, - backpressureStaleness = commandBackpressureStaleness).flow + backpressureStaleness = commandBackpressureStaleness + ).flow // for each worker, add an edge from the balancer to the worker, then wire // it to the merge element balancer ~> workerFlow.async ~> merge diff --git a/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala b/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala index 69e5551b8a9..5be9ad36d9c 100644 --- a/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala +++ b/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala @@ -4,8 +4,7 @@ import akka.actor.ActorSystem import akka.stream.scaladsl.Flow import cats.effect._ -import scala.util.Try -import cloud.nio.spi.{ChecksumFailure, ChecksumResult, ChecksumSkipped, ChecksumSuccess, FileHash, HashType} +import cloud.nio.spi.{ChecksumFailure, ChecksumResult, ChecksumSkipped, ChecksumSuccess, FileHash} import com.typesafe.config.Config import common.util.IORetry import cromwell.core.io._ @@ -15,10 +14,7 @@ import cromwell.engine.io.RetryableRequestSupport.{isInfinitelyRetryable, isRetr import cromwell.engine.io.{IoAttempts, IoCommandContext, IoCommandStalenessBackpressuring} import cromwell.filesystems.blob.BlobPath import cromwell.filesystems.drs.DrsPath -import cromwell.filesystems.gcs.GcsPath import cromwell.filesystems.http.HttpPath -import cromwell.filesystems.s3.S3Path -import cromwell.util.TryWithResource._ import net.ceedubs.ficus.Ficus._ import net.ceedubs.ficus.readers.ValueReader @@ -27,7 +23,6 @@ import java.nio.charset.StandardCharsets import scala.concurrent.ExecutionContext import scala.concurrent.duration.FiniteDuration - /** * Flow that executes IO operations by calling java.nio.Path methods */ @@ -36,7 +31,8 @@ class NioFlow(parallelism: Int, onBackpressure: Option[Double] => Unit, numberOfAttempts: Int, commandBackpressureStaleness: FiniteDuration - )(implicit system: ActorSystem) extends IoCommandStalenessBackpressuring { +)(implicit system: ActorSystem) + extends IoCommandStalenessBackpressuring { implicit private val ec: ExecutionContext = system.dispatcher implicit private val timer: Timer[IO] = IO.timer(ec) @@ -66,18 +62,19 @@ class NioFlow(parallelism: Int, result <- operationResult } yield (result, commandContext) - io handleErrorWith { - failure => IO.pure(commandContext.fail(failure)) + io handleErrorWith { failure => + IO.pure(commandContext.fail(failure)) } } - private [nio] def handleSingleCommand(ioSingleCommand: IoCommand[_]): IO[IoSuccess[_]] = { + private[nio] def handleSingleCommand(ioSingleCommand: IoCommand[_]): IO[IoSuccess[_]] = { val ret = ioSingleCommand match { case copyCommand: IoCopyCommand => copy(copyCommand) map copyCommand.success case writeCommand: IoWriteCommand => write(writeCommand) map writeCommand.success case deleteCommand: IoDeleteCommand => delete(deleteCommand) map deleteCommand.success case sizeCommand: IoSizeCommand => size(sizeCommand) map sizeCommand.success - case readAsStringCommand: IoContentAsStringCommand => readAsString(readAsStringCommand) map readAsStringCommand.success + case readAsStringCommand: IoContentAsStringCommand => + readAsString(readAsStringCommand) map readAsStringCommand.success case hashCommand: IoHashCommand => hash(hashCommand) map hashCommand.success case touchCommand: IoTouchCommand => touch(touchCommand) map touchCommand.success case existsCommand: IoExistsCommand => exists(existsCommand) map existsCommand.success @@ -131,9 +128,9 @@ class NioFlow(parallelism: Int, ) } - def readFileAndChecksum: IO[String] = { + def readFileAndChecksum: IO[String] = for { - fileHash <- getStoredHash(command.file) + fileHash <- NioHashing.getStoredHash(command.file) uncheckedValue <- readFile checksumResult <- fileHash match { case Some(hash) => checkHash(uncheckedValue, hash) @@ -145,23 +142,25 @@ class NioFlow(parallelism: Int, verifiedValue <- checksumResult match { case _: ChecksumSkipped => IO.pure(uncheckedValue) case _: ChecksumSuccess => IO.pure(uncheckedValue) - case failure: ChecksumFailure => IO.raiseError( - ChecksumFailedException( - fileHash match { - case Some(hash) => s"Failed checksum for '${command.file}'. Expected '${hash.hashType}' hash of '${hash.hash}'. Calculated hash '${failure.calculatedHash}'" - case None => s"Failed checksum for '${command.file}'. Couldn't find stored file hash." // This should never happen - } + case failure: ChecksumFailure => + IO.raiseError( + ChecksumFailedException( + fileHash match { + case Some(hash) => + s"Failed checksum for '${command.file}'. Expected '${hash.hashType}' hash of '${hash.hash}'. Calculated hash '${failure.calculatedHash}'" + case None => + s"Failed checksum for '${command.file}'. Couldn't find stored file hash." // This should never happen + } + ) ) - ) } } yield verifiedValue - } val fileContentIo = command.file match { - case _: DrsPath => readFileAndChecksum + case _: DrsPath => readFileAndChecksum // Temporarily disable since our hashing algorithm doesn't match the stored hash // https://broadworkbench.atlassian.net/browse/WX-1257 - case _: BlobPath => readFile//readFileAndChecksum + case _: BlobPath => readFile // readFileAndChecksum case _ => readFile } fileContentIo.map(_.replaceAll("\\r\\n", "\\\n")) @@ -173,30 +172,8 @@ class NioFlow(parallelism: Int, case nioPath => IO(nioPath.size) } - private def hash(hash: IoHashCommand): IO[String] = { - // If there is no hash accessible from the file storage system, - // we'll read the file and generate the hash ourselves. - getStoredHash(hash.file).flatMap { - case Some(storedHash) => IO.pure(storedHash) - case None => generateMd5FileHashForPath(hash.file) - }.map(_.hash) - } - - private def getStoredHash(file: Path): IO[Option[FileHash]] = { - file match { - case gcsPath: GcsPath => getFileHashForGcsPath(gcsPath).map(Option(_)) - case blobPath: BlobPath => getFileHashForBlobPath(blobPath) - case drsPath: DrsPath => IO { - // We assume all DRS files have a stored hash; this will throw - // if the file does not. - drsPath.getFileHash - }.map(Option(_)) - case s3Path: S3Path => IO { - Option(FileHash(HashType.S3Etag, s3Path.eTag)) - } - case _ => IO.pure(None) - } - } + private def hash(hash: IoHashCommand): IO[String] = + NioHashing.hash(hash.file) private def touch(touch: IoTouchCommand) = IO { touch.file.touch() @@ -217,28 +194,6 @@ class NioFlow(parallelism: Int, } private def createDirectories(path: Path) = path.parent.createDirectories() - - /** - * Lazy evaluation of a Try in a delayed IO. This avoids accidentally eagerly evaluating the Try. - * - * IMPORTANT: Use this instead of IO.fromTry to make sure the Try will be reevaluated if the - * IoCommand is retried. - */ - private def delayedIoFromTry[A](t: => Try[A]): IO[A] = IO[A] { t.get } - - private def getFileHashForGcsPath(gcsPath: GcsPath): IO[FileHash] = delayedIoFromTry { - gcsPath.objectBlobId.map(id => FileHash(HashType.GcsCrc32c, gcsPath.cloudStorage.get(id).getCrc32c)) - } - - private def getFileHashForBlobPath(blobPath: BlobPath): IO[Option[FileHash]] = delayedIoFromTry { - blobPath.md5HexString.map(md5 => md5.map(FileHash(HashType.Md5, _))) - } - - private def generateMd5FileHashForPath(path: Path): IO[FileHash] = delayedIoFromTry { - tryWithResource(() => path.newInputStream) { inputStream => - FileHash(HashType.Md5, org.apache.commons.codec.digest.DigestUtils.md5Hex(inputStream)) - } - } } object NioFlow { diff --git a/engine/src/main/scala/cromwell/engine/io/nio/NioHashing.scala b/engine/src/main/scala/cromwell/engine/io/nio/NioHashing.scala new file mode 100644 index 00000000000..7f371311006 --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/io/nio/NioHashing.scala @@ -0,0 +1,92 @@ +package cromwell.engine.io.nio + +import cats.effect.IO +import cloud.nio.spi.{FileHash, HashType} +import common.util.StringUtil.EnhancedString +import cromwell.core.path.Path +import cromwell.filesystems.blob.BlobPath +import cromwell.filesystems.drs.DrsPath +import cromwell.filesystems.gcs.GcsPath +import cromwell.filesystems.http.HttpPath +import cromwell.filesystems.s3.S3Path +import cromwell.util.TryWithResource.tryWithResource + +import scala.util.Try + +object NioHashing { + + def hash(file: Path): IO[String] = + // If there is no hash accessible from the file storage system, + // we'll read the file and generate the hash ourselves if we can. + getStoredHash(file) + .flatMap { + case Some(storedHash) => IO.pure(storedHash) + case None => + if (canHashLocally(file)) + generateMd5FileHashForPath(file) + else + IO.raiseError( + new Exception( + s"File of type ${file.getClass.getSimpleName} requires an associated hash, not present for ${file.pathAsString.maskSensitiveUri}" + ) + ) + } + .map(_.hash) + + def getStoredHash(file: Path): IO[Option[FileHash]] = + file match { + case gcsPath: GcsPath => getFileHashForGcsPath(gcsPath).map(Option(_)) + case blobPath: BlobPath => getFileHashForBlobPath(blobPath) + case drsPath: DrsPath => + IO { + // We assume all DRS files have a stored hash; this will throw + // if the file does not. + drsPath.getFileHash + }.map(Option(_)) + case s3Path: S3Path => + IO { + Option(FileHash(HashType.S3Etag, s3Path.eTag)) + } + case _ => IO.pure(None) + } + + /** + * In some scenarios like SFS it is appropriate for Cromwell to hash files using its own CPU power. + * + * In cloud scenarios, we don't want this because the files are huge, downloading them is slow & expensive, + * and the extreme CPU usage destabilizes the instance (WX-1566). For more context, see also comments + * on `cromwell.filesystems.blob.BlobPath#largeBlobFileMetadataKey()`. + * + * Cromwell is fundamentally supposed to be a job scheduler, and heavy computation should take place elsewhere. + * + * @param file The path to consider for local hashing + */ + private def canHashLocally(file: Path) = + file match { + case _: HttpPath => false + case _: BlobPath => false + case _ => true + } + + private def generateMd5FileHashForPath(path: Path): IO[FileHash] = delayedIoFromTry { + tryWithResource(() => path.newInputStream) { inputStream => + FileHash(HashType.Md5, org.apache.commons.codec.digest.DigestUtils.md5Hex(inputStream)) + } + } + + private def getFileHashForGcsPath(gcsPath: GcsPath): IO[FileHash] = delayedIoFromTry { + gcsPath.objectBlobId.map(id => FileHash(HashType.GcsCrc32c, gcsPath.cloudStorage.get(id).getCrc32c)) + } + + private def getFileHashForBlobPath(blobPath: BlobPath): IO[Option[FileHash]] = delayedIoFromTry { + blobPath.md5HexString.map(md5 => md5.map(FileHash(HashType.Md5, _))) + } + + /** + * Lazy evaluation of a Try in a delayed IO. This avoids accidentally eagerly evaluating the Try. + * + * IMPORTANT: Use this instead of IO.fromTry to make sure the Try will be reevaluated if the + * IoCommand is retried. + */ + private def delayedIoFromTry[A](t: => Try[A]): IO[A] = IO[A](t.get) +} diff --git a/engine/src/main/scala/cromwell/engine/package.scala b/engine/src/main/scala/cromwell/engine/package.scala index f4083286612..1bd7a13a244 100644 --- a/engine/src/main/scala/cromwell/engine/package.scala +++ b/engine/src/main/scala/cromwell/engine/package.scala @@ -14,8 +14,8 @@ package object engine { } implicit class EnhancedCallOutputMap[A](val m: Map[A, JobOutput]) extends AnyVal { - def mapToValues: Map[A, WomValue] = m map { - case (k, JobOutput(womValue)) => (k, womValue) + def mapToValues: Map[A, WomValue] = m map { case (k, JobOutput(womValue)) => + (k, womValue) } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala index 91be71fbf35..8b72d7eaaba 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala @@ -20,8 +20,13 @@ import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.SubmitWorkflow import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreSubmitActor} import cromwell.jobstore.EmptyJobStoreActor import cromwell.server.CromwellRootActor -import cromwell.services.{SuccessfulMetadataJsonResponse, FailedMetadataJsonResponse} -import cromwell.services.metadata.MetadataService.{GetSingleWorkflowMetadataAction, GetStatus, ListenToMetadataWriteActor, WorkflowOutputs} +import cromwell.services.{FailedMetadataJsonResponse, SuccessfulMetadataJsonResponse} +import cromwell.services.metadata.MetadataService.{ + GetSingleWorkflowMetadataAction, + GetStatus, + ListenToMetadataWriteActor, + WorkflowOutputs +} import cromwell.subworkflowstore.EmptySubWorkflowStoreActor import spray.json._ @@ -40,8 +45,8 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, gracefulShutdown: Boolean, abortJobsOnTerminate: Boolean, config: Config - )(implicit materializer: ActorMaterializer) - extends CromwellRootActor(terminator, gracefulShutdown, abortJobsOnTerminate, false, config) +)(implicit materializer: ActorMaterializer) + extends CromwellRootActor(terminator, gracefulShutdown, abortJobsOnTerminate, false, config) with LoggingFSM[RunnerState, SwraData] { import SingleWorkflowRunnerActor._ @@ -54,14 +59,13 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, log.info("{}: Version {}", Tag, VersionUtil.getVersion("cromwell-engine")) startWith(NotStarted, EmptySwraData) - when (NotStarted) { - case Event(RunWorkflow, EmptySwraData) => - log.info(s"$Tag: Submitting workflow") - workflowStoreActor ! SubmitWorkflow(source) - goto(SubmittedWorkflow) using SubmittedSwraData(sender()) + when(NotStarted) { case Event(RunWorkflow, EmptySwraData) => + log.info(s"$Tag: Submitting workflow") + workflowStoreActor ! SubmitWorkflow(source) + goto(SubmittedWorkflow) using SubmittedSwraData(sender()) } - when (SubmittedWorkflow) { + when(SubmittedWorkflow) { case Event(WorkflowStoreSubmitActor.WorkflowSubmittedToStore(id, WorkflowSubmitted), SubmittedSwraData(replyTo)) => log.info(s"$Tag: Workflow submitted UUID($id)") // Since we only have a single workflow, force the WorkflowManagerActor's hand in case the polling rate is long @@ -72,51 +76,57 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, goto(RunningWorkflow) using RunningSwraData(replyTo, id) } - when (RunningWorkflow) { + when(RunningWorkflow) { case Event(IssuePollRequest, RunningSwraData(_, id)) => requestStatus(id) stay() - case Event(SuccessfulMetadataJsonResponse(_, jsObject: JsObject), RunningSwraData(_, _)) if !jsObject.state.isTerminal => + case Event(SuccessfulMetadataJsonResponse(_, jsObject: JsObject), RunningSwraData(_, _)) + if !jsObject.state.isTerminal => schedulePollRequest() stay() - case Event(SuccessfulMetadataJsonResponse(_, jsObject: JsObject), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowSucceeded => + case Event(SuccessfulMetadataJsonResponse(_, jsObject: JsObject), RunningSwraData(replyTo, id)) + if jsObject.state == WorkflowSucceeded => log.info(s"$Tag workflow finished with status '$WorkflowSucceeded'.") serviceRegistryActor ! ListenToMetadataWriteActor goto(WaitingForFlushedMetadata) using SucceededSwraData(replyTo, id) - case Event(SuccessfulMetadataJsonResponse(_, jsObject: JsObject), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowFailed => + case Event(SuccessfulMetadataJsonResponse(_, jsObject: JsObject), RunningSwraData(replyTo, id)) + if jsObject.state == WorkflowFailed => log.info(s"$Tag workflow finished with status '$WorkflowFailed'.") serviceRegistryActor ! ListenToMetadataWriteActor - goto(WaitingForFlushedMetadata) using FailedSwraData(replyTo, id, new RuntimeException(s"Workflow $id transitioned to state $WorkflowFailed")) - case Event(SuccessfulMetadataJsonResponse(_, jsObject: JsObject), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowAborted => + goto(WaitingForFlushedMetadata) using FailedSwraData( + replyTo, + id, + new RuntimeException(s"Workflow $id transitioned to state $WorkflowFailed") + ) + case Event(SuccessfulMetadataJsonResponse(_, jsObject: JsObject), RunningSwraData(replyTo, id)) + if jsObject.state == WorkflowAborted => log.info(s"$Tag workflow finished with status '$WorkflowAborted'.") serviceRegistryActor ! ListenToMetadataWriteActor goto(WaitingForFlushedMetadata) using AbortedSwraData(replyTo, id) } - - when (WaitingForFlushedMetadata) { + + when(WaitingForFlushedMetadata) { case Event(QueueWeight(weight), _) if weight > 0 => stay() case Event(QueueWeight(_), data: SucceededSwraData) => - serviceRegistryActor ! WorkflowOutputs(data.id) goto(RequestingOutputs) - case Event(QueueWeight(_), data : TerminalSwraData) => + case Event(QueueWeight(_), data: TerminalSwraData) => requestMetadataOrIssueReply(data) } - when (RequestingOutputs) { - case Event(SuccessfulMetadataJsonResponse(_, outputs: JsObject), data: TerminalSwraData) => - outputOutputs(outputs) - requestMetadataOrIssueReply(data) + when(RequestingOutputs) { case Event(SuccessfulMetadataJsonResponse(_, outputs: JsObject), data: TerminalSwraData) => + outputOutputs(outputs) + requestMetadataOrIssueReply(data) } - when (RequestingMetadata) { + when(RequestingMetadata) { case Event(SuccessfulMetadataJsonResponse(_, metadata: JsObject), data: TerminalSwraData) => outputMetadata(metadata) issueReply(data) } - onTransition { - case NotStarted -> RunningWorkflow => schedulePollRequest() + onTransition { case NotStarted -> RunningWorkflow => + schedulePollRequest() } whenUnhandled { @@ -137,11 +147,12 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, stay() } - private def requestMetadataOrIssueReply(newData: TerminalSwraData) = if (metadataOutputPath.isDefined) requestMetadata(newData) else issueReply(newData) - + private def requestMetadataOrIssueReply(newData: TerminalSwraData) = + if (metadataOutputPath.isDefined) requestMetadata(newData) else issueReply(newData) + private def requestMetadata(newData: TerminalSwraData): State = { serviceRegistryActor ! GetSingleWorkflowMetadataAction(newData.id, None, None, expandSubWorkflows = true) - goto (RequestingMetadata) using newData + goto(RequestingMetadata) using newData } private def schedulePollRequest(): Unit = { @@ -150,12 +161,11 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, () } - private def requestStatus(id: WorkflowId): Unit = { + private def requestStatus(id: WorkflowId): Unit = // This requests status via the metadata service rather than instituting an FSM watch on the underlying workflow actor. // Cromwell's eventual consistency means it isn't safe to use an FSM transition to a terminal state as the signal for // when outputs or metadata have stabilized. serviceRegistryActor ! GetStatus(id) - } private def issueSuccessReply(replyTo: ActorRef): State = { replyTo.tell(msg = (), sender = self) // Because replyTo ! () is the parameterless call replyTo.!() @@ -168,17 +178,16 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, done() stay() } - - private [workflow] def done() = {} - private def issueReply(data: TerminalSwraData) = { + private[workflow] def done() = {} + + private def issueReply(data: TerminalSwraData) = data match { case s: SucceededSwraData => issueSuccessReply(s.replyTo) case f: FailedSwraData => issueFailureReply(f.replyTo, f.failure) case a: AbortedSwraData => issueSuccessReply(a.replyTo) } - } private def failAndFinish(e: Throwable, data: SwraData): State = { log.error(e, s"$Tag received Failure message: ${e.getMessage}") @@ -199,11 +208,10 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, /** * Outputs the outputs to stdout, and then requests the metadata. */ - private def outputOutputs(outputs: JsObject): Unit = { + private def outputOutputs(outputs: JsObject): Unit = println(outputs.prettyPrint) - } - private def outputMetadata(metadata: JsObject): Try[Unit] = { + private def outputMetadata(metadata: JsObject): Try[Unit] = Try { val path = metadataOutputPath.get if (path.isDirectory) { @@ -213,7 +221,6 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, path.createIfNotExists(createParents = true).write(metadata.prettyPrint) } } void - } } object SingleWorkflowRunnerActor { @@ -222,8 +229,8 @@ object SingleWorkflowRunnerActor { terminator: CromwellTerminator, gracefulShutdown: Boolean, abortJobsOnTerminate: Boolean, - config: Config) - (implicit materializer: ActorMaterializer): Props = { + config: Config + )(implicit materializer: ActorMaterializer): Props = Props( new SingleWorkflowRunnerActor( source = source, @@ -234,7 +241,6 @@ object SingleWorkflowRunnerActor { config = config ) ).withDispatcher(EngineDispatcher) - } sealed trait RunnerMessage // The message to actually run the workflow is made explicit so the non-actor Main can `ask` this actor to do the @@ -255,16 +261,20 @@ object SingleWorkflowRunnerActor { final case class SubmittedSwraData(replyTo: ActorRef) extends SwraData final case class RunningSwraData(replyTo: ActorRef, id: WorkflowId) extends SwraData - sealed trait TerminalSwraData extends SwraData { def replyTo: ActorRef; def terminalState: WorkflowState; def id: WorkflowId } - final case class SucceededSwraData(replyTo: ActorRef, - id: WorkflowId) extends TerminalSwraData { override val terminalState = WorkflowSucceeded } + sealed trait TerminalSwraData extends SwraData { + def replyTo: ActorRef; def terminalState: WorkflowState; def id: WorkflowId + } + final case class SucceededSwraData(replyTo: ActorRef, id: WorkflowId) extends TerminalSwraData { + override val terminalState = WorkflowSucceeded + } - final case class FailedSwraData(replyTo: ActorRef, - id: WorkflowId, - failure: Throwable) extends TerminalSwraData { override val terminalState = WorkflowFailed } + final case class FailedSwraData(replyTo: ActorRef, id: WorkflowId, failure: Throwable) extends TerminalSwraData { + override val terminalState = WorkflowFailed + } - final case class AbortedSwraData(replyTo: ActorRef, - id: WorkflowId) extends TerminalSwraData { override val terminalState = WorkflowAborted } + final case class AbortedSwraData(replyTo: ActorRef, id: WorkflowId) extends TerminalSwraData { + override val terminalState = WorkflowAborted + } implicit class EnhancedJsObject(val jsObject: JsObject) extends AnyVal { def state: WorkflowState = WorkflowState.withName(jsObject.fields("status").asInstanceOf[JsString].value) diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala index 3061e2e8a74..8bf936fa88b 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala @@ -19,19 +19,49 @@ import cromwell.engine.workflow.WorkflowActor._ import cromwell.engine.workflow.WorkflowManagerActor.WorkflowActorWorkComplete import cromwell.engine.workflow.lifecycle._ import cromwell.engine.workflow.lifecycle.deletion.DeleteWorkflowFilesActor -import cromwell.engine.workflow.lifecycle.deletion.DeleteWorkflowFilesActor.{DeleteWorkflowFilesFailedResponse, DeleteWorkflowFilesSucceededResponse, StartWorkflowFilesDeletion} +import cromwell.engine.workflow.lifecycle.deletion.DeleteWorkflowFilesActor.{ + DeleteWorkflowFilesFailedResponse, + DeleteWorkflowFilesSucceededResponse, + StartWorkflowFilesDeletion +} import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor._ import cromwell.engine.workflow.lifecycle.finalization.WorkflowCallbackActor.PerformCallbackCommand -import cromwell.engine.workflow.lifecycle.finalization.WorkflowFinalizationActor.{StartFinalizationCommand, WorkflowFinalizationFailedResponse, WorkflowFinalizationSucceededResponse} -import cromwell.engine.workflow.lifecycle.finalization.{CopyWorkflowLogsActor, CopyWorkflowOutputsActor, WorkflowFinalizationActor} +import cromwell.engine.workflow.lifecycle.finalization.WorkflowFinalizationActor.{ + StartFinalizationCommand, + WorkflowFinalizationFailedResponse, + WorkflowFinalizationSucceededResponse +} +import cromwell.engine.workflow.lifecycle.finalization.{ + CopyWorkflowLogsActor, + CopyWorkflowOutputsActor, + WorkflowFinalizationActor +} import cromwell.engine.workflow.lifecycle.initialization.WorkflowInitializationActor -import cromwell.engine.workflow.lifecycle.initialization.WorkflowInitializationActor.{StartInitializationCommand, WorkflowInitializationFailedResponse, WorkflowInitializationResponse, WorkflowInitializationSucceededResponse} +import cromwell.engine.workflow.lifecycle.initialization.WorkflowInitializationActor.{ + StartInitializationCommand, + WorkflowInitializationFailedResponse, + WorkflowInitializationResponse, + WorkflowInitializationSucceededResponse +} import cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor -import cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorCommand, MaterializeWorkflowDescriptorFailureResponse, MaterializeWorkflowDescriptorSuccessResponse} +import cromwell.engine.workflow.lifecycle.materialization.MaterializeWorkflowDescriptorActor.{ + MaterializeWorkflowDescriptorCommand, + MaterializeWorkflowDescriptorFailureResponse, + MaterializeWorkflowDescriptorSuccessResponse +} import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.WorkflowStoreWriteHeartbeatCommand -import cromwell.engine.workflow.workflowstore.{RestartableAborting, StartableState, WorkflowHeartbeatConfig, WorkflowToStart} -import cromwell.services.metadata.MetadataService.{MetadataWriteFailure, MetadataWriteSuccess, PutMetadataActionAndRespond} +import cromwell.engine.workflow.workflowstore.{ + RestartableAborting, + StartableState, + WorkflowHeartbeatConfig, + WorkflowToStart +} +import cromwell.services.metadata.MetadataService.{ + MetadataWriteFailure, + MetadataWriteSuccess, + PutMetadataActionAndRespond +} import cromwell.subworkflowstore.SubWorkflowStoreActor.WorkflowComplete import cromwell.webservice.EngineStatsActor import org.apache.commons.lang3.exception.ExceptionUtils @@ -152,14 +182,16 @@ object WorkflowActor { workflowFinalOutputs: Option[CallOutputs] = None, workflowAllOutputs: Set[WomValue] = Set.empty, rootAndSubworkflowIds: Set[WorkflowId] = Set.empty, - failedInitializationAttempts: Int = 0) + failedInitializationAttempts: Int = 0 + ) object WorkflowActorData { def apply(startableState: StartableState): WorkflowActorData = WorkflowActorData( currentLifecycleStateActor = None, workflowDescriptor = None, initializationData = AllBackendInitializationData.empty, lastStateReached = StateCheckpoint(WorkflowUnstartedState), - effectiveStartableState = startableState) + effectiveStartableState = startableState + ) } /** @@ -190,7 +222,8 @@ object WorkflowActor { workflowHeartbeatConfig: WorkflowHeartbeatConfig, totalJobsByRootWf: AtomicInteger, fileHashCacheActorProps: Option[Props], - blacklistCache: Option[BlacklistCache]): Props = { + blacklistCache: Option[BlacklistCache] + ): Props = Props( new WorkflowActor( workflowToStart = workflowToStart, @@ -214,8 +247,9 @@ object WorkflowActor { workflowHeartbeatConfig = workflowHeartbeatConfig, totalJobsByRootWf = totalJobsByRootWf, fileHashCacheActorProps = fileHashCacheActorProps, - blacklistCache = blacklistCache)).withDispatcher(EngineDispatcher) - } + blacklistCache = blacklistCache + ) + ).withDispatcher(EngineDispatcher) } /** @@ -224,7 +258,7 @@ object WorkflowActor { class WorkflowActor(workflowToStart: WorkflowToStart, conf: Config, callCachingEnabled: Boolean, - invalidateBadCacheResults:Boolean, + invalidateBadCacheResults: Boolean, ioActor: ActorRef, override val serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, @@ -245,9 +279,12 @@ class WorkflowActor(workflowToStart: WorkflowToStart, // child of this actor. The sbt subproject of `RootWorkflowFileHashCacheActor` is not visible from // the subproject this class belongs to so the `Props` are passed in. fileHashCacheActorProps: Option[Props], - blacklistCache: Option[BlacklistCache]) - extends LoggingFSM[WorkflowActorState, WorkflowActorData] with WorkflowLogging with WorkflowMetadataHelper - with WorkflowInstrumentation with Timers { + blacklistCache: Option[BlacklistCache] +) extends LoggingFSM[WorkflowActorState, WorkflowActorData] + with WorkflowLogging + with WorkflowMetadataHelper + with WorkflowInstrumentation + with Timers { implicit val ec = context.dispatcher private val WorkflowToStart(workflowId, submissionTime, sources, initialStartableState, hogGroup) = workflowToStart @@ -261,7 +298,9 @@ class WorkflowActor(workflowToStart: WorkflowToStart, private val deleteWorkflowFiles = conf.getBoolean("system.delete-workflow-files") private val workflowDockerLookupActor = context.actorOf( - WorkflowDockerLookupActor.props(workflowId, dockerHashActor, initialStartableState.restarted), s"WorkflowDockerLookupActor-$workflowId") + WorkflowDockerLookupActor.props(workflowId, dockerHashActor, initialStartableState.restarted), + s"WorkflowDockerLookupActor-$workflowId" + ) protected val pathBuilderFactories: List[PathBuilderFactory] = EngineFilesystems.configuredPathBuilderFactories @@ -297,8 +336,15 @@ class WorkflowActor(workflowToStart: WorkflowToStart, when(WorkflowUnstartedState) { case Event(StartWorkflowCommand, _) => - val actor = context.actorOf(MaterializeWorkflowDescriptorActor.props(serviceRegistryActor, workflowId, importLocalFilesystem = !serverMode, ioActorProxy = ioActor, hogGroup = hogGroup), - "MaterializeWorkflowDescriptorActor") + val actor = context.actorOf( + MaterializeWorkflowDescriptorActor.props(serviceRegistryActor, + workflowId, + importLocalFilesystem = !serverMode, + ioActorProxy = ioActor, + hogGroup = hogGroup + ), + "MaterializeWorkflowDescriptorActor" + ) pushWorkflowStart(workflowId) actor ! MaterializeWorkflowDescriptorCommand(sources, conf, callCachingEnabled, invalidateBadCacheResults) goto(MaterializingWorkflowDescriptorState) using stateData.copy(currentLifecycleStateActor = Option(actor)) @@ -315,7 +361,9 @@ class WorkflowActor(workflowToStart: WorkflowToStart, self ! StartInitializing goto(InitializingWorkflowState) using data.copy(workflowDescriptor = Option(workflowDescriptor)) case Event(MaterializeWorkflowDescriptorFailureResponse(reason: Throwable), data) => - goto(WorkflowFailedState) using data.copy(lastStateReached = StateCheckpoint(MaterializingWorkflowDescriptorState, Option(List(reason)))) + goto(WorkflowFailedState) using data.copy(lastStateReached = + StateCheckpoint(MaterializingWorkflowDescriptorState, Option(List(reason))) + ) // If the workflow is not being restarted then we can abort it immediately as nothing happened yet case Event(AbortWorkflowCommand, _) if !restarting => goto(WorkflowAbortedState) } @@ -323,25 +371,31 @@ class WorkflowActor(workflowToStart: WorkflowToStart, /* ************************** */ /* ****** Initializing ****** */ /* ************************** */ - protected def createInitializationActor(workflowDescriptor: EngineWorkflowDescriptor, name: String): ActorRef = { - context.actorOf( - WorkflowInitializationActor.props( - workflowIdForLogging, - rootWorkflowIdForLogging, - workflowDescriptor, - ioActor, - serviceRegistryActor, - restarting - ), - name) - } + protected def createInitializationActor(workflowDescriptor: EngineWorkflowDescriptor, name: String): ActorRef = + context.actorOf(WorkflowInitializationActor.props( + workflowIdForLogging, + rootWorkflowIdForLogging, + workflowDescriptor, + ioActor, + serviceRegistryActor, + restarting + ), + name + ) when(InitializingWorkflowState) { case Event(StartInitializing, data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _)) => - val initializerActor = createInitializationActor(workflowDescriptor, s"WorkflowInitializationActor-$workflowId-${data.failedInitializationAttempts + 1}") + val initializerActor = + createInitializationActor(workflowDescriptor, + s"WorkflowInitializationActor-$workflowId-${data.failedInitializationAttempts + 1}" + ) initializerActor ! StartInitializationCommand - goto(InitializingWorkflowState) using data.copy(currentLifecycleStateActor = Option(initializerActor), workflowDescriptor = Option(workflowDescriptor)) - case Event(WorkflowInitializationSucceededResponse(initializationData), data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _)) => + goto(InitializingWorkflowState) using data.copy(currentLifecycleStateActor = Option(initializerActor), + workflowDescriptor = Option(workflowDescriptor) + ) + case Event(WorkflowInitializationSucceededResponse(initializationData), + data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _) + ) => val dataWithInitializationData = data.copy(initializationData = initializationData) val executionActor = createWorkflowExecutionActor(workflowDescriptor, dataWithInitializationData) executionActor ! ExecuteWorkflowCommand @@ -350,18 +404,26 @@ class WorkflowActor(workflowToStart: WorkflowToStart, case _ => ExecutingWorkflowState } goto(nextState) using dataWithInitializationData.copy(currentLifecycleStateActor = Option(executionActor)) - case Event(WorkflowInitializationFailedResponse(reason), data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _)) => + case Event(WorkflowInitializationFailedResponse(reason), + data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _) + ) => val failedInitializationAttempts = data.failedInitializationAttempts + 1 if (failedInitializationAttempts < maxInitializationAttempts) { - workflowLogger.info(s"Initialization failed on attempt $failedInitializationAttempts. Will retry up to $maxInitializationAttempts times. Next retry is in $initializationRetryInterval", CromwellAggregatedException(reason, "Initialization Failure")) - context.system.scheduler.scheduleOnce(initializationRetryInterval) { self ! StartInitializing} - stay() using data.copy(currentLifecycleStateActor = None, failedInitializationAttempts = failedInitializationAttempts) + workflowLogger.info( + s"Initialization failed on attempt $failedInitializationAttempts. Will retry up to $maxInitializationAttempts times. Next retry is in $initializationRetryInterval", + CromwellAggregatedException(reason, "Initialization Failure") + ) + context.system.scheduler.scheduleOnce(initializationRetryInterval)(self ! StartInitializing) + stay() using data.copy(currentLifecycleStateActor = None, + failedInitializationAttempts = failedInitializationAttempts + ) } else { finalizeWorkflow(data, workflowDescriptor, Map.empty, CallOutputs.empty, Option(reason.toList)) } // If the workflow is not restarting, handle the Abort command normally and send an abort message to the init actor - case Event(AbortWorkflowCommand, data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _)) if !restarting => + case Event(AbortWorkflowCommand, data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _)) + if !restarting => handleAbortCommand(data, workflowDescriptor) } @@ -369,43 +431,51 @@ class WorkflowActor(workflowToStart: WorkflowToStart, /* ****** Running ****** */ /* ********************* */ - def createWorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, data: WorkflowActorData): ActorRef = { - context.actorOf(WorkflowExecutionActor.props( - workflowDescriptor, - ioActor = ioActor, - serviceRegistryActor = serviceRegistryActor, - jobStoreActor = jobStoreActor, - subWorkflowStoreActor = subWorkflowStoreActor, - callCacheReadActor = callCacheReadActor, - callCacheWriteActor = callCacheWriteActor, - workflowDockerLookupActor = workflowDockerLookupActor, - jobRestartCheckTokenDispenserActor = jobRestartCheckTokenDispenserActor, - jobExecutionTokenDispenserActor = jobExecutionTokenDispenserActor, - backendSingletonCollection, - data.initializationData, - startState = data.effectiveStartableState, - rootConfig = conf, - totalJobsByRootWf = totalJobsByRootWf, - fileHashCacheActor = fileHashCacheActorProps map context.system.actorOf, - blacklistCache = blacklistCache), name = s"WorkflowExecutionActor-$workflowId") - } + def createWorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, data: WorkflowActorData): ActorRef = + context.actorOf( + WorkflowExecutionActor.props( + workflowDescriptor, + ioActor = ioActor, + serviceRegistryActor = serviceRegistryActor, + jobStoreActor = jobStoreActor, + subWorkflowStoreActor = subWorkflowStoreActor, + callCacheReadActor = callCacheReadActor, + callCacheWriteActor = callCacheWriteActor, + workflowDockerLookupActor = workflowDockerLookupActor, + jobRestartCheckTokenDispenserActor = jobRestartCheckTokenDispenserActor, + jobExecutionTokenDispenserActor = jobExecutionTokenDispenserActor, + backendSingletonCollection, + data.initializationData, + startState = data.effectiveStartableState, + rootConfig = conf, + totalJobsByRootWf = totalJobsByRootWf, + fileHashCacheActor = fileHashCacheActorProps map context.system.actorOf, + blacklistCache = blacklistCache + ), + name = s"WorkflowExecutionActor-$workflowId" + ) // Handles workflow completion events from the WEA and abort command val executionResponseHandler: StateFunction = { // Workflow responses case Event(WorkflowExecutionSucceededResponse(jobKeys, rootAndSubworklowIds, finalOutputs, allOutputs), - data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _)) => + data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _) + ) => finalizeWorkflow(data, workflowDescriptor, jobKeys, finalOutputs, None, allOutputs, rootAndSubworklowIds) case Event(WorkflowExecutionFailedResponse(jobKeys, failures), - data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _)) => + data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _) + ) => finalizeWorkflow(data, workflowDescriptor, jobKeys, CallOutputs.empty, Option(List(failures))) case Event(WorkflowExecutionAbortedResponse(jobKeys), - data @ WorkflowActorData(_, Some(workflowDescriptor), _, StateCheckpoint(_, failures), _, _, _, _, _)) => + data @ WorkflowActorData(_, Some(workflowDescriptor), _, StateCheckpoint(_, failures), _, _, _, _, _) + ) => finalizeWorkflow(data, workflowDescriptor, jobKeys, CallOutputs.empty, failures) // Whether we're running or aborting, restarting or not, pass along the abort command. // Note that aborting a workflow multiple times will result in as many abort commands sent to the execution actor - case Event(AbortWorkflowWithExceptionCommand(ex), data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _)) => + case Event(AbortWorkflowWithExceptionCommand(ex), + data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _) + ) => handleAbortCommand(data, workflowDescriptor, Option(ex)) case Event(AbortWorkflowCommand, data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _)) => handleAbortCommand(data, workflowDescriptor) @@ -420,11 +490,15 @@ class WorkflowActor(workflowToStart: WorkflowToStart, // Handles initialization responses we can get if the abort came in when we were initializing the workflow val abortHandler: StateFunction = { // If the initialization failed, record the failure in the data and finalize the workflow - case Event(WorkflowInitializationFailedResponse(reason), data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _)) => + case Event(WorkflowInitializationFailedResponse(reason), + data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _) + ) => finalizeWorkflow(data, workflowDescriptor, Map.empty, CallOutputs.empty, Option(reason.toList)) // Otherwise (success or abort), finalize the workflow without failures - case Event(_: WorkflowInitializationResponse, data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _)) => + case Event(_: WorkflowInitializationResponse, + data @ WorkflowActorData(_, Some(workflowDescriptor), _, _, _, _, _, _, _) + ) => finalizeWorkflow(data, workflowDescriptor, Map.empty, CallOutputs.empty, failures = None) case Event(StartInitializing, _) => @@ -444,7 +518,9 @@ class WorkflowActor(workflowToStart: WorkflowToStart, case Event(WorkflowFinalizationSucceededResponse, data) => finalizationSucceeded(data) case Event(WorkflowFinalizationFailedResponse(finalizationFailures), data) => val failures = data.lastStateReached.failures.getOrElse(List.empty) ++ finalizationFailures - goto(WorkflowFailedState) using data.copy(lastStateReached = StateCheckpoint(FinalizingWorkflowState, Option(failures))) + goto(WorkflowFailedState) using data.copy(lastStateReached = + StateCheckpoint(FinalizingWorkflowState, Option(failures)) + ) case Event(AbortWorkflowCommand, _) => stay() case Event(StartInitializing, _) => // An initialization trigger we no longer need to action. Ignore: @@ -461,12 +537,18 @@ class WorkflowActor(workflowToStart: WorkflowToStart, // better luck. If we continue to be unable to write the completion message to the DB it's better to leave the // workflow in its current state in the DB than to let the WMA delete it // Note: this is an infinite retry right now, but it doesn't consume much in terms of resources and could help us successfully weather maintenance downtime on the DB - workflowLogger.error(reason, "Unable to complete workflow due to inability to write concluding metadata status. Retrying...") + workflowLogger.error( + reason, + "Unable to complete workflow due to inability to write concluding metadata status. Retrying..." + ) PutMetadataActionAndRespond(msgs, self) stay() } - def handleAbortCommand(data: WorkflowActorData, workflowDescriptor: EngineWorkflowDescriptor, exceptionCausedAbortOpt: Option[Throwable] = None) = { + def handleAbortCommand(data: WorkflowActorData, + workflowDescriptor: EngineWorkflowDescriptor, + exceptionCausedAbortOpt: Option[Throwable] = None + ) = { val updatedData = data.copy(lastStateReached = StateCheckpoint(stateName, exceptionCausedAbortOpt.map(List(_)))) data.currentLifecycleStateActor match { case Some(currentActor) => @@ -474,11 +556,21 @@ class WorkflowActor(workflowToStart: WorkflowToStart, goto(WorkflowAbortingState) using updatedData case None => if (stateName == InitializingWorkflowState) { - workflowLogger.info(s"Received an abort command in state $stateName (while awaiting an initialization retry). Finalizing the workflow.") + workflowLogger.info( + s"Received an abort command in state $stateName (while awaiting an initialization retry). Finalizing the workflow." + ) } else { - workflowLogger.warn(s"Received an abort command in state $stateName but there's no lifecycle actor associated. This is an abnormal state, finalizing the workflow anyway.") + workflowLogger.warn( + s"Received an abort command in state $stateName but there's no lifecycle actor associated. This is an abnormal state, finalizing the workflow anyway." + ) } - finalizeWorkflow(updatedData, workflowDescriptor, Map.empty, CallOutputs.empty, failures = None, lastStateOverride = Option(WorkflowAbortingState)) + finalizeWorkflow(updatedData, + workflowDescriptor, + Map.empty, + CallOutputs.empty, + failures = None, + lastStateOverride = Option(WorkflowAbortingState) + ) } } @@ -490,19 +582,23 @@ class WorkflowActor(workflowToStart: WorkflowToStart, // since deletion happens only if the workflow and finalization succeeded we can directly goto Succeeded state when(DeletingFilesState) { case Event(DeleteWorkflowFilesSucceededResponse(filesNotFound, callCacheInvalidationErrors), data) => - workflowLogger.info(s"Successfully deleted intermediate output file(s) for root workflow $rootWorkflowIdForLogging." + - deleteFilesAdditionalError(filesNotFound, callCacheInvalidationErrors)) + workflowLogger.info( + s"Successfully deleted intermediate output file(s) for root workflow $rootWorkflowIdForLogging." + + deleteFilesAdditionalError(filesNotFound, callCacheInvalidationErrors) + ) goto(WorkflowSucceededState) using data.copy(currentLifecycleStateActor = None) case Event(DeleteWorkflowFilesFailedResponse(errors, filesNotFound, callCacheInvalidationErrors), data) => - workflowLogger.info(s"Failed to delete ${errors.size} intermediate output file(s) for root workflow $rootWorkflowIdForLogging." + - deleteFilesAdditionalError(filesNotFound, callCacheInvalidationErrors) + s" Errors: ${errors.map(ExceptionUtils.getMessage)}") + workflowLogger.info( + s"Failed to delete ${errors.size} intermediate output file(s) for root workflow $rootWorkflowIdForLogging." + + deleteFilesAdditionalError(filesNotFound, callCacheInvalidationErrors) + s" Errors: ${errors.map(ExceptionUtils.getMessage)}" + ) goto(WorkflowSucceededState) using data.copy(currentLifecycleStateActor = None) } // Let these messages fall through to the whenUnhandled handler: - when(WorkflowAbortedState) { FSM.NullFunction } - when(WorkflowFailedState) { FSM.NullFunction } - when(WorkflowSucceededState) { FSM.NullFunction } + when(WorkflowAbortedState)(FSM.NullFunction) + when(WorkflowFailedState)(FSM.NullFunction) + when(WorkflowSucceededState)(FSM.NullFunction) whenUnhandled { case Event(SendWorkflowHeartbeatCommand, _) => @@ -514,11 +610,14 @@ class WorkflowActor(workflowToStart: WorkflowToStart, case Event(msg @ EngineStatsActor.JobCountQuery, data) => data.currentLifecycleStateActor match { case Some(a) => a forward msg - case None => sender() ! EngineStatsActor.NoJobs // This should be impossible, but if somehow here it's technically correct + case None => + sender() ! EngineStatsActor.NoJobs // This should be impossible, but if somehow here it's technically correct } stay() case Event(AwaitMetadataIntegrity, data) => - goto(MetadataIntegrityValidationState) using data.copy(lastStateReached = data.lastStateReached.copy(state = stateName)) + goto(MetadataIntegrityValidationState) using data.copy(lastStateReached = + data.lastStateReached.copy(state = stateName) + ) } onTransition { @@ -527,7 +626,11 @@ class WorkflowActor(workflowToStart: WorkflowToStart, setWorkflowTimePerState(terminalState.workflowState, (System.currentTimeMillis() - startTime).millis) workflowLogger.debug(s"transition from {} to {}. Stopping self.", arg1 = oldState, arg2 = terminalState) pushWorkflowEnd(workflowId) - WorkflowProcessingEventPublishing.publish(workflowId, workflowHeartbeatConfig.cromwellId, Finished, serviceRegistryActor) + WorkflowProcessingEventPublishing.publish(workflowId, + workflowHeartbeatConfig.cromwellId, + Finished, + serviceRegistryActor + ) subWorkflowStoreActor ! WorkflowComplete(workflowId) terminalState match { case WorkflowFailedState => @@ -546,10 +649,11 @@ class WorkflowActor(workflowToStart: WorkflowToStart, val system = context.system val ec = context.system.dispatcher - def bruteForcePathBuilders: Future[List[PathBuilder]] = { + def bruteForcePathBuilders: Future[List[PathBuilder]] = // Protect against path builders that may throw an exception instead of returning a failed future - Future(EngineFilesystems.pathBuildersForWorkflow(bruteForceWorkflowOptions, pathBuilderFactories)(system))(ec).flatten - } + Future(EngineFilesystems.pathBuildersForWorkflow(bruteForceWorkflowOptions, pathBuilderFactories)(system))( + ec + ).flatten val (workflowOptions, pathBuilders) = stateData.workflowDescriptor match { case Some(wd) => (wd.backendDescriptor.workflowOptions, Future.successful(wd.pathBuilders)) @@ -561,12 +665,17 @@ class WorkflowActor(workflowToStart: WorkflowToStart, workflowOptions.get(FinalWorkflowLogDir).toOption match { case Some(destinationDir) => pathBuilders - .map(pb => workflowLogCopyRouter ! CopyWorkflowLogsActor.Copy(workflowId, PathFactory.buildPath(destinationDir, pb)))(ec) + .map(pb => + workflowLogCopyRouter ! CopyWorkflowLogsActor.Copy(workflowId, + PathFactory.buildPath(destinationDir, pb) + ) + )(ec) .recover { case e => log.error(e, "Failed to copy workflow log") }(ec) - case None => workflowLogger.close(andDelete = WorkflowLogger.isTemporary) match { - case Failure(f) => log.error(f, "Failed to delete workflow log") - case _ => - } + case None => + workflowLogger.close(andDelete = WorkflowLogger.isTemporary) match { + case Failure(f) => log.error(f, "Failed to delete workflow log") + case _ => + } } } @@ -598,7 +707,9 @@ class WorkflowActor(workflowToStart: WorkflowToStart, } } - private def deleteFilesAdditionalError(filesNotFound: List[Path], callCacheInvalidationErrors: List[Throwable]): String = { + private def deleteFilesAdditionalError(filesNotFound: List[Path], + callCacheInvalidationErrors: List[Throwable] + ): String = { val filesNotFoundMsg = if (filesNotFound.nonEmpty) { s" File(s) not found during deletion: ${filesNotFound.mkString(",")}" @@ -634,30 +745,35 @@ class WorkflowActor(workflowToStart: WorkflowToStart, it instantiates the DeleteWorkflowFilesActor and waits for it to respond. Note: We can't start deleting files before finalization succeeds as we don't want to start deleting them as they are being copied to another location. - */ + */ private def deleteFilesOrGotoFinalState(data: WorkflowActorData) = { def deleteFiles() = { val rootWorkflowId = data.workflowDescriptor.get.rootWorkflowId - val deleteActor = context.actorOf(DeleteWorkflowFilesActor.props( - rootWorkflowId = rootWorkflowId, - rootWorkflowRootPaths = data.initializationData.getWorkflowRoots(), - rootAndSubworkflowIds = data.rootAndSubworkflowIds, - workflowFinalOutputs = data.workflowFinalOutputs.map(out => out.outputs.values.toSet).getOrElse(Set.empty), - workflowAllOutputs = data.workflowAllOutputs, - pathBuilders = data.workflowDescriptor.get.pathBuilders, - serviceRegistryActor = serviceRegistryActor, - ioActor = ioActor), - name = s"DeleteWorkflowFilesActor-${rootWorkflowId.id}") + val deleteActor = context.actorOf( + DeleteWorkflowFilesActor.props( + rootWorkflowId = rootWorkflowId, + rootWorkflowRootPaths = data.initializationData.getWorkflowRoots(), + rootAndSubworkflowIds = data.rootAndSubworkflowIds, + workflowFinalOutputs = data.workflowFinalOutputs.map(out => out.outputs.values.toSet).getOrElse(Set.empty), + workflowAllOutputs = data.workflowAllOutputs, + pathBuilders = data.workflowDescriptor.get.pathBuilders, + serviceRegistryActor = serviceRegistryActor, + ioActor = ioActor + ), + name = s"DeleteWorkflowFilesActor-${rootWorkflowId.id}" + ) deleteActor ! StartWorkflowFilesDeletion goto(DeletingFilesState) using data } - val userDeleteFileWfOption = data.workflowDescriptor.flatMap( - _.backendDescriptor.workflowOptions.getBoolean("delete_intermediate_output_files").toOption - ).getOrElse(false) + val userDeleteFileWfOption = data.workflowDescriptor + .flatMap( + _.backendDescriptor.workflowOptions.getBoolean("delete_intermediate_output_files").toOption + ) + .getOrElse(false) (deleteWorkflowFiles, userDeleteFileWfOption, data.workflowAllOutputs.nonEmpty) match { case (true, true, true) => deleteFiles() @@ -668,26 +784,42 @@ class WorkflowActor(workflowToStart: WorkflowToStart, // user has not enabled delete intermediate outputs option, so go to succeeded status goto(WorkflowSucceededState) using data.copy(currentLifecycleStateActor = None) case (false, true, _) => - log.info(s"User wants to delete intermediate files but it is not enabled in Cromwell config. To use it system.delete-workflow-files to true.") + log.info( + s"User wants to delete intermediate files but it is not enabled in Cromwell config. To use it system.delete-workflow-files to true." + ) goto(WorkflowSucceededState) using data.copy(currentLifecycleStateActor = None) case (false, false, _) => goto(WorkflowSucceededState) using data.copy(currentLifecycleStateActor = None) } } - private[workflow] def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs) = { + private[workflow] def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, + jobExecutionMap: JobExecutionMap, + workflowOutputs: CallOutputs + ) = { val copyWorkflowOutputsActorProps = stateName match { case InitializingWorkflowState => None - case _ => Option(CopyWorkflowOutputsActor.props(workflowIdForLogging, ioActor, workflowDescriptor, workflowOutputs, stateData.initializationData)) + case _ => + Option( + CopyWorkflowOutputsActor.props(workflowIdForLogging, + ioActor, + workflowDescriptor, + workflowOutputs, + stateData.initializationData + ) + ) } - context.actorOf(WorkflowFinalizationActor.props( - workflowDescriptor = workflowDescriptor, - ioActor = ioActor, - jobExecutionMap = jobExecutionMap, - workflowOutputs = workflowOutputs, - initializationData = stateData.initializationData, - copyWorkflowOutputsActor = copyWorkflowOutputsActorProps - ), name = s"WorkflowFinalizationActor") + context.actorOf( + WorkflowFinalizationActor.props( + workflowDescriptor = workflowDescriptor, + ioActor = ioActor, + jobExecutionMap = jobExecutionMap, + workflowOutputs = workflowOutputs, + initializationData = stateData.initializationData, + copyWorkflowOutputsActor = copyWorkflowOutputsActorProps + ), + name = s"WorkflowFinalizationActor" + ) } /** @@ -700,17 +832,19 @@ class WorkflowActor(workflowToStart: WorkflowToStart, failures: Option[List[Throwable]], workflowAllOutputs: Set[WomValue] = Set.empty, rootAndSubworkflowIds: Set[WorkflowId] = Set.empty, - lastStateOverride: Option[WorkflowActorState] = None) = { + lastStateOverride: Option[WorkflowActorState] = None + ) = { val finalizationActor = makeFinalizationActor(workflowDescriptor, jobExecutionMap, workflowFinalOutputs) finalizationActor ! StartFinalizationCommand goto(FinalizingWorkflowState) using data.copy( - lastStateReached = StateCheckpoint (lastStateOverride.getOrElse(stateName), failures), + lastStateReached = StateCheckpoint(lastStateOverride.getOrElse(stateName), failures), workflowFinalOutputs = Option(workflowFinalOutputs), workflowAllOutputs = workflowAllOutputs, rootAndSubworkflowIds = rootAndSubworkflowIds ) } - private def sendHeartbeat(): Unit = workflowStoreActor ! WorkflowStoreWriteHeartbeatCommand(workflowId, submissionTime) + private def sendHeartbeat(): Unit = + workflowStoreActor ! WorkflowStoreWriteHeartbeatCommand(workflowId, submissionTime) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowDockerLookupActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowDockerLookupActor.scala index 32889f7f849..8c3afe375a1 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowDockerLookupActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowDockerLookupActor.scala @@ -8,7 +8,12 @@ import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.{Dispatcher, WorkflowId} import cromwell.database.sql.EngineSqlDatabase import cromwell.database.sql.tables.DockerHashStoreEntry -import cromwell.docker.DockerInfoActor.{DockerHashFailureResponse, DockerInfoSuccessResponse, DockerInformation, DockerSize} +import cromwell.docker.DockerInfoActor.{ + DockerHashFailureResponse, + DockerInformation, + DockerInfoSuccessResponse, + DockerSize +} import cromwell.docker.{DockerClientHelper, DockerHashResult, DockerImageIdentifier, DockerInfoRequest} import cromwell.engine.workflow.WorkflowDockerLookupActor._ import cromwell.services.EngineServicesStore @@ -37,11 +42,12 @@ import scala.util.{Failure, Success} * for this tag will be attempted again. */ -class WorkflowDockerLookupActor private[workflow](workflowId: WorkflowId, - val dockerHashingActor: ActorRef, - isRestart: Boolean, - databaseInterface: EngineSqlDatabase) - extends LoggingFSM[WorkflowDockerLookupActorState, WorkflowDockerLookupActorData] with DockerClientHelper { +class WorkflowDockerLookupActor private[workflow] (workflowId: WorkflowId, + val dockerHashingActor: ActorRef, + isRestart: Boolean, + databaseInterface: EngineSqlDatabase +) extends LoggingFSM[WorkflowDockerLookupActorState, WorkflowDockerLookupActorData] + with DockerClientHelper { implicit val ec = context.system.dispatchers.lookup(Dispatcher.EngineDispatcher) @@ -55,10 +61,9 @@ class WorkflowDockerLookupActor private[workflow](workflowId: WorkflowId, // `AwaitingFirstRequestOnRestart` is only used in restart scenarios. This state waits until there's at least one hash // request before trying to load the docker hash mappings. This is so we'll have at least one `JobPreparationActor` // reference available to message with a terminal failure in case the reading or parsing of these mappings fails. - when(AwaitingFirstRequestOnRestart) { - case Event(request: DockerInfoRequest, data) => - loadDockerHashStoreEntries() - goto(LoadingCache) using data.addHashRequest(request, sender()) + when(AwaitingFirstRequestOnRestart) { case Event(request: DockerInfoRequest, data) => + loadDockerHashStoreEntries() + goto(LoadingCache) using data.addHashRequest(request, sender()) } // Waiting for a response from the database with the hash mapping for this workflow. @@ -89,7 +94,10 @@ class WorkflowDockerLookupActor private[workflow](workflowId: WorkflowId, case Event(DockerHashStoreSuccess(response), data) => recordMappingAndRespond(response, data) case Event(DockerHashStoreFailure(request, e), data) => - handleStoreFailure(request, new Exception(s"Failure storing docker hash for ${request.dockerImageID.fullName}", e), data) + handleStoreFailure(request, + new Exception(s"Failure storing docker hash for ${request.dockerImageID.fullName}", e), + data + ) } when(Terminal) { @@ -97,7 +105,10 @@ class WorkflowDockerLookupActor private[workflow](workflowId: WorkflowId, // In the Terminal state we reject all requests with the cause set in the state data. sender() ! WorkflowDockerLookupFailure(data.failureCause.orNull, request) stay() - case Event(_ @ (_: DockerInfoSuccessResponse | _: DockerHashFailureResponse | _: DockerHashStoreSuccess | _: DockerHashStoreFailure), _) => + case Event(_ @(_: DockerInfoSuccessResponse | _: DockerHashFailureResponse | _: DockerHashStoreSuccess | + _: DockerHashStoreFailure), + _ + ) => // Other expected message types are unsurprising in the Terminal state and can be swallowed. Unexpected message // types will be handled by `whenUnhandled`. stay() @@ -110,7 +121,8 @@ class WorkflowDockerLookupActor private[workflow](workflowId: WorkflowId, whenUnhandled { case Event(DockerHashActorTimeout(request), data) => - val reason = new Exception(s"Timeout looking up hash for Docker image ${request.dockerImageID} in state $stateName") + val reason = + new Exception(s"Timeout looking up hash for Docker image ${request.dockerImageID} in state $stateName") data.hashRequests.get(request.dockerImageID) match { case Some(requestsAndReplyTos) => requestsAndReplyTos foreach { case RequestAndReplyTo(_, replyTo) => @@ -119,9 +131,16 @@ class WorkflowDockerLookupActor private[workflow](workflowId: WorkflowId, val updatedData = data.copy(hashRequests = data.hashRequests - request.dockerImageID) stay() using updatedData case None => - val headline = s"Unable to find requesters for timed out lookup of Docker image '${request.dockerImageID}' in state $stateName" - val pendingImageIdsAndCounts = stateData.hashRequests.toList map { case (imageId, requestAndReplyTos) => s"$imageId -> ${requestAndReplyTos.size}" } - val message = pendingImageIdsAndCounts.mkString(headline + "\n" + "Pending image ID requests with requester counts: ", ", ", "") + val headline = + s"Unable to find requesters for timed out lookup of Docker image '${request.dockerImageID}' in state $stateName" + val pendingImageIdsAndCounts = stateData.hashRequests.toList map { case (imageId, requestAndReplyTos) => + s"$imageId -> ${requestAndReplyTos.size}" + } + val message = + pendingImageIdsAndCounts.mkString(headline + "\n" + "Pending image ID requests with requester counts: ", + ", ", + "" + ) fail(new RuntimeException(message) with NoStackTrace) } case Event(TransitionToFailed(cause), data) => @@ -137,27 +156,37 @@ class WorkflowDockerLookupActor private[workflow](workflowId: WorkflowId, * Load mappings from the database into the state data, reply to queued requests which have mappings, and initiate * hash lookups for requests which don't have mappings. */ - private def loadCacheAndHandleHashRequests(hashEntries: Map[String, DockerHashStoreEntry], data: WorkflowDockerLookupActorData): State = { - val dockerMappingsTry = hashEntries map { - case (dockerTag, entry) => ( + private def loadCacheAndHandleHashRequests(hashEntries: Map[String, DockerHashStoreEntry], + data: WorkflowDockerLookupActorData + ): State = { + val dockerMappingsTry = hashEntries map { case (dockerTag, entry) => + ( DockerImageIdentifier.fromString(dockerTag), - DockerHashResult.fromString(entry.dockerHash) map { hash => DockerInformation(hash, entry.dockerSize.map(DockerSize.apply)) } + DockerHashResult.fromString(entry.dockerHash) map { hash => + DockerInformation(hash, entry.dockerSize.map(DockerSize.apply)) + } ) } TryUtil.sequenceKeyValues(dockerMappingsTry) match { case Success(dockerMappings) => // Figure out which of the queued requests already have established mappings. - val (hasMappings, doesNotHaveMappings) = data.hashRequests.partition { case (dockerImageId, _) => dockerMappings.contains(dockerImageId) } + val (hasMappings, doesNotHaveMappings) = data.hashRequests.partition { case (dockerImageId, _) => + dockerMappings.contains(dockerImageId) + } // The requests which have mappings receive success responses. hasMappings foreach { case (dockerImageId, requestAndReplyTos) => val result = dockerMappings(dockerImageId) - requestAndReplyTos foreach { case RequestAndReplyTo(request, replyTo) => replyTo ! DockerInfoSuccessResponse(result, request)} + requestAndReplyTos foreach { case RequestAndReplyTo(request, replyTo) => + replyTo ! DockerInfoSuccessResponse(result, request) + } } // The requests without mappings need to be looked up. - doesNotHaveMappings foreach { case (_, requestAndReplyTos) => sendDockerCommand(requestAndReplyTos.head.request) } + doesNotHaveMappings foreach { case (_, requestAndReplyTos) => + sendDockerCommand(requestAndReplyTos.head.request) + } // Update state data accordingly. val newData = data.copy(hashRequests = doesNotHaveMappings, mappings = dockerMappings, failureCause = None) @@ -171,55 +200,83 @@ class WorkflowDockerLookupActor private[workflow](workflowId: WorkflowId, private def requestDockerHash(request: DockerInfoRequest, data: WorkflowDockerLookupActorData): State = { sendDockerCommand(request) val replyTo = sender() - val updatedData = data.copy(hashRequests = data.hashRequests + (request.dockerImageID -> NonEmptyList.of(RequestAndReplyTo(request, replyTo)))) + val updatedData = data.copy(hashRequests = + data.hashRequests + (request.dockerImageID -> NonEmptyList.of(RequestAndReplyTo(request, replyTo))) + ) stay() using updatedData } - private def recordMappingAndRespond(response: DockerInfoSuccessResponse, data: WorkflowDockerLookupActorData): State = { + private def recordMappingAndRespond(response: DockerInfoSuccessResponse, + data: WorkflowDockerLookupActorData + ): State = { // Add the new label to hash mapping to the current set of mappings. val request = response.request data.hashRequests.get(request.dockerImageID) match { - case Some(actors) => actors foreach { case RequestAndReplyTo(_, replyTo) => replyTo ! DockerInfoSuccessResponse(response.dockerInformation, request) } - case None => fail(new Exception(s"Could not find the actors associated with $request. Available requests are ${data.hashRequests.keys.mkString(", ")}") with NoStackTrace) + case Some(actors) => + actors foreach { case RequestAndReplyTo(_, replyTo) => + replyTo ! DockerInfoSuccessResponse(response.dockerInformation, request) + } + case None => + fail( + new Exception( + s"Could not find the actors associated with $request. Available requests are ${data.hashRequests.keys.mkString(", ")}" + ) with NoStackTrace + ) } - val updatedData = data.copy(hashRequests = data.hashRequests - request.dockerImageID, mappings = data.mappings + (request.dockerImageID -> response.dockerInformation)) + val updatedData = data.copy(hashRequests = data.hashRequests - request.dockerImageID, + mappings = data.mappings + (request.dockerImageID -> response.dockerInformation) + ) stay() using updatedData } private def respondToAllRequests(reason: Throwable, data: WorkflowDockerLookupActorData, - messageBuilder: (Throwable, DockerInfoRequest) => WorkflowDockerLookupResponse): WorkflowDockerLookupActorData = { + messageBuilder: (Throwable, DockerInfoRequest) => WorkflowDockerLookupResponse + ): WorkflowDockerLookupActorData = { data.hashRequests foreach { case (_, replyTos) => replyTos foreach { case RequestAndReplyTo(request, replyTo) => replyTo ! messageBuilder(reason, request) } } data.clearHashRequests } - private def respondToAllRequestsWithTerminalFailure(reason: Throwable, data: WorkflowDockerLookupActorData): WorkflowDockerLookupActorData = { + private def respondToAllRequestsWithTerminalFailure(reason: Throwable, + data: WorkflowDockerLookupActorData + ): WorkflowDockerLookupActorData = respondToAllRequests(reason, data, WorkflowDockerTerminalFailure.apply) - } private def persistDockerHash(response: DockerInfoSuccessResponse, data: WorkflowDockerLookupActorData): State = { // BA-6495 if there are actors awaiting for this data, then proceed, otherwise - don't bother to persist if (data.hashRequests.contains(response.request.dockerImageID)) { - val dockerHashStoreEntry = DockerHashStoreEntry(workflowId.toString, response.request.dockerImageID.fullName, response.dockerInformation.dockerHash.algorithmAndHash, response.dockerInformation.dockerCompressedSize.map(_.compressedSize)) + val dockerHashStoreEntry = DockerHashStoreEntry( + workflowId.toString, + response.request.dockerImageID.fullName, + response.dockerInformation.dockerHash.algorithmAndHash, + response.dockerInformation.dockerCompressedSize.map(_.compressedSize) + ) databaseInterface.addDockerHashStoreEntry(dockerHashStoreEntry) onComplete { case Success(_) => self ! DockerHashStoreSuccess(response) case Failure(ex) => self ! DockerHashStoreFailure(response.request, ex) } } else { - log.debug(s"Unable to find requesters for succeeded lookup of Docker image " + - s"'${response.request.dockerImageID}'. Most likely reason is that requesters have already been cleaned out " + - s"earlier by the timeout.") + log.debug( + s"Unable to find requesters for succeeded lookup of Docker image " + + s"'${response.request.dockerImageID}'. Most likely reason is that requesters have already been cleaned out " + + s"earlier by the timeout." + ) } stay() } - private def handleLookupFailure(dockerResponse: DockerHashFailureResponse, data: WorkflowDockerLookupActorData): State = { + private def handleLookupFailure(dockerResponse: DockerHashFailureResponse, + data: WorkflowDockerLookupActorData + ): State = { // Fail all pending requests. This logic does not blacklist the tag, which will allow lookups to be attempted // again in the future. - val exceptionMessage = s"Failed Docker lookup '${dockerResponse.request.dockerImageID}' '${dockerResponse.request.credentialDetails.mkString("[", ", ", "]")}'" - val failureResponse = WorkflowDockerLookupFailure(new Exception(dockerResponse.reason), dockerResponse.request, exceptionMessage) + val exceptionMessage = + s"Failed Docker lookup '${dockerResponse.request.dockerImageID}' '${dockerResponse.request.credentialDetails + .mkString("[", ", ", "]")}'" + val failureResponse = + WorkflowDockerLookupFailure(new Exception(dockerResponse.reason), dockerResponse.request, exceptionMessage) val request = dockerResponse.request data.hashRequests.get(request.dockerImageID) match { case Some(requestAndReplyTos) => @@ -227,27 +284,35 @@ class WorkflowDockerLookupActor private[workflow](workflowId: WorkflowId, val updatedData = data.copy(hashRequests = data.hashRequests - request.dockerImageID) stay() using updatedData case None => - log.debug(s"Unable to find requesters for failed lookup of Docker image '${request.dockerImageID}'. " + - s"Most likely reason is that requesters have already been cleaned out earlier by the timeout.") + log.debug( + s"Unable to find requesters for failed lookup of Docker image '${request.dockerImageID}'. " + + s"Most likely reason is that requesters have already been cleaned out earlier by the timeout." + ) stay() } } - private def handleStoreFailure(dockerHashRequest: DockerInfoRequest, reason: Throwable, data: WorkflowDockerLookupActorData): State = { + private def handleStoreFailure(dockerHashRequest: DockerInfoRequest, + reason: Throwable, + data: WorkflowDockerLookupActorData + ): State = data.hashRequests.get(dockerHashRequest.dockerImageID) match { case Some(requestAndReplyTos) => - requestAndReplyTos foreach { case RequestAndReplyTo(_, replyTo) => replyTo ! WorkflowDockerLookupFailure(reason, dockerHashRequest) } + requestAndReplyTos foreach { case RequestAndReplyTo(_, replyTo) => + replyTo ! WorkflowDockerLookupFailure(reason, dockerHashRequest) + } // Remove these requesters from the collection of those awaiting hashes. stay() using data.copy(hashRequests = data.hashRequests - dockerHashRequest.dockerImageID) case None => - log.debug(s"Unable to find requesters for failed store of hash for Docker image " + - s"'${dockerHashRequest.dockerImageID}'. Most likely reason is that requesters have already been cleaned " + - s"out earlier by the timeout.") + log.debug( + s"Unable to find requesters for failed store of hash for Docker image " + + s"'${dockerHashRequest.dockerImageID}'. Most likely reason is that requesters have already been cleaned " + + s"out earlier by the timeout." + ) stay() } - } - def loadDockerHashStoreEntries(): Unit = { + def loadDockerHashStoreEntries(): Unit = databaseInterface.queryDockerHashStoreEntries(workflowId.toString) onComplete { case Success(dockerHashEntries) => val dockerMappings = dockerHashEntries.map(entry => entry.dockerTag -> entry).toMap @@ -255,13 +320,11 @@ class WorkflowDockerLookupActor private[workflow](workflowId: WorkflowId, case Failure(ex) => fail(new RuntimeException("Failed to load docker tag -> hash mappings from DB", ex)) } - } - override protected def onTimeout(message: Any, to: ActorRef): Unit = { + override protected def onTimeout(message: Any, to: ActorRef): Unit = message match { case r: DockerInfoRequest => self ! DockerHashActorTimeout(r) } - } } object WorkflowDockerLookupActor { @@ -272,41 +335,52 @@ object WorkflowDockerLookupActor { case object Running extends WorkflowDockerLookupActorState case object Terminal extends WorkflowDockerLookupActorState private val FailedException = - new Exception(s"The WorkflowDockerLookupActor has failed. Subsequent docker tags for this workflow will not be resolved.") + new Exception( + s"The WorkflowDockerLookupActor has failed. Subsequent docker tags for this workflow will not be resolved." + ) /* Internal ADTs */ final case class DockerRequestContext(dockerHashRequest: DockerInfoRequest, replyTo: ActorRef) sealed trait DockerHashStoreResponse final case class DockerHashStoreSuccess(successResponse: DockerInfoSuccessResponse) extends DockerHashStoreResponse - final case class DockerHashStoreFailure(dockerHashRequest: DockerInfoRequest, reason: Throwable) extends DockerHashStoreResponse + final case class DockerHashStoreFailure(dockerHashRequest: DockerInfoRequest, reason: Throwable) + extends DockerHashStoreResponse final case class DockerHashStoreLoadingSuccess(dockerMappings: Map[String, DockerHashStoreEntry]) final case class DockerHashActorTimeout(request: DockerInfoRequest) /* Messages */ sealed trait WorkflowDockerLookupActorMessage - private final case class TransitionToFailed(cause: Throwable) extends WorkflowDockerLookupActorMessage + final private case class TransitionToFailed(cause: Throwable) extends WorkflowDockerLookupActorMessage /* Responses */ sealed trait WorkflowDockerLookupResponse - final case class WorkflowDockerLookupFailure(reason: Throwable, request: DockerInfoRequest, additionalLoggingMessage: String = "") extends WorkflowDockerLookupResponse - final case class WorkflowDockerTerminalFailure(reason: Throwable, request: DockerInfoRequest) extends WorkflowDockerLookupResponse + final case class WorkflowDockerLookupFailure(reason: Throwable, + request: DockerInfoRequest, + additionalLoggingMessage: String = "" + ) extends WorkflowDockerLookupResponse + final case class WorkflowDockerTerminalFailure(reason: Throwable, request: DockerInfoRequest) + extends WorkflowDockerLookupResponse case class RequestAndReplyTo(request: DockerInfoRequest, replyTo: ActorRef) def props(workflowId: WorkflowId, dockerHashingActor: ActorRef, isRestart: Boolean, - databaseInterface: EngineSqlDatabase = EngineServicesStore.engineDatabaseInterface): Props = { - Props(new WorkflowDockerLookupActor(workflowId, dockerHashingActor, isRestart, databaseInterface)).withDispatcher(EngineDispatcher) - } + databaseInterface: EngineSqlDatabase = EngineServicesStore.engineDatabaseInterface + ): Props = + Props(new WorkflowDockerLookupActor(workflowId, dockerHashingActor, isRestart, databaseInterface)) + .withDispatcher(EngineDispatcher) object WorkflowDockerLookupActorData { def empty = WorkflowDockerLookupActorData(hashRequests = Map.empty, mappings = Map.empty, failureCause = None) } - final case class WorkflowDockerLookupActorData(hashRequests: Map[DockerImageIdentifier, NonEmptyList[RequestAndReplyTo]], - mappings: Map[DockerImageIdentifier, DockerInformation], - failureCause: Option[Throwable]) { + final case class WorkflowDockerLookupActorData( + hashRequests: Map[DockerImageIdentifier, NonEmptyList[RequestAndReplyTo]], + mappings: Map[DockerImageIdentifier, DockerInformation], + failureCause: Option[Throwable] + ) { + /** * Add the specified request and replyTo to this state data. * diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index e84d9091ad0..abad4961211 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -37,6 +37,7 @@ object WorkflowManagerActor { class WorkflowNotFoundException(s: String) extends Exception(s) sealed trait WorkflowManagerActorMessage + /** * Commands */ @@ -48,7 +49,8 @@ object WorkflowManagerActor { final case class SubscribeToWorkflowCommand(id: WorkflowId) extends WorkflowManagerActorCommand case object EngineStatsCommand extends WorkflowManagerActorCommand case class AbortWorkflowsCommand(ids: Set[WorkflowId]) extends WorkflowManagerActorCommand - final case class WorkflowActorWorkComplete(id: WorkflowId, actor: ActorRef, finalState: WorkflowState) extends WorkflowManagerActorCommand + final case class WorkflowActorWorkComplete(id: WorkflowId, actor: ActorRef, finalState: WorkflowState) + extends WorkflowManagerActorCommand def props(config: Config, callCachingEnabled: Boolean, @@ -67,7 +69,8 @@ object WorkflowManagerActor { jobExecutionTokenDispenserActor: ActorRef, backendSingletonCollection: BackendSingletonCollection, serverMode: Boolean, - workflowHeartbeatConfig: WorkflowHeartbeatConfig): Props = { + workflowHeartbeatConfig: WorkflowHeartbeatConfig + ): Props = { val params = WorkflowManagerActorParams( config = config, callCachingEnabled = callCachingEnabled, @@ -86,7 +89,8 @@ object WorkflowManagerActor { jobExecutionTokenDispenserActor = jobExecutionTokenDispenserActor, backendSingletonCollection = backendSingletonCollection, serverMode = serverMode, - workflowHeartbeatConfig = workflowHeartbeatConfig) + workflowHeartbeatConfig = workflowHeartbeatConfig + ) Props(new WorkflowManagerActor(params)).withDispatcher(EngineDispatcher) } @@ -111,11 +115,10 @@ object WorkflowManagerActor { } def without(id: WorkflowId): WorkflowManagerData = this.copy(workflows = workflows - id) - def without(actor: ActorRef): WorkflowManagerData = { + def without(actor: ActorRef): WorkflowManagerData = // If the ID was found in the lookup return a modified copy of the state data, otherwise just return // the same state data. idFromActor(actor) map without getOrElse this - } } } @@ -136,25 +139,30 @@ case class WorkflowManagerActorParams(config: Config, jobExecutionTokenDispenserActor: ActorRef, backendSingletonCollection: BackendSingletonCollection, serverMode: Boolean, - workflowHeartbeatConfig: WorkflowHeartbeatConfig) + workflowHeartbeatConfig: WorkflowHeartbeatConfig +) class WorkflowManagerActor(params: WorkflowManagerActorParams) - extends LoggingFSM[WorkflowManagerState, WorkflowManagerData] with WorkflowMetadataHelper with Timers { + extends LoggingFSM[WorkflowManagerState, WorkflowManagerData] + with WorkflowMetadataHelper + with Timers { private val config = params.config private val callCachingEnabled: Boolean = params.callCachingEnabled private val invalidateBadCacheResults = params.invalidateBadCacheResults override val serviceRegistryActor = params.serviceRegistryActor - private val maxWorkflowsRunning = config.getConfig("system").as[Option[Int]]("max-concurrent-workflows").getOrElse(DefaultMaxWorkflowsToRun) - private val maxWorkflowsToLaunch = config.getConfig("system").as[Option[Int]]("max-workflow-launch-count").getOrElse(DefaultMaxWorkflowsToLaunch) - private val newWorkflowPollRate = config.getConfig("system").as[Option[Int]]("new-workflow-poll-rate").getOrElse(DefaultNewWorkflowPollRate).seconds + private val maxWorkflowsRunning = + config.getConfig("system").as[Option[Int]]("max-concurrent-workflows").getOrElse(DefaultMaxWorkflowsToRun) + private val maxWorkflowsToLaunch = + config.getConfig("system").as[Option[Int]]("max-workflow-launch-count").getOrElse(DefaultMaxWorkflowsToLaunch) + private val newWorkflowPollRate = + config.getConfig("system").as[Option[Int]]("new-workflow-poll-rate").getOrElse(DefaultNewWorkflowPollRate).seconds private val fileHashCacheEnabled = config.as[Option[Boolean]]("system.file-hash-cache").getOrElse(false) private val logger = Logging(context.system, this) private val tag = self.path.name - override def preStart(): Unit = { // Starts the workflow polling cycle timers.startSingleTimer(RetrieveNewWorkflowsKey, RetrieveNewWorkflows, Duration.Zero) @@ -179,11 +187,17 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams) params.jobExecutionTokenDispenserActor ! FetchLimitedGroups stay() case Event(ReplyLimitedGroups(groups), stateData) => + val wfCount = stateData.workflows.size + val swfCount = stateData.subWorkflows.size + val maxNewWorkflows = maxWorkflowsToLaunch min (maxWorkflowsRunning - wfCount - swfCount) + val fetchCountLog = + s"Fetching $maxNewWorkflows new workflows ($wfCount workflows and $swfCount subworkflows in flight)" if (groups.nonEmpty) - log.info(s"Excluding groups from workflow launch: ${groups.mkString(", ")}") + log.info(s"${fetchCountLog}, excluding groups: ${groups.mkString(", ")}") + else if (maxNewWorkflows < 1) + log.info(s"${fetchCountLog}, no groups excluded from workflow launch.") else - log.debug("No groups excluded from workflow launch.") - val maxNewWorkflows = maxWorkflowsToLaunch min (maxWorkflowsRunning - stateData.workflows.size - stateData.subWorkflows.size) + log.debug(s"${fetchCountLog}, no groups excluded from workflow launch.") params.workflowStore ! WorkflowStoreActor.FetchRunnableWorkflows(maxNewWorkflows, excludedGroups = groups) stay() case Event(WorkflowStoreEngineActor.NoNewWorkflowsToStart, _) => @@ -194,13 +208,13 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams) log.info("Retrieved {} workflows from the WorkflowStoreActor", newSubmissions.toList.size) stay() using stateData.withAddition(newSubmissions) case Event(SubscribeToWorkflowCommand(id), data) => - data.workflows.get(id) foreach {_ ! SubscribeTransitionCallBack(sender())} + data.workflows.get(id) foreach { _ ! SubscribeTransitionCallBack(sender()) } stay() case Event(AbortAllWorkflowsCommand, data) if data.workflows.isEmpty => goto(Done) case Event(AbortAllWorkflowsCommand, data) => log.info(s"$tag: Aborting all workflows") - data.workflows.values.foreach { _ ! WorkflowActor.AbortWorkflowCommand } + data.workflows.values.foreach(_ ! WorkflowActor.AbortWorkflowCommand) goto(Aborting) /* Responses from services @@ -209,7 +223,9 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams) log.info(s"$tag: Workflow $workflowId failed (during $inState): ${expandFailureReasons(reasons)}") stay() case Event(WorkflowActorWorkComplete(id: WorkflowId, workflowActor: ActorRef, finalState: WorkflowState), data) => - log.info(s"$tag: Workflow actor for $id completed with status '$finalState'. The workflow will be removed from the workflow store.") + log.info( + s"$tag: Workflow actor for $id completed with status '$finalState'. The workflow will be removed from the workflow store." + ) // This silently fails if idFromActor is None, but data.without call right below will as well data.idFromActor(workflowActor) foreach { workflowId => params.jobStoreActor ! RegisterWorkflowCompleted(workflowId) @@ -218,16 +234,18 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams) } val scheduleNextNewWorkflowPollStateFunction: StateFunction = { - case event @ Event(WorkflowStoreEngineActor.NoNewWorkflowsToStart | _: WorkflowStoreEngineActor.NewWorkflowsToStart, _) => + case event @ Event(WorkflowStoreEngineActor.NoNewWorkflowsToStart | _: WorkflowStoreEngineActor.NewWorkflowsToStart, + _ + ) => scheduleNextNewWorkflowPoll() runningAndNotStartingNewWorkflowsStateFunction(event) } - when (Running) (scheduleNextNewWorkflowPollStateFunction.orElse(runningAndNotStartingNewWorkflowsStateFunction)) + when(Running)(scheduleNextNewWorkflowPollStateFunction.orElse(runningAndNotStartingNewWorkflowsStateFunction)) - when (RunningAndNotStartingNewWorkflows) (runningAndNotStartingNewWorkflowsStateFunction) + when(RunningAndNotStartingNewWorkflows)(runningAndNotStartingNewWorkflowsStateFunction) - when (Aborting) { + when(Aborting) { case Event(Transition(workflowActor, _, _: WorkflowActorTerminalState), data) => // Remove this terminal actor from the workflowStore and log a progress message. val updatedData = data.without(workflowActor) @@ -244,7 +262,7 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams) case Event(_, _) => stay() } - when (Done) { FSM.NullFunction } + when(Done)(FSM.NullFunction) whenUnhandled { case Event(AbortWorkflowsCommand(ids), stateData) => @@ -275,7 +293,9 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams) stay() case Event(EngineStatsCommand, data) => val sndr = sender() - context.actorOf(EngineStatsActor.props(data.workflows.values.toList, sndr), s"EngineStatsActor-${sndr.hashCode()}") + context.actorOf(EngineStatsActor.props(data.workflows.values.toList, sndr), + s"EngineStatsActor-${sndr.hashCode()}" + ) stay() // Anything else certainly IS interesting: case Event(unhandled, _) => @@ -306,12 +326,13 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams) logger.info(s"$tag: Starting workflow UUID($workflowId)") } - val fileHashCacheActorProps: Option[Props] = fileHashCacheEnabled.option(RootWorkflowFileHashCacheActor.props(params.ioActor, workflowId)) + val fileHashCacheActorProps: Option[Props] = + fileHashCacheEnabled.option(RootWorkflowFileHashCacheActor.props(params.ioActor, workflowId)) val wfProps = WorkflowActor.props( workflowToStart = workflow, conf = config, - callCachingEnabled = callCachingEnabled , + callCachingEnabled = callCachingEnabled, invalidateBadCacheResults = invalidateBadCacheResults, ioActor = params.ioActor, serviceRegistryActor = params.serviceRegistryActor, @@ -330,7 +351,8 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams) workflowHeartbeatConfig = params.workflowHeartbeatConfig, totalJobsByRootWf = new AtomicInteger(), fileHashCacheActorProps = fileHashCacheActorProps, - blacklistCache = callCachingBlacklistManager.blacklistCacheFor(workflow)) + blacklistCache = callCachingBlacklistManager.blacklistCacheFor(workflow) + ) val wfActor = context.actorOf(wfProps, name = s"WorkflowActor-$workflowId") wfActor ! SubscribeTransitionCallBack(self) @@ -339,9 +361,8 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams) WorkflowIdToActorRef(workflowId, wfActor) } - private def scheduleNextNewWorkflowPoll() = { + private def scheduleNextNewWorkflowPoll() = timers.startSingleTimer(RetrieveNewWorkflowsKey, RetrieveNewWorkflows, newWorkflowPollRate) - } private def expandFailureReasons(reasons: Seq[Throwable]): String = { @@ -351,9 +372,9 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams) case reason: ThrowableAggregation => expandFailureReasons(reason.throwables.toSeq) case reason: KnownJobFailureException => val stderrMessage = reason.stderrPath map { path => - val content = Try(path.annotatedContentAsStringWithLimit(3000)).recover({ - case e => s"Could not retrieve content: ${e.getMessage}" - }).get + val content = Try(path.annotatedContentAsStringWithLimit(3000)).recover { case e => + s"Could not retrieve content: ${e.getMessage}" + }.get s"\nCheck the content of stderr for potential additional information: ${path.pathAsString}.\n $content" } getOrElse "" reason.getMessage + stderrMessage diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowMetadataHelper.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowMetadataHelper.scala index e26dfaf4eea..784604be614 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowMetadataHelper.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowMetadataHelper.scala @@ -10,7 +10,7 @@ import cromwell.services.metadata.MetadataService._ trait WorkflowMetadataHelper { def serviceRegistryActor: ActorRef - + def pushWorkflowStart(workflowId: WorkflowId) = { val startEvent = MetadataEvent( MetadataKey(workflowId, None, WorkflowMetadataKeys.StartTime), @@ -18,7 +18,7 @@ trait WorkflowMetadataHelper { ) serviceRegistryActor ! PutMetadataAction(startEvent) } - + def pushWorkflowEnd(workflowId: WorkflowId) = { val metadataEventMsg = MetadataEvent( MetadataKey(workflowId, None, WorkflowMetadataKeys.EndTime), @@ -26,19 +26,25 @@ trait WorkflowMetadataHelper { ) serviceRegistryActor ! PutMetadataAction(metadataEventMsg) } - + def pushWorkflowFailures(workflowId: WorkflowId, failures: List[Throwable]) = { - val failureEvents = failures flatMap { r => throwableToMetadataEvents(MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Failures}"), r) } + val failureEvents = failures flatMap { r => + throwableToMetadataEvents(MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Failures}"), r) + } serviceRegistryActor ! PutMetadataAction(failureEvents) } - - def pushCurrentStateToMetadataService(workflowId: WorkflowId, workflowState: WorkflowState, confirmTo: Option[ActorRef] = None): Unit = { - val metadataEventMsg = MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.Status), MetadataValue(workflowState)) + + def pushCurrentStateToMetadataService(workflowId: WorkflowId, + workflowState: WorkflowState, + confirmTo: Option[ActorRef] = None + ): Unit = { + val metadataEventMsg = + MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.Status), MetadataValue(workflowState)) confirmTo match { case None => serviceRegistryActor ! PutMetadataAction(metadataEventMsg) case Some(actorRef) => serviceRegistryActor ! PutMetadataActionAndRespond(List(metadataEventMsg), actorRef) } } - + } diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowProcessingEventPublishing.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowProcessingEventPublishing.scala index 67b5626f018..83bac473d26 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowProcessingEventPublishing.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowProcessingEventPublishing.scala @@ -17,7 +17,11 @@ import scala.util.Random object WorkflowProcessingEventPublishing { private lazy val cromwellVersion = VersionUtil.getVersion("cromwell") - def publish(workflowId: WorkflowId, cromwellId: String, descriptionValue: DescriptionEventValue.Value, serviceRegistry: ActorRef): Unit = { + def publish(workflowId: WorkflowId, + cromwellId: String, + descriptionValue: DescriptionEventValue.Value, + serviceRegistry: ActorRef + ): Unit = { def randomNumberString: String = Random.nextInt(Int.MaxValue).toString def metadataKey(workflowId: WorkflowId, randomNumberString: String, key: String) = @@ -41,18 +45,16 @@ object WorkflowProcessingEventPublishing { def publishLabelsToMetadata(workflowId: WorkflowId, labels: Map[String, String], - serviceRegistry: ActorRef): IOChecked[Unit] = { + serviceRegistry: ActorRef + ): IOChecked[Unit] = { val defaultLabel = "cromwell-workflow-id" -> s"cromwell-$workflowId" Monad[IOChecked].pure(labelsToMetadata(workflowId, labels + defaultLabel, serviceRegistry)) } - private def labelsToMetadata(workflowId: WorkflowId, - labels: Map[String, String], - serviceRegistry: ActorRef): Unit = { + private def labelsToMetadata(workflowId: WorkflowId, labels: Map[String, String], serviceRegistry: ActorRef): Unit = labels foreach { case (labelKey, labelValue) => val metadataKey = MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Labels}:$labelKey") val metadataValue = MetadataValue(labelValue) serviceRegistry ! PutMetadataAction(MetadataEvent(metadataKey, metadataValue)) } - } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/TimedFSM.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/TimedFSM.scala index 8b29fe561c5..9367ba6e3cc 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/TimedFSM.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/TimedFSM.scala @@ -11,10 +11,9 @@ trait TimedFSM[S] { this: FSM[S, _] => def currentStateDuration: FiniteDuration = (System.currentTimeMillis() - lastTransitionTime).milliseconds - onTransition { - case from -> to => - val now = System.currentTimeMillis() - onTimedTransition(from, to, (now - lastTransitionTime).milliseconds) - lastTransitionTime = now + onTransition { case from -> to => + val now = System.currentTimeMillis() + onTimedTransition(from, to, (now - lastTransitionTime).milliseconds) + lastTransitionTime = now } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowLifecycleActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowLifecycleActor.scala index c5657e9a60a..0e76c4fb900 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowLifecycleActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowLifecycleActor.scala @@ -27,17 +27,18 @@ object WorkflowLifecycleActor { case class WorkflowLifecycleActorData(actors: Set[ActorRef], successes: Seq[BackendActorAndInitializationData], failures: Map[ActorRef, Throwable], - aborted: Seq[ActorRef]) { + aborted: Seq[ActorRef] + ) { def withActors(actors: Set[ActorRef]) = this.copy( actors = this.actors ++ actors ) def withSuccess(successfulActor: ActorRef, data: Option[BackendInitializationData] = None) = this.copy( actors = this.actors - successfulActor, - successes = successes :+ BackendActorAndInitializationData(successfulActor, data)) - def withFailure(failedActor: ActorRef, reason: Throwable) = this.copy( - actors = this.actors - failedActor, - failures = failures + (failedActor -> reason)) + successes = successes :+ BackendActorAndInitializationData(successfulActor, data) + ) + def withFailure(failedActor: ActorRef, reason: Throwable) = + this.copy(actors = this.actors - failedActor, failures = failures + (failedActor -> reason)) def withAborted(abortedActor: ActorRef) = this.copy( actors = this.actors - abortedActor, aborted = aborted :+ abortedActor @@ -51,7 +52,7 @@ trait AbortableWorkflowLifecycleActor[S <: WorkflowLifecycleActorState] extends def abortedResponse: EngineLifecycleActorAbortedResponse - override protected def checkForDoneAndTransition(newData: WorkflowLifecycleActorData): State = { + override protected def checkForDoneAndTransition(newData: WorkflowLifecycleActorData): State = if (checkForDone(newData)) { if (stateName == abortingState) { context.parent ! abortedResponse @@ -60,10 +61,11 @@ trait AbortableWorkflowLifecycleActor[S <: WorkflowLifecycleActorState] extends } else { stay() using newData } - } } -trait WorkflowLifecycleActor[S <: WorkflowLifecycleActorState] extends LoggingFSM[S, WorkflowLifecycleActorData] with WorkflowLogging { +trait WorkflowLifecycleActor[S <: WorkflowLifecycleActorState] + extends LoggingFSM[S, WorkflowLifecycleActorData] + with WorkflowLogging { val successState: S val failureState: S @@ -78,10 +80,9 @@ trait WorkflowLifecycleActor[S <: WorkflowLifecycleActorState] extends LoggingFS case t => super.supervisorStrategy.decider.applyOrElse(t, (_: Any) => Escalate) } - whenUnhandled { - case unhandledMessage => - workflowLogger.warn(s"received an unhandled message: $unhandledMessage") - stay() + whenUnhandled { case unhandledMessage => + workflowLogger.warn(s"received an unhandled message: $unhandledMessage") + stay() } onTransition { @@ -92,7 +93,7 @@ trait WorkflowLifecycleActor[S <: WorkflowLifecycleActorState] extends LoggingFS workflowLogger.debug(s"State is transitioning from $fromState to $toState.") } - protected def checkForDoneAndTransition(newData: WorkflowLifecycleActorData): State = { + protected def checkForDoneAndTransition(newData: WorkflowLifecycleActorData): State = if (checkForDone(newData)) { if (newData.failures.isEmpty) { context.parent ! successResponse(newData) @@ -104,7 +105,6 @@ trait WorkflowLifecycleActor[S <: WorkflowLifecycleActorState] extends LoggingFS } else { stay() using newData } - } - protected final def checkForDone(stateData: WorkflowLifecycleActorData) = stateData.actors.isEmpty + final protected def checkForDone(stateData: WorkflowLifecycleActorData) = stateData.actors.isEmpty } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/deletion/DeleteWorkflowFilesActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/deletion/DeleteWorkflowFilesActor.scala index 56d6012125d..3b32aa1feaf 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/deletion/DeleteWorkflowFilesActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/deletion/DeleteWorkflowFilesActor.scala @@ -34,16 +34,15 @@ class DeleteWorkflowFilesActor(rootWorkflowId: RootWorkflowId, pathBuilders: List[PathBuilder], serviceRegistryActor: ActorRef, ioActorRef: ActorRef, - gcsCommandBuilder: IoCommandBuilder, - ) - extends LoggingFSM[DeleteWorkflowFilesActorState, DeleteWorkflowFilesActorStateData] with IoClientHelper { + gcsCommandBuilder: IoCommandBuilder +) extends LoggingFSM[DeleteWorkflowFilesActorState, DeleteWorkflowFilesActorStateData] + with IoClientHelper { implicit val ec: ExecutionContext = context.dispatcher val asyncIO = new AsyncIo(ioActorRef, gcsCommandBuilder) val callCache = new CallCache(EngineServicesStore.engineDatabaseInterface) - startWith(Pending, NoData) when(Pending) { @@ -55,8 +54,7 @@ class DeleteWorkflowFilesActor(rootWorkflowId: RootWorkflowId, if (intermediateOutputs.nonEmpty) { self ! DeleteFiles goto(DeleteIntermediateFiles) using DeletingIntermediateFilesData(intermediateOutputs) - } - else { + } else { log.info(s"Root workflow ${rootWorkflowId.id} does not have any intermediate output files to delete.") respondAndStop(Nil, Nil, Nil) } @@ -112,7 +110,8 @@ class DeleteWorkflowFilesActor(rootWorkflowId: RootWorkflowId, In both these cases, we consider the deletion process a success, but warn the users of such files not found. */ val newDataWithErrorUpdates = error match { - case EnhancedCromwellIoException(_, _: FileNotFoundException) => newData.copy(filesNotFound = newData.filesNotFound :+ command.file) + case EnhancedCromwellIoException(_, _: FileNotFoundException) => + newData.copy(filesNotFound = newData.filesNotFound :+ command.file) case _ => newData.copy(deleteErrors = newData.deleteErrors :+ error) } commandState match { @@ -120,7 +119,9 @@ class DeleteWorkflowFilesActor(rootWorkflowId: RootWorkflowId, case AllCommandsDone => // once deletion is complete, invalidate call cache entries self ! InvalidateCallCache - goto(InvalidatingCallCache) using InvalidateCallCacheData(newDataWithErrorUpdates.deleteErrors, newDataWithErrorUpdates.filesNotFound) + goto(InvalidatingCallCache) using InvalidateCallCacheData(newDataWithErrorUpdates.deleteErrors, + newDataWithErrorUpdates.filesNotFound + ) } } @@ -159,14 +160,17 @@ class DeleteWorkflowFilesActor(rootWorkflowId: RootWorkflowId, val (newData: WaitingForInvalidateCCResponsesData, invalidateState) = data.commandComplete(cacheId.id) invalidateState match { case StillWaiting => stay() using newData - case AllCommandsDone => respondAndStop(newData.deleteErrors, newData.filesNotFound, newData.callCacheInvalidationErrors) + case AllCommandsDone => + respondAndStop(newData.deleteErrors, newData.filesNotFound, newData.callCacheInvalidationErrors) } case Event(CallCacheInvalidatedFailure(cacheId, error), data: WaitingForInvalidateCCResponsesData) => val (newData: WaitingForInvalidateCCResponsesData, invalidateState) = data.commandComplete(cacheId.id) - val updatedDataWithError = newData.copy(callCacheInvalidationErrors = newData.callCacheInvalidationErrors :+ error) + val updatedDataWithError = + newData.copy(callCacheInvalidationErrors = newData.callCacheInvalidationErrors :+ error) invalidateState match { case StillWaiting => stay() using updatedDataWithError - case AllCommandsDone => respondAndStop(newData.deleteErrors, newData.filesNotFound, updatedDataWithError.callCacheInvalidationErrors) + case AllCommandsDone => + respondAndStop(newData.deleteErrors, newData.filesNotFound, updatedDataWithError.callCacheInvalidationErrors) } } @@ -176,28 +180,37 @@ class DeleteWorkflowFilesActor(rootWorkflowId: RootWorkflowId, stay() case Event(ShutdownCommand, _) => stopSelf() case other => - log.error(s"Programmer Error: Unexpected message to ${getClass.getSimpleName} ${self.path.name} in state $stateName with $stateData: ${other.toPrettyElidedString(1000)}") + log.error( + s"Programmer Error: Unexpected message to ${getClass.getSimpleName} ${self.path.name} in state $stateName with $stateData: ${other + .toPrettyElidedString(1000)}" + ) stay() } - private def stopSelf() = { context stop self stay() } - - private def respondAndStop(errors: List[Throwable], filesNotFound: List[Path], callCacheInvalidationErrors: List[Throwable]) = { + private def respondAndStop(errors: List[Throwable], + filesNotFound: List[Path], + callCacheInvalidationErrors: List[Throwable] + ) = { val (metadataEvent, response) = - if (errors.isEmpty) (metadataEventForDeletionStatus(Succeeded), DeleteWorkflowFilesSucceededResponse(filesNotFound, callCacheInvalidationErrors)) - else (metadataEventForDeletionStatus(Failed), DeleteWorkflowFilesFailedResponse(errors, filesNotFound, callCacheInvalidationErrors)) + if (errors.isEmpty) + (metadataEventForDeletionStatus(Succeeded), + DeleteWorkflowFilesSucceededResponse(filesNotFound, callCacheInvalidationErrors) + ) + else + (metadataEventForDeletionStatus(Failed), + DeleteWorkflowFilesFailedResponse(errors, filesNotFound, callCacheInvalidationErrors) + ) serviceRegistryActor ! PutMetadataAction(metadataEvent) context.parent ! response stopSelf() } - private def metadataEventForDeletionStatus(status: FileDeletionStatus): MetadataEvent = { val key = MetadataKey(rootWorkflowId, None, WorkflowMetadataKeys.FileDeletionStatus) val value = MetadataValue(FileDeletionStatus.toDatabaseValue(status)) @@ -205,27 +218,28 @@ class DeleteWorkflowFilesActor(rootWorkflowId: RootWorkflowId, MetadataEvent(key, value) } - private def fetchCallCacheEntries(callCache: CallCache): Future[Set[Long]] = { - val callCacheEntryIdsFuture = rootAndSubworkflowIds.map(x => callCache.callCacheEntryIdsForWorkflowId(x.toString)).map { f => - f.map { Success(_) }.recover { case t => Failure(t) }} - - Future.sequence(callCacheEntryIdsFuture).map { _.flatMap { - case Success(callCacheEntryIds) => - Option(callCacheEntryIds) - case Failure(e) => - log.error(s"Failed to fetch call cache entry ids for workflow. Error: ${ExceptionUtils.getMessage(e)}") - None - }.flatten} + val callCacheEntryIdsFuture = + rootAndSubworkflowIds.map(x => callCache.callCacheEntryIdsForWorkflowId(x.toString)).map { f => + f.map(Success(_)).recover { case t => Failure(t) } + } + + Future.sequence(callCacheEntryIdsFuture).map { + _.flatMap { + case Success(callCacheEntryIds) => + Option(callCacheEntryIds) + case Failure(e) => + log.error(s"Failed to fetch call cache entry ids for workflow. Error: ${ExceptionUtils.getMessage(e)}") + None + }.flatten + } } - private def toPath(womSingleFile: WomSingleFile): Option[Path] = { + private def toPath(womSingleFile: WomSingleFile): Option[Path] = Try(PathFactory.buildPath(womSingleFile.valueString, pathBuilders)).toOption - } - private def getWomSingleFiles(womValue: WomValue): Seq[WomSingleFile] = { - womValue.collectAsSeq({ case womSingleFile: WomSingleFile => womSingleFile }) - } + private def getWomSingleFiles(womValue: WomValue): Seq[WomSingleFile] = + womValue.collectAsSeq { case womSingleFile: WomSingleFile => womSingleFile } /** * Returns Paths for WomSingleFiles in allOutputs that are not in finalOutputs, verifying that the Paths are contained @@ -236,27 +250,33 @@ class DeleteWorkflowFilesActor(rootWorkflowId: RootWorkflowId, val allOutputFiles = allOutputs.flatMap(getWomSingleFiles) val finalOutputFiles = finalOutputs.flatMap(getWomSingleFiles) val potentialIntermediaries = allOutputFiles.diff(finalOutputFiles).flatMap(toPath) - val checkedIntermediaries = potentialIntermediaries.filter(p => rootWorkflowRootPaths.exists(r => p.toAbsolutePath.startsWith(r.toAbsolutePath))) - for ( path <- potentialIntermediaries.diff(checkedIntermediaries) ) { - log.info(s"Did not delete $path because it is not contained within a workflow root directory for $rootWorkflowId.") - } + val checkedIntermediaries = potentialIntermediaries.filter(p => + rootWorkflowRootPaths.exists(r => p.toAbsolutePath.startsWith(r.toAbsolutePath)) + ) + for (path <- potentialIntermediaries.diff(checkedIntermediaries)) + log.info( + s"Did not delete $path because it is not contained within a workflow root directory for $rootWorkflowId." + ) checkedIntermediaries } override def ioActor: ActorRef = ioActorRef - - override protected def onTimeout(message: Any, to: ActorRef): Unit = { + override protected def onTimeout(message: Any, to: ActorRef): Unit = message match { - case delete: IoDeleteCommand => log.error(s"The DeleteWorkflowFilesActor for root workflow $rootWorkflowId timed out " + - s"waiting for a response for deleting file ${delete.file}.") - case other => log.error(s"The DeleteWorkflowFilesActor for root workflow $rootWorkflowId timed out " + - s"waiting for a response for unknown operation: $other.") + case delete: IoDeleteCommand => + log.error( + s"The DeleteWorkflowFilesActor for root workflow $rootWorkflowId timed out " + + s"waiting for a response for deleting file ${delete.file}." + ) + case other => + log.error( + s"The DeleteWorkflowFilesActor for root workflow $rootWorkflowId timed out " + + s"waiting for a response for unknown operation: $other." + ) } - } } - object DeleteWorkflowFilesActor { //@formatter:off @@ -289,63 +309,68 @@ object DeleteWorkflowFilesActor { def setCommandsToWaitFor(updatedCommandsToWaitFor: Set[A]): WaitingForResponseFromActorData[A] - def commandComplete(command: A): (WaitingForResponseFromActorData[A], WaitingForResponseState) = { + def commandComplete(command: A): (WaitingForResponseFromActorData[A], WaitingForResponseState) = if (commandsToWaitFor.isEmpty) (this, AllCommandsDone) else { val updatedCommandsSet = commandsToWaitFor - command val expectedCommandSetSize = updatedCommandsSet.size val requiredCommandSetSize = commandsToWaitFor.size - 1 - require(expectedCommandSetSize == requiredCommandSetSize, assertionFailureMsg(expectedCommandSetSize, requiredCommandSetSize)) + require(expectedCommandSetSize == requiredCommandSetSize, + assertionFailureMsg(expectedCommandSetSize, requiredCommandSetSize) + ) if (updatedCommandsSet.isEmpty) (setCommandsToWaitFor(Set.empty), AllCommandsDone) else (setCommandsToWaitFor(updatedCommandsSet), StillWaiting) } - } } case class WaitingForIoResponsesData(commandsToWaitFor: Set[IoDeleteCommand], deleteErrors: List[Throwable] = List.empty, - filesNotFound: List[Path] = List.empty) - extends WaitingForResponseFromActorData[IoDeleteCommand](commandsToWaitFor) with DeleteWorkflowFilesActorStateData { + filesNotFound: List[Path] = List.empty + ) extends WaitingForResponseFromActorData[IoDeleteCommand](commandsToWaitFor) + with DeleteWorkflowFilesActorStateData { - override def assertionFailureMsg(expectedSize: Int, requiredSize: Int): String = { + override def assertionFailureMsg(expectedSize: Int, requiredSize: Int): String = s"Found updated command set size as $expectedSize instead of $requiredSize. The updated set of commands that " + s"DeleteWorkflowFilesActor has to wait for should be 1 less after removing a completed command." - } - override def setCommandsToWaitFor(updatedCommandsToWaitFor: Set[IoDeleteCommand]): WaitingForResponseFromActorData[IoDeleteCommand] = { + override def setCommandsToWaitFor( + updatedCommandsToWaitFor: Set[IoDeleteCommand] + ): WaitingForResponseFromActorData[IoDeleteCommand] = this.copy(commandsToWaitFor = updatedCommandsToWaitFor) - } } case class WaitingForInvalidateCCResponsesData(commandsToWaitFor: Set[Long], deleteErrors: List[Throwable], filesNotFound: List[Path], - callCacheInvalidationErrors: List[Throwable] = List.empty) - extends WaitingForResponseFromActorData[Long](commandsToWaitFor) with DeleteWorkflowFilesActorStateData { + callCacheInvalidationErrors: List[Throwable] = List.empty + ) extends WaitingForResponseFromActorData[Long](commandsToWaitFor) + with DeleteWorkflowFilesActorStateData { - override def assertionFailureMsg(expectedSize: Int, requiredSize: Int): String = { + override def assertionFailureMsg(expectedSize: Int, requiredSize: Int): String = s"Found updated call cache entries set size as $expectedSize instead of $requiredSize. The updated set of call cache entries" + s" that DeleteWorkflowFilesActor has to wait for should be 1 less after a call cache entry is invalidated." - } - override def setCommandsToWaitFor(updatedCommandsToWaitFor: Set[Long]): WaitingForResponseFromActorData[Long] = { + override def setCommandsToWaitFor(updatedCommandsToWaitFor: Set[Long]): WaitingForResponseFromActorData[Long] = this.copy(commandsToWaitFor = updatedCommandsToWaitFor) - } } // Responses sealed trait DeleteWorkflowFilesResponse - case class DeleteWorkflowFilesSucceededResponse(filesNotFound: List[Path], callCacheInvalidationErrors: List[Throwable]) extends DeleteWorkflowFilesResponse - case class DeleteWorkflowFilesFailedResponse(errors: List[Throwable], filesNotFound: List[Path], callCacheInvalidationErrors: List[Throwable]) extends DeleteWorkflowFilesResponse + case class DeleteWorkflowFilesSucceededResponse(filesNotFound: List[Path], + callCacheInvalidationErrors: List[Throwable] + ) extends DeleteWorkflowFilesResponse + case class DeleteWorkflowFilesFailedResponse(errors: List[Throwable], + filesNotFound: List[Path], + callCacheInvalidationErrors: List[Throwable] + ) extends DeleteWorkflowFilesResponse // internal state to keep track of deletion of files and call cache invalidation sealed trait WaitingForResponseState private[deletion] case object StillWaiting extends WaitingForResponseState private[deletion] case object AllCommandsDone extends WaitingForResponseState - def props(rootWorkflowId: RootWorkflowId, rootAndSubworkflowIds: Set[WorkflowId], rootWorkflowRootPaths: Set[Path], @@ -354,18 +379,19 @@ object DeleteWorkflowFilesActor { pathBuilders: List[PathBuilder], serviceRegistryActor: ActorRef, ioActor: ActorRef, - gcsCommandBuilder: IoCommandBuilder = GcsBatchCommandBuilder, - ): Props = { - Props(new DeleteWorkflowFilesActor( - rootWorkflowId = rootWorkflowId, - rootAndSubworkflowIds = rootAndSubworkflowIds, - rootWorkflowRootPaths = rootWorkflowRootPaths, - workflowFinalOutputs = workflowFinalOutputs, - workflowAllOutputs = workflowAllOutputs, - pathBuilders = pathBuilders, - serviceRegistryActor = serviceRegistryActor, - ioActorRef = ioActor, - gcsCommandBuilder = gcsCommandBuilder, - )) - } + gcsCommandBuilder: IoCommandBuilder = GcsBatchCommandBuilder + ): Props = + Props( + new DeleteWorkflowFilesActor( + rootWorkflowId = rootWorkflowId, + rootAndSubworkflowIds = rootAndSubworkflowIds, + rootWorkflowRootPaths = rootWorkflowRootPaths, + workflowFinalOutputs = workflowFinalOutputs, + workflowAllOutputs = workflowAllOutputs, + pathBuilders = pathBuilders, + serviceRegistryActor = serviceRegistryActor, + ioActorRef = ioActor, + gcsCommandBuilder = gcsCommandBuilder + ) + ) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala index 0a7e3760a67..7df49b106e3 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala @@ -21,15 +21,23 @@ trait CallMetadataHelper { def pushNewCallMetadata(callKey: CallKey, backendName: Option[String], serviceRegistryActor: ActorRef) = { val startEvents = List( Option(MetadataEvent(metadataKeyForCall(callKey, CallMetadataKeys.Start), MetadataValue(OffsetDateTime.now))), - Option(MetadataEvent(metadataKeyForCall(callKey, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.QueuedInCromwell))), - backendName map { name => MetadataEvent(metadataKeyForCall(callKey, CallMetadataKeys.Backend), MetadataValue(name)) } + Option( + MetadataEvent(metadataKeyForCall(callKey, CallMetadataKeys.ExecutionStatus), + MetadataValue(ExecutionStatus.QueuedInCromwell) + ) + ), + backendName map { name => + MetadataEvent(metadataKeyForCall(callKey, CallMetadataKeys.Backend), MetadataValue(name)) + } ).flatten serviceRegistryActor ! PutMetadataAction(startEvents) } def pushStartingCallMetadata(callKey: CallKey) = { - val statusChange = MetadataEvent(metadataKeyForCall(callKey, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.Starting)) + val statusChange = MetadataEvent(metadataKeyForCall(callKey, CallMetadataKeys.ExecutionStatus), + MetadataValue(ExecutionStatus.Starting) + ) serviceRegistryActor ! PutMetadataAction(statusChange) } @@ -42,13 +50,16 @@ trait CallMetadataHelper { case empty if empty.isEmpty => List(MetadataEvent.empty(metadataKeyForCall(key, s"${CallMetadataKeys.Inputs}"))) case inputs => - inputs flatMap { - case (inputName, inputValue) => - womValueToMetadataEvents(metadataKeyForCall(key, s"${CallMetadataKeys.Inputs}:${inputName.name}"), inputValue) + inputs flatMap { case (inputName, inputValue) => + womValueToMetadataEvents(metadataKeyForCall(key, s"${CallMetadataKeys.Inputs}:${inputName.name}"), + inputValue + ) } } - val runningEvent = List(MetadataEvent(metadataKeyForCall(key, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.Running))) + val runningEvent = List( + MetadataEvent(metadataKeyForCall(key, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.Running)) + ) serviceRegistryActor ! PutMetadataAction(runningEvent ++ inputEvents) alreadyPushedRunningCallMetadata += metadataKeyForUniqueness } @@ -59,7 +70,10 @@ trait CallMetadataHelper { List(MetadataEvent.empty(MetadataKey(workflowIdForCallMetadata, None, WorkflowMetadataKeys.Outputs))) } else { outputs flatMap { case (outputName, outputValue) => - womValueToMetadataEvents(MetadataKey(workflowIdForCallMetadata, None, s"${WorkflowMetadataKeys.Outputs}:$outputName"), outputValue) + womValueToMetadataEvents( + MetadataKey(workflowIdForCallMetadata, None, s"${WorkflowMetadataKeys.Outputs}:$outputName"), + outputValue + ) } } @@ -74,7 +88,10 @@ trait CallMetadataHelper { List(MetadataEvent.empty(metadataKeyForCall(jobKey, s"${CallMetadataKeys.Outputs}"))) case _ => outputs.outputs flatMap { case (outputPort, outputValue) => - womValueToMetadataEvents(metadataKeyForCall(jobKey, s"${CallMetadataKeys.Outputs}:${outputPort.internalName}"), outputValue) + womValueToMetadataEvents( + metadataKeyForCall(jobKey, s"${CallMetadataKeys.Outputs}:${outputPort.internalName}"), + outputValue + ) } } @@ -84,13 +101,15 @@ trait CallMetadataHelper { def pushFailedCallMetadata(jobKey: JobKey, returnCode: Option[Int], failure: Throwable, retryableFailure: Boolean) = { val failedState = if (retryableFailure) ExecutionStatus.RetryableFailure else ExecutionStatus.Failed val completionEvents = completedCallMetadataEvents(jobKey, failedState, returnCode) - val retryableFailureEvent = MetadataEvent(metadataKeyForCall(jobKey, CallMetadataKeys.RetryableFailure), MetadataValue(retryableFailure)) + val retryableFailureEvent = + MetadataEvent(metadataKeyForCall(jobKey, CallMetadataKeys.RetryableFailure), MetadataValue(retryableFailure)) val failureEvents = failure match { // If the job was already failed, don't republish the failure reasons, they're already there case _: JobAlreadyFailedInJobStore => List.empty case _ => - throwableToMetadataEvents(metadataKeyForCall(jobKey, s"${CallMetadataKeys.Failures}"), failure).+:(retryableFailureEvent) + throwableToMetadataEvents(metadataKeyForCall(jobKey, s"${CallMetadataKeys.Failures}"), failure) + .+:(retryableFailureEvent) } serviceRegistryActor ! PutMetadataAction(completionEvents ++ failureEvents) @@ -117,14 +136,13 @@ trait CallMetadataHelper { val now = OffsetDateTime.now.withOffsetSameInstant(offset) val lastEvent = ExecutionEvent("!!Bring Back the Monarchy!!", now) val tailedEventList = sortedEvents :+ lastEvent - val events = tailedEventList.sliding(2) flatMap { - case Seq(eventCurrent, eventNext) => - val eventKey = s"${CallMetadataKeys.ExecutionEvents}[$randomNumberString]" - List( - metadataEvent(s"$eventKey:description", eventCurrent.name), - metadataEvent(s"$eventKey:startTime", eventCurrent.offsetDateTime), - metadataEvent(s"$eventKey:endTime", eventNext.offsetDateTime) - ) ++ (eventCurrent.grouping map { g => metadataEvent(s"$eventKey:grouping", g) }) + val events = tailedEventList.sliding(2) flatMap { case Seq(eventCurrent, eventNext) => + val eventKey = s"${CallMetadataKeys.ExecutionEvents}[$randomNumberString]" + List( + metadataEvent(s"$eventKey:description", eventCurrent.name), + metadataEvent(s"$eventKey:startTime", eventCurrent.offsetDateTime), + metadataEvent(s"$eventKey:endTime", eventNext.offsetDateTime) + ) ++ (eventCurrent.grouping map { g => metadataEvent(s"$eventKey:grouping", g) }) } serviceRegistryActor ! PutMetadataAction(events.toList) @@ -144,5 +162,9 @@ trait CallMetadataHelper { private def randomNumberString: String = Random.nextInt().toString.stripPrefix("-") - def metadataKeyForCall(jobKey: JobKey, myKey: String) = MetadataKey(workflowIdForCallMetadata, Option(MetadataJobKey(jobKey.node.fullyQualifiedName, jobKey.index, jobKey.attempt)), myKey) + def metadataKeyForCall(jobKey: JobKey, myKey: String) = MetadataKey( + workflowIdForCallMetadata, + Option(MetadataJobKey(jobKey.node.fullyQualifiedName, jobKey.index, jobKey.attempt)), + myKey + ) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala index 1cd48695ba2..75649ebf547 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala @@ -48,7 +48,11 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey, rootConfig: Config, totalJobsByRootWf: AtomicInteger, fileHashCacheActor: Option[ActorRef], - blacklistCache: Option[BlacklistCache]) extends LoggingFSM[SubWorkflowExecutionActorState, SubWorkflowExecutionActorData] with JobLogging with WorkflowMetadataHelper with CallMetadataHelper { + blacklistCache: Option[BlacklistCache] +) extends LoggingFSM[SubWorkflowExecutionActorState, SubWorkflowExecutionActorData] + with JobLogging + with WorkflowMetadataHelper + with CallMetadataHelper { override def supervisorStrategy: SupervisorStrategy = OneForOneStrategy() { case _ => Escalate } @@ -66,14 +70,13 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey, private var eventList: Seq[ExecutionEvent] = Seq(ExecutionEvent(stateName.toString)) - when(SubWorkflowPendingState) { - case Event(Execute, _) => - if (startState.restarted) { - subWorkflowStoreActor ! QuerySubWorkflow(parentWorkflow.id, key) - goto(SubWorkflowCheckingStoreState) - } else { - requestValueStore(createSubWorkflowId()) - } + when(SubWorkflowPendingState) { case Event(Execute, _) => + if (startState.restarted) { + subWorkflowStoreActor ! QuerySubWorkflow(parentWorkflow.id, key) + goto(SubWorkflowCheckingStoreState) + } else { + requestValueStore(createSubWorkflowId()) + } } when(SubWorkflowCheckingStoreState) { @@ -87,18 +90,23 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey, } /* - * ! Hot Potato Warning ! - * We ask explicitly for the `ValueStore` so we can use it on the fly and more importantly not store it as a - * variable in this actor, which would prevent it from being garbage collected for the duration of the - * subworkflow and would lead to memory leaks. - */ + * ! Hot Potato Warning ! + * We ask explicitly for the `ValueStore` so we can use it on the fly and more importantly not store it as a + * variable in this actor, which would prevent it from being garbage collected for the duration of the + * subworkflow and would lead to memory leaks. + */ when(WaitingForValueStore) { case Event(valueStore: ValueStore, SubWorkflowExecutionActorLiveData(Some(subWorkflowId), _)) => prepareSubWorkflow(subWorkflowId, valueStore) case Event(_: ValueStore, _) => - context.parent ! SubWorkflowFailedResponse(key, Map.empty, new IllegalStateException( - "This is a programmer error, we're ready to prepare the job and should have" + - " a SubWorkflowId to use by now but somehow haven't. Failing the workflow.")) + context.parent ! SubWorkflowFailedResponse( + key, + Map.empty, + new IllegalStateException( + "This is a programmer error, we're ready to prepare the job and should have" + + " a SubWorkflowId to use by now but somehow haven't. Failing the workflow." + ) + ) context stop self stay() } @@ -108,16 +116,36 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey, startSubWorkflow(subWorkflowEngineDescriptor, inputs, data) case Event(failure: CallPreparationFailed, data) => // No subworkflow ID yet, so no need to record the status. Fail here and let the parent handle the fallout: - recordTerminalState(SubWorkflowFailedState, SubWorkflowExecutionActorTerminalData(data.subWorkflowId, SubWorkflowFailedResponse(key, Map.empty, failure.throwable))) + recordTerminalState( + SubWorkflowFailedState, + SubWorkflowExecutionActorTerminalData(data.subWorkflowId, + SubWorkflowFailedResponse(key, Map.empty, failure.throwable) + ) + ) } when(SubWorkflowRunningState) { - case Event(WorkflowExecutionSucceededResponse(executedJobKeys, rootAndSubworklowIds, outputs, cumulativeOutputs), data) => - recordTerminalState(SubWorkflowSucceededState, SubWorkflowExecutionActorTerminalData(data.subWorkflowId, SubWorkflowSucceededResponse(key, executedJobKeys, rootAndSubworklowIds, outputs, cumulativeOutputs))) + case Event(WorkflowExecutionSucceededResponse(executedJobKeys, rootAndSubworklowIds, outputs, cumulativeOutputs), + data + ) => + recordTerminalState( + SubWorkflowSucceededState, + SubWorkflowExecutionActorTerminalData( + data.subWorkflowId, + SubWorkflowSucceededResponse(key, executedJobKeys, rootAndSubworklowIds, outputs, cumulativeOutputs) + ) + ) case Event(WorkflowExecutionFailedResponse(executedJobKeys, reason), data) => - recordTerminalState(SubWorkflowFailedState, SubWorkflowExecutionActorTerminalData(data.subWorkflowId, SubWorkflowFailedResponse(key, executedJobKeys, reason))) + recordTerminalState(SubWorkflowFailedState, + SubWorkflowExecutionActorTerminalData(data.subWorkflowId, + SubWorkflowFailedResponse(key, executedJobKeys, reason) + ) + ) case Event(WorkflowExecutionAbortedResponse(executedJobKeys), data) => - recordTerminalState(SubWorkflowAbortedState, SubWorkflowExecutionActorTerminalData(data.subWorkflowId, SubWorkflowAbortedResponse(key, executedJobKeys))) + recordTerminalState( + SubWorkflowAbortedState, + SubWorkflowExecutionActorTerminalData(data.subWorkflowId, SubWorkflowAbortedResponse(key, executedJobKeys)) + ) case Event(EngineLifecycleActorAbortCommand, SubWorkflowExecutionActorLiveData(_, Some(actorRef))) => actorRef ! EngineLifecycleActorAbortCommand stay() @@ -134,14 +162,22 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey, // If the final state can't be written, it's fairly likely that other metadata has also been lost, so // the best answer may unfortunately be to fail the workflow rather than retry. Assuming the call cache is working // correctly, a retry of the workflow should at least cache-hit instead of having to re-run the work: - recordTerminalState(SubWorkflowFailedState, terminalData.copy(terminalStateResponse = SubWorkflowFailedResponse(key, - terminalData.terminalStateResponse.jobExecutionMap, - new Exception("Sub workflow execution actor unable to write final state to metadata", reason) with NoStackTrace))) + recordTerminalState( + SubWorkflowFailedState, + terminalData.copy(terminalStateResponse = + SubWorkflowFailedResponse( + key, + terminalData.terminalStateResponse.jobExecutionMap, + new Exception("Sub workflow execution actor unable to write final state to metadata", reason) + with NoStackTrace + ) + ) + ) } - when(SubWorkflowSucceededState) { terminalMetadataWriteResponseHandler } - when(SubWorkflowFailedState) { terminalMetadataWriteResponseHandler } - when(SubWorkflowAbortedState) { terminalMetadataWriteResponseHandler } + when(SubWorkflowSucceededState)(terminalMetadataWriteResponseHandler) + when(SubWorkflowFailedState)(terminalMetadataWriteResponseHandler) + when(SubWorkflowAbortedState)(terminalMetadataWriteResponseHandler) whenUnhandled { case Event(SubWorkflowStoreRegisterSuccess(_), _) => @@ -151,10 +187,15 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey, jobLogger.error(reason, s"SubWorkflowStore failure for command $command") stay() case Event(EngineLifecycleActorAbortCommand, data) => - recordTerminalState(SubWorkflowAbortedState, SubWorkflowExecutionActorTerminalData(data.subWorkflowId, SubWorkflowAbortedResponse(key, Map.empty))) + recordTerminalState( + SubWorkflowAbortedState, + SubWorkflowExecutionActorTerminalData(data.subWorkflowId, SubWorkflowAbortedResponse(key, Map.empty)) + ) } - def recordTerminalState(terminalState: SubWorkflowTerminalState, newStateData: SubWorkflowExecutionActorTerminalData): State = { + def recordTerminalState(terminalState: SubWorkflowTerminalState, + newStateData: SubWorkflowExecutionActorTerminalData + ): State = newStateData.subWorkflowId match { case Some(id) => pushWorkflowEnd(id) @@ -165,23 +206,27 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey, jobLogger.error("Programmer Error: Sub workflow completed without ever having a Sub Workflow UUID.") // Same situation as if we fail to write the final state metadata. Bail out and hope the workflow is more // successful next time: - context.parent ! SubWorkflowFailedResponse(key, + context.parent ! SubWorkflowFailedResponse( + key, Map.empty, - new Exception("Programmer Error: Sub workflow completed without ever having a Sub Workflow UUID") with NoStackTrace) + new Exception("Programmer Error: Sub workflow completed without ever having a Sub Workflow UUID") + with NoStackTrace + ) context.stop(self) stay() } - } - onTransition { - case (_, toState) => - eventList :+= ExecutionEvent(toState.toString) - if (!toState.isInstanceOf[SubWorkflowTerminalState]) { - stateData.subWorkflowId foreach { id => pushCurrentStateToMetadataService(id, toState.workflowState) } - } + onTransition { case (_, toState) => + eventList :+= ExecutionEvent(toState.toString) + if (!toState.isInstanceOf[SubWorkflowTerminalState]) { + stateData.subWorkflowId foreach { id => pushCurrentStateToMetadataService(id, toState.workflowState) } + } } - private def startSubWorkflow(subWorkflowEngineDescriptor: EngineWorkflowDescriptor, inputs: WomEvaluatedCallInputs, data: SubWorkflowExecutionActorData) = { + private def startSubWorkflow(subWorkflowEngineDescriptor: EngineWorkflowDescriptor, + inputs: WomEvaluatedCallInputs, + data: SubWorkflowExecutionActorData + ) = { val subWorkflowActor = createSubWorkflowActor(subWorkflowEngineDescriptor) subWorkflowActor ! WorkflowExecutionActor.ExecuteWorkflowCommand @@ -204,14 +249,13 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey, goto(WaitingForValueStore) using SubWorkflowExecutionActorLiveData(Option(workflowId), None) } - def createSubWorkflowPreparationActor(subWorkflowId: WorkflowId) = { + def createSubWorkflowPreparationActor(subWorkflowId: WorkflowId) = context.actorOf( SubWorkflowPreparationActor.props(parentWorkflow, expressionLanguageFunctions, key, subWorkflowId), s"$subWorkflowId-SubWorkflowPreparationActor-${key.tag}" ) - } - def createSubWorkflowActor(subWorkflowEngineDescriptor: EngineWorkflowDescriptor) = { + def createSubWorkflowActor(subWorkflowEngineDescriptor: EngineWorkflowDescriptor) = context.actorOf( WorkflowExecutionActor.props( subWorkflowEngineDescriptor, @@ -234,28 +278,42 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey, ), s"${subWorkflowEngineDescriptor.id}-SubWorkflowActor-${key.tag}" ) - } - private def pushWorkflowRunningMetadata(subWorkflowDescriptor: BackendWorkflowDescriptor, workflowInputs: WomEvaluatedCallInputs) = { + private def pushWorkflowRunningMetadata(subWorkflowDescriptor: BackendWorkflowDescriptor, + workflowInputs: WomEvaluatedCallInputs + ) = { val subWorkflowId = subWorkflowDescriptor.id - val parentWorkflowMetadataKey = MetadataKey(parentWorkflow.id, Option(MetadataJobKey(key.node.fullyQualifiedName, key.index, key.attempt)), CallMetadataKeys.SubWorkflowId) + val parentWorkflowMetadataKey = MetadataKey( + parentWorkflow.id, + Option(MetadataJobKey(key.node.fullyQualifiedName, key.index, key.attempt)), + CallMetadataKeys.SubWorkflowId + ) val events = List( MetadataEvent(parentWorkflowMetadataKey, MetadataValue(subWorkflowId)), MetadataEvent(MetadataKey(subWorkflowId, None, WorkflowMetadataKeys.Name), MetadataValue(key.node.localName)), - MetadataEvent(MetadataKey(subWorkflowId, None, WorkflowMetadataKeys.ParentWorkflowId), MetadataValue(parentWorkflow.id)), - MetadataEvent(MetadataKey(subWorkflowId, None, WorkflowMetadataKeys.RootWorkflowId), MetadataValue(parentWorkflow.rootWorkflow.id)) + MetadataEvent(MetadataKey(subWorkflowId, None, WorkflowMetadataKeys.ParentWorkflowId), + MetadataValue(parentWorkflow.id) + ), + MetadataEvent(MetadataKey(subWorkflowId, None, WorkflowMetadataKeys.RootWorkflowId), + MetadataValue(parentWorkflow.rootWorkflow.id) + ) ) val inputEvents = workflowInputs match { case empty if empty.isEmpty => - List(MetadataEvent.empty(MetadataKey(subWorkflowId, None,WorkflowMetadataKeys.Inputs))) + List(MetadataEvent.empty(MetadataKey(subWorkflowId, None, WorkflowMetadataKeys.Inputs))) case inputs => inputs flatMap { case (inputName, womValue) => - womValueToMetadataEvents(MetadataKey(subWorkflowId, None, s"${WorkflowMetadataKeys.Inputs}:${inputName.name}"), womValue) + womValueToMetadataEvents( + MetadataKey(subWorkflowId, None, s"${WorkflowMetadataKeys.Inputs}:${inputName.name}"), + womValue + ) } } + jobLogger.info(s"Running subworkflow: ${subWorkflowDescriptor.id}, root: ${parentWorkflow.rootWorkflow.id}") + val workflowRootEvents = buildWorkflowRootMetadataEvents(subWorkflowDescriptor) serviceRegistryActor ! PutMetadataAction(events ++ inputEvents ++ workflowRootEvents) @@ -264,14 +322,17 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey, private def buildWorkflowRootMetadataEvents(subWorkflowDescriptor: BackendWorkflowDescriptor) = { val subWorkflowId = subWorkflowDescriptor.id - factories flatMap { - case (backendName, factory) => - BackendConfiguration.backendConfigurationDescriptor(backendName).toOption map { config => - backendName -> factory.getWorkflowExecutionRootPath(subWorkflowDescriptor, config.backendConfig, initializationData.get(backendName)) - } - } map { - case (backend, wfRoot) => - MetadataEvent(MetadataKey(subWorkflowId, None, s"${WorkflowMetadataKeys.WorkflowRoot}[$backend]"), MetadataValue(wfRoot.toAbsolutePath)) + factories flatMap { case (backendName, factory) => + BackendConfiguration.backendConfigurationDescriptor(backendName).toOption map { config => + backendName -> factory.getWorkflowExecutionRootPath(subWorkflowDescriptor, + config.backendConfig, + initializationData.get(backendName) + ) + } + } map { case (backend, wfRoot) => + MetadataEvent(MetadataKey(subWorkflowId, None, s"${WorkflowMetadataKeys.WorkflowRoot}[$backend]"), + MetadataValue(wfRoot.toAbsolutePath) + ) } } @@ -324,8 +385,12 @@ object SubWorkflowExecutionActor { sealed trait SubWorkflowExecutionActorData { def subWorkflowId: Option[WorkflowId] } - final case class SubWorkflowExecutionActorLiveData(subWorkflowId: Option[WorkflowId], subWorkflowActor: Option[ActorRef]) extends SubWorkflowExecutionActorData - final case class SubWorkflowExecutionActorTerminalData(subWorkflowId: Option[WorkflowId], terminalStateResponse: SubWorkflowTerminalStateResponse) extends SubWorkflowExecutionActorData + final case class SubWorkflowExecutionActorLiveData(subWorkflowId: Option[WorkflowId], + subWorkflowActor: Option[ActorRef] + ) extends SubWorkflowExecutionActorData + final case class SubWorkflowExecutionActorTerminalData(subWorkflowId: Option[WorkflowId], + terminalStateResponse: SubWorkflowTerminalStateResponse + ) extends SubWorkflowExecutionActorData sealed trait EngineWorkflowExecutionActorCommand case object Execute @@ -349,28 +414,30 @@ object SubWorkflowExecutionActor { rootConfig: Config, totalJobsByRootWf: AtomicInteger, fileHashCacheActor: Option[ActorRef], - blacklistCache: Option[BlacklistCache]) = { - Props(new SubWorkflowExecutionActor( - key, - parentWorkflow, - expressionLanguageFunctions, - factories, - ioActor = ioActor, - serviceRegistryActor = serviceRegistryActor, - jobStoreActor = jobStoreActor, - subWorkflowStoreActor = subWorkflowStoreActor, - callCacheReadActor = callCacheReadActor, - callCacheWriteActor = callCacheWriteActor, - workflowDockerLookupActor = workflowDockerLookupActor, - jobRestartCheckTokenDispenserActor = jobRestartCheckTokenDispenserActor, - jobExecutionTokenDispenserActor = jobExecutionTokenDispenserActor, - backendSingletonCollection, - initializationData, - startState, - rootConfig, - totalJobsByRootWf, - fileHashCacheActor = fileHashCacheActor, - blacklistCache = blacklistCache) + blacklistCache: Option[BlacklistCache] + ) = + Props( + new SubWorkflowExecutionActor( + key, + parentWorkflow, + expressionLanguageFunctions, + factories, + ioActor = ioActor, + serviceRegistryActor = serviceRegistryActor, + jobStoreActor = jobStoreActor, + subWorkflowStoreActor = subWorkflowStoreActor, + callCacheReadActor = callCacheReadActor, + callCacheWriteActor = callCacheWriteActor, + workflowDockerLookupActor = workflowDockerLookupActor, + jobRestartCheckTokenDispenserActor = jobRestartCheckTokenDispenserActor, + jobExecutionTokenDispenserActor = jobExecutionTokenDispenserActor, + backendSingletonCollection, + initializationData, + startState, + rootConfig, + totalJobsByRootWf, + fileHashCacheActor = fileHashCacheActor, + blacklistCache = blacklistCache + ) ).withDispatcher(EngineDispatcher) - } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index dd6f0a42835..106b14a9a0c 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -28,7 +28,10 @@ import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends} import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor._ import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActorData.DataStoreUpdate import cromwell.engine.workflow.lifecycle.execution.job.EngineJobExecutionActor -import cromwell.engine.workflow.lifecycle.execution.keys.ExpressionKey.{ExpressionEvaluationFailedResponse, ExpressionEvaluationSucceededResponse} +import cromwell.engine.workflow.lifecycle.execution.keys.ExpressionKey.{ + ExpressionEvaluationFailedResponse, + ExpressionEvaluationSucceededResponse +} import cromwell.engine.workflow.lifecycle.execution.keys._ import cromwell.engine.workflow.lifecycle.execution.stores.{ActiveExecutionStore, ExecutionStore} import cromwell.engine.workflow.lifecycle.{EngineLifecycleActorAbortCommand, EngineLifecycleActorAbortedResponse} @@ -52,7 +55,7 @@ import scala.language.postfixOps import scala.util.control.NoStackTrace case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) - extends LoggingFSM[WorkflowExecutionActorState, WorkflowExecutionActorData] + extends LoggingFSM[WorkflowExecutionActorState, WorkflowExecutionActorData] with WorkflowLogging with CallMetadataHelper with StopAndLogSupervisor @@ -71,23 +74,25 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) private val DefaultTotalMaxJobsPerRootWf = 1000000 private val DefaultMaxScatterSize = 1000000 - private val TotalMaxJobsPerRootWf = params.rootConfig.getOrElse("system.total-max-jobs-per-root-workflow", DefaultTotalMaxJobsPerRootWf) - private val MaxScatterWidth = params.rootConfig.getOrElse("system.max-scatter-width-per-scatter", DefaultMaxScatterSize) + private val TotalMaxJobsPerRootWf = + params.rootConfig.getOrElse("system.total-max-jobs-per-root-workflow", DefaultTotalMaxJobsPerRootWf) + private val MaxScatterWidth = + params.rootConfig.getOrElse("system.max-scatter-width-per-scatter", DefaultMaxScatterSize) private val FileHashBatchSize: Int = params.rootConfig.as[Int]("system.file-hash-batch-size") private val backendFactories: Map[String, BackendLifecycleActorFactory] = { val factoriesValidation = workflowDescriptor.backendAssignments.values.toList - .traverse[ErrorOr, (String, BackendLifecycleActorFactory)] { - backendName => CromwellBackends.backendLifecycleFactoryActorByName(backendName) map { backendName -> _ } - } + .traverse[ErrorOr, (String, BackendLifecycleActorFactory)] { backendName => + CromwellBackends.backendLifecycleFactoryActorByName(backendName) map { backendName -> _ } + } factoriesValidation .map(_.toMap) .valueOr(errors => throw AggregatedMessageException("Could not instantiate backend factories", errors.toList)) } - - val executionStore: ErrorOr[ActiveExecutionStore] = ExecutionStore(workflowDescriptor.callable, params.totalJobsByRootWf, TotalMaxJobsPerRootWf) + val executionStore: ErrorOr[ActiveExecutionStore] = + ExecutionStore(workflowDescriptor.callable, params.totalJobsByRootWf, TotalMaxJobsPerRootWf) // If executionStore returns a Failure about root workflow creating jobs more than total jobs per root workflow limit, // the WEA will fail by sending WorkflowExecutionFailedResponse to its parent and kill itself @@ -95,7 +100,12 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) case Valid(validExecutionStore) => startWith( WorkflowExecutionPendingState, - WorkflowExecutionActorData(workflowDescriptor, ioEc, new AsyncIo(params.ioActor, GcsBatchCommandBuilder), params.totalJobsByRootWf, validExecutionStore) + WorkflowExecutionActorData(workflowDescriptor, + ioEc, + new AsyncIo(params.ioActor, GcsBatchCommandBuilder), + params.totalJobsByRootWf, + validExecutionStore + ) ) case Invalid(e) => val errorMsg = s"Failed to initialize WorkflowExecutionActor. Error: $e" @@ -106,7 +116,12 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) // it throws NullPointerException as FSM.goto can't find the currentState startWith( WorkflowExecutionFailedState, - WorkflowExecutionActorData(workflowDescriptor, ioEc, new AsyncIo(params.ioActor, GcsBatchCommandBuilder), params.totalJobsByRootWf, ExecutionStore.empty) + WorkflowExecutionActorData(workflowDescriptor, + ioEc, + new AsyncIo(params.ioActor, GcsBatchCommandBuilder), + params.totalJobsByRootWf, + ExecutionStore.empty + ) ) workflowLogger.debug("Actor failed to initialize. Stopping self.") @@ -124,14 +139,14 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) sendHeartBeat() /* - * Note that we don't record the fact that a workflow was failing, therefore we either restart in running or aborting state. - * If the workflow was failing, it means at least one job failed in which case it'll still be failed when we get to it. - * When that happens, we'll go to failing state. - * - * An effect of that is that up until the moment when we come across the failed job, - * all backend jobs will be restarted with a Recover command which could potentially re-execute the job if the backend doesn't support - * job recovery. A better way would be to record that the workflow was failing an restart in failing mode. However there will always be a gap - * between when a job has failed and Cromwell is aware of it, or has time to persist that information. + * Note that we don't record the fact that a workflow was failing, therefore we either restart in running or aborting state. + * If the workflow was failing, it means at least one job failed in which case it'll still be failed when we get to it. + * When that happens, we'll go to failing state. + * + * An effect of that is that up until the moment when we come across the failed job, + * all backend jobs will be restarted with a Recover command which could potentially re-execute the job if the backend doesn't support + * job recovery. A better way would be to record that the workflow was failing an restart in failing mode. However there will always be a gap + * between when a job has failed and Cromwell is aware of it, or has time to persist that information. */ params.startState match { case RestartableAborting => goto(WorkflowExecutionAbortingState) @@ -165,8 +180,9 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) // A job not found here means we were trying to reconnect to a job that was likely never started. Indicate this in the message. case Event(JobFailedNonRetryableResponse(jobKey, _: JobNotFoundException, _), _) if restarting => val benignException = - new Exception("Cromwell server was restarted while this workflow was running. As part of the restart process, Cromwell attempted to reconnect to this job, however it was never started in the first place. This is a benign failure and not the cause of failure for this workflow, it can be safely ignored.") - with NoStackTrace + new Exception( + "Cromwell server was restarted while this workflow was running. As part of the restart process, Cromwell attempted to reconnect to this job, however it was never started in the first place. This is a benign failure and not the cause of failure for this workflow, it can be safely ignored." + ) with NoStackTrace handleNonRetryableFailure(stateData, jobKey, benignException) } @@ -187,20 +203,28 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) handleCallAborted(stateData, jobKey, executedKeys) // Here we can't really know what the status of the job is. For now declare it aborted anyway but add some info in the metadata - case Event(JobFailedNonRetryableResponse(jobKey: BackendJobDescriptorKey, failure: JobReconnectionNotSupportedException, _), _) if restarting => + case Event(JobFailedNonRetryableResponse(jobKey: BackendJobDescriptorKey, + failure: JobReconnectionNotSupportedException, + _ + ), + _ + ) if restarting => pushBackendStatusUnknown(jobKey) handleNonRetryableFailure(stateData, jobKey, failure, None) } - when(WorkflowExecutionSuccessfulState) { FSM.NullFunction } - when(WorkflowExecutionFailedState) { FSM.NullFunction } - when(WorkflowExecutionAbortedState) { FSM.NullFunction } + when(WorkflowExecutionSuccessfulState)(FSM.NullFunction) + when(WorkflowExecutionFailedState)(FSM.NullFunction) + when(WorkflowExecutionAbortedState)(FSM.NullFunction) var previousHeartbeatTime: Option[OffsetDateTime] = None def measureTimeBetweenHeartbeats(): Unit = { val now = OffsetDateTime.now previousHeartbeatTime foreach { previous => - sendGauge(NonEmptyList("workflows", List("workflowexecutionactor", "heartbeat", "interval_millis", "set")), now.toInstant.toEpochMilli - previous.toInstant.toEpochMilli) + sendGauge( + NonEmptyList("workflows", List("workflowexecutionactor", "heartbeat", "interval_millis", "set")), + now.toInstant.toEpochMilli - previous.toInstant.toEpochMilli + ) } previousHeartbeatTime = Option(now) } @@ -225,15 +249,20 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) stay() using stateData .mergeExecutionDiff(WorkflowExecutionDiff(Map(key -> ExecutionStatus.Running))) - //Success + // Success // Job case Event(r: JobSucceededResponse, stateData) => if (r.resultGenerationMode != RunOnBackend) { - workflowLogger.info(s"Job results retrieved (${r.resultGenerationMode}): '${r.jobKey.call.fullyQualifiedName}' (scatter index: ${r.jobKey.index}, attempt ${r.jobKey.attempt})") + workflowLogger.info( + s"Job results retrieved (${r.resultGenerationMode}): '${r.jobKey.call.fullyQualifiedName}' (scatter index: ${r.jobKey.index}, attempt ${r.jobKey.attempt})" + ) } handleCallSuccessful(r.jobKey, r.jobOutputs, r.returnCode, stateData, Map.empty, Set(workflowDescriptor.id)) // Sub Workflow - case Event(SubWorkflowSucceededResponse(jobKey, descendantJobKeys, rootAndSubworklowIds, callOutputs, cumulativeOutputs), currentStateData) => + case Event( + SubWorkflowSucceededResponse(jobKey, descendantJobKeys, rootAndSubworklowIds, callOutputs, cumulativeOutputs), + currentStateData + ) => // Update call outputs to come from sub-workflow output ports: val subworkflowOutputs: Map[OutputPort, WomValue] = callOutputs.outputs flatMap { case (port, value) => jobKey.node.subworkflowCallOutputPorts collectFirst { @@ -241,16 +270,30 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) } } - if(subworkflowOutputs.size == callOutputs.outputs.size) { - handleCallSuccessful(jobKey, CallOutputs(subworkflowOutputs), None, currentStateData, descendantJobKeys, rootAndSubworklowIds, cumulativeOutputs) + if (subworkflowOutputs.size == callOutputs.outputs.size) { + handleCallSuccessful(jobKey, + CallOutputs(subworkflowOutputs), + None, + currentStateData, + descendantJobKeys, + rootAndSubworklowIds, + cumulativeOutputs + ) } else { - handleNonRetryableFailure(currentStateData, jobKey, new Exception(s"Subworkflow produced outputs: [${callOutputs.outputs.keys.mkString(", ")}], but we expected all of [${jobKey.node.subworkflowCallOutputPorts.map(_.internalName)}]")) + handleNonRetryableFailure( + currentStateData, + jobKey, + new Exception(s"Subworkflow produced outputs: [${callOutputs.outputs.keys + .mkString(", ")}], but we expected all of [${jobKey.node.subworkflowCallOutputPorts.map(_.internalName)}]") + ) } // Expression case Event(ExpressionEvaluationSucceededResponse(expressionKey, callOutputs), stateData) => expressionKey.node match { case _: ExposedExpressionNode | _: ExpressionBasedGraphOutputNode => - workflowLogger.debug(s"Expression evaluation succeeded: '${expressionKey.node.fullyQualifiedName}' (scatter index: ${expressionKey.index}, attempt: ${expressionKey.attempt})") + workflowLogger.debug( + s"Expression evaluation succeeded: '${expressionKey.node.fullyQualifiedName}' (scatter index: ${expressionKey.index}, attempt: ${expressionKey.attempt})" + ) case _ => // No logging; anonymous node } @@ -314,11 +357,23 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) override protected def onFailure(actorRef: ActorRef, throwable: => Throwable) = { // Both of these Should Never Happen (tm), assuming the state data is set correctly on EJEA creation. // If they do, it's a big programmer error and the workflow execution fails. - val jobKey = stateData.jobKeyActorMappings.getOrElse(actorRef, throw new RuntimeException("Programmer Error: A job or sub workflow actor has terminated but was not assigned a jobKey")) - val jobStatus = stateData.executionStore.jobStatus(jobKey).getOrElse(throw new RuntimeException(s"Programmer Error: An actor representing ${jobKey.tag} which this workflow is not running has sent up a terminated message.")) + val jobKey = stateData.jobKeyActorMappings.getOrElse( + actorRef, + throw new RuntimeException( + "Programmer Error: A job or sub workflow actor has terminated but was not assigned a jobKey" + ) + ) + val jobStatus = stateData.executionStore + .jobStatus(jobKey) + .getOrElse( + throw new RuntimeException( + s"Programmer Error: An actor representing ${jobKey.tag} which this workflow is not running has sent up a terminated message." + ) + ) if (!jobStatus.isTerminalOrRetryable) { - val terminationException = new RuntimeException(s"Unexpected failure or termination of the actor monitoring ${jobKey.tag}", throwable) + val terminationException = + new RuntimeException(s"Unexpected failure or termination of the actor monitoring ${jobKey.tag}", throwable) self ! JobFailedNonRetryableResponse(jobKey, terminationException, None) } } @@ -328,7 +383,9 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) // Send the abort to all children context.children foreach { _ ! EngineLifecycleActorAbortCommand } // As well as all backend singleton actors - params.backendSingletonCollection.backendSingletonActors.values.flatten.foreach { _ ! BackendSingletonActorAbortWorkflow(workflowIdForLogging) } + params.backendSingletonCollection.backendSingletonActors.values.flatten.foreach { + _ ! BackendSingletonActorAbortWorkflow(workflowIdForLogging) + } // Only seal the execution store if we're not restarting, otherwise we could miss some jobs that have been started before the // restart but are not started yet at this point @@ -344,8 +401,8 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) import spray.json._ def handleSuccessfulWorkflowOutputs(outputs: Map[GraphOutputNode, WomValue]) = { - val fullyQualifiedOutputs = outputs map { - case (outputNode, value) => outputNode.identifier.fullyQualifiedName.value -> value + val fullyQualifiedOutputs = outputs map { case (outputNode, value) => + outputNode.identifier.fullyQualifiedName.value -> value } // Publish fully qualified workflow outputs to log and metadata workflowLogger.info( @@ -354,15 +411,19 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) ) pushWorkflowOutputMetadata(fullyQualifiedOutputs) - val localOutputs = CallOutputs(outputs map { - case (outputNode, value) => outputNode.graphOutputPort -> value + val localOutputs = CallOutputs(outputs map { case (outputNode, value) => + outputNode.graphOutputPort -> value }) val currentCumulativeOutputs = data.cumulativeOutputs ++ localOutputs.outputs.values val currentRootAndSubworkflowIds = data.rootAndSubworkflowIds + workflowDescriptor.id - context.parent ! WorkflowExecutionSucceededResponse(data.jobExecutionMap, currentRootAndSubworkflowIds, localOutputs, currentCumulativeOutputs) + context.parent ! WorkflowExecutionSucceededResponse(data.jobExecutionMap, + currentRootAndSubworkflowIds, + localOutputs, + currentCumulativeOutputs + ) goto(WorkflowExecutionSuccessfulState) using data } @@ -380,14 +441,13 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) val workflowOutputValuesValidation = workflowOutputNodes // Try to find a value for each port in the value store - .map(outputNode => - outputNode -> data.valueStore.get(outputNode.graphOutputPort, None) - ) - .toList.traverse[IOChecked, (GraphOutputNode, WomValue)]({ - case (name, Some(value)) => value.initialize(data.expressionLanguageFunctions).map(name -> _) - case (name, None) => - s"Cannot find an output value for ${name.identifier.fullyQualifiedName.value}".invalidIOChecked - }) + .map(outputNode => outputNode -> data.valueStore.get(outputNode.graphOutputPort, None)) + .toList + .traverse[IOChecked, (GraphOutputNode, WomValue)] { + case (name, Some(value)) => value.initialize(data.expressionLanguageFunctions).map(name -> _) + case (name, None) => + s"Cannot find an output value for ${name.identifier.fullyQualifiedName.value}".invalidIOChecked + } // Convert the list of tuples to a Map .map(_.toMap) @@ -410,7 +470,8 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) failedJobKey: JobKey, reason: Throwable, returnCode: Option[Int] = None, - jobExecutionMap: JobExecutionMap = Map.empty) = { + jobExecutionMap: JobExecutionMap = Map.empty + ) = { pushFailedCallMetadata(failedJobKey, returnCode, reason, retryableFailure = false) val dataWithFailure = stateData.executionFailure(failedJobKey, reason, jobExecutionMap) @@ -442,7 +503,7 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) * and for which reconnection will fail. When that happens we'll fail the job (see failing state). * * This guarantees that we'll try to reconnect to all potentially running jobs on restart. - */ + */ val newData = if (workflowDescriptor.failureMode.allowNewCallsAfterFailure || restarting) { dataWithFailure } else { @@ -458,7 +519,10 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) goto(nextState) using newData } - private def handleCallAborted(data: WorkflowExecutionActorData, jobKey: JobKey, jobExecutionMap: JobExecutionMap = Map.empty) = { + private def handleCallAborted(data: WorkflowExecutionActorData, + jobKey: JobKey, + jobExecutionMap: JobExecutionMap = Map.empty + ) = { pushAbortedCallMetadata(jobKey) workflowLogger.info(s"$tag aborted: ${jobKey.tag}") val newStateData = data @@ -469,19 +533,20 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) } private def pushBackendStatusUnknown(jobKey: BackendJobDescriptorKey): Unit = { - val unknownBackendStatus = MetadataEvent(metadataKeyForCall(jobKey, CallMetadataKeys.BackendStatus), MetadataValue("Unknown")) + val unknownBackendStatus = + MetadataEvent(metadataKeyForCall(jobKey, CallMetadataKeys.BackendStatus), MetadataValue("Unknown")) serviceRegistryActor ! PutMetadataAction(unknownBackendStatus) } - private def handleRetryableFailure(jobKey: BackendJobDescriptorKey, - reason: Throwable, - returnCode: Option[Int]) = { + private def handleRetryableFailure(jobKey: BackendJobDescriptorKey, reason: Throwable, returnCode: Option[Int]) = { pushFailedCallMetadata(jobKey, returnCode, reason, retryableFailure = true) val newJobKey = jobKey.copy(attempt = jobKey.attempt + 1) workflowLogger.info(s"Retrying job execution for ${newJobKey.tag}") // Update current key to RetryableFailure status and add new key with attempt incremented and NotStarted status - val executionDiff = WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.RetryableFailure, newJobKey -> ExecutionStatus.NotStarted)) + val executionDiff = WorkflowExecutionDiff( + Map(jobKey -> ExecutionStatus.RetryableFailure, newJobKey -> ExecutionStatus.NotStarted) + ) stay() using stateData.mergeExecutionDiff(executionDiff) } @@ -492,20 +557,30 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) data: WorkflowExecutionActorData, jobExecutionMap: JobExecutionMap, rootAndSubworkflowIds: Set[WorkflowId], - cumulativeOutputs: Set[WomValue] = Set.empty) = { + cumulativeOutputs: Set[WomValue] = Set.empty + ) = { pushSuccessfulCallMetadata(jobKey, returnCode, outputs) - stay() using data.callExecutionSuccess(jobKey, outputs, cumulativeOutputs, rootAndSubworkflowIds).addExecutions(jobExecutionMap) + stay() using data + .callExecutionSuccess(jobKey, outputs, cumulativeOutputs, rootAndSubworkflowIds) + .addExecutions(jobExecutionMap) } - private def handleDeclarationEvaluationSuccessful(key: ExpressionKey, values: Map[OutputPort, WomValue], data: WorkflowExecutionActorData) = { + private def handleDeclarationEvaluationSuccessful(key: ExpressionKey, + values: Map[OutputPort, WomValue], + data: WorkflowExecutionActorData + ) = stay() using data.expressionEvaluationSuccess(key, values) - } - override def receive: Receive = { - case msg => - val starttime = OffsetDateTime.now - super[LoggingFSM].receive(msg) - sendGauge(NonEmptyList("workflows", List("workflowexecutionactor", this.stateName.toString, msg.getClass.getSimpleName , "processing_millis", "set")), OffsetDateTime.now.toInstant.toEpochMilli - starttime.toInstant.toEpochMilli) + override def receive: Receive = { case msg => + val starttime = OffsetDateTime.now + super[LoggingFSM].receive(msg) + sendGauge( + NonEmptyList( + "workflows", + List("workflowexecutionactor", this.stateName.toString, msg.getClass.getSimpleName, "processing_millis", "set") + ), + OffsetDateTime.now.toInstant.toEpochMilli - starttime.toInstant.toEpochMilli + ) } /** @@ -517,10 +592,14 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) val startRunnableStartTimestamp = OffsetDateTime.now - def updateExecutionStore(diffs: List[WorkflowExecutionDiff], updatedData: WorkflowExecutionActorData): WorkflowExecutionActorData = { - val notStartedBackendJobs = diffs.flatMap(d => d.executionStoreChanges.collect{ - case (key: BackendJobDescriptorKey, status: ExecutionStatus.NotStarted.type) => (key, status) - }.keys) + def updateExecutionStore(diffs: List[WorkflowExecutionDiff], + updatedData: WorkflowExecutionActorData + ): WorkflowExecutionActorData = { + val notStartedBackendJobs = diffs.flatMap(d => + d.executionStoreChanges.collect { + case (key: BackendJobDescriptorKey, status: ExecutionStatus.NotStarted.type) => (key, status) + }.keys + ) val notStartedBackendJobsCt = notStartedBackendJobs.size // this limits the total max jobs that can be created by a root workflow @@ -528,56 +607,66 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) // Since the root workflow tried creating jobs more than the total max jobs allowed per root workflow // we fail all the BackendJobDescriptorKey which are in 'Not Started' state, and update the execution // store with the status update of remaining keys - val updatedDiffs = diffs.map(d => d.copy(executionStoreChanges = d.executionStoreChanges -- notStartedBackendJobs)) + val updatedDiffs = + diffs.map(d => d.copy(executionStoreChanges = d.executionStoreChanges -- notStartedBackendJobs)) - notStartedBackendJobs.foreach(key => { - val errorMsg = s"Job $key failed to be created! Error: Root workflow tried creating ${data.totalJobsByRootWf.get} jobs, which is more than $TotalMaxJobsPerRootWf, the max cumulative jobs allowed per root workflow" + notStartedBackendJobs.foreach { key => + val errorMsg = + s"Job $key failed to be created! Error: Root workflow tried creating ${data.totalJobsByRootWf.get} jobs, which is more than $TotalMaxJobsPerRootWf, the max cumulative jobs allowed per root workflow" workflowLogger.error(errorMsg) self ! JobFailedNonRetryableResponse(key, new Exception(errorMsg), None) - }) + } updatedData.mergeExecutionDiffs(updatedDiffs) - } - else updatedData.mergeExecutionDiffs(diffs) + } else updatedData.mergeExecutionDiffs(diffs) } val DataStoreUpdate(runnableKeys, _, updatedData) = data.executionStoreUpdate val runnableCalls = runnableKeys.view - .collect({ case k: BackendJobDescriptorKey => k }) + .collect { case k: BackendJobDescriptorKey => k } .groupBy(_.node) - .map({ - case (node, keys) => - val tag = node.fullyQualifiedName - val shardCount = keys.map(_.index).toList.distinct.size - if (shardCount == 1) tag - else s"$tag ($shardCount shards)" - }) + .map { case (node, keys) => + val tag = node.fullyQualifiedName + val shardCount = keys.map(_.index).toList.distinct.size + if (shardCount == 1) tag + else s"$tag ($shardCount shards)" + } val mode = if (restarting) "Restarting" else "Starting" if (runnableCalls.nonEmpty) workflowLogger.info(s"$mode " + runnableCalls.mkString(", ")) - val keyStartDiffs: List[WorkflowExecutionDiff] = runnableKeys map { k => k -> (k match { - case key: BackendJobDescriptorKey => processRunnableJob(key, data) - case key: SubWorkflowKey => processRunnableSubWorkflow(key, data) - case key: ConditionalCollectorKey => key.processRunnable(data) - case key: ConditionalKey => key.processRunnable(data, workflowLogger) - case key @ ExpressionKey(expr: TaskCallInputExpressionNode, _) => processRunnableTaskCallInputExpression(key, data, expr) - case key: ExpressionKey => key.processRunnable(data.expressionLanguageFunctions, data.valueStore, self) - case key: ScatterCollectorKey => key.processRunnable(data) - case key: ScatteredCallCompletionKey => key.processRunnable(data) - case key: ScatterKey => key.processRunnable(data, self, MaxScatterWidth) - case other => - workflowLogger.error(s"${other.tag} is not a runnable key") - WorkflowExecutionDiff.empty.validNel - })} map { - case (key: JobKey, value: ErrorOr[WorkflowExecutionDiff]) => value.valueOr(errors => { - self ! JobFailedNonRetryableResponse(key, new Exception(errors.toList.mkString(System.lineSeparator)) with NoStackTrace, None) + val keyStartDiffs: List[WorkflowExecutionDiff] = runnableKeys map { k => + k -> (k match { + case key: BackendJobDescriptorKey => processRunnableJob(key, data) + case key: SubWorkflowKey => processRunnableSubWorkflow(key, data) + case key: ConditionalCollectorKey => key.processRunnable(data) + case key: ConditionalKey => key.processRunnable(data, workflowLogger) + case key @ ExpressionKey(expr: TaskCallInputExpressionNode, _) => + processRunnableTaskCallInputExpression(key, data, expr) + case key: ExpressionKey => key.processRunnable(data.expressionLanguageFunctions, data.valueStore, self) + case key: ScatterCollectorKey => key.processRunnable(data) + case key: ScatteredCallCompletionKey => key.processRunnable(data) + case key: ScatterKey => key.processRunnable(data, self, MaxScatterWidth) + case other => + workflowLogger.error(s"${other.tag} is not a runnable key") + WorkflowExecutionDiff.empty.validNel + }) + } map { case (key: JobKey, value: ErrorOr[WorkflowExecutionDiff]) => + value.valueOr { errors => + self ! JobFailedNonRetryableResponse(key, + new Exception(errors.toList.mkString(System.lineSeparator)) + with NoStackTrace, + None + ) // Don't update the execution store now - the failure message we just sent to ourselves will take care of that: WorkflowExecutionDiff.empty - }) + } } // Merge the execution diffs upon success val result = updateExecutionStore(keyStartDiffs, updatedData) - sendGauge(NonEmptyList("workflows", List("workflowexecutionactor", "startRunnableNodes", "duration_millis", "set")), OffsetDateTime.now.toInstant.toEpochMilli - startRunnableStartTimestamp.toInstant.toEpochMilli) + sendGauge( + NonEmptyList("workflows", List("workflowexecutionactor", "startRunnableNodes", "duration_millis", "set")), + OffsetDateTime.now.toInstant.toEpochMilli - startRunnableStartTimestamp.toInstant.toEpochMilli + ) result } @@ -587,35 +676,45 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) */ private def processRunnableTaskCallInputExpression(key: ExpressionKey, data: WorkflowExecutionActorData, - expressionNode: TaskCallInputExpressionNode): ErrorOr[WorkflowExecutionDiff] = { + expressionNode: TaskCallInputExpressionNode + ): ErrorOr[WorkflowExecutionDiff] = { import cats.syntax.either._ val taskCallNode: CommandCallNode = expressionNode.taskCallNodeReceivingInput.get(()) (for { - backendJobDescriptorKey <- data.executionStore.backendJobDescriptorKeyForNode(taskCallNode) toChecked s"No BackendJobDescriptorKey found for call node ${taskCallNode.identifier.fullyQualifiedName}" + backendJobDescriptorKey <- data.executionStore.backendJobDescriptorKeyForNode( + taskCallNode + ) toChecked s"No BackendJobDescriptorKey found for call node ${taskCallNode.identifier.fullyQualifiedName}" factory <- backendFactoryForTaskCallNode(taskCallNode) backendInitializationData = params.initializationData.get(factory.name) - functions = factory.expressionLanguageFunctions(workflowDescriptor.backendDescriptor, backendJobDescriptorKey, backendInitializationData, params.ioActor, ioEc) + functions = factory.expressionLanguageFunctions(workflowDescriptor.backendDescriptor, + backendJobDescriptorKey, + backendInitializationData, + params.ioActor, + ioEc + ) diff <- key.processRunnable(functions, data.valueStore, self).toEither } yield diff).toValidated } - private def backendFactoryForTaskCallNode(taskCallNode: CommandCallNode): Checked[BackendLifecycleActorFactory] = { + private def backendFactoryForTaskCallNode(taskCallNode: CommandCallNode): Checked[BackendLifecycleActorFactory] = for { - name <- workflowDescriptor - .backendAssignments.get(taskCallNode).toChecked(s"Cannot find an assigned backend for call ${taskCallNode.fullyQualifiedName}") + name <- workflowDescriptor.backendAssignments + .get(taskCallNode) + .toChecked(s"Cannot find an assigned backend for call ${taskCallNode.fullyQualifiedName}") factory <- backendFactories.get(name).toChecked(s"Cannot find a backend factory for backend $name") } yield factory - } /* - * Job and Sub Workflow processing - * - * Unlike other job keys, those methods are not embedded in the key class itself because they require creating a child actor. - * While it would be possible to extract those methods from the WEA as well and provide them with an actor factory, the majority of the objects needed to create - * the children actors are attributes of this class, so it makes more sense to keep the functions here. + * Job and Sub Workflow processing + * + * Unlike other job keys, those methods are not embedded in the key class itself because they require creating a child actor. + * While it would be possible to extract those methods from the WEA as well and provide them with an actor factory, the majority of the objects needed to create + * the children actors are attributes of this class, so it makes more sense to keep the functions here. */ - private def processRunnableJob(key: BackendJobDescriptorKey, data: WorkflowExecutionActorData): ErrorOr[WorkflowExecutionDiff] = { + private def processRunnableJob(key: BackendJobDescriptorKey, + data: WorkflowExecutionActorData + ): ErrorOr[WorkflowExecutionDiff] = { import cats.syntax.either._ import common.validation.Checked._ @@ -650,7 +749,8 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) private def startEJEA(jobKey: BackendJobDescriptorKey, backendLifecycleActorFactory: BackendLifecycleActorFactory, - command: BackendJobExecutionActorCommand): WorkflowExecutionDiff = { + command: BackendJobExecutionActorCommand + ): WorkflowExecutionDiff = { val ejeaName = s"${workflowDescriptor.id}-EngineJobExecutionActor-${jobKey.tag}" val backendName = backendLifecycleActorFactory.name val backendSingleton = params.backendSingletonCollection.backendSingletonActors(backendName) @@ -695,9 +795,11 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) } /* - * Creates another WEA to process the SubWorkflowKey + * Creates another WEA to process the SubWorkflowKey */ - private def processRunnableSubWorkflow(key:SubWorkflowKey, data: WorkflowExecutionActorData): ErrorOr[WorkflowExecutionDiff] = { + private def processRunnableSubWorkflow(key: SubWorkflowKey, + data: WorkflowExecutionActorData + ): ErrorOr[WorkflowExecutionDiff] = { val sweaRef = context.actorOf( SubWorkflowExecutionActor.props( key, @@ -719,7 +821,9 @@ case class WorkflowExecutionActor(params: WorkflowExecutionActorParams) params.rootConfig, params.totalJobsByRootWf, fileHashCacheActor = params.fileHashCacheActor, - blacklistCache = params.blacklistCache), s"$workflowIdForLogging-SubWorkflowExecutionActor-${key.tag}" + blacklistCache = params.blacklistCache + ), + s"$workflowIdForLogging-SubWorkflowExecutionActor-${key.tag}" ) context watch sweaRef @@ -784,17 +888,19 @@ object WorkflowExecutionActor { case class WorkflowExecutionSucceededResponse(jobExecutionMap: JobExecutionMap, rootAndSubworklowIds: Set[WorkflowId], outputs: CallOutputs, - cumulativeOutputs: Set[WomValue] = Set.empty) - extends WorkflowExecutionActorResponse { + cumulativeOutputs: Set[WomValue] = Set.empty + ) extends WorkflowExecutionActorResponse { override def toString = "WorkflowExecutionSucceededResponse" } case class WorkflowExecutionAbortedResponse(jobExecutionMap: JobExecutionMap) - extends WorkflowExecutionActorResponse with EngineLifecycleActorAbortedResponse { + extends WorkflowExecutionActorResponse + with EngineLifecycleActorAbortedResponse { override def toString = "WorkflowExecutionAbortedResponse" } - final case class WorkflowExecutionFailedResponse(jobExecutionMap: JobExecutionMap, reason: Throwable) extends WorkflowExecutionActorResponse { + final case class WorkflowExecutionFailedResponse(jobExecutionMap: JobExecutionMap, reason: Throwable) + extends WorkflowExecutionActorResponse { override def toString = "WorkflowExecutionFailedResponse" } @@ -822,31 +928,34 @@ object WorkflowExecutionActor { jobExecutionMap: JobExecutionMap, rootAndSubworklowIds: Set[WorkflowId], outputs: CallOutputs, - cumulativeOutputs: Set[WomValue] = Set.empty) extends SubWorkflowTerminalStateResponse + cumulativeOutputs: Set[WomValue] = Set.empty + ) extends SubWorkflowTerminalStateResponse - case class SubWorkflowFailedResponse(key: SubWorkflowKey, jobExecutionMap: JobExecutionMap, reason: Throwable) extends SubWorkflowTerminalStateResponse + case class SubWorkflowFailedResponse(key: SubWorkflowKey, jobExecutionMap: JobExecutionMap, reason: Throwable) + extends SubWorkflowTerminalStateResponse - case class SubWorkflowAbortedResponse(key: SubWorkflowKey, jobExecutionMap: JobExecutionMap) extends SubWorkflowTerminalStateResponse + case class SubWorkflowAbortedResponse(key: SubWorkflowKey, jobExecutionMap: JobExecutionMap) + extends SubWorkflowTerminalStateResponse case class WorkflowExecutionActorParams( - workflowDescriptor: EngineWorkflowDescriptor, - ioActor: ActorRef, - serviceRegistryActor: ActorRef, - jobStoreActor: ActorRef, - subWorkflowStoreActor: ActorRef, - callCacheReadActor: ActorRef, - callCacheWriteActor: ActorRef, - workflowDockerLookupActor: ActorRef, - jobRestartCheckTokenDispenserActor: ActorRef, - jobExecutionTokenDispenserActor: ActorRef, - backendSingletonCollection: BackendSingletonCollection, - initializationData: AllBackendInitializationData, - startState: StartableState, - rootConfig: Config, - totalJobsByRootWf: AtomicInteger, - fileHashCacheActor: Option[ActorRef], - blacklistCache: Option[BlacklistCache] - ) + workflowDescriptor: EngineWorkflowDescriptor, + ioActor: ActorRef, + serviceRegistryActor: ActorRef, + jobStoreActor: ActorRef, + subWorkflowStoreActor: ActorRef, + callCacheReadActor: ActorRef, + callCacheWriteActor: ActorRef, + workflowDockerLookupActor: ActorRef, + jobRestartCheckTokenDispenserActor: ActorRef, + jobExecutionTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection, + initializationData: AllBackendInitializationData, + startState: StartableState, + rootConfig: Config, + totalJobsByRootWf: AtomicInteger, + fileHashCacheActor: Option[ActorRef], + blacklistCache: Option[BlacklistCache] + ) def props(workflowDescriptor: EngineWorkflowDescriptor, ioActor: ActorRef, @@ -864,7 +973,8 @@ object WorkflowExecutionActor { rootConfig: Config, totalJobsByRootWf: AtomicInteger, fileHashCacheActor: Option[ActorRef], - blacklistCache: Option[BlacklistCache]): Props = { + blacklistCache: Option[BlacklistCache] + ): Props = Props( WorkflowExecutionActor( WorkflowExecutionActorParams( @@ -888,7 +998,6 @@ object WorkflowExecutionActor { ) ) ).withDispatcher(EngineDispatcher) - } implicit class EnhancedWorkflowOutputs(val outputs: Map[LocallyQualifiedName, WomValue]) extends AnyVal { def maxStringLength = 1000 diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala index ce00523a6e9..75e29db9c2f 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala @@ -20,15 +20,16 @@ import scala.concurrent.ExecutionContext object WorkflowExecutionDiff { def empty = WorkflowExecutionDiff(Map.empty) } + /** Data differential between current execution data, and updates performed in a method that needs to be merged. */ final case class WorkflowExecutionDiff(executionStoreChanges: Map[JobKey, ExecutionStatus], jobKeyActorMappings: Map[ActorRef, JobKey] = Map.empty, valueStoreAdditions: Map[ValueKey, WomValue] = Map.empty, cumulativeOutputsChanges: Set[WomValue] = Set.empty, - rootAndSubworkflowIds: Set[WorkflowId] = Set.empty) { - def containsNewEntry: Boolean = { + rootAndSubworkflowIds: Set[WorkflowId] = Set.empty +) { + def containsNewEntry: Boolean = executionStoreChanges.exists(esc => esc._2 == NotStarted) || valueStoreAdditions.nonEmpty - } } object WorkflowExecutionActorData { @@ -36,7 +37,8 @@ object WorkflowExecutionActorData { ec: ExecutionContext, asyncIo: AsyncIo, totalJobsByRootWf: AtomicInteger, - executionStore: ExecutionStore): WorkflowExecutionActorData = { + executionStore: ExecutionStore + ): WorkflowExecutionActorData = WorkflowExecutionActorData( workflowDescriptor, executionStore, @@ -46,9 +48,11 @@ object WorkflowExecutionActorData { totalJobsByRootWf = totalJobsByRootWf, rootAndSubworkflowIds = Set(workflowDescriptor.id) ) - } - final case class DataStoreUpdate(runnableKeys: List[JobKey], statusChanges: Map[JobKey, ExecutionStatus], newData: WorkflowExecutionActorData) + final case class DataStoreUpdate(runnableKeys: List[JobKey], + statusChanges: Map[JobKey, ExecutionStatus], + newData: WorkflowExecutionActorData + ) } case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescriptor, @@ -61,7 +65,8 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript downstreamExecutionMap: JobExecutionMap = Map.empty, totalJobsByRootWf: AtomicInteger, cumulativeOutputs: Set[WomValue] = Set.empty, - rootAndSubworkflowIds: Set[WorkflowId]) { + rootAndSubworkflowIds: Set[WorkflowId] +) { val expressionLanguageFunctions = new EngineIoFunctions(workflowDescriptor.pathBuilders, asyncIo, ec) @@ -69,54 +74,63 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript executionStore = executionStore.seal ) - def callExecutionSuccess(jobKey: JobKey, outputs: CallOutputs, cumulativeOutputs: Set[WomValue], rootAndSubworkflowIds: Set[WorkflowId]): WorkflowExecutionActorData = { - mergeExecutionDiff(WorkflowExecutionDiff( - executionStoreChanges = Map(jobKey -> Done), - valueStoreAdditions = toValuesMap(jobKey, outputs), - cumulativeOutputsChanges = cumulativeOutputs ++ outputs.outputs.values, - rootAndSubworkflowIds = rootAndSubworkflowIds - )) - } - - final def expressionEvaluationSuccess(expressionKey: ExpressionKey, values: Map[OutputPort, WomValue]): WorkflowExecutionActorData = { - val valueStoreAdditions = values.map({ - case (outputPort, value) => ValueKey(outputPort, expressionKey.index) -> value - }) - mergeExecutionDiff(WorkflowExecutionDiff( - executionStoreChanges = Map(expressionKey -> Done), - valueStoreAdditions =valueStoreAdditions - )) - } + def callExecutionSuccess(jobKey: JobKey, + outputs: CallOutputs, + cumulativeOutputs: Set[WomValue], + rootAndSubworkflowIds: Set[WorkflowId] + ): WorkflowExecutionActorData = + mergeExecutionDiff( + WorkflowExecutionDiff( + executionStoreChanges = Map(jobKey -> Done), + valueStoreAdditions = toValuesMap(jobKey, outputs), + cumulativeOutputsChanges = cumulativeOutputs ++ outputs.outputs.values, + rootAndSubworkflowIds = rootAndSubworkflowIds + ) + ) - def executionFailure(failedJobKey: JobKey, reason: Throwable, jobExecutionMap: JobExecutionMap): WorkflowExecutionActorData = { - mergeExecutionDiff(WorkflowExecutionDiff( - executionStoreChanges = Map(failedJobKey -> ExecutionStatus.Failed)) - ).addExecutions(jobExecutionMap) - .copy( - jobFailures = jobFailures + (failedJobKey -> reason) + final def expressionEvaluationSuccess(expressionKey: ExpressionKey, + values: Map[OutputPort, WomValue] + ): WorkflowExecutionActorData = { + val valueStoreAdditions = values.map { case (outputPort, value) => + ValueKey(outputPort, expressionKey.index) -> value + } + mergeExecutionDiff( + WorkflowExecutionDiff( + executionStoreChanges = Map(expressionKey -> Done), + valueStoreAdditions = valueStoreAdditions + ) ) } - def executionFailed(jobKey: JobKey): WorkflowExecutionActorData = mergeExecutionDiff(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Failed))) + def executionFailure(failedJobKey: JobKey, + reason: Throwable, + jobExecutionMap: JobExecutionMap + ): WorkflowExecutionActorData = + mergeExecutionDiff(WorkflowExecutionDiff(executionStoreChanges = Map(failedJobKey -> ExecutionStatus.Failed))) + .addExecutions(jobExecutionMap) + .copy( + jobFailures = jobFailures + (failedJobKey -> reason) + ) + + def executionFailed(jobKey: JobKey): WorkflowExecutionActorData = mergeExecutionDiff( + WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Failed)) + ) /** Converts call outputs to a ValueStore entries */ - private def toValuesMap(jobKey: JobKey, outputs: CallOutputs): Map[ValueKey, WomValue] = { - outputs.outputs.map({ - case (outputPort, jobOutput) => ValueKey(outputPort, jobKey.index) -> jobOutput - }) - } + private def toValuesMap(jobKey: JobKey, outputs: CallOutputs): Map[ValueKey, WomValue] = + outputs.outputs.map { case (outputPort, jobOutput) => + ValueKey(outputPort, jobKey.index) -> jobOutput + } - def addExecutions(jobExecutionMap: JobExecutionMap): WorkflowExecutionActorData = { + def addExecutions(jobExecutionMap: JobExecutionMap): WorkflowExecutionActorData = this.copy(downstreamExecutionMap = downstreamExecutionMap ++ jobExecutionMap) - } - def removeJobKeyActor(actorRef: ActorRef): WorkflowExecutionActorData = { + def removeJobKeyActor(actorRef: ActorRef): WorkflowExecutionActorData = this.copy( jobKeyActorMappings = jobKeyActorMappings - actorRef ) - } - def mergeExecutionDiff(diff: WorkflowExecutionDiff): WorkflowExecutionActorData = { + def mergeExecutionDiff(diff: WorkflowExecutionDiff): WorkflowExecutionActorData = this.copy( executionStore = executionStore.updateKeys(diff.executionStoreChanges), valueStore = valueStore.add(diff.valueStoreAdditions), @@ -124,15 +138,12 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript cumulativeOutputs = cumulativeOutputs ++ diff.cumulativeOutputsChanges, rootAndSubworkflowIds = rootAndSubworkflowIds ++ diff.rootAndSubworkflowIds ) - } - def mergeExecutionDiffs(diffs: Iterable[WorkflowExecutionDiff]): WorkflowExecutionActorData = { + def mergeExecutionDiffs(diffs: Iterable[WorkflowExecutionDiff]): WorkflowExecutionActorData = diffs.foldLeft(this)((newData, diff) => newData.mergeExecutionDiff(diff)) - } - def jobExecutionMap: JobExecutionMap = { + def jobExecutionMap: JobExecutionMap = downstreamExecutionMap updated (workflowDescriptor.backendDescriptor, executionStore.startedJobs) - } def executionStoreUpdate: DataStoreUpdate = { val update = executionStore.update diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala index 5e23a69d0c7..d904333bc22 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala @@ -33,7 +33,8 @@ class CallCache(database: CallCachingSqlDatabase) { jobIndex = b.jobIndex.fromIndex, jobAttempt = b.jobAttempt, returnCode = b.returnCode, - allowResultReuse = b.allowResultReuse) + allowResultReuse = b.allowResultReuse + ) val result = b.callOutputs.outputs.simplify val jobDetritus = b.jobDetritusFiles.getOrElse(Map.empty) buildCallCachingJoin(metaInfo, b.callCacheHashes, result, jobDetritus) @@ -42,70 +43,81 @@ class CallCache(database: CallCachingSqlDatabase) { database.addCallCaching(joins, batchSize) } - private def buildCallCachingJoin(callCachingEntry: CallCachingEntry, callCacheHashes: CallCacheHashes, - result: Iterable[WomValueSimpleton], jobDetritus: Map[String, Path]): CallCachingJoin = { + private def buildCallCachingJoin(callCachingEntry: CallCachingEntry, + callCacheHashes: CallCacheHashes, + result: Iterable[WomValueSimpleton], + jobDetritus: Map[String, Path] + ): CallCachingJoin = { - val hashesToInsert: Iterable[CallCachingHashEntry] = { + val hashesToInsert: Iterable[CallCachingHashEntry] = callCacheHashes.hashes map { hash => CallCachingHashEntry(hash.hashKey.key, hash.hashValue.value) } - } - val aggregatedHashesToInsert: Option[CallCachingAggregationEntry] = { - Option(CallCachingAggregationEntry( - baseAggregation = callCacheHashes.aggregatedInitialHash, - inputFilesAggregation = callCacheHashes.fileHashes.map(_.aggregatedHash) - )) - } + val aggregatedHashesToInsert: Option[CallCachingAggregationEntry] = + Option( + CallCachingAggregationEntry( + baseAggregation = callCacheHashes.aggregatedInitialHash, + inputFilesAggregation = callCacheHashes.fileHashes.map(_.aggregatedHash) + ) + ) - val resultToInsert: Iterable[CallCachingSimpletonEntry] = { - result map { - case WomValueSimpleton(simpletonKey, wdlPrimitive) => - CallCachingSimpletonEntry(simpletonKey, wdlPrimitive.valueString.toClobOption, wdlPrimitive.womType.stableName) + val resultToInsert: Iterable[CallCachingSimpletonEntry] = + result map { case WomValueSimpleton(simpletonKey, wdlPrimitive) => + CallCachingSimpletonEntry(simpletonKey, wdlPrimitive.valueString.toClobOption, wdlPrimitive.womType.stableName) } - } - val jobDetritusToInsert: Iterable[CallCachingDetritusEntry] = { - jobDetritus map { - case (fileName, filePath) => CallCachingDetritusEntry(fileName, filePath.pathAsString.toClobOption) + val jobDetritusToInsert: Iterable[CallCachingDetritusEntry] = + jobDetritus map { case (fileName, filePath) => + CallCachingDetritusEntry(fileName, filePath.pathAsString.toClobOption) } - } - CallCachingJoin(callCachingEntry, hashesToInsert.toSeq, aggregatedHashesToInsert, resultToInsert.toSeq, jobDetritusToInsert.toSeq) + CallCachingJoin(callCachingEntry, + hashesToInsert.toSeq, + aggregatedHashesToInsert, + resultToInsert.toSeq, + jobDetritusToInsert.toSeq + ) } - def hasBaseAggregatedHashMatch(baseAggregatedHash: String, hints: List[CacheHitHint])(implicit ec: ExecutionContext): Future[Boolean] = { + def hasBaseAggregatedHashMatch(baseAggregatedHash: String, hints: List[CacheHitHint])(implicit + ec: ExecutionContext + ): Future[Boolean] = { val ccpp = hints collectFirst { case h: CallCachePathPrefixes => h.prefixes } database.hasMatchingCallCachingEntriesForBaseAggregation(baseAggregatedHash, ccpp) } - def callCachingHitForAggregatedHashes(aggregatedCallHashes: AggregatedCallHashes, prefixesHint: Option[CallCachePathPrefixes], excludedIds: Set[CallCachingEntryId]) - (implicit ec: ExecutionContext): Future[Option[CallCachingEntryId]] = { - database.findCacheHitForAggregation( - baseAggregationHash = aggregatedCallHashes.baseAggregatedHash, - inputFilesAggregationHash = aggregatedCallHashes.inputFilesAggregatedHash, - callCachePathPrefixes = prefixesHint.map(_.prefixes), - excludedIds.map(_.id)).map(_ map CallCachingEntryId.apply) - } + def callCachingHitForAggregatedHashes(aggregatedCallHashes: AggregatedCallHashes, + prefixesHint: Option[CallCachePathPrefixes], + excludedIds: Set[CallCachingEntryId] + )(implicit ec: ExecutionContext): Future[Option[CallCachingEntryId]] = + database + .findCacheHitForAggregation( + baseAggregationHash = aggregatedCallHashes.baseAggregatedHash, + inputFilesAggregationHash = aggregatedCallHashes.inputFilesAggregatedHash, + callCachePathPrefixes = prefixesHint.map(_.prefixes), + excludedIds.map(_.id) + ) + .map(_ map CallCachingEntryId.apply) - def fetchCachedResult(callCachingEntryId: CallCachingEntryId)(implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] = { + def fetchCachedResult(callCachingEntryId: CallCachingEntryId)(implicit + ec: ExecutionContext + ): Future[Option[CallCachingJoin]] = database.queryResultsForCacheId(callCachingEntryId.id) - } - def callCachingJoinForCall(workflowUuid: String, callFqn: String, index: Int)(implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] = { + def callCachingJoinForCall(workflowUuid: String, callFqn: String, index: Int)(implicit + ec: ExecutionContext + ): Future[Option[CallCachingJoin]] = database.callCacheJoinForCall(workflowUuid, callFqn, index) - } - def invalidate(callCachingEntryId: CallCachingEntryId)(implicit ec: ExecutionContext) = { + def invalidate(callCachingEntryId: CallCachingEntryId)(implicit ec: ExecutionContext) = database.invalidateCall(callCachingEntryId.id) - } - def callCacheEntryIdsForWorkflowId(workflowId: String)(implicit ec: ExecutionContext) = { + def callCacheEntryIdsForWorkflowId(workflowId: String)(implicit ec: ExecutionContext) = database.callCacheEntryIdsForWorkflowId(workflowId) - } } object CallCache { object CallCacheHashBundle { - def apply(workflowId: WorkflowId, callCacheHashes: CallCacheHashes, jobSucceededResponse: JobSucceededResponse) = { + def apply(workflowId: WorkflowId, callCacheHashes: CallCacheHashes, jobSucceededResponse: JobSucceededResponse) = new CallCacheHashBundle( workflowId = workflowId, callCacheHashes = callCacheHashes, @@ -117,9 +129,11 @@ object CallCache { callOutputs = jobSucceededResponse.jobOutputs, jobDetritusFiles = jobSucceededResponse.jobDetritusFiles ) - } - def apply(workflowId: WorkflowId, callCacheHashes: CallCacheHashes, jobFailedNonRetryableResponse: JobFailedNonRetryableResponse) = { + def apply(workflowId: WorkflowId, + callCacheHashes: CallCacheHashes, + jobFailedNonRetryableResponse: JobFailedNonRetryableResponse + ) = new CallCacheHashBundle( workflowId = workflowId, callCacheHashes = callCacheHashes, @@ -131,47 +145,58 @@ object CallCache { callOutputs = CallOutputs.empty, jobDetritusFiles = None ) - } } case class CallCacheHashBundle private ( - workflowId: WorkflowId, - callCacheHashes: CallCacheHashes, - fullyQualifiedName: FullyQualifiedName, - jobIndex: ExecutionIndex, - jobAttempt: Option[Int], - returnCode: Option[Int], - allowResultReuse: Boolean, - callOutputs: CallOutputs, - jobDetritusFiles: Option[Map[String, Path]] - ) + workflowId: WorkflowId, + callCacheHashes: CallCacheHashes, + fullyQualifiedName: FullyQualifiedName, + jobIndex: ExecutionIndex, + jobAttempt: Option[Int], + returnCode: Option[Int], + allowResultReuse: Boolean, + callOutputs: CallOutputs, + jobDetritusFiles: Option[Map[String, Path]] + ) implicit class EnhancedCallCachingJoin(val callCachingJoin: CallCachingJoin) extends AnyVal { def toJobSuccess(key: BackendJobDescriptorKey, pathBuilders: List[PathBuilder]): JobSucceededResponse = { import cromwell.Simpletons._ import cromwell.core.path.PathFactory._ - val detritus = callCachingJoin.callCachingDetritusEntries.map({ jobDetritusEntry => + val detritus = callCachingJoin.callCachingDetritusEntries.map { jobDetritusEntry => jobDetritusEntry.detritusKey -> buildPath(jobDetritusEntry.detritusValue.toRawString, pathBuilders) - }).toMap - - val outputs = if (callCachingJoin.callCachingSimpletonEntries.isEmpty) CallOutputs(Map.empty) - else WomValueBuilder.toJobOutputs(key.call.outputPorts, callCachingJoin.callCachingSimpletonEntries map toSimpleton) - - JobSucceededResponse(key, callCachingJoin.callCachingEntry.returnCode,outputs, Option(detritus), Seq.empty, None, resultGenerationMode = CallCached) + }.toMap + + val outputs = + if (callCachingJoin.callCachingSimpletonEntries.isEmpty) CallOutputs(Map.empty) + else + WomValueBuilder.toJobOutputs(key.call.outputPorts, + callCachingJoin.callCachingSimpletonEntries map toSimpleton + ) + + JobSucceededResponse(key, + callCachingJoin.callCachingEntry.returnCode, + outputs, + Option(detritus), + Seq.empty, + None, + resultGenerationMode = CallCached + ) } def callCacheHashes: Set[HashResult] = { - val hashResults = callCachingJoin.callCachingHashEntries.map({ - case CallCachingHashEntry(k, v, _, _) => HashResult(HashKey.deserialize(k), HashValue(v)) - }) ++ callCachingJoin.callCachingAggregationEntry.collect({ - case CallCachingAggregationEntry(k, Some(v), _, _) => HashResult(HashKey.deserialize(k), HashValue(v)) - }) + val hashResults = callCachingJoin.callCachingHashEntries.map { case CallCachingHashEntry(k, v, _, _) => + HashResult(HashKey.deserialize(k), HashValue(v)) + } ++ callCachingJoin.callCachingAggregationEntry.collect { case CallCachingAggregationEntry(k, Some(v), _, _) => + HashResult(HashKey.deserialize(k), HashValue(v)) + } hashResults.toSet } } sealed trait CacheHitHint - case class CallCachePathPrefixes(callCacheRootPrefix: Option[String], workflowOptionPrefixes: List[String]) extends CacheHitHint { + case class CallCachePathPrefixes(callCacheRootPrefix: Option[String], workflowOptionPrefixes: List[String]) + extends CacheHitHint { lazy val prefixes: List[String] = (callCacheRootPrefix.toList ++ workflowOptionPrefixes) map { _.ensureSlashed } } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActor.scala index 22a1612c672..f99ff6ee675 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActor.scala @@ -14,35 +14,43 @@ import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheDiffAct import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheDiffQueryParameter.CallCacheDiffQueryCall import cromwell.services.metadata.MetadataService.GetMetadataAction import cromwell.services.metadata._ -import cromwell.services.{SuccessfulMetadataJsonResponse, FailedMetadataJsonResponse} +import cromwell.services.{FailedMetadataJsonResponse, SuccessfulMetadataJsonResponse} import spray.json.{JsArray, JsBoolean, JsNumber, JsObject, JsString, JsValue} -class CallCacheDiffActor(serviceRegistryActor: ActorRef) extends LoggingFSM[CallCacheDiffActorState, CallCacheDiffActorData] { +class CallCacheDiffActor(serviceRegistryActor: ActorRef) + extends LoggingFSM[CallCacheDiffActorState, CallCacheDiffActorData] { startWith(Idle, CallCacheDiffNoData) - when(Idle) { - case Event(CallCacheDiffQueryParameter(callA, callB), CallCacheDiffNoData) => - val queryA = makeMetadataQuery(callA) - val queryB = makeMetadataQuery(callB) - serviceRegistryActor ! GetMetadataAction(queryA) - serviceRegistryActor ! GetMetadataAction(queryB) - goto(WaitingForMetadata) using CallCacheDiffWithRequest(queryA, queryB, None, None, sender()) + when(Idle) { case Event(CallCacheDiffQueryParameter(callA, callB), CallCacheDiffNoData) => + val queryA = makeMetadataQuery(callA) + val queryB = makeMetadataQuery(callB) + serviceRegistryActor ! GetMetadataAction(queryA) + serviceRegistryActor ! GetMetadataAction(queryB) + goto(WaitingForMetadata) using CallCacheDiffWithRequest(queryA, queryB, None, None, sender()) } when(WaitingForMetadata) { // First Response // Response A - case Event(SuccessfulMetadataJsonResponse(GetMetadataAction(originalQuery, _), responseJson), data@CallCacheDiffWithRequest(queryA, _, None, None, _)) if queryA == originalQuery => + case Event(SuccessfulMetadataJsonResponse(GetMetadataAction(originalQuery, _), responseJson), + data @ CallCacheDiffWithRequest(queryA, _, None, None, _) + ) if queryA == originalQuery => stay() using data.copy(responseA = Option(WorkflowMetadataJson(responseJson))) // Response B - case Event(SuccessfulMetadataJsonResponse(GetMetadataAction(originalQuery, _), responseJson), data@CallCacheDiffWithRequest(_, queryB, None, None, _)) if queryB == originalQuery => + case Event(SuccessfulMetadataJsonResponse(GetMetadataAction(originalQuery, _), responseJson), + data @ CallCacheDiffWithRequest(_, queryB, None, None, _) + ) if queryB == originalQuery => stay() using data.copy(responseB = Option(WorkflowMetadataJson(responseJson))) // Second Response // Response A - case Event(SuccessfulMetadataJsonResponse(GetMetadataAction(originalQuery, _), responseJson), CallCacheDiffWithRequest(queryA, queryB, None, Some(responseB), replyTo)) if queryA == originalQuery => + case Event(SuccessfulMetadataJsonResponse(GetMetadataAction(originalQuery, _), responseJson), + CallCacheDiffWithRequest(queryA, queryB, None, Some(responseB), replyTo) + ) if queryA == originalQuery => buildDiffAndRespond(queryA, queryB, WorkflowMetadataJson(responseJson), responseB, replyTo) // Response B - case Event(SuccessfulMetadataJsonResponse(GetMetadataAction(originalQuery, _), responseJson), CallCacheDiffWithRequest(queryA, queryB, Some(responseA), None, replyTo)) if queryB == originalQuery => + case Event(SuccessfulMetadataJsonResponse(GetMetadataAction(originalQuery, _), responseJson), + CallCacheDiffWithRequest(queryA, queryB, Some(responseA), None, replyTo) + ) if queryB == originalQuery => buildDiffAndRespond(queryA, queryB, responseA, WorkflowMetadataJson(responseJson), replyTo) case Event(FailedMetadataJsonResponse(_, failure), data: CallCacheDiffWithRequest) => data.replyTo ! FailedCallCacheDiffResponse(failure) @@ -50,10 +58,11 @@ class CallCacheDiffActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Call stay() } - whenUnhandled { - case Event(oops, oopsData) => - log.error(s"Programmer Error: Unexpected event received by ${this.getClass.getSimpleName}: $oops / $oopsData (in state $stateName)") - stay() + whenUnhandled { case Event(oops, oopsData) => + log.error( + s"Programmer Error: Unexpected event received by ${this.getClass.getSimpleName}: $oops / $oopsData (in state $stateName)" + ) + stay() } @@ -61,17 +70,22 @@ class CallCacheDiffActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Call queryB: MetadataQuery, responseA: WorkflowMetadataJson, responseB: WorkflowMetadataJson, - replyTo: ActorRef) = { + replyTo: ActorRef + ) = { - def describeCallFromQuery(query: MetadataQuery): String = s"${query.workflowId} / ${query.jobKey.map(_.callFqn).getOrElse("<>")}:${query.jobKey.map(_.index.getOrElse(-1)).getOrElse("<>")}" + def describeCallFromQuery(query: MetadataQuery): String = + s"${query.workflowId} / ${query.jobKey.map(_.callFqn).getOrElse("<>")}:${query.jobKey.map(_.index.getOrElse(-1)).getOrElse("<>")}" - val callACachingMetadata = extractCallMetadata(queryA, responseA).contextualizeErrors(s"extract relevant metadata for call A (${describeCallFromQuery(queryA)})") - val callBCachingMetadata = extractCallMetadata(queryB, responseB).contextualizeErrors(s"extract relevant metadata for call B (${describeCallFromQuery(queryB)})") + val callACachingMetadata = extractCallMetadata(queryA, responseA).contextualizeErrors( + s"extract relevant metadata for call A (${describeCallFromQuery(queryA)})" + ) + val callBCachingMetadata = extractCallMetadata(queryB, responseB).contextualizeErrors( + s"extract relevant metadata for call B (${describeCallFromQuery(queryB)})" + ) val response = (callACachingMetadata, callBCachingMetadata) flatMapN { case (callA, callB) => - - val callADetails = extractCallDetails(queryA, callA) - val callBDetails = extractCallDetails(queryB, callB) + val callADetails = extractCallDetails(queryA, callA) + val callBDetails = extractCallDetails(queryB, callB) (callADetails, callBDetails) mapN { (cad, cbd) => val callAHashes = callA.callCachingMetadataJson.hashes @@ -79,8 +93,10 @@ class CallCacheDiffActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Call SuccessfulCallCacheDiffResponse(cad, cbd, calculateHashDifferential(callAHashes, callBHashes)) } - } valueOr { - e => FailedCallCacheDiffResponse(AggregatedMessageException("Failed to calculate diff for call A and call B", e.toList)) + } valueOr { e => + FailedCallCacheDiffResponse( + AggregatedMessageException("Failed to calculate diff for call A and call B", e.toList) + ) } replyTo ! response @@ -90,7 +106,6 @@ class CallCacheDiffActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Call } } - object CallCacheDiffActor { final case class CachedCallNotFoundException(message: String) extends Exception { @@ -108,18 +123,26 @@ object CallCacheDiffActor { responseA: Option[WorkflowMetadataJson], responseB: Option[WorkflowMetadataJson], replyTo: ActorRef - ) extends CallCacheDiffActorData + ) extends CallCacheDiffActorData sealed abstract class CallCacheDiffActorResponse case class FailedCallCacheDiffResponse(reason: Throwable) extends CallCacheDiffActorResponse - final case class SuccessfulCallCacheDiffResponse(callA: CallDetails, callB: CallDetails, hashDifferential: List[HashDifference]) extends CallCacheDiffActorResponse - def props(serviceRegistryActor: ActorRef) = Props(new CallCacheDiffActor(serviceRegistryActor)).withDispatcher(EngineDispatcher) - - final case class CallDetails(executionStatus: String, allowResultReuse: Boolean, callFqn: String, jobIndex: Int, workflowId: String) + final case class SuccessfulCallCacheDiffResponse(callA: CallDetails, + callB: CallDetails, + hashDifferential: List[HashDifference] + ) extends CallCacheDiffActorResponse + def props(serviceRegistryActor: ActorRef) = + Props(new CallCacheDiffActor(serviceRegistryActor)).withDispatcher(EngineDispatcher) + + final case class CallDetails(executionStatus: String, + allowResultReuse: Boolean, + callFqn: String, + jobIndex: Int, + workflowId: String + ) final case class HashDifference(hashKey: String, callA: Option[String], callB: Option[String]) - /** * Create a Metadata query from a CallCacheDiffQueryCall */ @@ -135,15 +158,16 @@ object CallCacheDiffActor { // These simple case classes are just to help apply a little type safety to input and output types: final case class WorkflowMetadataJson(value: JsObject) extends AnyVal - final case class CallMetadataJson(rawValue: JsObject, jobKey: MetadataQueryJobKey, callCachingMetadataJson: CallCachingMetadataJson) + final case class CallMetadataJson(rawValue: JsObject, + jobKey: MetadataQueryJobKey, + callCachingMetadataJson: CallCachingMetadataJson + ) final case class CallCachingMetadataJson(rawValue: JsObject, hashes: Map[String, String]) - /* * Takes in the JsObject returned from a metadata query and filters out only the appropriate call's callCaching section */ - def extractCallMetadata(query: MetadataQuery, response: WorkflowMetadataJson): ErrorOr[CallMetadataJson] = { - + def extractCallMetadata(query: MetadataQuery, response: WorkflowMetadataJson): ErrorOr[CallMetadataJson] = for { // Sanity Checks: _ <- response.value.validateNonEmptyResponse() @@ -158,17 +182,18 @@ object CallCacheDiffActor { callCachingElement <- onlyShardElement.fieldAsObject(CallMetadataKeys.CallCaching) hashes <- extractHashes(callCachingElement) } yield CallMetadataJson(onlyShardElement, jobKey, CallCachingMetadataJson(callCachingElement, hashes)) - } def extractHashes(callCachingMetadataJson: JsObject): ErrorOr[Map[String, String]] = { - def processField(keyPrefix: String)(fieldValue: (String, JsValue)): ErrorOr[Map[String, String]] = fieldValue match { - case (key, hashString: JsString) => Map(keyPrefix + key -> hashString.value).validNel - case (key, subObject: JsObject) => extractHashEntries(s"$keyPrefix$key:", subObject) - case (key, jsArray: JsArray) => - val subObjectElements = jsArray.elements.zipWithIndex.map { case (element, index) => (s"[$index]", element) } - extractHashEntries(keyPrefix + key, JsObject(subObjectElements: _*)) - case (key, otherValue) => s"Cannot extract hashes for $key. Expected JsString, JsObject, or JsArray but got ${otherValue.getClass.getSimpleName} $otherValue".invalidNel - } + def processField(keyPrefix: String)(fieldValue: (String, JsValue)): ErrorOr[Map[String, String]] = + fieldValue match { + case (key, hashString: JsString) => Map(keyPrefix + key -> hashString.value).validNel + case (key, subObject: JsObject) => extractHashEntries(s"$keyPrefix$key:", subObject) + case (key, jsArray: JsArray) => + val subObjectElements = jsArray.elements.zipWithIndex.map { case (element, index) => (s"[$index]", element) } + extractHashEntries(keyPrefix + key, JsObject(subObjectElements: _*)) + case (key, otherValue) => + s"Cannot extract hashes for $key. Expected JsString, JsObject, or JsArray but got ${otherValue.getClass.getSimpleName} $otherValue".invalidNel + } def extractHashEntries(keyPrefix: String, jsObject: JsObject): ErrorOr[Map[String, String]] = { val traversed = jsObject.fields.toList.traverse(processField(keyPrefix)) @@ -215,10 +240,12 @@ object CallCacheDiffActor { def fieldAsBoolean(field: String): ErrorOr[JsBoolean] = jsObject.getField(field) flatMap { _.mapToJsBoolean } def checkFieldValue(field: String, expectation: String): ErrorOr[Unit] = jsObject.getField(field) flatMap { case v: JsValue if v.toString == expectation => ().validNel - case other => s"Unexpected value '${other.toString}' for metadata field '$field', should have been '$expectation'".invalidNel + case other => + s"Unexpected value '${other.toString}' for metadata field '$field', should have been '$expectation'".invalidNel } - def validateNonEmptyResponse(): ErrorOr[Unit] = if (jsObject.fields.nonEmpty) { ().validNel } else { + def validateNonEmptyResponse(): ErrorOr[Unit] = if (jsObject.fields.nonEmpty) { ().validNel } + else { "No metadata was found for that workflow/call/index combination. Check that the workflow ID is correct, that the call name is formatted like 'workflowname.callname' and that an index is provided if this was a scattered task. (NOTE: the default index is -1, i.e. non-scattered)".invalidNel } } @@ -231,18 +258,20 @@ object CallCacheDiffActor { attempt <- asObject.fieldAsNumber("attempt") } yield (attempt.value.intValue, asObject) - def foldFunction(accumulator: ErrorOr[(Int, JsObject)], nextElement: JsValue): ErrorOr[(Int, JsObject)] = { - (accumulator, extractAttemptAndObject(nextElement)) mapN { case ((previousHighestAttempt, previousJsObject), (nextAttempt, nextJsObject)) => - if (previousHighestAttempt > nextAttempt) { - (previousHighestAttempt, previousJsObject) - } else { - (nextAttempt, nextJsObject) - } + def foldFunction(accumulator: ErrorOr[(Int, JsObject)], nextElement: JsValue): ErrorOr[(Int, JsObject)] = + (accumulator, extractAttemptAndObject(nextElement)) mapN { + case ((previousHighestAttempt, previousJsObject), (nextAttempt, nextJsObject)) => + if (previousHighestAttempt > nextAttempt) { + (previousHighestAttempt, previousJsObject) + } else { + (nextAttempt, nextJsObject) + } } - } for { - attemptListNel <- NonEmptyList.fromList(jsArray.elements.toList).toErrorOr("Expected at least one attempt but found 0") + attemptListNel <- NonEmptyList + .fromList(jsArray.elements.toList) + .toErrorOr("Expected at least one attempt but found 0") highestAttempt <- attemptListNel.toList.foldLeft(extractAttemptAndObject(attemptListNel.head))(foldFunction) } yield highestAttempt._2 } @@ -251,23 +280,28 @@ object CallCacheDiffActor { implicit class EnhancedJsValue(val jsValue: JsValue) extends AnyVal { def mapToJsObject: ErrorOr[JsObject] = jsValue match { case obj: JsObject => obj.validNel - case other => s"Invalid value type. Expected JsObject but got ${other.getClass.getSimpleName}: ${other.prettyPrint}".invalidNel + case other => + s"Invalid value type. Expected JsObject but got ${other.getClass.getSimpleName}: ${other.prettyPrint}".invalidNel } def mapToJsArray: ErrorOr[JsArray] = jsValue match { case arr: JsArray => arr.validNel - case other => s"Invalid value type. Expected JsArray but got ${other.getClass.getSimpleName}: ${other.prettyPrint}".invalidNel + case other => + s"Invalid value type. Expected JsArray but got ${other.getClass.getSimpleName}: ${other.prettyPrint}".invalidNel } def mapToJsString: ErrorOr[JsString] = jsValue match { case str: JsString => str.validNel - case other => s"Invalid value type. Expected JsString but got ${other.getClass.getSimpleName}: ${other.prettyPrint}".invalidNel + case other => + s"Invalid value type. Expected JsString but got ${other.getClass.getSimpleName}: ${other.prettyPrint}".invalidNel } def mapToJsBoolean: ErrorOr[JsBoolean] = jsValue match { case boo: JsBoolean => boo.validNel - case other => s"Invalid value type. Expected JsBoolean but got ${other.getClass.getSimpleName}: ${other.prettyPrint}".invalidNel + case other => + s"Invalid value type. Expected JsBoolean but got ${other.getClass.getSimpleName}: ${other.prettyPrint}".invalidNel } def mapToJsNumber: ErrorOr[JsNumber] = jsValue match { case boo: JsNumber => boo.validNel - case other => s"Invalid value type. Expected JsNumber but got ${other.getClass.getSimpleName}: ${other.prettyPrint}".invalidNel + case other => + s"Invalid value type. Expected JsNumber but got ${other.getClass.getSimpleName}: ${other.prettyPrint}".invalidNel } } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActorJsonFormatting.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActorJsonFormatting.scala index 11ba8bfc3a4..8012c8951d8 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActorJsonFormatting.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActorJsonFormatting.scala @@ -1,6 +1,10 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching -import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheDiffActor.{CallDetails, HashDifference, SuccessfulCallCacheDiffResponse} +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheDiffActor.{ + CallDetails, + HashDifference, + SuccessfulCallCacheDiffResponse +} import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport import org.apache.commons.lang3.NotImplementedException import spray.json._ @@ -13,14 +17,18 @@ object CallCacheDiffActorJsonFormatting extends SprayJsonSupport with DefaultJso implicit val hashDifferenceJsonFormatter = new RootJsonFormat[HashDifference] { override def write(hashDifference: HashDifference): JsValue = { def fromOption(opt: Option[String]) = opt.map(JsString.apply).getOrElse(JsNull) - JsObject(Map( - "hashKey" -> JsString(hashDifference.hashKey), - "callA" -> fromOption(hashDifference.callA), - "callB" -> fromOption(hashDifference.callB) - )) + JsObject( + Map( + "hashKey" -> JsString(hashDifference.hashKey), + "callA" -> fromOption(hashDifference.callA), + "callB" -> fromOption(hashDifference.callB) + ) + ) } override def read(json: JsValue): HashDifference = - throw new NotImplementedException("Programmer Error: No reader for HashDifferentials written. It was not expected to be required") + throw new NotImplementedException( + "Programmer Error: No reader for HashDifferentials written. It was not expected to be required" + ) } implicit val successfulResponseJsonFormatter = jsonFormat3(SuccessfulCallCacheDiffResponse) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffQueryParameter.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffQueryParameter.scala index ca094105988..6b32632a5a9 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffQueryParameter.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffQueryParameter.scala @@ -15,30 +15,29 @@ object CallCacheDiffQueryParameter { private def missingWorkflowError(attribute: String) = s"missing $attribute query parameter".invalidNel def fromParameters(parameters: Seq[(String, String)]): ErrorOr[CallCacheDiffQueryParameter] = { - def extractIndex(parameter: String): ErrorOr[Option[Int]] = { + def extractIndex(parameter: String): ErrorOr[Option[Int]] = parameters.find(_._1 == parameter) match { - case Some((_, value)) => Try(value.trim.toInt) match { - case Success(index) => Option(index).validNel - case Failure(f) => f.getMessage.invalidNel - } + case Some((_, value)) => + Try(value.trim.toInt) match { + case Success(index) => Option(index).validNel + case Failure(f) => f.getMessage.invalidNel + } case None => None.validNel } - } - def extractAttribute(parameter: String): ErrorOr[String] = { + def extractAttribute(parameter: String): ErrorOr[String] = parameters.find(_._1 == parameter) match { case Some((_, value)) => value.validNel case None => missingWorkflowError(parameter) } - } - + def validateWorkflowId(parameter: String): ErrorOr[WorkflowId] = for { workflowIdString <- extractAttribute(parameter) workflowId <- fromTry(Try(WorkflowId.fromString(workflowIdString.trim))) .leftMap(_.getMessage) .toValidatedNel[String, WorkflowId] } yield workflowId - + val workflowAValidation = validateWorkflowId("workflowA") val workflowBValidation = validateWorkflowId("workflowB") @@ -49,15 +48,16 @@ object CallCacheDiffQueryParameter { val indexBValidation: ErrorOr[Option[Int]] = extractIndex("indexB") (workflowAValidation, - callAValidation, - indexAValidation, - workflowBValidation, - callBValidation, - indexBValidation) mapN { (workflowA, callA, indexA, workflowB, callB, indexB) => - CallCacheDiffQueryParameter( - CallCacheDiffQueryCall(workflowA, callA, indexA), - CallCacheDiffQueryCall(workflowB, callB, indexB) - ) + callAValidation, + indexAValidation, + workflowBValidation, + callBValidation, + indexBValidation + ) mapN { (workflowA, callA, indexA, workflowB, callB, indexB) => + CallCacheDiffQueryParameter( + CallCacheDiffQueryCall(workflowA, callA, indexA), + CallCacheDiffQueryCall(workflowB, callB, indexB) + ) } } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala index 2329ad1331b..ff36d6deb2f 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala @@ -17,7 +17,6 @@ import wom.values._ import java.security.MessageDigest import javax.xml.bind.DatatypeConverter - /** * Actor responsible for calculating individual as well as aggregated hashes for a job. * First calculate the initial hashes (individual and aggregated), and send them to its parent @@ -42,7 +41,7 @@ class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor, callCachingActivity: CallCachingActivity, callCachePathPrefixes: Option[CallCachePathPrefixes], fileHashBatchSize: Int - ) extends LoggingFSM[CallCacheHashingJobActorState, CallCacheHashingJobActorData] { +) extends LoggingFSM[CallCacheHashingJobActorState, CallCacheHashingJobActorData] { val fileHashingActor: ActorRef = makeFileHashingActor() @@ -103,9 +102,8 @@ class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor, } // In its own function so it can be overridden in the test - private [callcaching] def addFileHash(hashResult: HashResult, data: CallCacheHashingJobActorData) = { + private[callcaching] def addFileHash(hashResult: HashResult, data: CallCacheHashingJobActorData) = data.withFileHash(hashResult) - } private def stopAndStay(fileHashResult: Option[FinalFileHashingResult]) = { fileHashResult foreach { context.parent ! _ } @@ -114,14 +112,15 @@ class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor, stay() } - private def sendToCallCacheReadingJobActor(message: Any, data: CallCacheHashingJobActorData): Unit = { + private def sendToCallCacheReadingJobActor(message: Any, data: CallCacheHashingJobActorData): Unit = data.callCacheReadingJobActor foreach { _ ! message } - } private def initializeCCHJA(): Unit = { import cromwell.core.simpleton.WomValueSimpleton._ - val unqualifiedInputs = jobDescriptor.evaluatedTaskInputs map { case (declaration, value) => declaration.name -> value } + val unqualifiedInputs = jobDescriptor.evaluatedTaskInputs map { case (declaration, value) => + declaration.name -> value + } val inputSimpletons = unqualifiedInputs.simplifyForCaching val (fileInputSimpletons, nonFileInputSimpletons) = inputSimpletons partition { @@ -131,11 +130,12 @@ class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor, val initialHashes = calculateInitialHashes(nonFileInputSimpletons, fileInputSimpletons) - val fileHashRequests = fileInputSimpletons collect { - case WomValueSimpleton(name, x: WomFile) => SingleFileHashRequest(jobDescriptor.key, HashKey(true, "input", s"File $name"), x, initializationData) + val fileHashRequests = fileInputSimpletons collect { case WomValueSimpleton(name, x: WomFile) => + SingleFileHashRequest(jobDescriptor.key, HashKey(true, "input", s"File $name"), x, initializationData) } - val hashingJobActorData = CallCacheHashingJobActorData(fileHashRequests.toList, callCacheReadingJobActor, fileHashBatchSize) + val hashingJobActorData = + CallCacheHashingJobActorData(fileHashRequests.toList, callCacheReadingJobActor, fileHashBatchSize) startWith(WaitingForHashFileRequest, hashingJobActorData) val aggregatedBaseHash = calculateHashAggregation(initialHashes, MessageDigest.getInstance("MD5")) @@ -149,39 +149,65 @@ class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor, if (hashingJobActorData.callCacheReadingJobActor.isEmpty) self ! NextBatchOfFileHashesRequest } - private def calculateInitialHashes(nonFileInputs: Iterable[WomValueSimpleton], fileInputs: Iterable[WomValueSimpleton]): Set[HashResult] = { + private def calculateInitialHashes(nonFileInputs: Iterable[WomValueSimpleton], + fileInputs: Iterable[WomValueSimpleton] + ): Set[HashResult] = { - val commandTemplateHash = HashResult(HashKey("command template"), jobDescriptor.taskCall.callable.commandTemplateString(jobDescriptor.evaluatedTaskInputs).md5HashValue) + val commandTemplateHash = HashResult( + HashKey("command template"), + jobDescriptor.taskCall.callable.commandTemplateString(jobDescriptor.evaluatedTaskInputs).md5HashValue + ) val backendNameHash = HashResult(HashKey("backend name"), backendNameForCallCachingPurposes.md5HashValue) - val inputCountHash = HashResult(HashKey("input count"), (nonFileInputs.size + fileInputs.size).toString.md5HashValue) - val outputCountHash = HashResult(HashKey("output count"), jobDescriptor.taskCall.callable.outputs.size.toString.md5HashValue) - - val runtimeAttributeHashes = runtimeAttributeDefinitions map { definition => jobDescriptor.runtimeAttributes.get(definition.name) match { - case Some(_) if definition.name == RuntimeAttributesKeys.DockerKey && callCachingEligible.dockerHash.isDefined => - HashResult(HashKey(definition.usedInCallCaching, "runtime attribute", definition.name), callCachingEligible.dockerHash.get.md5HashValue) - case Some(womValue) => HashResult(HashKey(definition.usedInCallCaching, "runtime attribute", definition.name), womValue.valueString.md5HashValue) - case None => HashResult(HashKey(definition.usedInCallCaching, "runtime attribute", definition.name), UnspecifiedRuntimeAttributeHashValue) - }} - - val inputHashResults = nonFileInputs map { - case WomValueSimpleton(name, value) => - val womTypeHashKeyString = value.womType.toHashKeyString - log.debug("Hashing input expression as {} {}", womTypeHashKeyString, name) - HashResult(HashKey("input", s"$womTypeHashKeyString $name"), value.toWomString.md5HashValue) + val inputCountHash = + HashResult(HashKey("input count"), (nonFileInputs.size + fileInputs.size).toString.md5HashValue) + val outputCountHash = + HashResult(HashKey("output count"), jobDescriptor.taskCall.callable.outputs.size.toString.md5HashValue) + + val runtimeAttributeHashes = runtimeAttributeDefinitions map { definition => + jobDescriptor.runtimeAttributes.get(definition.name) match { + case Some(_) + if definition.name == RuntimeAttributesKeys.DockerKey && callCachingEligible.dockerHash.isDefined => + HashResult(HashKey(definition.usedInCallCaching, "runtime attribute", definition.name), + callCachingEligible.dockerHash.get.md5HashValue + ) + case Some(womValue) => + HashResult(HashKey(definition.usedInCallCaching, "runtime attribute", definition.name), + womValue.valueString.md5HashValue + ) + case None => + HashResult(HashKey(definition.usedInCallCaching, "runtime attribute", definition.name), + UnspecifiedRuntimeAttributeHashValue + ) + } + } + + val inputHashResults = nonFileInputs map { case WomValueSimpleton(name, value) => + val womTypeHashKeyString = value.womType.toHashKeyString + log.debug("Hashing input expression as {} {}", womTypeHashKeyString, name) + HashResult(HashKey("input", s"$womTypeHashKeyString $name"), value.toWomString.md5HashValue) } val outputExpressionHashResults = jobDescriptor.taskCall.callable.outputs map { output => val womTypeHashKeyString = output.womType.toHashKeyString val outputExpressionCacheString = output.expression.cacheString - log.debug("Hashing output expression type as '{}' and value as '{}'", womTypeHashKeyString, outputExpressionCacheString) - HashResult(HashKey("output expression", s"$womTypeHashKeyString ${output.name}"), outputExpressionCacheString.md5HashValue) + log.debug("Hashing output expression type as '{}' and value as '{}'", + womTypeHashKeyString, + outputExpressionCacheString + ) + HashResult(HashKey("output expression", s"$womTypeHashKeyString ${output.name}"), + outputExpressionCacheString.md5HashValue + ) } // Build these all together for the final set of initial hashes: - Set(commandTemplateHash, backendNameHash, inputCountHash, outputCountHash) ++ runtimeAttributeHashes ++ inputHashResults ++ outputExpressionHashResults + Set(commandTemplateHash, + backendNameHash, + inputCountHash, + outputCountHash + ) ++ runtimeAttributeHashes ++ inputHashResults ++ outputExpressionHashResults } - private [callcaching] def makeFileHashingActor() = { + private[callcaching] def makeFileHashingActor() = { val fileHashingActorName = s"FileHashingActor_for_${jobDescriptor.key.tag}" context.actorOf(fileHashingActorProps, fileHashingActorName) } @@ -199,18 +225,20 @@ object CallCacheHashingJobActor { callCachingActivity: CallCachingActivity, callCachePathPrefixes: Option[CallCachePathPrefixes], fileHashBatchSize: Int - ): Props = Props(new CallCacheHashingJobActor( - jobDescriptor, - callCacheReadingJobActor, - initializationData, - runtimeAttributeDefinitions, - backendNameForCallCachingPurposes, - fileHashingActorProps, - callCachingEligible, - callCachingActivity, - callCachePathPrefixes, - fileHashBatchSize - )).withDispatcher(EngineDispatcher) + ): Props = Props( + new CallCacheHashingJobActor( + jobDescriptor, + callCacheReadingJobActor, + initializationData, + runtimeAttributeDefinitions, + backendNameForCallCachingPurposes, + fileHashingActorProps, + callCachingEligible, + callCachingActivity, + callCachePathPrefixes, + fileHashBatchSize + ) + ).withDispatcher(EngineDispatcher) sealed trait CallCacheHashingJobActorState case object WaitingForHashFileRequest extends CallCacheHashingJobActorState @@ -227,22 +255,29 @@ object CallCacheHashingJobActor { val sortedHashes = hashes.toList .filter(_.hashKey.checkForHitOrMiss) .sortBy(_.hashKey.key) - .map({ case HashResult(hashKey, HashValue(hashValue)) => hashKey.key + hashValue }) + .map { case HashResult(hashKey, HashValue(hashValue)) => hashKey.key + hashValue } .map(_.getBytes) sortedHashes foreach messageDigest.update DatatypeConverter.printHexBinary(messageDigest.digest()) } object CallCacheHashingJobActorData { - def apply(fileHashRequestsRemaining: List[SingleFileHashRequest], callCacheReadingJobActor: Option[ActorRef], batchSize: Int): CallCacheHashingJobActorData = { - new CallCacheHashingJobActorData(fileHashRequestsRemaining.grouped(batchSize).toList, List.empty, callCacheReadingJobActor, batchSize) - } + def apply(fileHashRequestsRemaining: List[SingleFileHashRequest], + callCacheReadingJobActor: Option[ActorRef], + batchSize: Int + ): CallCacheHashingJobActorData = + new CallCacheHashingJobActorData(fileHashRequestsRemaining.grouped(batchSize).toList, + List.empty, + callCacheReadingJobActor, + batchSize + ) } final case class CallCacheHashingJobActorData(fileHashRequestsRemaining: List[List[SingleFileHashRequest]], fileHashResults: List[HashResult], callCacheReadingJobActor: Option[ActorRef], - batchSize: Int) { + batchSize: Int + ) { private val md5Digest = MessageDigest.getInstance("MD5") /** @@ -259,7 +294,14 @@ object CallCacheHashingJobActor { val updatedBatch = lastBatch.filterNot(_.hashKey == hashResult.hashKey) // If we're processing the last batch, and it's now empty, then we're done // In that case compute the aggregated hash and send that - if (updatedBatch.isEmpty) (List.empty, Option(CompleteFileHashingResult(newFileHashResults.toSet, calculateHashAggregation(newFileHashResults, md5Digest)))) + if (updatedBatch.isEmpty) + (List.empty, + Option( + CompleteFileHashingResult(newFileHashResults.toSet, + calculateHashAggregation(newFileHashResults, md5Digest) + ) + ) + ) // Otherwise just return the updated batch and no message else (List(updatedBatch), None) case currentBatch :: otherBatches => @@ -275,7 +317,9 @@ object CallCacheHashingJobActor { else (updatedBatch :: otherBatches, None) } - (this.copy(fileHashRequestsRemaining = updatedRequestsList, fileHashResults = newFileHashResults), responseMessage) + (this.copy(fileHashRequestsRemaining = updatedRequestsList, fileHashResults = newFileHashResults), + responseMessage + ) } } @@ -285,14 +329,18 @@ object CallCacheHashingJobActor { case object NextBatchOfFileHashesRequest extends CCHJARequest sealed trait CCHJAResponse - case class InitialHashingResult(initialHashes: Set[HashResult], aggregatedBaseHash: String, cacheHitHints: List[CacheHitHint] = List.empty) extends CCHJAResponse + case class InitialHashingResult(initialHashes: Set[HashResult], + aggregatedBaseHash: String, + cacheHitHints: List[CacheHitHint] = List.empty + ) extends CCHJAResponse // File Hashing responses sealed trait CCHJAFileHashResponse extends CCHJAResponse case class PartialFileHashingResult(initialHashes: NonEmptyList[HashResult]) extends CCHJAFileHashResponse sealed trait FinalFileHashingResult extends CCHJAFileHashResponse - case class CompleteFileHashingResult(fileHashes: Set[HashResult], aggregatedFileHash: String) extends FinalFileHashingResult + case class CompleteFileHashingResult(fileHashes: Set[HashResult], aggregatedFileHash: String) + extends FinalFileHashingResult case object NoFileHashesResult extends FinalFileHashingResult implicit class StringMd5er(val unhashedString: String) extends AnyVal { @@ -303,11 +351,11 @@ object CallCacheHashingJobActor { } implicit class WomTypeHashString(val womType: WomType) extends AnyVal { - def toHashKeyString: String = { + def toHashKeyString: String = womType match { case c: WomCompositeType => - val fieldTypes = c.typeMap map { - case (key, value) => s"$key -> ${value.stableName}" + val fieldTypes = c.typeMap map { case (key, value) => + s"$key -> ${value.stableName}" } "CompositeType_digest_" + fieldTypes.mkString("\n").md5Sum case a: WomArrayType => @@ -321,6 +369,5 @@ object CallCacheHashingJobActor { s"Coproduct($hashStrings)" case o => o.stableName } - } } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala index 16bb0fb9caa..f61a15cf3de 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala @@ -23,18 +23,19 @@ class CallCacheInvalidateActor(callCache: CallCache, cacheId: CallCachingEntryId context.stop(self) } - override def receive: Receive = { - case any => log.error("Unexpected message to InvalidateCallCacheActor: " + any) + override def receive: Receive = { case any => + log.error("Unexpected message to InvalidateCallCacheActor: " + any) } } object CallCacheInvalidateActor { - def props(callCache: CallCache, cacheId: CallCachingEntryId) = { - Props(new CallCacheInvalidateActor(callCache: CallCache, cacheId: CallCachingEntryId)).withDispatcher(EngineDispatcher) - } + def props(callCache: CallCache, cacheId: CallCachingEntryId) = + Props(new CallCacheInvalidateActor(callCache: CallCache, cacheId: CallCachingEntryId)) + .withDispatcher(EngineDispatcher) } sealed trait CallCacheInvalidatedResponse -case class CallCacheInvalidatedSuccess(cacheId: CallCachingEntryId, maybeEntry: Option[CallCachingEntry]) extends CallCacheInvalidatedResponse +case class CallCacheInvalidatedSuccess(cacheId: CallCachingEntryId, maybeEntry: Option[CallCachingEntry]) + extends CallCacheInvalidatedResponse case object CallCacheInvalidationUnnecessary extends CallCacheInvalidatedResponse case class CallCacheInvalidatedFailure(cacheId: CallCachingEntryId, t: Throwable) extends CallCacheInvalidatedResponse diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala index 16c4ce39ed7..63cc33d76b5 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala @@ -20,10 +20,8 @@ import cromwell.services.CallCaching.CallCachingEntryId * * Would be nice if instead there was a pull- rather than push-based mailbox but I can't find one... */ -class CallCacheReadActor(cache: CallCache, - override val serviceRegistryActor: ActorRef, - override val threshold: Int) - extends EnhancedThrottlerActor[CommandAndReplyTo[CallCacheReadActorRequest]] +class CallCacheReadActor(cache: CallCache, override val serviceRegistryActor: ActorRef, override val threshold: Int) + extends EnhancedThrottlerActor[CommandAndReplyTo[CallCacheReadActorRequest]] with ActorLogging { override def routed = true override def processHead(request: CommandAndReplyTo[CallCacheReadActorRequest]): Future[Int] = instrumentedProcess { @@ -58,29 +56,34 @@ class CallCacheReadActor(cache: CallCache, override def receive: Receive = enhancedReceive.orElse(super.receive) override protected def instrumentationPath = NonEmptyList.of("callcaching", "read") override protected def instrumentationPrefix = InstrumentationPrefixes.JobPrefix - override def commandToData(snd: ActorRef) = { - case request: CallCacheReadActorRequest => CommandAndReplyTo(request, snd) + override def commandToData(snd: ActorRef) = { case request: CallCacheReadActorRequest => + CommandAndReplyTo(request, snd) } } object CallCacheReadActor { - def props(callCache: CallCache, serviceRegistryActor: ActorRef): Props = { - Props(new CallCacheReadActor(callCache, serviceRegistryActor, LoadConfig.CallCacheReadThreshold)).withDispatcher(EngineDispatcher) - } + def props(callCache: CallCache, serviceRegistryActor: ActorRef): Props = + Props(new CallCacheReadActor(callCache, serviceRegistryActor, LoadConfig.CallCacheReadThreshold)) + .withDispatcher(EngineDispatcher) private[CallCacheReadActor] case class RequestTuple(requester: ActorRef, request: CallCacheReadActorRequest) object AggregatedCallHashes { - def apply(baseAggregatedHash: String, inputFilesAggregatedHash: String) = { + def apply(baseAggregatedHash: String, inputFilesAggregatedHash: String) = new AggregatedCallHashes(baseAggregatedHash, Option(inputFilesAggregatedHash)) - } } case class AggregatedCallHashes(baseAggregatedHash: String, inputFilesAggregatedHash: Option[String]) sealed trait CallCacheReadActorRequest - final case class CacheLookupRequest(aggregatedCallHashes: AggregatedCallHashes, excludedIds: Set[CallCachingEntryId], prefixesHint: Option[CallCachePathPrefixes]) extends CallCacheReadActorRequest - final case class HasMatchingInitialHashLookup(aggregatedTaskHash: String, cacheHitHints: List[CacheHitHint] = List.empty) extends CallCacheReadActorRequest - final case class CallCacheEntryForCall(workflowId: WorkflowId, jobKey: BackendJobDescriptorKey) extends CallCacheReadActorRequest + final case class CacheLookupRequest(aggregatedCallHashes: AggregatedCallHashes, + excludedIds: Set[CallCachingEntryId], + prefixesHint: Option[CallCachePathPrefixes] + ) extends CallCacheReadActorRequest + final case class HasMatchingInitialHashLookup(aggregatedTaskHash: String, + cacheHitHints: List[CacheHitHint] = List.empty + ) extends CallCacheReadActorRequest + final case class CallCacheEntryForCall(workflowId: WorkflowId, jobKey: BackendJobDescriptorKey) + extends CallCacheReadActorRequest sealed trait CallCacheReadActorResponse // Responses on whether or not there is at least one matching entry (can for initial matches of file matches) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActor.scala index 953da95042f..dee9dd2aeea 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActor.scala @@ -4,7 +4,12 @@ import akka.actor.{ActorRef, LoggingFSM, Props} import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.callcaching.HashingFailedMessage import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCache.CallCachePathPrefixes -import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{CompleteFileHashingResult, InitialHashingResult, NextBatchOfFileHashesRequest, NoFileHashesResult} +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{ + CompleteFileHashingResult, + InitialHashingResult, + NextBatchOfFileHashesRequest, + NoFileHashesResult +} import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor._ import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadingJobActor._ import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, HashError} @@ -24,16 +29,16 @@ import cromwell.services.CallCaching.CallCachingEntryId * Sends the response to its parent. * In case of a CacheHit, stays alive in case using the hit fails and it needs to fetch the next one. Otherwise just dies. */ -class CallCacheReadingJobActor(callCacheReadActor: ActorRef, prefixesHint: Option[CallCachePathPrefixes]) extends LoggingFSM[CallCacheReadingJobActorState, CCRJAData] { - +class CallCacheReadingJobActor(callCacheReadActor: ActorRef, prefixesHint: Option[CallCachePathPrefixes]) + extends LoggingFSM[CallCacheReadingJobActorState, CCRJAData] { + startWith(WaitingForInitialHash, CCRJANoData) - - when(WaitingForInitialHash) { - case Event(InitialHashingResult(_, aggregatedBaseHash, hints), CCRJANoData) => - callCacheReadActor ! HasMatchingInitialHashLookup(aggregatedBaseHash, hints) - goto(WaitingForHashCheck) using CCRJAWithData(sender(), aggregatedBaseHash, fileHash = None, seenCaches = Set.empty) + + when(WaitingForInitialHash) { case Event(InitialHashingResult(_, aggregatedBaseHash, hints), CCRJANoData) => + callCacheReadActor ! HasMatchingInitialHashLookup(aggregatedBaseHash, hints) + goto(WaitingForHashCheck) using CCRJAWithData(sender(), aggregatedBaseHash, fileHash = None, seenCaches = Set.empty) } - + when(WaitingForHashCheck) { case Event(HasMatchingEntries, CCRJAWithData(hashingActor, _, _, _)) => hashingActor ! NextBatchOfFileHashesRequest @@ -41,16 +46,22 @@ class CallCacheReadingJobActor(callCacheReadActor: ActorRef, prefixesHint: Optio case Event(NoMatchingEntries, _) => cacheMiss } - + when(WaitingForFileHashes) { case Event(CompleteFileHashingResult(_, aggregatedFileHash), data: CCRJAWithData) => - callCacheReadActor ! CacheLookupRequest(AggregatedCallHashes(data.initialHash, aggregatedFileHash), data.seenCaches, prefixesHint) + callCacheReadActor ! CacheLookupRequest(AggregatedCallHashes(data.initialHash, aggregatedFileHash), + data.seenCaches, + prefixesHint + ) goto(WaitingForCacheHitOrMiss) using data.withFileHash(aggregatedFileHash) case Event(NoFileHashesResult, data: CCRJAWithData) => - callCacheReadActor ! CacheLookupRequest(AggregatedCallHashes(data.initialHash, None), data.seenCaches, prefixesHint) + callCacheReadActor ! CacheLookupRequest(AggregatedCallHashes(data.initialHash, None), + data.seenCaches, + prefixesHint + ) goto(WaitingForCacheHitOrMiss) } - + when(WaitingForCacheHitOrMiss) { case Event(CacheLookupNextHit(hit), data: CCRJAWithData) => context.parent ! CacheHit(hit) @@ -58,7 +69,10 @@ class CallCacheReadingJobActor(callCacheReadActor: ActorRef, prefixesHint: Optio case Event(CacheLookupNoHit, _) => cacheMiss case Event(NextHit, CCRJAWithData(_, aggregatedInitialHash, aggregatedFileHash, seenCaches)) => - callCacheReadActor ! CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, aggregatedFileHash), seenCaches, prefixesHint) + callCacheReadActor ! CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, aggregatedFileHash), + seenCaches, + prefixesHint + ) stay() } @@ -80,9 +94,8 @@ class CallCacheReadingJobActor(callCacheReadActor: ActorRef, prefixesHint: Optio object CallCacheReadingJobActor { - def props(callCacheReadActor: ActorRef, prefixesHint: Option[CallCachePathPrefixes]) = { + def props(callCacheReadActor: ActorRef, prefixesHint: Option[CallCachePathPrefixes]) = Props(new CallCacheReadingJobActor(callCacheReadActor, prefixesHint)).withDispatcher(EngineDispatcher) - } sealed trait CallCacheReadingJobActorState case object WaitingForInitialHash extends CallCacheReadingJobActorState @@ -92,7 +105,11 @@ object CallCacheReadingJobActor { sealed trait CCRJAData case object CCRJANoData extends CCRJAData - case class CCRJAWithData(hashingActor: ActorRef, initialHash: String, fileHash: Option[String], seenCaches: Set[CallCachingEntryId]) extends CCRJAData { + case class CCRJAWithData(hashingActor: ActorRef, + initialHash: String, + fileHash: Option[String], + seenCaches: Set[CallCachingEntryId] + ) extends CCRJAData { def withSeenCache(id: CallCachingEntryId): CCRJAWithData = this.copy(seenCaches = seenCaches + id) def withFileHash(aggregatedFileHash: String): CCRJAWithData = this.copy(fileHash = Option(aggregatedFileHash)) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala index 06d48fcdea0..012f2b0a7c0 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala @@ -16,18 +16,22 @@ import scala.concurrent.duration._ import scala.language.postfixOps case class CallCacheWriteActor(callCache: CallCache, serviceRegistryActor: ActorRef, threshold: Int) - extends EnhancedBatchActor[CommandAndReplyTo[SaveCallCacheHashes]]( - CallCacheWriteActor.dbFlushRate, - CallCacheWriteActor.dbBatchSize) { + extends EnhancedBatchActor[CommandAndReplyTo[SaveCallCacheHashes]](CallCacheWriteActor.dbFlushRate, + CallCacheWriteActor.dbBatchSize + ) { override protected def process(data: NonEmptyVector[CommandAndReplyTo[SaveCallCacheHashes]]) = instrumentedProcess { log.debug("Flushing {} call cache hashes sets to the DB", data.length) // Collect all the bundles of hashes that should be written and all the senders which should be informed of // success or failure. - val (bundles, replyTos) = data.toList.foldMap { case CommandAndReplyTo(s: SaveCallCacheHashes, r: ActorRef) => (List(s.bundle), List(r)) } + val (bundles, replyTos) = data.toList.foldMap { case CommandAndReplyTo(s: SaveCallCacheHashes, r: ActorRef) => + (List(s.bundle), List(r)) + } if (bundles.nonEmpty) { - val futureMessage = callCache.addToCache(bundles, batchSize) map { _ => CallCacheWriteSuccess } recover { case t => CallCacheWriteFailure(t) } + val futureMessage = callCache.addToCache(bundles, batchSize) map { _ => CallCacheWriteSuccess } recover { + case t => CallCacheWriteFailure(t) + } futureMessage map { message => replyTos foreach { _ ! message } } @@ -46,9 +50,9 @@ case class CallCacheWriteActor(callCache: CallCache, serviceRegistryActor: Actor } object CallCacheWriteActor { - def props(callCache: CallCache, registryActor: ActorRef): Props = { - Props(CallCacheWriteActor(callCache, registryActor, LoadConfig.CallCacheWriteThreshold)).withDispatcher(EngineDispatcher) - } + def props(callCache: CallCache, registryActor: ActorRef): Props = + Props(CallCacheWriteActor(callCache, registryActor, LoadConfig.CallCacheWriteThreshold)) + .withDispatcher(EngineDispatcher) case class SaveCallCacheHashes(bundle: CallCacheHashBundle) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala index 698f1d20d4d..d1d5a9a6db0 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala @@ -8,7 +8,12 @@ import cromwell.core.callcaching._ import cromwell.core.logging.JobLogging import cromwell.engine.workflow.lifecycle.execution.CallMetadataHelper import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCache.CallCachePathPrefixes -import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{CompleteFileHashingResult, FinalFileHashingResult, InitialHashingResult, NoFileHashesResult} +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{ + CompleteFileHashingResult, + FinalFileHashingResult, + InitialHashingResult, + NoFileHashesResult +} import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadingJobActor.NextHit import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor._ import cromwell.services.CallCaching.CallCachingEntryId @@ -31,7 +36,11 @@ class EngineJobHashingActor(receiver: ActorRef, activity: CallCachingActivity, callCachingEligible: CallCachingEligible, callCachePathPrefixes: Option[CallCachePathPrefixes], - fileHashBatchSize: Int) extends Actor with ActorLogging with JobLogging with CallMetadataHelper { + fileHashBatchSize: Int +) extends Actor + with ActorLogging + with JobLogging + with CallMetadataHelper { override val jobTag = jobDescriptor.key.tag val workflowId = jobDescriptor.workflowDescriptor.id @@ -39,25 +48,28 @@ class EngineJobHashingActor(receiver: ActorRef, override val rootWorkflowIdForLogging = jobDescriptor.workflowDescriptor.rootWorkflowId override val workflowIdForCallMetadata: WorkflowId = workflowId - private [callcaching] var initialHash: Option[InitialHashingResult] = None + private[callcaching] var initialHash: Option[InitialHashingResult] = None - private [callcaching] val callCacheReadingJobActor = if (activity.readFromCache) { + private[callcaching] val callCacheReadingJobActor = if (activity.readFromCache) { Option(context.actorOf(callCacheReadingJobActorProps, s"CCReadingJobActor-${workflowId.shortString}-$jobTag")) } else None override def preStart(): Unit = { - context.actorOf(CallCacheHashingJobActor.props( - jobDescriptor, - callCacheReadingJobActor, - initializationData, - runtimeAttributeDefinitions, - backendNameForCallCachingPurposes, - fileHashingActorProps, - callCachingEligible, - activity, - callCachePathPrefixes, - fileHashBatchSize - ), s"CCHashingJobActor-${workflowId.shortString}-$jobTag") + context.actorOf( + CallCacheHashingJobActor.props( + jobDescriptor, + callCacheReadingJobActor, + initializationData, + runtimeAttributeDefinitions, + backendNameForCallCachingPurposes, + fileHashingActorProps, + callCachingEligible, + activity, + callCachePathPrefixes, + fileHashBatchSize + ), + s"CCHashingJobActor-${workflowId.shortString}-$jobTag" + ) super.preStart() } @@ -81,7 +93,10 @@ class EngineJobHashingActor(receiver: ActorRef, private def publishHashFailure(failure: Throwable) = { import cromwell.services.metadata.MetadataService._ - val failureAsEvents = throwableToMetadataEvents(metadataKeyForCall(jobDescriptor.key, CallMetadataKeys.CallCachingKeys.HashFailuresKey), failure) + val failureAsEvents = throwableToMetadataEvents( + metadataKeyForCall(jobDescriptor.key, CallMetadataKeys.CallCachingKeys.HashFailuresKey), + failure + ) serviceRegistryActor ! PutMetadataAction(failureAsEvents) } @@ -125,7 +140,10 @@ object EngineJobHashingActor { case class CacheHit(cacheResultId: CallCachingEntryId) extends EJHAResponse case class HashError(reason: Throwable) extends EJHAResponse case class FileHashes(hashes: Set[HashResult], aggregatedHash: String) - case class CallCacheHashes(initialHashes: Set[HashResult], aggregatedInitialHash: String, fileHashes: Option[FileHashes]) extends EJHAResponse { + case class CallCacheHashes(initialHashes: Set[HashResult], + aggregatedInitialHash: String, + fileHashes: Option[FileHashes] + ) extends EJHAResponse { val hashes = initialHashes ++ fileHashes.map(_.hashes).getOrElse(Set.empty) def aggregatedHashString: String = { val file = fileHashes match { @@ -147,17 +165,21 @@ object EngineJobHashingActor { activity: CallCachingActivity, callCachingEligible: CallCachingEligible, callCachePathPrefixes: Option[CallCachePathPrefixes], - fileHashBatchSize: Int): Props = Props(new EngineJobHashingActor( - receiver = receiver, - serviceRegistryActor = serviceRegistryActor, - jobDescriptor = jobDescriptor, - initializationData = initializationData, - fileHashingActorProps = fileHashingActorProps, - callCacheReadingJobActorProps = callCacheReadingJobActorProps, - runtimeAttributeDefinitions = runtimeAttributeDefinitions, - backendNameForCallCachingPurposes = backendNameForCallCachingPurposes, - activity = activity, - callCachingEligible = callCachingEligible, - callCachePathPrefixes = callCachePathPrefixes, - fileHashBatchSize = fileHashBatchSize)).withDispatcher(EngineDispatcher) + fileHashBatchSize: Int + ): Props = Props( + new EngineJobHashingActor( + receiver = receiver, + serviceRegistryActor = serviceRegistryActor, + jobDescriptor = jobDescriptor, + initializationData = initializationData, + fileHashingActorProps = fileHashingActorProps, + callCacheReadingJobActorProps = callCacheReadingJobActorProps, + runtimeAttributeDefinitions = runtimeAttributeDefinitions, + backendNameForCallCachingPurposes = backendNameForCallCachingPurposes, + activity = activity, + callCachingEligible = callCachingEligible, + callCachePathPrefixes = callCachePathPrefixes, + fileHashBatchSize = fileHashBatchSize + ) + ).withDispatcher(EngineDispatcher) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala index 89745e6523d..5d30adf9ce0 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala @@ -5,7 +5,10 @@ import cromwell.Simpletons._ import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.simpleton.WomValueSimpleton import cromwell.database.sql.SqlConverters._ -import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{CachedOutputLookupFailed, CachedOutputLookupSucceeded} +import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{ + CachedOutputLookupFailed, + CachedOutputLookupSucceeded +} import cromwell.services.CallCaching.CallCachingEntryId import scala.concurrent.ExecutionContext @@ -16,14 +19,19 @@ object FetchCachedResultsActor { Props(new FetchCachedResultsActor(callCachingEntryId, replyTo, callCache)).withDispatcher(EngineDispatcher) sealed trait CachedResultResponse - case class CachedOutputLookupFailed(callCachingEntryId: CallCachingEntryId, failure: Throwable) extends CachedResultResponse - case class CachedOutputLookupSucceeded(simpletons: Seq[WomValueSimpleton], callOutputFiles: Map[String,String], - returnCode: Option[Int], cacheHit: CallCachingEntryId, cacheHitDetails: String) extends CachedResultResponse + case class CachedOutputLookupFailed(callCachingEntryId: CallCachingEntryId, failure: Throwable) + extends CachedResultResponse + case class CachedOutputLookupSucceeded(simpletons: Seq[WomValueSimpleton], + callOutputFiles: Map[String, String], + returnCode: Option[Int], + cacheHit: CallCachingEntryId, + cacheHitDetails: String + ) extends CachedResultResponse } - class FetchCachedResultsActor(cacheResultId: CallCachingEntryId, replyTo: ActorRef, callCache: CallCache) - extends Actor with ActorLogging { + extends Actor + with ActorLogging { { implicit val ec: ExecutionContext = context.dispatcher @@ -36,12 +44,16 @@ class FetchCachedResultsActor(cacheResultId: CallCachingEntryId, replyTo: ActorR } val sourceCacheDetails = Seq(result.callCachingEntry.workflowExecutionUuid, - result.callCachingEntry.callFullyQualifiedName, - result.callCachingEntry.jobIndex.toString).mkString(":") + result.callCachingEntry.callFullyQualifiedName, + result.callCachingEntry.jobIndex.toString + ).mkString(":") - CachedOutputLookupSucceeded(simpletons, jobDetritusFiles.toMap, - result.callCachingEntry.returnCode, - cacheResultId, sourceCacheDetails) + CachedOutputLookupSucceeded(simpletons, + jobDetritusFiles.toMap, + result.callCachingEntry.returnCode, + cacheResultId, + sourceCacheDetails + ) case None => val reason = new RuntimeException(s"Cache hit vanished between discovery and retrieval: $cacheResultId") CachedOutputLookupFailed(cacheResultId, reason) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/package.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/package.scala index 1913e64c9a9..917b559693f 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/package.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/package.scala @@ -1,6 +1,3 @@ package cromwell.engine.workflow.lifecycle.execution -package object callcaching { - - -} +package object callcaching {} diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/EngineJobExecutionActor.scala index 4423de202cc..b6db1f05fcd 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/EngineJobExecutionActor.scala @@ -3,7 +3,13 @@ package cromwell.engine.workflow.lifecycle.execution.job import akka.actor.SupervisorStrategy.{Escalate, Stop} import akka.actor.{ActorInitializationException, ActorRef, LoggingFSM, OneForOneStrategy, Props} import cats.data.NonEmptyList -import cromwell.backend.BackendCacheHitCopyingActor.{CacheCopyFailure, CopyOutputsCommand, CopyingOutputsFailedResponse, CopyAttemptError, BlacklistSkip} +import cromwell.backend.BackendCacheHitCopyingActor.{ + BlacklistSkip, + CacheCopyFailure, + CopyAttemptError, + CopyingOutputsFailedResponse, + CopyOutputsCommand +} import cromwell.backend.BackendJobExecutionActor._ import cromwell.backend.BackendLifecycleActor.AbortJobCommand import cromwell.backend.MetricableCacheCopyErrorCategory.MetricableCacheCopyErrorCategory @@ -27,10 +33,16 @@ import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadAct import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadingJobActor.NextHit import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheWriteActor._ import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor._ -import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{CachedOutputLookupFailed, CachedOutputLookupSucceeded} +import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{ + CachedOutputLookupFailed, + CachedOutputLookupSucceeded +} import cromwell.engine.workflow.lifecycle.execution.callcaching._ import cromwell.engine.workflow.lifecycle.execution.job.EngineJobExecutionActor._ -import cromwell.engine.workflow.lifecycle.execution.job.preparation.CallPreparation.{BackendJobPreparationSucceeded, CallPreparationFailed} +import cromwell.engine.workflow.lifecycle.execution.job.preparation.CallPreparation.{ + BackendJobPreparationSucceeded, + CallPreparationFailed +} import cromwell.engine.workflow.lifecycle.execution.job.preparation.{CallPreparation, JobPreparationActor} import cromwell.engine.workflow.lifecycle.execution.stores.ValueStore import cromwell.engine.workflow.lifecycle.{EngineLifecycleActorAbortCommand, TimedFSM} @@ -63,12 +75,13 @@ class EngineJobExecutionActor(replyTo: ActorRef, jobExecutionTokenDispenserActor: ActorRef, backendSingletonActor: Option[ActorRef], command: BackendJobExecutionActorCommand, - callCachingParameters: CallCachingParameters) extends LoggingFSM[EngineJobExecutionActorState, EJEAData] - with WorkflowLogging - with CallMetadataHelper - with JobInstrumentation - with CromwellInstrumentation - with TimedFSM[EngineJobExecutionActorState] { + callCachingParameters: CallCachingParameters +) extends LoggingFSM[EngineJobExecutionActorState, EJEAData] + with WorkflowLogging + with CallMetadataHelper + with JobInstrumentation + with CromwellInstrumentation + with TimedFSM[EngineJobExecutionActorState] { override val workflowIdForLogging = workflowDescriptor.possiblyNotRootWorkflowId override val rootWorkflowIdForLogging = workflowDescriptor.rootWorkflowId @@ -85,21 +98,21 @@ class EngineJobExecutionActor(replyTo: ActorRef, super.supervisorStrategy.decider.applyOrElse(t, (_: Any) => Escalate) } - val jobTag = s"${workflowIdForLogging.shortString}:${jobDescriptorKey.call.fullyQualifiedName}:${jobDescriptorKey.index.fromIndex}:${jobDescriptorKey.attempt}" + val jobTag = + s"${workflowIdForLogging.shortString}:${jobDescriptorKey.call.fullyQualifiedName}:${jobDescriptorKey.index.fromIndex}:${jobDescriptorKey.attempt}" val tag = s"EJEA_$jobTag" - //noinspection ActorMutableStateInspection + // noinspection ActorMutableStateInspection // There's no need to check for a cache hit again if we got preempted, or if there's no result copying actor defined // NB: this can also change (e.g. if we have a HashError we just force this to CallCachingOff) - private[execution] var effectiveCallCachingMode = { + private[execution] var effectiveCallCachingMode = if (backendLifecycleActorFactory.fileHashingActorProps.isEmpty) CallCachingOff else if (jobDescriptorKey.node.callable.meta.get("volatile").contains(MetaValueElementBoolean(true))) CallCachingOff else if (backendLifecycleActorFactory.cacheHitCopyingActorProps.isEmpty || jobDescriptorKey.attempt > 1) { callCachingParameters.mode.withoutRead } else callCachingParameters.mode - } - //noinspection ActorMutableStateInspection + // noinspection ActorMutableStateInspection // If this actor is currently holding a job token, the token dispenser to which the token should be returned. private var currentTokenDispenser: Option[ActorRef] = None @@ -128,48 +141,46 @@ class EngineJobExecutionActor(replyTo: ActorRef, implicit val ec: ExecutionContext = context.dispatcher - override def preStart() = { + override def preStart() = log.debug(s"$tag: $effectiveCallCachingKey: $effectiveCallCachingMode") - } startWith(Pending, NoData) - //noinspection ActorMutableStateInspection + // noinspection ActorMutableStateInspection private var eventList: Seq[ExecutionEvent] = Seq(ExecutionEvent(stateName.toString)) - override def onTimedTransition(from: EngineJobExecutionActorState, to: EngineJobExecutionActorState, duration: FiniteDuration) = { + override def onTimedTransition(from: EngineJobExecutionActorState, + to: EngineJobExecutionActorState, + duration: FiniteDuration + ) = // Send to StatsD recordExecutionStepTiming(from.toString, duration) - } // When Pending, the FSM always has NoData - when(Pending) { - case Event(Execute, NoData) => - increment(NonEmptyList("jobs", List("ejea", "executing", "starting"))) - if (restarting) { - requestRestartCheckToken() - goto(RequestingRestartCheckToken) - } else { - requestExecutionToken() - goto(RequestingExecutionToken) - } + when(Pending) { case Event(Execute, NoData) => + increment(NonEmptyList("jobs", List("ejea", "executing", "starting"))) + if (restarting) { + requestRestartCheckToken() + goto(RequestingRestartCheckToken) + } else { + requestExecutionToken() + goto(RequestingExecutionToken) + } } // This condition only applies for restarts - when(RequestingRestartCheckToken) { - case Event(JobTokenDispensed, NoData) => - currentTokenDispenser = Option(jobRestartCheckTokenDispenserActor) - replyTo ! JobStarting(jobDescriptorKey) - val jobStoreKey = jobDescriptorKey.toJobStoreKey(workflowIdForLogging) - jobStoreActor ! QueryJobCompletion(jobStoreKey, jobDescriptorKey.call.outputPorts.toSeq) - goto(CheckingJobStore) + when(RequestingRestartCheckToken) { case Event(JobTokenDispensed, NoData) => + currentTokenDispenser = Option(jobRestartCheckTokenDispenserActor) + replyTo ! JobStarting(jobDescriptorKey) + val jobStoreKey = jobDescriptorKey.toJobStoreKey(workflowIdForLogging) + jobStoreActor ! QueryJobCompletion(jobStoreKey, jobDescriptorKey.call.outputPorts.toSeq) + goto(CheckingJobStore) } - when(RequestingExecutionToken) { - case Event(JobTokenDispensed, NoData) => - currentTokenDispenser = Option(jobExecutionTokenDispenserActor) - if (!restarting) - replyTo ! JobStarting(jobDescriptorKey) - requestValueStore() + when(RequestingExecutionToken) { case Event(JobTokenDispensed, NoData) => + currentTokenDispenser = Option(jobExecutionTokenDispenserActor) + if (!restarting) + replyTo ! JobStarting(jobDescriptorKey) + requestValueStore() } // When CheckingJobStore, the FSM always has NoData @@ -196,37 +207,45 @@ class EngineJobExecutionActor(replyTo: ActorRef, when(CheckingCacheEntryExistence) { // There was already a cache entry for this job case Event(join: CallCachingJoin, NoData) => - Try(join.toJobSuccess(jobDescriptorKey, backendLifecycleActorFactory.pathBuilders(initializationData))).map({ jobSuccess => - // We can't create a CallCacheHashes to give to the SucceededResponseData here because it involves knowledge of - // which hashes are file hashes and which are not. We can't know that (nor do we care) when pulling them from the - // database. So instead manually publish the hashes here. - publishHashResultsToMetadata(Option(Success(join.callCacheHashes))) - saveJobCompletionToJobStore(SucceededResponseData(jobSuccess, None)) - }).recover({ - case f => + Try(join.toJobSuccess(jobDescriptorKey, backendLifecycleActorFactory.pathBuilders(initializationData))) + .map { jobSuccess => + // We can't create a CallCacheHashes to give to the SucceededResponseData here because it involves knowledge of + // which hashes are file hashes and which are not. We can't know that (nor do we care) when pulling them from the + // database. So instead manually publish the hashes here. + publishHashResultsToMetadata(Option(Success(join.callCacheHashes))) + saveJobCompletionToJobStore(SucceededResponseData(jobSuccess, None)) + } + .recover { case f => // If for some reason the above fails, fail the job cleanly - saveJobCompletionToJobStore(FailedResponseData(JobFailedNonRetryableResponse(jobDescriptorKey, f, None), None)) - }).get + saveJobCompletionToJobStore( + FailedResponseData(JobFailedNonRetryableResponse(jobDescriptorKey, f, None), None) + ) + } + .get // No cache entry for this job - keep going case Event(NoCallCacheEntry(_), NoData) => returnCurrentToken() requestExecutionToken() goto(RequestingExecutionToken) case Event(CacheResultLookupFailure(reason), NoData) => - log.error(reason, "{}: Failure checking for cache entry existence: {}. Attempting to resume job anyway.", jobTag, reason.getMessage) + log.error(reason, + "{}: Failure checking for cache entry existence: {}. Attempting to resume job anyway.", + jobTag, + reason.getMessage + ) returnCurrentToken() requestExecutionToken() goto(RequestingExecutionToken) } /* - * ! Hot Potato Warning ! - * We ask explicitly for the output store so we can use it on the fly and more importantly not store it as a - * variable in this actor, which would prevent it from being garbage collected for the duration of the - * job and would lead to memory leaks. - */ - when(WaitingForValueStore) { - case Event(valueStore: ValueStore, NoData) => prepareJob(valueStore) + * ! Hot Potato Warning ! + * We ask explicitly for the output store so we can use it on the fly and more importantly not store it as a + * variable in this actor, which would prevent it from being garbage collected for the duration of the + * job and would lead to memory leaks. + */ + when(WaitingForValueStore) { case Event(valueStore: ValueStore, NoData) => + prepareJob(valueStore) } // When PreparingJob, the FSM always has NoData @@ -234,7 +253,8 @@ class EngineJobExecutionActor(replyTo: ActorRef, case Event(BackendJobPreparationSucceeded(jobDescriptor, bjeaProps), NoData) => val updatedData = ResponsePendingData(jobDescriptor, bjeaProps) effectiveCallCachingMode match { - case activity: CallCachingActivity if activity.readFromCache => handleReadFromCacheOn(jobDescriptor, activity, updatedData) + case activity: CallCachingActivity if activity.readFromCache => + handleReadFromCacheOn(jobDescriptor, activity, updatedData) case activity: CallCachingActivity => handleReadFromCacheOff(jobDescriptor, activity, updatedData) case CallCachingOff => runJob(updatedData) } @@ -244,9 +264,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, when(CheckingCallCache) { case Event(CacheMiss, data: ResponsePendingData) => - writeToMetadata(Map( - callCachingHitResultMetadataKey -> false, - callCachingReadResultMetadataKey -> "Cache Miss")) + writeToMetadata(Map(callCachingHitResultMetadataKey -> false, callCachingReadResultMetadataKey -> "Cache Miss")) if (data.cacheHitFailureCount > 0) { val totalHits = data.cacheHitFailureCount @@ -259,7 +277,12 @@ class EngineJobExecutionActor(replyTo: ActorRef, s"Falling back to running job." ) val template = s"BT-322 {} cache hit copying failure: {} failed copy attempts of maximum {} with {}." - log.info(template, jobTag, data.failedCopyAttempts, callCachingParameters.maxFailedCopyAttempts, data.aggregatedHashString) + log.info(template, + jobTag, + data.failedCopyAttempts, + callCachingParameters.maxFailedCopyAttempts, + data.aggregatedHashString + ) } else { log.info(s"BT-322 {} cache hit copying nomatch: could not find a suitable cache hit.", jobTag) workflowLogger.info("Could not copy a suitable cache hit for {}. No copy attempts were made.", arg = jobTag) @@ -280,17 +303,25 @@ class EngineJobExecutionActor(replyTo: ActorRef, when(FetchingCachedOutputsFromDatabase) { case Event( - CachedOutputLookupSucceeded(womValueSimpletons, jobDetritus, returnCode, cacheResultId, cacheHitDetails), - data@ResponsePendingData(_, _, _, _, Some(ejeaCacheHit), _, _, _), - ) => + CachedOutputLookupSucceeded(womValueSimpletons, jobDetritus, returnCode, cacheResultId, cacheHitDetails), + data @ ResponsePendingData(_, _, _, _, Some(ejeaCacheHit), _, _, _) + ) => if (cacheResultId != ejeaCacheHit.hit.cacheResultId) { // Sanity check: was this the right set of results (a false here is a BAD thing!): - log.error(s"Received incorrect call cache results from FetchCachedResultsActor. Expected ${ejeaCacheHit.hit} but got $cacheResultId. Running job") + log.error( + s"Received incorrect call cache results from FetchCachedResultsActor. Expected ${ejeaCacheHit.hit} but got $cacheResultId. Running job" + ) // Treat this like the "CachedOutputLookupFailed" event: runJob(data) } else { log.debug("Cache hit for {}! Fetching cached result {}", jobTag, cacheResultId) - makeBackendCopyCacheHit(womValueSimpletons, jobDetritus, returnCode, data, cacheResultId, ejeaCacheHit.hitNumber) using data.withCacheDetails(cacheHitDetails) + makeBackendCopyCacheHit(womValueSimpletons, + jobDetritus, + returnCode, + data, + cacheResultId, + ejeaCacheHit.hitNumber + ) using data.withCacheDetails(cacheHitDetails) } case Event(CachedOutputLookupFailed(_, error), data: ResponsePendingData) => log.warning("Can't fetch a list of cached outputs to copy for {} due to {}. Running job.", jobTag, error) @@ -306,12 +337,13 @@ class EngineJobExecutionActor(replyTo: ActorRef, when(BackendIsCopyingCachedOutputs) { // Backend copying response: case Event( - response: JobSucceededResponse, - data@ResponsePendingData(_, _, Some(Success(hashes)), _, _, _, _, _), - ) => + response: JobSucceededResponse, + data @ ResponsePendingData(_, _, Some(Success(hashes)), _, _, _, _, _) + ) => logCacheHitSuccessAndNotifyMetadata(data) saveCacheResults(hashes, data.withSuccessResponse(response)) - case Event(response: JobSucceededResponse, data: ResponsePendingData) if effectiveCallCachingMode.writeToCache && data.hashes.isEmpty => + case Event(response: JobSucceededResponse, data: ResponsePendingData) + if effectiveCallCachingMode.writeToCache && data.hashes.isEmpty => logCacheHitSuccessAndNotifyMetadata(data) // Wait for the CallCacheHashes stay() using data.withSuccessResponse(response) @@ -319,9 +351,9 @@ class EngineJobExecutionActor(replyTo: ActorRef, logCacheHitSuccessAndNotifyMetadata(data) saveJobCompletionToJobStore(data.withSuccessResponse(response)) case Event( - CopyingOutputsFailedResponse(_, cacheCopyAttempt, reason), - data@ResponsePendingData(_, _, _, _, Some(cacheHit), _, _, _) - ) if cacheCopyAttempt == cacheHit.hitNumber => + CopyingOutputsFailedResponse(_, cacheCopyAttempt, reason), + data @ ResponsePendingData(_, _, _, _, Some(cacheHit), _, _, _) + ) if cacheCopyAttempt == cacheHit.hitNumber => invalidateCacheHitAndTransition(cacheHit, data, reason) // Hashes arrive: @@ -359,16 +391,17 @@ class EngineJobExecutionActor(replyTo: ActorRef, val jobSuccessHandler: StateFunction = { // writeToCache is true and all hashes have already been retrieved - save to the cache case Event( - response: JobSucceededResponse, - data@ResponsePendingData(_, _, Some(Success(hashes)), _, _, _, _, _) - ) if effectiveCallCachingMode.writeToCache => + response: JobSucceededResponse, + data @ ResponsePendingData(_, _, Some(Success(hashes)), _, _, _, _, _) + ) if effectiveCallCachingMode.writeToCache => eventList ++= response.executionEvents // Publish the image used now that we have it as we might lose the information if Cromwell is restarted // in between writing to the cache and writing to the job store response.dockerImageUsed foreach publishDockerImageUsed saveCacheResults(hashes, data.withSuccessResponse(response)) // Hashes are still missing and we want them (writeToCache is true) - wait for them - case Event(response: JobSucceededResponse, data: ResponsePendingData) if effectiveCallCachingMode.writeToCache && data.hashes.isEmpty => + case Event(response: JobSucceededResponse, data: ResponsePendingData) + if effectiveCallCachingMode.writeToCache && data.hashes.isEmpty => eventList ++= response.executionEvents stay() using data.withSuccessResponse(response) // Hashes are missing but writeToCache is OFF - complete the job @@ -381,12 +414,13 @@ class EngineJobExecutionActor(replyTo: ActorRef, val jobFailedHandler: StateFunction = { // writeToCache is true and all hashes already retrieved - save to job store case Event( - response: BackendJobFailedResponse, - data@ResponsePendingData(_, _, Some(Success(_)), _, _, _, _, _) - ) if effectiveCallCachingMode.writeToCache => + response: BackendJobFailedResponse, + data @ ResponsePendingData(_, _, Some(Success(_)), _, _, _, _, _) + ) if effectiveCallCachingMode.writeToCache => saveJobCompletionToJobStore(data.withFailedResponse(response)) // Hashes are still missing and we want them (writeToCache is true) - wait for them - case Event(response: BackendJobFailedResponse, data: ResponsePendingData) if effectiveCallCachingMode.writeToCache && data.hashes.isEmpty => + case Event(response: BackendJobFailedResponse, data: ResponsePendingData) + if effectiveCallCachingMode.writeToCache && data.hashes.isEmpty => stay() using data.withFailedResponse(response) // Hashes are missing but writeToCache is OFF - complete the job case Event(response: BackendJobFailedResponse, data: ResponsePendingData) => @@ -448,7 +482,13 @@ class EngineJobExecutionActor(replyTo: ActorRef, stay() using data.copy(hashes = Option(Failure(t))) } - when(RunningJob)(jobSuccessHandler.orElse(jobFailedHandler).orElse(jobAbortedHandler).orElse(hashSuccessResponseHandler).orElse(hashFailureResponseHandler)) + when(RunningJob)( + jobSuccessHandler + .orElse(jobFailedHandler) + .orElse(jobAbortedHandler) + .orElse(hashSuccessResponseHandler) + .orElse(hashFailureResponseHandler) + ) // When UpdatingCallCache, the FSM always has SucceededResponseData. when(UpdatingCallCache) { @@ -464,16 +504,20 @@ class EngineJobExecutionActor(replyTo: ActorRef, case Event(JobStoreWriteSuccess(_), data: ResponseData) => forwardAndStop(data.response) case Event(JobStoreWriteFailure(t), _: ResponseData) => - respondAndStop(JobFailedNonRetryableResponse(jobDescriptorKey, new Exception(s"JobStore write failure: ${t.getMessage}", t), None)) + respondAndStop( + JobFailedNonRetryableResponse(jobDescriptorKey, + new Exception(s"JobStore write failure: ${t.getMessage}", t), + None + ) + ) } - onTransition { - case fromState -> toState => - log.debug("Transitioning from {}({}) to {}({})", fromState, stateData, toState, nextStateData) + onTransition { case fromState -> toState => + log.debug("Transitioning from {}({}) to {}({})", fromState, stateData, toState, nextStateData) - EngineJobExecutionActorState.transitionEventString(fromState, toState) foreach { - eventList :+= ExecutionEvent(_) - } + EngineJobExecutionActorState.transitionEventString(fromState, toState) foreach { + eventList :+= ExecutionEvent(_) + } } @@ -502,23 +546,35 @@ class EngineJobExecutionActor(replyTo: ActorRef, // due to timeouts). That's ok, we just ignore this message in any other situation: stay() case Event(msg, _) => - log.error("Bad message from {} to EngineJobExecutionActor in state {}(with data {}): {}", sender(), stateName, stateData, msg) + log.error("Bad message from {} to EngineJobExecutionActor in state {}(with data {}): {}", + sender(), + stateName, + stateData, + msg + ) stay() } - private def publishHashesToMetadata(maybeHashes: Option[Try[CallCacheHashes]]) = publishHashResultsToMetadata(maybeHashes.map(_.map(_.hashes))) + private def publishHashesToMetadata(maybeHashes: Option[Try[CallCacheHashes]]) = publishHashResultsToMetadata( + maybeHashes.map(_.map(_.hashes)) + ) private def publishDockerImageUsed(image: String) = writeToMetadata(Map("dockerImageUsed" -> image)) private def publishHashResultsToMetadata(maybeHashes: Option[Try[Set[HashResult]]]) = maybeHashes match { case Some(Success(hashes)) => - val hashMap = hashes.collect({ + val hashMap = hashes.collect { case HashResult(HashKey(useInCallCaching, keyComponents), HashValue(value)) if useInCallCaching => - (callCachingHashes + MetadataKey.KeySeparator + keyComponents.mkString(MetadataKey.KeySeparator.toString)) -> value - }).toMap + (callCachingHashes + MetadataKey.KeySeparator + keyComponents.mkString( + MetadataKey.KeySeparator.toString + )) -> value + }.toMap writeToMetadata(hashMap) case _ => } - private def handleReadFromCacheOn(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity, updatedData: ResponsePendingData) = { + private def handleReadFromCacheOn(jobDescriptor: BackendJobDescriptor, + activity: CallCachingActivity, + updatedData: ResponsePendingData + ) = jobDescriptor.maybeCallCachingEligible match { // If the job is eligible, initialize job hashing and go to CheckingCallCache state case eligible: CallCachingEligible => @@ -538,21 +594,24 @@ class EngineJobExecutionActor(replyTo: ActorRef, disableCallCaching() runJob(updatedData) } - } - private def handleReadFromCacheOff(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity, updatedData: ResponsePendingData) = { + private def handleReadFromCacheOff(jobDescriptor: BackendJobDescriptor, + activity: CallCachingActivity, + updatedData: ResponsePendingData + ) = { jobDescriptor.maybeCallCachingEligible match { // If the job is eligible, initialize job hashing so it can be written to the cache - case eligible: CallCachingEligible => initializeJobHashing(jobDescriptor, activity, eligible) match { - case Failure(failure) => - log.warning(s"BT-322 {} failed to initialize job hashing", jobTag) - // This condition in `handleReadFromCacheOn` ends in a `respondAndStop(JobFailedNonRetryableResponse(...))`, - // but with cache reading off Cromwell instead logs this condition and runs the job. - log.error(failure, "Failed to initialize job hashing. The job will not be written to the cache") - case _ => - val template = s"BT-322 {} is eligible for call caching with read = {} and write = {}" - log.info(template, jobTag, activity.readFromCache, activity.writeToCache) - } + case eligible: CallCachingEligible => + initializeJobHashing(jobDescriptor, activity, eligible) match { + case Failure(failure) => + log.warning(s"BT-322 {} failed to initialize job hashing", jobTag) + // This condition in `handleReadFromCacheOn` ends in a `respondAndStop(JobFailedNonRetryableResponse(...))`, + // but with cache reading off Cromwell instead logs this condition and runs the job. + log.error(failure, "Failed to initialize job hashing. The job will not be written to the cache") + case _ => + val template = s"BT-322 {} is eligible for call caching with read = {} and write = {}" + log.info(template, jobTag, activity.readFromCache, activity.writeToCache) + } // Don't even initialize hashing to write to the cache if the job is ineligible case _ => log.info(s"BT-322 {} is not eligible for call caching", jobTag) @@ -562,16 +621,20 @@ class EngineJobExecutionActor(replyTo: ActorRef, runJob(updatedData) } - private def requestRestartCheckToken(): Unit = { - jobRestartCheckTokenDispenserActor ! JobTokenRequest(workflowDescriptor.backendDescriptor.hogGroup, backendLifecycleActorFactory.jobRestartCheckTokenType) - } + private def requestRestartCheckToken(): Unit = + jobRestartCheckTokenDispenserActor ! JobTokenRequest(workflowDescriptor.backendDescriptor.hogGroup, + backendLifecycleActorFactory.jobRestartCheckTokenType + ) - private def requestExecutionToken(): Unit = { - jobExecutionTokenDispenserActor ! JobTokenRequest(workflowDescriptor.backendDescriptor.hogGroup, backendLifecycleActorFactory.jobExecutionTokenType) - } + private def requestExecutionToken(): Unit = + jobExecutionTokenDispenserActor ! JobTokenRequest(workflowDescriptor.backendDescriptor.hogGroup, + backendLifecycleActorFactory.jobExecutionTokenType + ) // Return any currently held job restart check or execution token. - private def returnCurrentToken(): Unit = if (stateName != Pending && stateName != RequestingRestartCheckToken && stateName != RequestingExecutionToken) { + private def returnCurrentToken(): Unit = if ( + stateName != Pending && stateName != RequestingRestartCheckToken && stateName != RequestingExecutionToken + ) { currentTokenDispenser foreach { _ ! JobTokenReturn } currentTokenDispenser = None } @@ -597,9 +660,8 @@ class EngineJobExecutionActor(replyTo: ActorRef, } // Note: StatsD will automatically add a counter value so ne need to separately increment a counter. - private def instrumentJobComplete(response: BackendJobExecutionResponse) = { + private def instrumentJobComplete(response: BackendJobExecutionResponse) = setJobTimePerState(response, (System.currentTimeMillis() - jobStartTime).millis) - } private def disableCallCaching(reason: Option[Throwable] = None) = { log.warning(s"BT-322 {} disabling call caching due to error", jobTag) @@ -626,8 +688,16 @@ class EngineJobExecutionActor(replyTo: ActorRef, def prepareJob(valueStore: ValueStore) = { writeCallCachingModeToMetadata() val jobPreparationActorName = s"BackendPreparationActor_for_$jobTag" - val jobPrepProps = JobPreparationActor.props(workflowDescriptor, jobDescriptorKey, backendLifecycleActorFactory, workflowDockerLookupActor = workflowDockerLookupActor, - initializationData, serviceRegistryActor = serviceRegistryActor, ioActor = ioActor, backendSingletonActor = backendSingletonActor) + val jobPrepProps = JobPreparationActor.props( + workflowDescriptor, + jobDescriptorKey, + backendLifecycleActorFactory, + workflowDockerLookupActor = workflowDockerLookupActor, + initializationData, + serviceRegistryActor = serviceRegistryActor, + ioActor = ioActor, + backendSingletonActor = backendSingletonActor + ) val jobPreparationActor = createJobPreparationActor(jobPrepProps, jobPreparationActorName) jobPreparationActor ! CallPreparation.Start(valueStore) goto(PreparingJob) @@ -638,9 +708,17 @@ class EngineJobExecutionActor(replyTo: ActorRef, goto(WaitingForValueStore) } - def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity, callCachingEligible: CallCachingEligible): Try[ActorRef] = { + def initializeJobHashing(jobDescriptor: BackendJobDescriptor, + activity: CallCachingActivity, + callCachingEligible: CallCachingEligible + ): Try[ActorRef] = { val maybeFileHashingActorProps = backendLifecycleActorFactory.fileHashingActorProps map { - _.apply(jobDescriptor, initializationData, serviceRegistryActor, ioActor, callCachingParameters.fileHashCacheActor) + _.apply(jobDescriptor, + initializationData, + serviceRegistryActor, + ioActor, + callCachingParameters.fileHashCacheActor + ) } maybeFileHashingActorProps match { @@ -667,41 +745,55 @@ class EngineJobExecutionActor(replyTo: ActorRef, } def makeFetchCachedResultsActor(callCachingEntryId: CallCachingEntryId): Unit = { - context.actorOf(FetchCachedResultsActor.props(callCachingEntryId, self, - new CallCache(EngineServicesStore.engineDatabaseInterface))) + context.actorOf( + FetchCachedResultsActor.props(callCachingEntryId, + self, + new CallCache(EngineServicesStore.engineDatabaseInterface) + ) + ) () } - private def fetchCachedResults( callCachingEntryId: CallCachingEntryId, data: ResponsePendingData) = { + private def fetchCachedResults(callCachingEntryId: CallCachingEntryId, data: ResponsePendingData) = { makeFetchCachedResultsActor(callCachingEntryId) goto(FetchingCachedOutputsFromDatabase) using data } private def makeBackendCopyCacheHit(womValueSimpletons: Seq[WomValueSimpleton], - jobDetritusFiles: Map[String,String], + jobDetritusFiles: Map[String, String], returnCode: Option[Int], data: ResponsePendingData, cacheResultId: CallCachingEntryId, - cacheCopyAttempt: Int) = { + cacheCopyAttempt: Int + ) = backendLifecycleActorFactory.cacheHitCopyingActorProps match { case Some(propsMaker) => - val backendCacheHitCopyingActorProps = propsMaker(data.jobDescriptor, initializationData, serviceRegistryActor, ioActor, cacheCopyAttempt, callCachingParameters.blacklistCache) - val cacheHitCopyActor = context.actorOf(backendCacheHitCopyingActorProps, buildCacheHitCopyingActorName(data.jobDescriptor, cacheResultId)) + val backendCacheHitCopyingActorProps = propsMaker(data.jobDescriptor, + initializationData, + serviceRegistryActor, + ioActor, + cacheCopyAttempt, + callCachingParameters.blacklistCache + ) + val cacheHitCopyActor = context.actorOf(backendCacheHitCopyingActorProps, + buildCacheHitCopyingActorName(data.jobDescriptor, cacheResultId) + ) cacheHitCopyActor ! CopyOutputsCommand(womValueSimpletons, jobDetritusFiles, cacheResultId, returnCode) replyTo ! JobRunning(data.jobDescriptor.key, data.jobDescriptor.evaluatedTaskInputs) goto(BackendIsCopyingCachedOutputs) case None => // This should be impossible with the FSM, but luckily, we CAN recover if some foolish future programmer makes this happen: - val errorMessage = "Call caching copying should never have even been attempted with no copy actor props! (Programmer error!)" + val errorMessage = + "Call caching copying should never have even been attempted with no copy actor props! (Programmer error!)" log.error(errorMessage) self ! JobFailedNonRetryableResponse(data.jobDescriptor.key, new RuntimeException(errorMessage), None) goto(BackendIsCopyingCachedOutputs) } - } - private [job] def createBackendJobExecutionActor(data: ResponsePendingData) = { - context.actorOf(data.bjeaProps, BackendJobExecutionActor.buildJobExecutionActorName(workflowIdForLogging, data.jobDescriptor.key)) - } + private[job] def createBackendJobExecutionActor(data: ResponsePendingData) = + context.actorOf(data.bjeaProps, + BackendJobExecutionActor.buildJobExecutionActorName(workflowIdForLogging, data.jobDescriptor.key) + ) private def runJob(data: ResponsePendingData) = { val backendJobExecutionActor = createBackendJobExecutionActor(data) @@ -724,7 +816,8 @@ class EngineJobExecutionActor(replyTo: ActorRef, } response match { - case CallCacheInvalidatedFailure(_, failure) => log.error(failure, "Failed to invalidate cache entry for job: {}", jobDescriptorKey) + case CallCacheInvalidatedFailure(_, failure) => + log.error(failure, "Failed to invalidate cache entry for job: {}", jobDescriptorKey) case CallCacheInvalidatedSuccess(_, Some(entry)) => updateMetadataForInvalidatedEntry(entry) case _ => } @@ -735,25 +828,36 @@ class EngineJobExecutionActor(replyTo: ActorRef, ejha ! NextHit goto(CheckingCallCache) using data case Some(_) => - writeToMetadata(Map( - callCachingHitResultMetadataKey -> false, - callCachingReadResultMetadataKey -> s"Cache Miss (${callCachingParameters.maxFailedCopyAttempts} failed copy attempts)")) - log.warning("BT-322 {} cache hit copying maxfail: Cache miss due to exceeding the maximum of {} failed copy attempts.", jobTag, callCachingParameters.maxFailedCopyAttempts) + writeToMetadata( + Map( + callCachingHitResultMetadataKey -> false, + callCachingReadResultMetadataKey -> s"Cache Miss (${callCachingParameters.maxFailedCopyAttempts} failed copy attempts)" + ) + ) + log.warning( + "BT-322 {} cache hit copying maxfail: Cache miss due to exceeding the maximum of {} failed copy attempts.", + jobTag, + callCachingParameters.maxFailedCopyAttempts + ) publishCopyAttemptAbandonedMetrics(data) runJob(data) case _ => - workflowLogger.error("Programmer error: We got a cache failure but there was no hashing actor scanning for hits. Falling back to running job") + workflowLogger.error( + "Programmer error: We got a cache failure but there was no hashing actor scanning for hits. Falling back to running job" + ) runJob(data) } } - private def buildCacheHitCopyingActorName(jobDescriptor: BackendJobDescriptor, cacheResultId: CallCachingEntryId) = { + private def buildCacheHitCopyingActorName(jobDescriptor: BackendJobDescriptor, cacheResultId: CallCachingEntryId) = s"$workflowIdForLogging-BackendCacheHitCopyingActor-$jobTag-${cacheResultId.id}" - } private def logCacheHitSuccessAndNotifyMetadata(data: ResponsePendingData): Unit = { - val metadataMap = Map[String, Any](callCachingHitResultMetadataKey -> true) ++ data.ejeaCacheHit.flatMap(_.details).map(details => callCachingReadResultMetadataKey -> s"Cache Hit: $details").toMap + val metadataMap = Map[String, Any](callCachingHitResultMetadataKey -> true) ++ data.ejeaCacheHit + .flatMap(_.details) + .map(details => callCachingReadResultMetadataKey -> s"Cache Hit: $details") + .toMap writeToMetadata(metadataMap) @@ -786,19 +890,15 @@ class EngineJobExecutionActor(replyTo: ActorRef, workflowLogger.info( s"Failure copying cache results for job $jobDescriptorKey (${reason.getClass.getSimpleName}: ${reason.getMessage})" - + multipleFailuresContext + + multipleFailuresContext ) } private def publishCopyAttemptFailuresMetrics(data: ResponsePendingData): Unit = { val copyErrorsPerHitPath: NonEmptyList[String] = - NonEmptyList.of( - "job", - "callcaching", "read", "error", "invalidhits", "copyerrors") + NonEmptyList.of("job", "callcaching", "read", "error", "invalidhits", "copyerrors") val copyBlacklistsPerHitPath: NonEmptyList[String] = - NonEmptyList.of( - "job", - "callcaching", "read", "error", "invalidhits", "blacklisted") + NonEmptyList.of("job", "callcaching", "read", "error", "invalidhits", "blacklisted") sendGauge(copyErrorsPerHitPath, data.failedCopyAttempts.longValue) sendGauge(copyBlacklistsPerHitPath, data.cacheHitFailureCount - data.failedCopyAttempts.longValue) @@ -806,24 +906,33 @@ class EngineJobExecutionActor(replyTo: ActorRef, private def publishCopyAttemptAbandonedMetrics(data: ResponsePendingData): Unit = { val cacheCopyAttemptAbandonedPath: NonEmptyList[String] = - NonEmptyList.of( - "job", - "callcaching", "read", "error", "invalidhits", "abandonments") + NonEmptyList.of("job", "callcaching", "read", "error", "invalidhits", "abandonments") increment(cacheCopyAttemptAbandonedPath) // Also publish the attempt failure metrics publishCopyAttemptFailuresMetrics(data) } - private def publishBlacklistReadMetrics(data: ResponsePendingData, failureCategory: MetricableCacheCopyErrorCategory): Unit = { + private def publishBlacklistReadMetrics(data: ResponsePendingData, + failureCategory: MetricableCacheCopyErrorCategory + ): Unit = { val callCachingErrorsMetricPath: NonEmptyList[String] = NonEmptyList.of( "job", - "callcaching", "read", "error", failureCategory.toString, data.jobDescriptor.taskCall.localName, data.jobDescriptor.workflowDescriptor.hogGroup.value) + "callcaching", + "read", + "error", + failureCategory.toString, + data.jobDescriptor.taskCall.localName, + data.jobDescriptor.workflowDescriptor.hogGroup.value + ) increment(callCachingErrorsMetricPath) } - private def invalidateCacheHitAndTransition(ejeaCacheHit: EJEACacheHit, data: ResponsePendingData, reason: CacheCopyFailure) = { + private def invalidateCacheHitAndTransition(ejeaCacheHit: EJEACacheHit, + data: ResponsePendingData, + reason: CacheCopyFailure + ) = { val copyAttemptIncrement = reason match { case CopyAttemptError(failure) => logCacheHitFailure(data, failure) @@ -837,10 +946,13 @@ class EngineJobExecutionActor(replyTo: ActorRef, // Increment the total failure count and actual copy failure count as appropriate. val updatedData = data.copy(cacheHitFailureCount = data.cacheHitFailureCount + 1, - failedCopyAttempts = data.failedCopyAttempts + copyAttemptIncrement) + failedCopyAttempts = data.failedCopyAttempts + copyAttemptIncrement + ) if (invalidationRequired) { - workflowLogger.warn(s"Invalidating cache entry ${ejeaCacheHit.hit.cacheResultId} (Cache entry details: ${ejeaCacheHit.details})") + workflowLogger.warn( + s"Invalidating cache entry ${ejeaCacheHit.hit.cacheResultId} (Cache entry details: ${ejeaCacheHit.details})" + ) invalidateCacheHit(ejeaCacheHit.hit.cacheResultId) goto(InvalidatingCacheEntry) using updatedData } else { @@ -860,7 +972,9 @@ class EngineJobExecutionActor(replyTo: ActorRef, } private def saveCacheResults(hashes: CallCacheHashes, data: SucceededResponseData) = { - callCachingParameters.writeActor ! SaveCallCacheHashes(CallCacheHashBundle(workflowIdForLogging, hashes, data.response)) + callCachingParameters.writeActor ! SaveCallCacheHashes( + CallCacheHashBundle(workflowIdForLogging, hashes, data.response) + ) val updatedData = data.copy(hashes = Option(Success(hashes))) goto(UpdatingCallCache) using updatedData } @@ -890,7 +1004,11 @@ class EngineJobExecutionActor(replyTo: ActorRef, jobStoreActor ! RegisterJobCompleted(jobStoreKey, jobStoreResult) } - private def saveUnsuccessfulJobResults(jobKey: JobKey, returnCode: Option[Int], reason: Throwable, retryable: Boolean) = { + private def saveUnsuccessfulJobResults(jobKey: JobKey, + returnCode: Option[Int], + reason: Throwable, + retryable: Boolean + ) = { val jobStoreKey = jobKey.toJobStoreKey(workflowIdForLogging) val jobStoreResult = JobResultFailure(returnCode, reason, retryable) jobStoreActor ! RegisterJobCompleted(jobStoreKey, jobStoreResult) @@ -912,6 +1030,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, } object EngineJobExecutionActor { + /** States */ sealed trait EngineJobExecutionActorState case object Pending extends EngineJobExecutionActorState @@ -930,7 +1049,9 @@ object EngineJobExecutionActor { case object InvalidatingCacheEntry extends EngineJobExecutionActorState object EngineJobExecutionActorState { - def transitionEventString(fromState: EngineJobExecutionActorState, toState: EngineJobExecutionActorState): Option[String] = { + def transitionEventString(fromState: EngineJobExecutionActorState, + toState: EngineJobExecutionActorState + ): Option[String] = { def callCacheStateGroup: Set[EngineJobExecutionActorState] = Set( CheckingCallCache, @@ -948,14 +1069,14 @@ object EngineJobExecutionActor { } case class CallCachingParameters( - mode: CallCachingMode, - readActor: ActorRef, - writeActor: ActorRef, - fileHashCacheActor: Option[ActorRef], - maxFailedCopyAttempts: Int, - blacklistCache: Option[BlacklistCache], - fileHashBatchSize: Int - ) + mode: CallCachingMode, + readActor: ActorRef, + writeActor: ActorRef, + fileHashCacheActor: Option[ActorRef], + maxFailedCopyAttempts: Int, + blacklistCache: Option[BlacklistCache], + fileHashBatchSize: Int + ) /** Commands */ sealed trait EngineJobExecutionActorCommand @@ -977,29 +1098,31 @@ object EngineJobExecutionActor { jobExecutionTokenDispenserActor: ActorRef, backendSingletonActor: Option[ActorRef], command: BackendJobExecutionActorCommand, - callCachingParameters: EngineJobExecutionActor.CallCachingParameters) = { - - Props(new EngineJobExecutionActor( - replyTo = replyTo, - jobDescriptorKey = jobDescriptorKey, - workflowDescriptor = workflowDescriptor, - backendLifecycleActorFactory = backendLifecycleActorFactory, - initializationData = initializationData, - restarting = restarting, - serviceRegistryActor = serviceRegistryActor, - ioActor = ioActor, - jobStoreActor = jobStoreActor, - workflowDockerLookupActor = workflowDockerLookupActor, - jobRestartCheckTokenDispenserActor = jobRestartCheckTokenDispenserActor, - jobExecutionTokenDispenserActor = jobExecutionTokenDispenserActor, - backendSingletonActor = backendSingletonActor, - command = command, - callCachingParameters = callCachingParameters)).withDispatcher(EngineDispatcher) - } + callCachingParameters: EngineJobExecutionActor.CallCachingParameters + ) = + Props( + new EngineJobExecutionActor( + replyTo = replyTo, + jobDescriptorKey = jobDescriptorKey, + workflowDescriptor = workflowDescriptor, + backendLifecycleActorFactory = backendLifecycleActorFactory, + initializationData = initializationData, + restarting = restarting, + serviceRegistryActor = serviceRegistryActor, + ioActor = ioActor, + jobStoreActor = jobStoreActor, + workflowDockerLookupActor = workflowDockerLookupActor, + jobRestartCheckTokenDispenserActor = jobRestartCheckTokenDispenserActor, + jobExecutionTokenDispenserActor = jobExecutionTokenDispenserActor, + backendSingletonActor = backendSingletonActor, + command = command, + callCachingParameters = callCachingParameters + ) + ).withDispatcher(EngineDispatcher) case class EJEACacheHit(hit: CacheHit, hitNumber: Int, details: Option[String]) - private[execution] sealed trait EJEAData { + sealed private[execution] trait EJEAData { override def toString = getClass.getSimpleName } @@ -1013,13 +1136,14 @@ object EngineJobExecutionActor { backendJobActor: Option[ActorRef] = None, cacheHitFailureCount: Int = 0, failedCopyAttempts: Int = 0 - ) extends EJEAData { + ) extends EJEAData { def withEJHA(ejha: ActorRef): EJEAData = this.copy(ejha = Option(ejha)) def withBackendActor(actorRef: ActorRef) = this.copy(backendJobActor = Option(actorRef)) - def withSuccessResponse(success: JobSucceededResponse): SucceededResponseData = SucceededResponseData(success, hashes) + def withSuccessResponse(success: JobSucceededResponse): SucceededResponseData = + SucceededResponseData(success, hashes) def withFailedResponse(failed: BackendJobFailedResponse): FailedResponseData = FailedResponseData(failed, hashes) def withAbortedResponse(aborted: JobAbortedResponse): AbortedResponseData = AbortedResponseData(aborted, hashes) @@ -1031,7 +1155,8 @@ object EngineJobExecutionActor { this.copy(ejeaCacheHit = Option(newEjeaCacheHit)) } - def withCacheDetails(details: String) = this.copy(ejeaCacheHit = ejeaCacheHit.map(_.copy(details = Option(details)))) + def withCacheDetails(details: String) = + this.copy(ejeaCacheHit = ejeaCacheHit.map(_.copy(details = Option(details)))) def aggregatedHashString: String = hashes match { case Some(Success(hashes)) => hashes.aggregatedHashString @@ -1051,14 +1176,16 @@ object EngineJobExecutionActor { private[execution] trait ShouldBeSavedToJobStoreResponseData extends ResponseData private[execution] case class SucceededResponseData(successResponse: JobSucceededResponse, - hashes: Option[Try[CallCacheHashes]] = None) extends ShouldBeSavedToJobStoreResponseData { + hashes: Option[Try[CallCacheHashes]] = None + ) extends ShouldBeSavedToJobStoreResponseData { override def response = successResponse override def dockerImageUsed = successResponse.dockerImageUsed override def withHashes(hashes: Option[Try[CallCacheHashes]]) = copy(hashes = hashes) } private[execution] case class FailedResponseData(failedResponse: BackendJobFailedResponse, - hashes: Option[Try[CallCacheHashes]] = None) extends ShouldBeSavedToJobStoreResponseData { + hashes: Option[Try[CallCacheHashes]] = None + ) extends ShouldBeSavedToJobStoreResponseData { override def response = failedResponse // Seems like we should be able to get the docker image used even if the job failed override def dockerImageUsed = None @@ -1066,7 +1193,8 @@ object EngineJobExecutionActor { } private[execution] case class AbortedResponseData(abortedResponse: JobAbortedResponse, - hashes: Option[Try[CallCacheHashes]] = None) extends ResponseData { + hashes: Option[Try[CallCacheHashes]] = None + ) extends ResponseData { override def response = abortedResponse override def dockerImageUsed = None override def withHashes(hashes: Option[Try[CallCacheHashes]]) = copy(hashes = hashes) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/CallPreparation.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/CallPreparation.scala index e9665aa01b6..efdee8df7a0 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/CallPreparation.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/CallPreparation.scala @@ -15,15 +15,15 @@ object CallPreparation { trait CallPreparationActorResponse - case class BackendJobPreparationSucceeded(jobDescriptor: BackendJobDescriptor, bjeaProps: Props) extends CallPreparationActorResponse + case class BackendJobPreparationSucceeded(jobDescriptor: BackendJobDescriptor, bjeaProps: Props) + extends CallPreparationActorResponse case class JobCallPreparationFailed(jobKey: JobKey, throwable: Throwable) extends CallPreparationActorResponse case class CallPreparationFailed(jobKey: JobKey, throwable: Throwable) extends CallPreparationActorResponse def resolveAndEvaluateInputs(callKey: CallKey, expressionLanguageFunctions: IoFunctionSet, - valueStore: ValueStore): ErrorOr[WomEvaluatedCallInputs] = { - + valueStore: ValueStore + ): ErrorOr[WomEvaluatedCallInputs] = CallNode.resolveAndEvaluateInputs(callKey.node, expressionLanguageFunctions, valueStore.resolve(callKey.index)) - } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/JobPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/JobPreparationActor.scala index 8540717a073..c39590ebf03 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/JobPreparationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/JobPreparationActor.scala @@ -13,7 +13,7 @@ import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.callcaching._ import cromwell.core.logging.WorkflowLogging import cromwell.core.{Dispatcher, DockerConfiguration} -import cromwell.docker.DockerInfoActor.{DockerInfoSuccessResponse, DockerInformation, DockerSize} +import cromwell.docker.DockerInfoActor.{DockerInformation, DockerInfoSuccessResponse, DockerSize} import cromwell.docker._ import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.workflow.WorkflowDockerLookupActor.{WorkflowDockerLookupFailure, WorkflowDockerTerminalFailure} @@ -51,8 +51,10 @@ class JobPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, initializationData: Option[BackendInitializationData], val serviceRegistryActor: ActorRef, ioActor: ActorRef, - backendSingletonActor: Option[ActorRef]) - extends FSM[JobPreparationActorState, JobPreparationActorData] with WorkflowLogging with CallMetadataHelper { + backendSingletonActor: Option[ActorRef] +) extends FSM[JobPreparationActorState, JobPreparationActorData] + with WorkflowLogging + with CallMetadataHelper { override lazy val workflowIdForLogging = workflowDescriptor.possiblyNotRootWorkflowId override lazy val workflowIdForCallMetadata = workflowDescriptor.possiblyNotRootWorkflowId @@ -62,29 +64,39 @@ class JobPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, private[preparation] val ioEc = context.system.dispatchers.lookup(Dispatcher.IoDispatcher) private[preparation] lazy val expressionLanguageFunctions = { - val ioFunctionSet: IoFunctionSet = factory.expressionLanguageFunctions(workflowDescriptor.backendDescriptor, jobKey, initializationData, ioActor, ioEc) + val ioFunctionSet: IoFunctionSet = factory.expressionLanguageFunctions(workflowDescriptor.backendDescriptor, + jobKey, + initializationData, + ioActor, + ioEc + ) ioFunctionSet.makeInputSpecificFunctions() } - private[preparation] lazy val dockerHashCredentials = factory.dockerHashCredentials(workflowDescriptor.backendDescriptor, initializationData) + private[preparation] lazy val dockerHashCredentials = + factory.dockerHashCredentials(workflowDescriptor.backendDescriptor, initializationData) private[preparation] lazy val runtimeAttributeDefinitions = factory.runtimeAttributeDefinitions(initializationData) - private[preparation] lazy val hasDockerDefinition = runtimeAttributeDefinitions.exists(_.name == DockerValidation.instance.key) + private[preparation] lazy val hasDockerDefinition = + runtimeAttributeDefinitions.exists(_.name == DockerValidation.instance.key) startWith(Idle, JobPreparationActorNoData) - when(Idle) { - case Event(Start(valueStore), JobPreparationActorNoData) => - evaluateInputsAndAttributes(valueStore) match { - case Valid((inputs, attributes)) => fetchDockerHashesIfNecessary(inputs, attributes) - case Invalid(failure) => sendFailureAndStop(new MessageAggregation with NoStackTrace { - override def exceptionContext: String = s"Call input and runtime attributes evaluation failed for ${jobKey.call.localName}" + when(Idle) { case Event(Start(valueStore), JobPreparationActorNoData) => + evaluateInputsAndAttributes(valueStore) match { + case Valid((inputs, attributes)) => fetchDockerHashesIfNecessary(inputs, attributes) + case Invalid(failure) => + sendFailureAndStop(new MessageAggregation with NoStackTrace { + override def exceptionContext: String = + s"Call input and runtime attributes evaluation failed for ${jobKey.call.localName}" override def errorMessages: Iterable[String] = failure.toList }) - } + } } when(WaitingForDockerHash) { - case Event(DockerInfoSuccessResponse(DockerInformation(dockerHash, dockerSize), _), data: JobPreparationDockerLookupData) => + case Event(DockerInfoSuccessResponse(DockerInformation(dockerHash, dockerSize), _), + data: JobPreparationDockerLookupData + ) => handleDockerHashSuccess(dockerHash, dockerSize, data) case Event(WorkflowDockerLookupFailure(reason, _, _), data: JobPreparationDockerLookupData) => workflowLogger.warn("Docker lookup failed", reason) @@ -94,32 +106,45 @@ class JobPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, } when(FetchingKeyValueStoreEntries) { - case Event(kvResponse: KvResponse, data @ JobPreparationKeyLookupData(keyLookups, maybeCallCachingEligible, dockerSize, inputs, attributes)) => + case Event(kvResponse: KvResponse, + data @ JobPreparationKeyLookupData(keyLookups, maybeCallCachingEligible, dockerSize, inputs, attributes) + ) => keyLookups.withResponse(kvResponse.key, kvResponse) match { case newPartialLookup: PartialKeyValueLookups => stay() using data.copy(keyLookups = newPartialLookup) case finished: KeyValueLookupResults => - sendResponseAndStop(prepareBackendDescriptor(inputs, attributes, maybeCallCachingEligible, finished.unscoped, dockerSize)) + sendResponseAndStop( + prepareBackendDescriptor(inputs, attributes, maybeCallCachingEligible, finished.unscoped, dockerSize) + ) } } - whenUnhandled { - case Event(unexpectedMessage, _) => - workflowLogger.warn(s"JobPreparation actor received an unexpected message in state $stateName: $unexpectedMessage") - stay() + whenUnhandled { case Event(unexpectedMessage, _) => + workflowLogger.warn(s"JobPreparation actor received an unexpected message in state $stateName: $unexpectedMessage") + stay() } - private[preparation] lazy val kvStoreKeysToPrefetch: Seq[String] = factory.requestedKeyValueStoreKeys ++ factory.defaultKeyValueStoreKeys + private[preparation] lazy val kvStoreKeysToPrefetch: Seq[String] = + factory.requestedKeyValueStoreKeys ++ factory.defaultKeyValueStoreKeys private[preparation] def scopedKey(key: String) = ScopedKey(workflowDescriptor.id, KvJobKey(jobKey), key) private[preparation] def lookupKeyValueEntries(inputs: WomEvaluatedCallInputs, attributes: Map[LocallyQualifiedName, WomValue], maybeCallCachingEligible: MaybeCallCachingEligible, - dockerSize: Option[DockerSize]) = { + dockerSize: Option[DockerSize] + ) = { val keysToLookup = kvStoreKeysToPrefetch map scopedKey keysToLookup foreach { serviceRegistryActor ! KvGet(_) } - goto(FetchingKeyValueStoreEntries) using JobPreparationKeyLookupData(PartialKeyValueLookups(Map.empty, keysToLookup), maybeCallCachingEligible, dockerSize, inputs, attributes) + goto(FetchingKeyValueStoreEntries) using JobPreparationKeyLookupData( + PartialKeyValueLookups(Map.empty, keysToLookup), + maybeCallCachingEligible, + dockerSize, + inputs, + attributes + ) } - private [preparation] def evaluateInputsAndAttributes(valueStore: ValueStore): ErrorOr[(WomEvaluatedCallInputs, Map[LocallyQualifiedName, WomValue])] = { + private[preparation] def evaluateInputsAndAttributes( + valueStore: ValueStore + ): ErrorOr[(WomEvaluatedCallInputs, Map[LocallyQualifiedName, WomValue])] = { import common.validation.ErrorOr.{ShortCircuitingFlatMap, NestedErrorOr} for { evaluatedInputs <- ErrorOr(resolveAndEvaluateInputs(jobKey, expressionLanguageFunctions, valueStore)).flatten @@ -127,7 +152,9 @@ class JobPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, } yield (evaluatedInputs, runtimeAttributes) } - private def fetchDockerHashesIfNecessary(inputs: WomEvaluatedCallInputs, attributes: Map[LocallyQualifiedName, WomValue]) = { + private def fetchDockerHashesIfNecessary(inputs: WomEvaluatedCallInputs, + attributes: Map[LocallyQualifiedName, WomValue] + ) = { def sendDockerRequest(dockerImageId: DockerImageIdentifier) = { val dockerHashRequest = DockerInfoRequest(dockerImageId, dockerHashCredentials) val newData = JobPreparationDockerLookupData(dockerHashRequest, inputs, attributes) @@ -136,12 +163,17 @@ class JobPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, } def handleDockerValue(value: String) = DockerImageIdentifier.fromString(value) match { - // If the backend supports docker, lookup is enabled, and we got a tag - we need to lookup the hash - case Success(dockerImageId: DockerImageIdentifierWithoutHash) if hasDockerDefinition && DockerConfiguration.instance.enabled => + // If the backend supports docker, lookup is enabled, and we got a tag - we need to lookup the hash + case Success(dockerImageId: DockerImageIdentifierWithoutHash) + if hasDockerDefinition && DockerConfiguration.instance.enabled => sendDockerRequest(dockerImageId) - // If the backend supports docker, we got a tag but lookup is disabled, continue with no call caching and no hash + // If the backend supports docker, we got a tag but lookup is disabled, continue with no call caching and no hash case Success(dockerImageId: DockerImageIdentifierWithoutHash) if hasDockerDefinition => - lookupKvsOrBuildDescriptorAndStop(inputs, attributes, FloatingDockerTagWithoutHash(dockerImageId.fullName), None) + lookupKvsOrBuildDescriptorAndStop(inputs, + attributes, + FloatingDockerTagWithoutHash(dockerImageId.fullName), + None + ) // If the backend doesn't support docker - no need to lookup and we're ok for call caching case Success(_: DockerImageIdentifierWithoutHash) if !hasDockerDefinition => @@ -165,8 +197,9 @@ class JobPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, } private def updateRuntimeMemory(runtimeAttributes: Map[LocallyQualifiedName, WomValue], - memoryMultiplierOption: Option[Double]): Map[LocallyQualifiedName, WomValue] = { - def multiplyRuntimeMemory(multiplier: Double): Map[LocallyQualifiedName, WomValue] = { + memoryMultiplierOption: Option[Double] + ): Map[LocallyQualifiedName, WomValue] = { + def multiplyRuntimeMemory(multiplier: Double): Map[LocallyQualifiedName, WomValue] = runtimeAttributes.get(RuntimeAttributesKeys.MemoryKey) match { case Some(WomString(memory)) => MemorySize.parse(memory) match { @@ -177,7 +210,6 @@ class JobPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, } case _ => runtimeAttributes } - } memoryMultiplierOption match { case None => runtimeAttributes @@ -192,12 +224,16 @@ class JobPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, private def lookupKvsOrBuildDescriptorAndStop(inputs: WomEvaluatedCallInputs, attributes: Map[LocallyQualifiedName, WomValue], maybeCallCachingEligible: MaybeCallCachingEligible, - dockerSize: Option[DockerSize]) = { + dockerSize: Option[DockerSize] + ) = if (kvStoreKeysToPrefetch.nonEmpty) lookupKeyValueEntries(inputs, attributes, maybeCallCachingEligible, dockerSize) - else sendResponseAndStop(prepareBackendDescriptor(inputs, attributes, maybeCallCachingEligible, Map.empty, dockerSize)) - } + else + sendResponseAndStop(prepareBackendDescriptor(inputs, attributes, maybeCallCachingEligible, Map.empty, dockerSize)) - private def handleDockerHashSuccess(dockerHashResult: DockerHashResult, dockerSize: Option[DockerSize], data: JobPreparationDockerLookupData) = { + private def handleDockerHashSuccess(dockerHashResult: DockerHashResult, + dockerSize: Option[DockerSize], + data: JobPreparationDockerLookupData + ) = { val hashValue = data.dockerHashRequest.dockerImageID match { case withoutHash: DockerImageIdentifierWithoutHash => withoutHash.withHash(dockerHashResult) case withHash => withHash @@ -207,13 +243,19 @@ class JobPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, } private def sendCompressedDockerSizeToMetadata(dockerSize: DockerSize) = { - val event = MetadataEvent(metadataKeyForCall(jobKey, CallMetadataKeys.CompressedDockerSize), MetadataValue(dockerSize.compressedSize)) + val event = MetadataEvent(metadataKeyForCall(jobKey, CallMetadataKeys.CompressedDockerSize), + MetadataValue(dockerSize.compressedSize) + ) serviceRegistryActor ! PutMetadataAction(event) } private def handleDockerHashFailed(data: JobPreparationDockerLookupData) = { val floatingDockerTag = data.dockerHashRequest.dockerImageID.fullName - lookupKvsOrBuildDescriptorAndStop(data.inputs, data.attributes, FloatingDockerTagWithoutHash(floatingDockerTag), None) + lookupKvsOrBuildDescriptorAndStop(data.inputs, + data.attributes, + FloatingDockerTagWithoutHash(floatingDockerTag), + None + ) } private def sendResponseAndStop(response: CallPreparationActorResponse) = { @@ -221,44 +263,71 @@ class JobPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, stay() } - private def sendFailureAndStop(failure: Throwable) = { + private def sendFailureAndStop(failure: Throwable) = sendResponseAndStop(CallPreparationFailed(jobKey, failure)) - } // 'jobExecutionProps' is broken into a separate function for TestJobPreparationActor to override: private[preparation] def jobExecutionProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef, - backendSingletonActor: Option[ActorRef]) = factory.jobExecutionActorProps(jobDescriptor, initializationData, serviceRegistryActor, ioActor, backendSingletonActor) + backendSingletonActor: Option[ActorRef] + ) = factory.jobExecutionActorProps(jobDescriptor, + initializationData, + serviceRegistryActor, + ioActor, + backendSingletonActor + ) private[preparation] def prepareBackendDescriptor(inputEvaluation: WomEvaluatedCallInputs, runtimeAttributes: Map[LocallyQualifiedName, WomValue], maybeCallCachingEligible: MaybeCallCachingEligible, prefetchedJobStoreEntries: Map[String, KvResponse], - dockerSize: Option[DockerSize]): BackendJobPreparationSucceeded = { + dockerSize: Option[DockerSize] + ): BackendJobPreparationSucceeded = { val memoryMultiplier: Option[Double] = prefetchedJobStoreEntries.get(MemoryMultiplierKey) match { - case Some(KvPair(_,v)) => Try(v.toDouble) match { - case Success(m) => Option(m) - case Failure(e) => - // should not happen as we are converting a value that Cromwell put in DB after validation - log.error(e, s"Programmer error: unexpected failure attempting to convert value of MemoryMultiplierKey from JOB_KEY_VALUE_ENTRY table to Double.") - None - } + case Some(KvPair(_, v)) => + Try(v.toDouble) match { + case Success(m) => Option(m) + case Failure(e) => + // should not happen as we are converting a value that Cromwell put in DB after validation + log.error( + e, + s"Programmer error: unexpected failure attempting to convert value of MemoryMultiplierKey from JOB_KEY_VALUE_ENTRY table to Double." + ) + None + } case _ => None } val updatedRuntimeAttributes = updateRuntimeMemory(runtimeAttributes, memoryMultiplier) - val jobDescriptor = BackendJobDescriptor(workflowDescriptor.backendDescriptor, jobKey, updatedRuntimeAttributes, inputEvaluation, maybeCallCachingEligible, dockerSize, prefetchedJobStoreEntries) - BackendJobPreparationSucceeded(jobDescriptor, jobExecutionProps(jobDescriptor, initializationData, serviceRegistryActor, ioActor, backendSingletonActor)) + val jobDescriptor = BackendJobDescriptor(workflowDescriptor.backendDescriptor, + jobKey, + updatedRuntimeAttributes, + inputEvaluation, + maybeCallCachingEligible, + dockerSize, + prefetchedJobStoreEntries + ) + BackendJobPreparationSucceeded( + jobDescriptor, + jobExecutionProps(jobDescriptor, initializationData, serviceRegistryActor, ioActor, backendSingletonActor) + ) } - private [preparation] def prepareRuntimeAttributes(inputEvaluation: Map[InputDefinition, WomValue]): ErrorOr[Map[LocallyQualifiedName, WomValue]] = { + private[preparation] def prepareRuntimeAttributes( + inputEvaluation: Map[InputDefinition, WomValue] + ): ErrorOr[Map[LocallyQualifiedName, WomValue]] = { import RuntimeAttributeDefinition.{addDefaultsToAttributes, evaluateRuntimeAttributes} - val curriedAddDefaultsToAttributes = addDefaultsToAttributes(runtimeAttributeDefinitions, workflowDescriptor.backendDescriptor.workflowOptions) _ + val curriedAddDefaultsToAttributes = + addDefaultsToAttributes(runtimeAttributeDefinitions, workflowDescriptor.backendDescriptor.workflowOptions) _ val unevaluatedRuntimeAttributes = jobKey.call.callable.runtimeAttributes - evaluateRuntimeAttributes(unevaluatedRuntimeAttributes, expressionLanguageFunctions, inputEvaluation) map curriedAddDefaultsToAttributes + evaluateRuntimeAttributes(unevaluatedRuntimeAttributes, + expressionLanguageFunctions, + inputEvaluation, + factory.platform + ) map curriedAddDefaultsToAttributes } } @@ -266,14 +335,16 @@ object JobPreparationActor { sealed trait JobPreparationActorData case object JobPreparationActorNoData extends JobPreparationActorData - private final case class JobPreparationKeyLookupData(keyLookups: PartialKeyValueLookups, + final private case class JobPreparationKeyLookupData(keyLookups: PartialKeyValueLookups, maybeCallCachingEligible: MaybeCallCachingEligible, dockerSize: Option[DockerSize], inputs: WomEvaluatedCallInputs, - attributes: Map[LocallyQualifiedName, WomValue]) extends JobPreparationActorData - private final case class JobPreparationDockerLookupData(dockerHashRequest: DockerInfoRequest, + attributes: Map[LocallyQualifiedName, WomValue] + ) extends JobPreparationActorData + final private case class JobPreparationDockerLookupData(dockerHashRequest: DockerInfoRequest, inputs: WomEvaluatedCallInputs, - attributes: Map[LocallyQualifiedName, WomValue]) extends JobPreparationActorData + attributes: Map[LocallyQualifiedName, WomValue] + ) extends JobPreparationActorData sealed trait JobPreparationActorState case object Idle extends JobPreparationActorState @@ -287,16 +358,20 @@ object JobPreparationActor { initializationData: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef, - backendSingletonActor: Option[ActorRef]) = { + backendSingletonActor: Option[ActorRef] + ) = // Note that JobPreparationActor doesn't run on the engine dispatcher as it mostly executes backend-side code // (WDL expression evaluation using Backend's expressionLanguageFunctions) - Props(new JobPreparationActor(workflowDescriptor, - jobKey, - factory, - workflowDockerLookupActor = workflowDockerLookupActor, - initializationData, - serviceRegistryActor = serviceRegistryActor, - ioActor = ioActor, - backendSingletonActor = backendSingletonActor)).withDispatcher(EngineDispatcher) - } + Props( + new JobPreparationActor( + workflowDescriptor, + jobKey, + factory, + workflowDockerLookupActor = workflowDockerLookupActor, + initializationData, + serviceRegistryActor = serviceRegistryActor, + ioActor = ioActor, + backendSingletonActor = backendSingletonActor + ) + ).withDispatcher(EngineDispatcher) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/KeyValueLookups.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/KeyValueLookups.scala index f4d5786223b..73014807d26 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/KeyValueLookups.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/KeyValueLookups.scala @@ -5,9 +5,11 @@ import cromwell.services.keyvalue.KeyValueServiceActor.{KvResponse, ScopedKey} /** * Handles the determination of when we know key lookups are successful. */ -private sealed trait KeyValueLookups +sealed private trait KeyValueLookups -private[preparation] final case class PartialKeyValueLookups(responses: Map[ScopedKey, KvResponse], awaiting: Seq[ScopedKey]) { +final private[preparation] case class PartialKeyValueLookups(responses: Map[ScopedKey, KvResponse], + awaiting: Seq[ScopedKey] +) { def withResponse(key: ScopedKey, response: KvResponse) = { val newResponses = responses + (key -> response) val newAwaiting = awaiting diff List(key) @@ -19,6 +21,6 @@ private[preparation] final case class PartialKeyValueLookups(responses: Map[Scop } } -private final case class KeyValueLookupResults(values: Map[ScopedKey, KvResponse]) { +final private case class KeyValueLookupResults(values: Map[ScopedKey, KvResponse]) { def unscoped: Map[String, KvResponse] = values map { case (k, v) => k.key -> v } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/SubWorkflowPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/SubWorkflowPreparationActor.scala index f14c00965d7..8165b13128b 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/SubWorkflowPreparationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/SubWorkflowPreparationActor.scala @@ -20,7 +20,9 @@ import wom.values.{WomEvaluatedCallInputs, WomValue} class SubWorkflowPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, expressionLanguageFunctions: EngineIoFunctions, callKey: SubWorkflowKey, - subWorkflowId: WorkflowId) extends Actor with WorkflowLogging { + subWorkflowId: WorkflowId +) extends Actor + with WorkflowLogging { override lazy val workflowIdForLogging = workflowDescriptor.possiblyNotRootWorkflowId override lazy val rootWorkflowIdForLogging = workflowDescriptor.rootWorkflowId @@ -33,9 +35,13 @@ class SubWorkflowPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, id = subWorkflowId, callable = callKey.node.callable, knownValues = startingValues, - breadCrumbs = oldBackendDescriptor.breadCrumbs :+ BackendJobBreadCrumb(workflowDescriptor.callable, workflowDescriptor.id, callKey) + breadCrumbs = oldBackendDescriptor.breadCrumbs :+ BackendJobBreadCrumb(workflowDescriptor.callable, + workflowDescriptor.id, + callKey + ) ) - val engineDescriptor = workflowDescriptor.copy(backendDescriptor = newBackendDescriptor, parentWorkflow = Option(workflowDescriptor)) + val engineDescriptor = + workflowDescriptor.copy(backendDescriptor = newBackendDescriptor, parentWorkflow = Option(workflowDescriptor)) SubWorkflowPreparationSucceeded(engineDescriptor, inputEvaluation) } } @@ -43,7 +49,9 @@ class SubWorkflowPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, /** * Work out a set of "workflow inputs" to pass to this subworkflow as though it were a top-level workflow receiving inputs */ - private def evaluateStartingKnownValues(inputEvaluation: WomEvaluatedCallInputs, workflowInputs: Set[GraphInputNode]): ErrorOr[Map[OutputPort, WomValue]] = { + private def evaluateStartingKnownValues(inputEvaluation: WomEvaluatedCallInputs, + workflowInputs: Set[GraphInputNode] + ): ErrorOr[Map[OutputPort, WomValue]] = { // Find the values in the provided inputs that match up with subworkflow inputs. Silently drop the rest on the floor. val providedInputs: Map[OutputPort, WomValue] = inputEvaluation.toList.flatMap { case (name, value) => @@ -59,9 +67,14 @@ class SubWorkflowPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, } // Make sure the subworkflow will be getting a value for every required input: - NonEmptyList.fromList((workflowInputs.toSet[GraphNode] diff providedInputs.keySet.map(_.graphNode) diff optionalsAndDefaults).toList) match { + NonEmptyList.fromList( + (workflowInputs.toSet[GraphNode] diff providedInputs.keySet.map(_.graphNode) diff optionalsAndDefaults).toList + ) match { case None => Valid(providedInputs) - case Some(missingNodeNel) => Invalid(missingNodeNel map (n => s"Couldn't find starting value for subworkflow input: ${n.identifier.localName}")) + case Some(missingNodeNel) => + Invalid( + missingNodeNel map (n => s"Couldn't find starting value for subworkflow input: ${n.identifier.localName}") + ) } } @@ -71,10 +84,14 @@ class SubWorkflowPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, import common.validation.ErrorOr._ evaluatedInputs.flatMap(prepareExecutionActor) match { case Valid(response) => context.parent ! response - case Invalid(f) => context.parent ! CallPreparationFailed(callKey, new MessageAggregation { - override def exceptionContext: String = "Failed to evaluate inputs for sub workflow" - override def errorMessages: Iterable[String] = f.toList - }) + case Invalid(f) => + context.parent ! CallPreparationFailed( + callKey, + new MessageAggregation { + override def exceptionContext: String = "Failed to evaluate inputs for sub workflow" + override def errorMessages: Iterable[String] = f.toList + } + ) } context stop self @@ -83,14 +100,17 @@ class SubWorkflowPreparationActor(workflowDescriptor: EngineWorkflowDescriptor, } object SubWorkflowPreparationActor { - case class SubWorkflowPreparationSucceeded(workflowDescriptor: EngineWorkflowDescriptor, inputs: WomEvaluatedCallInputs) extends CallPreparationActorResponse + case class SubWorkflowPreparationSucceeded(workflowDescriptor: EngineWorkflowDescriptor, + inputs: WomEvaluatedCallInputs + ) extends CallPreparationActorResponse def props(workflowDescriptor: EngineWorkflowDescriptor, expressionLanguageFunctions: EngineIoFunctions, key: SubWorkflowKey, - subWorkflowId: WorkflowId) = { + subWorkflowId: WorkflowId + ) = // Note that JobPreparationActor doesn't run on the engine dispatcher as it mostly executes backend-side code // (WDL expression evaluation using Backend's expressionLanguageFunctions) - Props(new SubWorkflowPreparationActor(workflowDescriptor, expressionLanguageFunctions, key, subWorkflowId)).withDispatcher(EngineDispatcher) - } + Props(new SubWorkflowPreparationActor(workflowDescriptor, expressionLanguageFunctions, key, subWorkflowId)) + .withDispatcher(EngineDispatcher) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ConditionalCollectorKey.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ConditionalCollectorKey.scala index 8645ce0f6a5..244c1dba99b 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ConditionalCollectorKey.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ConditionalCollectorKey.scala @@ -11,19 +11,19 @@ import wom.graph.GraphNodePort.ConditionalOutputPort * Key that becomes runnable when a node inside a conditional node is complete. * This is needed so that the ConditionalOutputPort of the conditional can be given a value. */ -private [execution] case class ConditionalCollectorKey(conditionalOutputPort: ConditionalOutputPort, - index: ExecutionIndex) extends JobKey { +private[execution] case class ConditionalCollectorKey(conditionalOutputPort: ConditionalOutputPort, + index: ExecutionIndex +) extends JobKey { val outputNodeToCollect: PortBasedGraphOutputNode = conditionalOutputPort.outputToExpose override val node: GraphNode = conditionalOutputPort.outputToExpose override val attempt = 1 override val tag = s"Collector-${node.localName}" - def processRunnable(data: WorkflowExecutionActorData): ErrorOr[WorkflowExecutionDiff] = { + def processRunnable(data: WorkflowExecutionActorData): ErrorOr[WorkflowExecutionDiff] = data.valueStore.collectConditional(this) map { outputs => WorkflowExecutionDiff( executionStoreChanges = Map(this -> ExecutionStatus.Done), valueStoreAdditions = outputs ) } - } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ConditionalKey.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ConditionalKey.scala index a263601f488..7cc18c3b964 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ConditionalKey.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ConditionalKey.scala @@ -16,7 +16,7 @@ import wom.values.{WomBoolean, WomValue} * Represents a conditional node in the execution store. * Runnable when the associated expression (represented by an expression node in the graph) is done. */ -private [execution] case class ConditionalKey(node: ConditionalNode, index: ExecutionIndex) extends JobKey { +private[execution] case class ConditionalKey(node: ConditionalNode, index: ExecutionIndex) extends JobKey { override val tag = node.localName override val attempt = 1 @@ -30,10 +30,10 @@ private [execution] case class ConditionalKey(node: ConditionalNode, index: Exec * @return ExecutionStore of scattered children. */ def populate(bypassed: Boolean): Map[JobKey, ExecutionStatus.Value] = { - val conditionalKeys = node.innerGraph.nodes.flatMap({ node => keyify(node) }) + val conditionalKeys = node.innerGraph.nodes.flatMap(node => keyify(node)) val finalStatus = if (bypassed) ExecutionStatus.NotStarted else ExecutionStatus.Bypassed - (conditionalKeys ++ collectors).map({ _ -> finalStatus }).toMap + (conditionalKeys ++ collectors).map(_ -> finalStatus).toMap } /** @@ -48,12 +48,16 @@ private [execution] case class ConditionalKey(node: ConditionalNode, index: Exec case _: GraphInputNode => None case _: PortBasedGraphOutputNode => None case _: ScatterNode => - throw new UnsupportedOperationException("Nested Scatters are not supported (yet) ... but you might try a sub workflow to achieve the same effect!") + throw new UnsupportedOperationException( + "Nested Scatters are not supported (yet) ... but you might try a sub workflow to achieve the same effect!" + ) case e => throw new UnsupportedOperationException(s"Scope ${e.getClass.getName} is not supported in an If block.") } - def processRunnable(data: WorkflowExecutionActorData, workflowLogger: WorkflowLogger): ErrorOr[WorkflowExecutionDiff] = { + def processRunnable(data: WorkflowExecutionActorData, + workflowLogger: WorkflowLogger + ): ErrorOr[WorkflowExecutionDiff] = { // This is the output port from the conditional's 'condition' input: val conditionOutputPort = node.conditionExpression.singleOutputPort data.valueStore.get(conditionOutputPort, index) match { @@ -64,9 +68,9 @@ private [execution] case class ConditionalKey(node: ConditionalNode, index: Exec node.conditionalOutputPorts.map(op => ValueKey(op, index) -> op.womType.none).toMap } else Map.empty - WorkflowExecutionDiff( - executionStoreChanges = populate(b.value) + (this -> conditionalStatus), - valueStoreAdditions = valueStoreAdditions).validNel + WorkflowExecutionDiff(executionStoreChanges = populate(b.value) + (this -> conditionalStatus), + valueStoreAdditions = valueStoreAdditions + ).validNel case Some(v: WomValue) => s"'if' condition ${node.conditionExpression.womExpression.sourceString} must evaluate to a boolean but instead got ${v.womType.stableName}".invalidNel case None => diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ExpressionKey.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ExpressionKey.scala index 5978ec8c254..51183af9f04 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ExpressionKey.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ExpressionKey.scala @@ -6,8 +6,11 @@ import common.validation.ErrorOr._ import common.validation.Validation._ import cromwell.core.ExecutionIndex._ import cromwell.core.{ExecutionStatus, JobKey} -import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionDiff -import cromwell.engine.workflow.lifecycle.execution.keys.ExpressionKey.{ExpressionEvaluationFailedResponse, ExpressionEvaluationSucceededResponse} +import cromwell.engine.workflow.lifecycle.execution.{WdlRuntimeException, WorkflowExecutionDiff} +import cromwell.engine.workflow.lifecycle.execution.keys.ExpressionKey.{ + ExpressionEvaluationFailedResponse, + ExpressionEvaluationSucceededResponse +} import cromwell.engine.workflow.lifecycle.execution.stores.ValueStore import wom.expression.IoFunctionSet import wom.graph.GraphNodePort.OutputPort @@ -18,15 +21,18 @@ final case class ExpressionKey(node: ExpressionNodeLike, index: ExecutionIndex) override val attempt = 1 override lazy val tag = s"Expression-${node.localName}:${index.fromIndex}:$attempt" - def processRunnable(ioFunctionSet: IoFunctionSet, valueStore: ValueStore, workflowExecutionActor: ActorRef): ErrorOr[WorkflowExecutionDiff] = { + def processRunnable(ioFunctionSet: IoFunctionSet, + valueStore: ValueStore, + workflowExecutionActor: ActorRef + ): ErrorOr[WorkflowExecutionDiff] = { // Send a message to self in case we decide to change evaluate to return asynchronously, if we don't we could // directly add the value to the value store in the execution diff node .evaluate(valueStore.resolve(index), ioFunctionSet) .contextualizeErrors(s"evaluate '${node.fullyQualifiedName}'") match { case Right(result) => workflowExecutionActor ! ExpressionEvaluationSucceededResponse(this, result) - case Left(f) => - workflowExecutionActor ! ExpressionEvaluationFailedResponse(this, new RuntimeException(f.toList.mkString(", "))) + case Left(f) => + workflowExecutionActor ! ExpressionEvaluationFailedResponse(this, WdlRuntimeException(f.toList.mkString(", "))) } WorkflowExecutionDiff(Map(this -> ExecutionStatus.Running)).validNel @@ -34,6 +40,8 @@ final case class ExpressionKey(node: ExpressionNodeLike, index: ExecutionIndex) } object ExpressionKey { - private [execution] case class ExpressionEvaluationSucceededResponse(expressionKey: ExpressionKey, values: Map[OutputPort, WomValue]) - private [execution] case class ExpressionEvaluationFailedResponse(expressionKey: ExpressionKey, reason: Throwable) + private[execution] case class ExpressionEvaluationSucceededResponse(expressionKey: ExpressionKey, + values: Map[OutputPort, WomValue] + ) + private[execution] case class ExpressionEvaluationFailedResponse(expressionKey: ExpressionKey, reason: Throwable) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatterCollectorKey.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatterCollectorKey.scala index 353971bd652..aa50be93144 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatterCollectorKey.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatterCollectorKey.scala @@ -11,21 +11,21 @@ import wom.graph.{GraphNode, PortBasedGraphOutputNode} * Key that becomes runnable when all shards of a collectible node are complete and need to be collected to form the output of this * call outside the scatter block. */ -private [execution] case class ScatterCollectorKey(scatterGatherPort: ScatterGathererPort, - scatterWidth: Int, - scatterCollectionFunction: ScatterCollectionFunction) extends JobKey { +private[execution] case class ScatterCollectorKey(scatterGatherPort: ScatterGathererPort, + scatterWidth: Int, + scatterCollectionFunction: ScatterCollectionFunction +) extends JobKey { val outputNodeToGather: PortBasedGraphOutputNode = scatterGatherPort.outputToGather override val node: GraphNode = outputNodeToGather override val index = None override val attempt = 1 override val tag = s"Collector-${node.localName}" - def processRunnable(data: WorkflowExecutionActorData): ErrorOr[WorkflowExecutionDiff] = { + def processRunnable(data: WorkflowExecutionActorData): ErrorOr[WorkflowExecutionDiff] = data.valueStore.collectShards(this) map { outputs => WorkflowExecutionDiff( executionStoreChanges = Map(this -> ExecutionStatus.Done), valueStoreAdditions = outputs ) } - } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatterKey.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatterKey.scala index b3e4529b9d3..0bd8a14f3a3 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatterKey.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatterKey.scala @@ -20,18 +20,19 @@ import wom.values.WomValue import scala.language.postfixOps -private [execution] case class ScatterKey(node: ScatterNode) extends JobKey { +private[execution] case class ScatterKey(node: ScatterNode) extends JobKey { // When scatters are nested, this might become Some(_) override val index = None override val attempt = 1 override val tag = node.localName - def makeCollectors(count: Int, scatterCollectionFunction: ScatterCollectionFunction): Set[ScatterCollectorKey] = (node.outputMapping.groupBy(_.outputToGather.source.graphNode) flatMap { - case (_: CallNode | _: ExposedExpressionNode | _: ConditionalNode, scatterGatherPorts) => - scatterGatherPorts.map(sgp => ScatterCollectorKey(sgp, count, scatterCollectionFunction)) - case _ => Set.empty[ScatterCollectorKey] - }).toSet + def makeCollectors(count: Int, scatterCollectionFunction: ScatterCollectionFunction): Set[ScatterCollectorKey] = + (node.outputMapping.groupBy(_.outputToGather.source.graphNode) flatMap { + case (_: CallNode | _: ExposedExpressionNode | _: ConditionalNode, scatterGatherPorts) => + scatterGatherPorts.map(sgp => ScatterCollectorKey(sgp, count, scatterCollectionFunction)) + case _ => Set.empty[ScatterCollectorKey] + }).toSet /** * Creates a sub-ExecutionStore with Starting entries for each of the scoped children. @@ -54,12 +55,17 @@ private [execution] case class ScatterKey(node: ScatterNode) extends JobKey { case _: GraphInputNode => List.empty case _: PortBasedGraphOutputNode => List.empty case _: ScatterNode => - throw new UnsupportedOperationException("Nested Scatters are not supported (yet) ... but you might try a sub workflow to achieve the same effect!") + throw new UnsupportedOperationException( + "Nested Scatters are not supported (yet) ... but you might try a sub workflow to achieve the same effect!" + ) case e => throw new UnsupportedOperationException(s"Scope ${e.getClass.getName} is not supported.") } - def processRunnable(data: WorkflowExecutionActorData, workflowExecutionActor: ActorRef, maxScatterWidth: Int): ErrorOr[WorkflowExecutionDiff] = { + def processRunnable(data: WorkflowExecutionActorData, + workflowExecutionActor: ActorRef, + maxScatterWidth: Int + ): ErrorOr[WorkflowExecutionDiff] = { import cats.syntax.traverse._ def getScatterArray(scatterVariableNode: ScatterVariableNode): ErrorOr[ScatterVariableAndValue] = { @@ -84,26 +90,29 @@ private [execution] case class ScatterKey(node: ScatterNode) extends JobKey { // Execution changes (for execution store and value store) generated by the scatter iteration nodes def buildExecutionChanges(scatterVariableAndValues: List[ScatterVariableAndValue]) = { - val (executionStoreChanges, valueStoreChanges) = scatterVariableAndValues.map({ + val (executionStoreChanges, valueStoreChanges) = scatterVariableAndValues.map { case ScatterVariableAndValue(scatterVariableNode, arrayValue) => val executionStoreChange = ScatterVariableInputKey(scatterVariableNode, arrayValue) -> ExecutionStatus.Done val valueStoreChange = ValueKey(scatterVariableNode.singleOutputPort, None) -> arrayValue executionStoreChange -> valueStoreChange - }).unzip + }.unzip executionStoreChanges.toMap -> valueStoreChanges.toMap } // Checks the scatter width of a scatter node and builds WorkflowExecutionDiff accordingly // If scatter width is more than max allowed limit, it fails the ScatterNode key - def buildExecutionDiff(scatterSize: Int, arrays: List[ScatterVariableAndValue]): WorkflowExecutionDiff = { - if(scatterSize > maxScatterWidth) { - workflowExecutionActor ! JobFailedNonRetryableResponse(this, new Exception(s"Workflow scatter width of $scatterSize exceeds configured limit of $maxScatterWidth."), None) + def buildExecutionDiff(scatterSize: Int, arrays: List[ScatterVariableAndValue]): WorkflowExecutionDiff = + if (scatterSize > maxScatterWidth) { + workflowExecutionActor ! JobFailedNonRetryableResponse( + this, + new Exception(s"Workflow scatter width of $scatterSize exceeds configured limit of $maxScatterWidth."), + None + ) WorkflowExecutionDiff(Map(this -> ExecutionStatus.Failed)) - } - else { + } else { val (scatterVariablesExecutionChanges, valueStoreChanges) = buildExecutionChanges(arrays) val executionStoreChanges = populate( scatterSize, @@ -115,8 +124,6 @@ private [execution] case class ScatterKey(node: ScatterNode) extends JobKey { valueStoreAdditions = valueStoreChanges ) } - } - (for { arrays <- scatterArraysValuesCheck diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatterVariableInputKey.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatterVariableInputKey.scala index 202e1e2db7b..6480c643ea0 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatterVariableInputKey.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatterVariableInputKey.scala @@ -4,7 +4,7 @@ import cromwell.core.JobKey import wom.graph.GraphInputNode import wom.values.WomArray.WomArrayLike -private [execution] case class ScatterVariableInputKey(node: GraphInputNode, womArrayLike: WomArrayLike) extends JobKey { +private[execution] case class ScatterVariableInputKey(node: GraphInputNode, womArrayLike: WomArrayLike) extends JobKey { override def index: Option[Int] = None override def attempt: Int = 1 override def tag: String = node.localName diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatteredCallCompletionKey.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatteredCallCompletionKey.scala index c58982e807c..e418cbe1d43 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatteredCallCompletionKey.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/ScatteredCallCompletionKey.scala @@ -9,15 +9,13 @@ import wom.graph.{CallNode, GraphNode} /** * Key that should become runnable when all shards of a scattered call are complete. */ -private [execution] case class ScatteredCallCompletionKey(call: CallNode, - scatterWidth: Int) extends JobKey { +private[execution] case class ScatteredCallCompletionKey(call: CallNode, scatterWidth: Int) extends JobKey { override val node: GraphNode = call override val index = None override val attempt = 1 override val totalIndices = scatterWidth override val tag = s"CallCompletion-${node.localName}" - def processRunnable(data: WorkflowExecutionActorData): ErrorOr[WorkflowExecutionDiff] = { + def processRunnable(data: WorkflowExecutionActorData): ErrorOr[WorkflowExecutionDiff] = WorkflowExecutionDiff(executionStoreChanges = Map(this -> ExecutionStatus.Done)).validNel - } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/SubWorkflowKey.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/SubWorkflowKey.scala index 44c2f0a57e2..f66f7f2fade 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/SubWorkflowKey.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/SubWorkflowKey.scala @@ -4,6 +4,7 @@ import cromwell.core.CallKey import cromwell.core.ExecutionIndex._ import wom.graph.WorkflowCallNode -private [execution] case class SubWorkflowKey(node: WorkflowCallNode, index: ExecutionIndex, attempt: Int) extends CallKey { +private[execution] case class SubWorkflowKey(node: WorkflowCallNode, index: ExecutionIndex, attempt: Int) + extends CallKey { override val tag = s"SubWorkflow-${node.localName}:${index.fromIndex}:$attempt" } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/package.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/package.scala index 95d00892c4a..4ddef0c980d 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/package.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/keys/package.scala @@ -15,9 +15,8 @@ package object keys { ).validNel } - private def bypassedScopeResults(jobKey: JobKey): Map[ValueKey, WomOptionalValue] = { - jobKey.node.outputPorts.map({ outputPort => - ValueKey(outputPort, jobKey.index) -> WomOptionalValue.none(outputPort.womType) - }).toMap - } + private def bypassedScopeResults(jobKey: JobKey): Map[ValueKey, WomOptionalValue] = + jobKey.node.outputPorts.map { outputPort => + ValueKey(outputPort, jobKey.index) -> WomOptionalValue.none(outputPort.womType) + }.toMap } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/package.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/package.scala index 52fbe868f1e..e1695e1d355 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/package.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/package.scala @@ -5,6 +5,16 @@ package execution { import cromwell.core.CallKey import wom.values.WomEvaluatedCallInputs + import scala.util.control.NoStackTrace + final case class JobRunning(key: CallKey, inputs: WomEvaluatedCallInputs) final case class JobStarting(callKey: CallKey) + + /** + * An exception specific to conditions inside the executing WDL, as opposed to one that is "Cromwell's fault" + * @param message Description suitable for user display + */ + final case class WdlRuntimeException(message: String) extends RuntimeException with NoStackTrace { + override def getMessage: String = message + } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/stores/ExecutionStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/stores/ExecutionStore.scala index 32d9aa2ca4c..e6445ffda23 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/stores/ExecutionStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/stores/ExecutionStore.scala @@ -25,6 +25,7 @@ object ExecutionStore { val MaxJobsToStartPerTick = 1000 implicit class EnhancedJobKey(val key: JobKey) extends AnyVal { + /** * Given a StatusStable, return true if all dependencies of this key are in the table (and therefore are in this status), * false otherwise. @@ -42,26 +43,30 @@ object ExecutionStore { case scatterCollector: ScatterCollectorKey => // The outputToGather is the PortBasedGraphOutputNode of the inner graph that we're collecting. Go one step upstream and then // find the node which will have entries in the execution store. If that has 'n' entries, then we're good to start collecting, - statusTable.row(scatterCollector.outputNodeToGather.singleUpstreamPort.executionNode).size == scatterCollector.scatterWidth + statusTable + .row(scatterCollector.outputNodeToGather.singleUpstreamPort.executionNode) + .size == scatterCollector.scatterWidth case conditionalCollector: ConditionalCollectorKey => val upstreamPort = conditionalCollector.outputNodeToCollect.singleUpstreamPort upstreamPort.executionNode.isInStatus(chooseIndex(upstreamPort), statusTable) // In the general case, the dependencies are held by the upstreamPorts case _ => key.node.upstreamPorts forall { p => - p.executionNode.isInStatus(chooseIndex(p), statusTable) + p.executionNode.isInStatus(chooseIndex(p), statusTable) } } } def nonStartableOutputKeys: Set[JobKey] = key match { - case scatterKey: ScatterKey => scatterKey.makeCollectors(0, scatterKey.node.scatterCollectionFunctionBuilder(List.empty)).toSet[JobKey] + case scatterKey: ScatterKey => + scatterKey.makeCollectors(0, scatterKey.node.scatterCollectionFunctionBuilder(List.empty)).toSet[JobKey] case conditionalKey: ConditionalKey => conditionalKey.collectors.toSet[JobKey] case _ => Set.empty[JobKey] } } implicit class EnhancedOutputPort(val outputPort: OutputPort) extends AnyVal { + /** * Node that should be considered to determine upstream dependencies */ @@ -78,18 +83,25 @@ object ExecutionStore { case svn: ScatterVariableNode => table.contains(svn.linkToOuterGraph.graphNode, None) // OuterGraphInputNodes signal that an input comes from outside the graph. // Depending on whether or not this input is outside of a scatter graph will change the index which we need to look at - case ogin: OuterGraphInputNode if !ogin.preserveScatterIndex => ogin.linkToOuterGraph.executionNode.isInStatus(None, table) + case ogin: OuterGraphInputNode if !ogin.preserveScatterIndex => + ogin.linkToOuterGraph.executionNode.isInStatus(None, table) case ogin: OuterGraphInputNode => ogin.linkToOuterGraph.executionNode.isInStatus(index, table) case _: GraphInputNode => true case _ => table.contains(graphNode, index) } } - case class ExecutionStoreUpdate(runnableKeys: List[JobKey], updatedStore: ExecutionStore, statusChanges: Map[JobKey, ExecutionStatus]) + case class ExecutionStoreUpdate(runnableKeys: List[JobKey], + updatedStore: ExecutionStore, + statusChanges: Map[JobKey, ExecutionStatus] + ) def empty = ActiveExecutionStore(Map.empty[JobKey, ExecutionStatus], needsUpdate = false) - def apply(callable: ExecutableCallable, totalJobsByRootWf: AtomicInteger, totalMaxJobsPerRootWf: Int): ErrorOr[ActiveExecutionStore] = { + def apply(callable: ExecutableCallable, + totalJobsByRootWf: AtomicInteger, + totalMaxJobsPerRootWf: Int + ): ErrorOr[ActiveExecutionStore] = { // Keys that are added in a NotStarted Status val notStartedKeys = callable.graph.nodes collect { case call: CommandCallNode => BackendJobDescriptorKey(call, None, attempt = 1) @@ -118,21 +130,25 @@ object ExecutionStore { /** * Execution store in its nominal state */ -final case class ActiveExecutionStore private[stores](private val statusStore: Map[JobKey, ExecutionStatus], override val needsUpdate: Boolean) extends ExecutionStore(statusStore, needsUpdate) { +final case class ActiveExecutionStore private[stores] (private val statusStore: Map[JobKey, ExecutionStatus], + override val needsUpdate: Boolean +) extends ExecutionStore(statusStore, needsUpdate) { override def toString: String = { import io.circe.syntax._ import io.circe.Printer - statusStore.map { - case (k, v) if k.isShard => s"${k.node.fullyQualifiedName}:${k.index.get}" -> v.toString - case (k, v) => k.node.fullyQualifiedName -> v.toString - }.asJson.printWith(Printer.spaces2.copy(dropNullValues = true, colonLeft = "")) + statusStore + .map { + case (k, v) if k.isShard => s"${k.node.fullyQualifiedName}:${k.index.get}" -> v.toString + case (k, v) => k.node.fullyQualifiedName -> v.toString + } + .asJson + .printWith(Printer.spaces2.copy(dropNullValues = true, colonLeft = "")) } - override def updateKeys(values: Map[JobKey, ExecutionStatus], needsUpdate: Boolean): ActiveExecutionStore = { + override def updateKeys(values: Map[JobKey, ExecutionStatus], needsUpdate: Boolean): ActiveExecutionStore = this.copy(statusStore = statusStore ++ values, needsUpdate = needsUpdate) - } override def seal: SealedExecutionStore = SealedExecutionStore(statusStore.filterNot(_._2 == NotStarted), needsUpdate) override def withNeedsUpdateFalse: ExecutionStore = if (!needsUpdate) this else this.copy(needsUpdate = false) override def withNeedsUpdateTrue: ExecutionStore = if (needsUpdate) this else this.copy(needsUpdate = true) @@ -142,12 +158,13 @@ final case class ActiveExecutionStore private[stores](private val statusStore: M * Execution store when the workflow is in either Failing or Aborting state. Keys in NotStarted state have been removed and * no new NotStarted key can be added. Other statuses can still be updated. */ -final case class SealedExecutionStore private[stores](private val statusStore: Map[JobKey, ExecutionStatus], override val needsUpdate: Boolean) extends ExecutionStore(statusStore, false) { +final case class SealedExecutionStore private[stores] (private val statusStore: Map[JobKey, ExecutionStatus], + override val needsUpdate: Boolean +) extends ExecutionStore(statusStore, false) { - override def updateKeys(values: Map[JobKey, ExecutionStatus], needsUpdate: Boolean): SealedExecutionStore = { + override def updateKeys(values: Map[JobKey, ExecutionStatus], needsUpdate: Boolean): SealedExecutionStore = // Don't allow NotStarted keys in sealed mode this.copy(statusStore = statusStore ++ values.filterNot(_._2 == NotStarted), needsUpdate = needsUpdate) - } override def seal: SealedExecutionStore = this override def withNeedsUpdateFalse: ExecutionStore = this.copy(needsUpdate = false) override def withNeedsUpdateTrue: ExecutionStore = this.copy(needsUpdate = true) @@ -162,13 +179,14 @@ final case class SealedExecutionStore private[stores](private val statusStore: M * when true, something happened since the last update that could yield new runnable keys, so update should be called * when false, nothing happened between the last update and now that will yield different results so no need to call the update method */ -sealed abstract class ExecutionStore private[stores](statusStore: Map[JobKey, ExecutionStatus], val needsUpdate: Boolean) { +sealed abstract class ExecutionStore private[stores] (statusStore: Map[JobKey, ExecutionStatus], + val needsUpdate: Boolean +) { // View of the statusStore more suited for lookup based on status lazy val store: Map[ExecutionStatus, List[JobKey]] = statusStore.groupBy(_._2).safeMapValues(_.keys.toList) - def backendJobDescriptorKeyForNode(node: GraphNode): Option[BackendJobDescriptorKey] = { + def backendJobDescriptorKeyForNode(node: GraphNode): Option[BackendJobDescriptorKey] = statusStore.keys collectFirst { case k: BackendJobDescriptorKey if k.node eq node => k } - } /** * Number of queued jobs @@ -183,10 +201,9 @@ sealed abstract class ExecutionStore private[stores](statusStore: Map[JobKey, Ex /** * Update key statuses */ - def updateKeys(values: Map[JobKey, ExecutionStatus]): ExecutionStore = { + def updateKeys(values: Map[JobKey, ExecutionStatus]): ExecutionStore = // The store might newly need updating now if a job has completed because downstream jobs might now be runnable updateKeys(values, needsUpdate || values.values.exists(_.isTerminalOrRetryable)) - } /** * Returns a SealedExecutionStore: all NotStarted keys will be removed and no new NotStarted keys can be added after that @@ -204,23 +221,23 @@ sealed abstract class ExecutionStore private[stores](statusStore: Map[JobKey, Ex protected def withNeedsUpdateFalse: ExecutionStore /* - * Create 2 Tables, one for keys in done status and one for keys in terminal status. - * A Table is nothing more than a Map[R, Map[C, V]], see Table trait for more details - * In this case, rows are GraphNodes, columns are ExecutionIndexes, and values are JobKeys - * This allows for quick lookup of all shards for a node, as well as accessing a specific key with a - * (node, index) pair + * Create 2 Tables, one for keys in done status and one for keys in terminal status. + * A Table is nothing more than a Map[R, Map[C, V]], see Table trait for more details + * In this case, rows are GraphNodes, columns are ExecutionIndexes, and values are JobKeys + * This allows for quick lookup of all shards for a node, as well as accessing a specific key with a + * (node, index) pair */ lazy val (doneStatus, terminalStatus) = { def toTableEntry(key: JobKey) = (key.node, key.index, key) - store.foldLeft((Table.empty[GraphNode, ExecutionIndex, JobKey], Table.empty[GraphNode, ExecutionIndex, JobKey]))({ - case ((done, terminal), (status, keys)) => + store.foldLeft((Table.empty[GraphNode, ExecutionIndex, JobKey], Table.empty[GraphNode, ExecutionIndex, JobKey])) { + case ((done, terminal), (status, keys)) => lazy val newMapEntries = keys map toTableEntry val newDone = if (status.isDoneOrBypassed) done.addAll(newMapEntries) else done val newTerminal = if (status.isTerminal) terminal.addAll(newMapEntries) else terminal newDone -> newTerminal - }) + } } private def keysWithStatus(status: ExecutionStatus) = store.getOrElse(status, List.empty) @@ -229,29 +246,28 @@ sealed abstract class ExecutionStore private[stores](statusStore: Map[JobKey, Ex * We're done when all the keys have a terminal status, * which is equivalent to non of them being in a non-terminal status and faster to verify */ - def isDone: Boolean = { + def isDone: Boolean = NonTerminalStatuses.toList.map(keysWithStatus).forall(_.isEmpty) - } - def isStalled: Boolean = { + def isStalled: Boolean = !isDone && !needsUpdate && ActiveStatuses.map(keysWithStatus).forall(_.isEmpty) - } def unstarted = keysWithStatus(NotStarted) def jobStatus(jobKey: JobKey): Option[ExecutionStatus] = statusStore.get(jobKey) - def startedJobs: List[BackendJobDescriptorKey] = { - store.filterNot({ case (s, _) => s == NotStarted}).values.toList.flatten collect { + def startedJobs: List[BackendJobDescriptorKey] = + store.filterNot { case (s, _) => s == NotStarted }.values.toList.flatten collect { case k: BackendJobDescriptorKey => k } - } override def toString: String = s""" |ExecutionStore( | statusStore = { - | ${store.map { case (j, s) => s"$j -> ${s.mkString(System.lineSeparator + " ", ", " + System.lineSeparator + " ", "")}" } mkString("," + System.lineSeparator + " ")} + | ${store.map { case (j, s) => + s"$j -> ${s.mkString(System.lineSeparator + " ", ", " + System.lineSeparator + " ", "")}" + } mkString ("," + System.lineSeparator + " ")} | }, | needsUpdate = $needsUpdate |)""".stripMargin diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/stores/ValueStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/stores/ValueStore.scala index a5c1d6fc995..778e434f634 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/stores/ValueStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/stores/ValueStore.scala @@ -39,9 +39,8 @@ case class ValueStore(store: Table[OutputPort, ExecutionIndex, WomValue]) { values.asJson.printWith(Printer.spaces2.copy(dropNullValues = true, colonLeft = "")) } - final def add(values: Map[ValueKey, WomValue]): ValueStore = { - this.copy(store = store.addAll(values.map({ case (key, value) => (key.port, key.index, value) }))) - } + final def add(values: Map[ValueKey, WomValue]): ValueStore = + this.copy(store = store.addAll(values.map { case (key, value) => (key.port, key.index, value) })) final def get(outputKey: ValueKey): Option[WomValue] = store.getValue(outputKey.port, outputKey.index) @@ -76,7 +75,8 @@ case class ValueStore(store: Table[OutputPort, ExecutionIndex, WomValue]) { } } else { // If we don't find enough shards, this collector was found "runnable" when it shouldn't have - s"Some shards are missing from the value store for node ${collector.node.fullyQualifiedName}, expected ${collector.scatterWidth} shards but only got ${collectedValue.size}: ${collectedValue.mkString(", ")}".invalidNel + s"Some shards are missing from the value store for node ${collector.node.fullyQualifiedName}, expected ${collector.scatterWidth} shards but only got ${collectedValue.size}: ${collectedValue + .mkString(", ")}".invalidNel } case None if collector.scatterWidth == 0 => // If there's nothing, the scatter was empty, let the scatterCollectionFunction deal with it and just pass it an empty shard list @@ -92,8 +92,10 @@ case class ValueStore(store: Table[OutputPort, ExecutionIndex, WomValue]) { val conditionalPort = collector.conditionalOutputPort val sourcePort = conditionalPort.outputToExpose.source store.getValue(sourcePort, collector.index) match { - case Some(womValue) => Map(ValueKey(conditionalPort, collector.index) -> WomOptionalValue(womValue).flattenOptional).validNel - case None => s"Conditional collector cannot find a value for output port ${sourcePort.identifier.fullyQualifiedName.value} in value store: $this".invalidNel + case Some(womValue) => + Map(ValueKey(conditionalPort, collector.index) -> WomOptionalValue(womValue).flattenOptional).validNel + case None => + s"Conditional collector cannot find a value for output port ${sourcePort.identifier.fullyQualifiedName.value} in value store: $this".invalidNel } } @@ -107,12 +109,16 @@ case class ValueStore(store: Table[OutputPort, ExecutionIndex, WomValue]) { case Some(womValue: WomArrayLike) => index match { case Some(jobIndex) => - womValue.asArray.value.lift(svn.indexForShard(jobIndex, womValue.asArray.value.size)) + womValue.asArray.value + .lift(svn.indexForShard(jobIndex, womValue.asArray.value.size)) .toValidNel(s"Shard index $jobIndex exceeds scatter array length: ${womValue.asArray.value.size}") - case None => s"Unsharded execution key references a scatter variable: ${p.identifier.fullyQualifiedName}".invalidNel + case None => + s"Unsharded execution key references a scatter variable: ${p.identifier.fullyQualifiedName}".invalidNel } - case Some(other) => s"Value for scatter collection ${p.identifier.fullyQualifiedName} is not an array: ${other.womType.stableName}".invalidNel - case None => s"Can't find a value for scatter collection ${p.identifier.fullyQualifiedName} (looking for index $index)".invalidNel + case Some(other) => + s"Value for scatter collection ${p.identifier.fullyQualifiedName} is not an array: ${other.womType.stableName}".invalidNel + case None => + s"Can't find a value for scatter collection ${p.identifier.fullyQualifiedName} (looking for index $index)".invalidNel } } @@ -123,7 +129,7 @@ case class ValueStore(store: Table[OutputPort, ExecutionIndex, WomValue]) { s"Can't find a ValueStore value for $p at index $index in $this".invalidNel } - def findValueStorePort(p: OutputPort, index: ExecutionIndex): ErrorOr[WomValue] = { + def findValueStorePort(p: OutputPort, index: ExecutionIndex): ErrorOr[WomValue] = p.graphNode match { case svn: ScatterVariableNode => forScatterVariable(svn) case ogin: OuterGraphInputNode if ogin.preserveScatterIndex => findValueStorePort(ogin.linkToOuterGraph, index) @@ -131,7 +137,6 @@ case class ValueStore(store: Table[OutputPort, ExecutionIndex, WomValue]) { case _: GraphInputNode => forGraphNodePort(p, None) // Must be a workflow input, which never have indices case _ => forGraphNodePort(p, index) } - } findValueStorePort(outputPort, index) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/CopyWorkflowLogsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/CopyWorkflowLogsActor.scala index 467cc1c2586..f29482cb75e 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/CopyWorkflowLogsActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/CopyWorkflowLogsActor.scala @@ -22,14 +22,13 @@ object CopyWorkflowLogsActor { // Commands case class Copy(workflowId: WorkflowId, destinationDirPath: Path) - val strategy: OneForOneStrategy = OneForOneStrategy(maxNrOfRetries = 3) { - case _: IOException => Restart + val strategy: OneForOneStrategy = OneForOneStrategy(maxNrOfRetries = 3) { case _: IOException => + Restart } def props(serviceRegistryActor: ActorRef, ioActor: ActorRef, - workflowLogConfigurationOption: Option[WorkflowLogConfiguration] = - WorkflowLogger.workflowLogConfiguration, + workflowLogConfigurationOption: Option[WorkflowLogConfiguration] = WorkflowLogger.workflowLogConfiguration, /* The theory is that the `GcsBatchCommandBuilder` copies the temporary workflow logs from the local disk to GCS. Then later, the separate `DefaultIOCommandBuilder` deletes files from the local disk. @@ -47,16 +46,17 @@ object CopyWorkflowLogsActor { implemented, It Works (TM), and I'm not changing it for now. */ copyCommandBuilder: IoCommandBuilder = GcsBatchCommandBuilder, - deleteCommandBuilder: IoCommandBuilder = DefaultIoCommandBuilder, - ): Props = { - Props(new CopyWorkflowLogsActor( - serviceRegistryActor = serviceRegistryActor, - ioActor = ioActor, - workflowLogConfigurationOption = workflowLogConfigurationOption, - copyCommandBuilder = copyCommandBuilder, - deleteCommandBuilder = deleteCommandBuilder, - )).withDispatcher(IoDispatcher) - } + deleteCommandBuilder: IoCommandBuilder = DefaultIoCommandBuilder + ): Props = + Props( + new CopyWorkflowLogsActor( + serviceRegistryActor = serviceRegistryActor, + ioActor = ioActor, + workflowLogConfigurationOption = workflowLogConfigurationOption, + copyCommandBuilder = copyCommandBuilder, + deleteCommandBuilder = deleteCommandBuilder + ) + ).withDispatcher(IoDispatcher) } // This could potentially be turned into a more generic "Copy/Move something from A to B" @@ -65,9 +65,12 @@ class CopyWorkflowLogsActor(override val serviceRegistryActor: ActorRef, override val ioActor: ActorRef, workflowLogConfigurationOption: Option[WorkflowLogConfiguration], copyCommandBuilder: IoCommandBuilder, - deleteCommandBuilder: IoCommandBuilder, - ) extends Actor - with ActorLogging with IoClientHelper with WorkflowMetadataHelper with MonitoringCompanionHelper { + deleteCommandBuilder: IoCommandBuilder +) extends Actor + with ActorLogging + with IoClientHelper + with WorkflowMetadataHelper + with MonitoringCompanionHelper { implicit val ec: ExecutionContext = context.dispatcher @@ -87,9 +90,10 @@ class CopyWorkflowLogsActor(override val serviceRegistryActor: ActorRef, removeWork() } } else removeWork() - + private def updateLogsPathInMetadata(workflowId: WorkflowId, path: Path): Unit = { - val metadataEventMsg = MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.WorkflowLog), MetadataValue(path.pathAsString)) + val metadataEventMsg = + MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.WorkflowLog), MetadataValue(path.pathAsString)) serviceRegistryActor ! PutMetadataAction(metadataEventMsg) } @@ -111,30 +115,31 @@ class CopyWorkflowLogsActor(override val serviceRegistryActor: ActorRef, case Failure(failure) => log.error( cause = failure, - message = - s"Failed to copy workflow logs from ${src.pathAsString} to ${destPath.pathAsString}: " + - s"${failure.getMessage}", + message = s"Failed to copy workflow logs from ${src.pathAsString} to ${destPath.pathAsString}: " + + s"${failure.getMessage}" ) deleteLog(src) case Success(_) => - // Deliberately not deleting the file here, that will be done in batch in `deleteLog` - // after the copy is terminal. + // Deliberately not deleting the file here, that will be done in batch in `deleteLog` + // after the copy is terminal. } workflowLogger.close() } } - + case (workflowId: WorkflowId, IoSuccess(copy: IoCopyCommand, _)) => updateLogsPathInMetadata(workflowId, copy.destination) deleteLog(copy.source) - + case (workflowId: WorkflowId, IoFailAck(copy: IoCopyCommand, failure)) => pushWorkflowFailures(workflowId, List(new IOException("Could not copy workflow logs", failure))) - log.error(failure, s"Failed to copy workflow logs from ${copy.source.pathAsString} to ${copy.destination.pathAsString}") + log.error(failure, + s"Failed to copy workflow logs from ${copy.source.pathAsString} to ${copy.destination.pathAsString}" + ) deleteLog(copy.source) - + case IoSuccess(_: IoDeleteCommand, _) => removeWork() - + case IoFailAck(delete: IoDeleteCommand, failure) => removeWork() log.error(failure, s"Failed to delete workflow logs from ${delete.file.pathAsString}") @@ -148,13 +153,14 @@ class CopyWorkflowLogsActor(override val serviceRegistryActor: ActorRef, override def receive: Receive = monitoringReceive orElse ioReceive orElse copyLogsReceive /*_*/ - override def preRestart(t: Throwable, message: Option[Any]): Unit = { + override def preRestart(t: Throwable, message: Option[Any]): Unit = message foreach self.forward - } override protected def onTimeout(message: Any, to: ActorRef): Unit = message match { case copy: IoCopyCommand => - log.error(s"Failed to copy workflow logs from ${copy.source.pathAsString} to ${copy.destination.pathAsString}: Timeout") + log.error( + s"Failed to copy workflow logs from ${copy.source.pathAsString} to ${copy.destination.pathAsString}: Timeout" + ) deleteLog(copy.source) case delete: IoDeleteCommand => log.error(s"Failed to delete workflow logs from ${delete.file.pathAsString}: Timeout") diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/CopyWorkflowOutputsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/CopyWorkflowOutputsActor.scala index b6fa6b285a6..142a6623fcd 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/CopyWorkflowOutputsActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/CopyWorkflowOutputsActor.scala @@ -3,8 +3,18 @@ package cromwell.engine.workflow.lifecycle.finalization import akka.actor.{Actor, ActorLogging, ActorRef, Props} import akka.event.LoggingReceive import cromwell.backend.BackendLifecycleActor.BackendWorkflowLifecycleActorResponse -import cromwell.backend.BackendWorkflowFinalizationActor.{FinalizationFailed, FinalizationResponse, FinalizationSuccess, Finalize} -import cromwell.backend.{AllBackendInitializationData, BackendConfigurationDescriptor, BackendInitializationData, BackendLifecycleActorFactory} +import cromwell.backend.BackendWorkflowFinalizationActor.{ + FinalizationFailed, + FinalizationResponse, + FinalizationSuccess, + Finalize +} +import cromwell.backend.{ + AllBackendInitializationData, + BackendConfigurationDescriptor, + BackendInitializationData, + BackendLifecycleActorFactory +} import cromwell.core.Dispatcher.IoDispatcher import cromwell.core.WorkflowOptions._ import cromwell.core._ @@ -19,26 +29,36 @@ import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} object CopyWorkflowOutputsActor { - def props(workflowId: WorkflowId, ioActor: ActorRef, workflowDescriptor: EngineWorkflowDescriptor, workflowOutputs: CallOutputs, - initializationData: AllBackendInitializationData) = Props( + def props(workflowId: WorkflowId, + ioActor: ActorRef, + workflowDescriptor: EngineWorkflowDescriptor, + workflowOutputs: CallOutputs, + initializationData: AllBackendInitializationData + ) = Props( new CopyWorkflowOutputsActor(workflowId, ioActor, workflowDescriptor, workflowOutputs, initializationData) ).withDispatcher(IoDispatcher) } -class CopyWorkflowOutputsActor(workflowId: WorkflowId, override val ioActor: ActorRef, val workflowDescriptor: EngineWorkflowDescriptor, workflowOutputs: CallOutputs, - initializationData: AllBackendInitializationData) - extends Actor with ActorLogging with PathFactory with AsyncIoActorClient { +class CopyWorkflowOutputsActor(workflowId: WorkflowId, + override val ioActor: ActorRef, + val workflowDescriptor: EngineWorkflowDescriptor, + workflowOutputs: CallOutputs, + initializationData: AllBackendInitializationData +) extends Actor + with ActorLogging + with PathFactory + with AsyncIoActorClient { override lazy val ioCommandBuilder = GcsBatchCommandBuilder implicit val ec = context.dispatcher override val pathBuilders = workflowDescriptor.pathBuilders - override def receive = LoggingReceive { - case Finalize => performActionThenRespond(afterAll()(context.dispatcher), FinalizationFailed)(context.dispatcher) + override def receive = LoggingReceive { case Finalize => + performActionThenRespond(afterAll()(context.dispatcher), FinalizationFailed)(context.dispatcher) } private def performActionThenRespond(operation: => Future[BackendWorkflowLifecycleActorResponse], - onFailure: (Throwable) => BackendWorkflowLifecycleActorResponse) - (implicit ec: ExecutionContext) = { + onFailure: (Throwable) => BackendWorkflowLifecycleActorResponse + )(implicit ec: ExecutionContext) = { val respondTo: ActorRef = sender() operation onComplete { case Success(r) => respondTo ! r @@ -53,34 +73,36 @@ class CopyWorkflowOutputsActor(workflowId: WorkflowId, override val ioActor: Act // Check if there are duplicated destination paths and throw an exception if that is the case. // This creates a map of destinations and source paths which point to them in cases where there are multiple // source paths that point to the same destination. - val duplicatedDestPaths: Map[Path, List[Path]] = outputFilePaths.groupBy{ case (_, destPath) => destPath}.collect { - case (destPath, list) if list.size > 1 => destPath -> list.map {case (source, _) => source} - } + val duplicatedDestPaths: Map[Path, List[Path]] = + outputFilePaths.groupBy { case (_, destPath) => destPath }.collect { + case (destPath, list) if list.size > 1 => destPath -> list.map { case (source, _) => source } + } if (duplicatedDestPaths.nonEmpty) { val formattedCollidingCopyOptions = duplicatedDestPaths.toList - .sortBy{case(dest, _) => dest.pathAsString} // Sort by destination path + .sortBy { case (dest, _) => dest.pathAsString } // Sort by destination path // Make a '/my/src -> /my/dest' copy tape string for each source and destination. Use flat map to get a single list // srcList is also sorted to get a deterministic output order. This is necessary for making sure the tests // for the error always succeed. - .flatMap{ case (dest, srcList) => srcList.sortBy(_.pathAsString).map(_.pathAsString + s" -> $dest")} + .flatMap { case (dest, srcList) => srcList.sortBy(_.pathAsString).map(_.pathAsString + s" -> $dest") } throw new IllegalStateException( "Cannot copy output files to given final_workflow_outputs_dir" + - s" as multiple files will be copied to the same path: \n${formattedCollidingCopyOptions.mkString("\n")}")} + s" as multiple files will be copied to the same path: \n${formattedCollidingCopyOptions.mkString("\n")}" + ) + } - val copies = outputFilePaths map { - case (srcPath, dstPath) => asyncIo.copyAsync(srcPath, dstPath) + val copies = outputFilePaths map { case (srcPath, dstPath) => + asyncIo.copyAsync(srcPath, dstPath) } Future.sequence(copies) } - private def findFiles(values: Seq[WomValue]): Seq[WomSingleFile] = { + private def findFiles(values: Seq[WomValue]): Seq[WomSingleFile] = values flatMap { - _.collectAsSeq { - case file: WomSingleFile => file + _.collectAsSeq { case file: WomSingleFile => + file } } - } private def getOutputFilePaths(workflowOutputsPath: Path): List[(Path, Path)] = { @@ -100,42 +122,37 @@ class CopyWorkflowOutputsActor(workflowId: WorkflowId, override val ioActor: Act // "execution" should be optional, because its not created on AWS. // Also cacheCopy or attempt- folders are optional. lazy val truncateRegex = ".*/call-[^/]*/(shard-[0-9]+/)?(cacheCopy/)?(attempt-[0-9]+/)?(execution/)?".r - val outputFileDestinations = rootAndFiles flatMap { - case (workflowRoot, outputs) => - outputs map { output => - val outputPath = PathFactory.buildPath(output, pathBuilders) - outputPath -> { - if (useRelativeOutputPaths) { - val pathRelativeToExecDir = truncateRegex.replaceFirstIn(outputPath.pathAsString, "") - workflowOutputsPath.resolve(pathRelativeToExecDir) - } - else PathCopier.getDestinationFilePath(workflowRoot, outputPath, workflowOutputsPath) - } + val outputFileDestinations = rootAndFiles flatMap { case (workflowRoot, outputs) => + outputs map { output => + val outputPath = PathFactory.buildPath(output, pathBuilders) + outputPath -> { + if (useRelativeOutputPaths) { + val pathRelativeToExecDir = truncateRegex.replaceFirstIn(outputPath.pathAsString, "") + workflowOutputsPath.resolve(pathRelativeToExecDir) + } else PathCopier.getDestinationFilePath(workflowRoot, outputPath, workflowOutputsPath) } + } } outputFileDestinations.distinct.toList } - private def getBackendRootPath(backend: String, config: BackendConfigurationDescriptor): Option[Path] = { + private def getBackendRootPath(backend: String, config: BackendConfigurationDescriptor): Option[Path] = getBackendFactory(backend) map getRootPath(config, initializationData.get(backend)) - } - private def getBackendFactory(backend: String): Option[BackendLifecycleActorFactory] = { + private def getBackendFactory(backend: String): Option[BackendLifecycleActorFactory] = CromwellBackends.backendLifecycleFactoryActorByName(backend).toOption - } - private def getRootPath(config: BackendConfigurationDescriptor, initializationData: Option[BackendInitializationData]) - (backendFactory: BackendLifecycleActorFactory): Path = { + private def getRootPath(config: BackendConfigurationDescriptor, + initializationData: Option[BackendInitializationData] + )(backendFactory: BackendLifecycleActorFactory): Path = backendFactory.getExecutionRootPath(workflowDescriptor.backendDescriptor, config.backendConfig, initializationData) - } /** * Happens after everything else runs */ - final def afterAll()(implicit ec: ExecutionContext): Future[FinalizationResponse] = { + final def afterAll()(implicit ec: ExecutionContext): Future[FinalizationResponse] = workflowDescriptor.getWorkflowOption(FinalWorkflowOutputsDir) match { case Some(outputs) => copyWorkflowOutputs(outputs) map { _ => FinalizationSuccess } case None => Future.successful(FinalizationSuccess) } - } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackActor.scala index 516184d023a..726066b09ee 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackActor.scala @@ -11,7 +11,7 @@ import akka.http.scaladsl.model.headers.RawHeader import akka.routing.Broadcast import akka.util.ByteString import cats.data.Validated.{Invalid, Valid} -import cats.implicits.toTraverseOps +import cats.implicits.{catsSyntaxValidatedId, toTraverseOps} import com.typesafe.config.Config import com.typesafe.scalalogging.LazyLogging import common.validation.ErrorOr @@ -36,32 +36,42 @@ import java.time.Instant import java.util.concurrent.Executors import scala.util.{Failure, Success} - case class WorkflowCallbackConfig(enabled: Boolean, numThreads: Int, retryBackoff: SimpleExponentialBackoff, maxRetries: Int, defaultUri: Option[URI], // May be overridden by workflow options - authMethod: Option[WorkflowCallbackConfig.AuthMethod]) + authMethod: Option[WorkflowCallbackConfig.AuthMethod] +) object WorkflowCallbackConfig extends LazyLogging { - sealed trait AuthMethod { def getAccessToken: ErrorOr.ErrorOr[String] } + sealed trait AuthMethod { def getAccessToken: ErrorOr.ErrorOr[String] } case object AzureAuth extends AuthMethod { override def getAccessToken: ErrorOr.ErrorOr[String] = AzureCredentials.getAccessToken() } + case class StaticTokenAuth(token: String) extends AuthMethod { + override def getAccessToken: ErrorOr.ErrorOr[String] = token.validNel + } + private lazy val defaultNumThreads = 5 private lazy val defaultRetryBackoff = SimpleExponentialBackoff(3.seconds, 5.minutes, 1.1) private lazy val defaultMaxRetries = 10 def empty: WorkflowCallbackConfig = WorkflowCallbackConfig( - false, defaultNumThreads, defaultRetryBackoff, defaultMaxRetries, None, None + false, + defaultNumThreads, + defaultRetryBackoff, + defaultMaxRetries, + None, + None ) def apply(config: Config): WorkflowCallbackConfig = { val enabled = config.as[Boolean]("enabled") val numThreads = config.as[Option[Int]]("num-threads").getOrElse(defaultNumThreads) - val backoff = config.as[Option[Config]]("request-backoff").map(SimpleExponentialBackoff(_)).getOrElse(defaultRetryBackoff) + val backoff = + config.as[Option[Config]]("request-backoff").map(SimpleExponentialBackoff(_)).getOrElse(defaultRetryBackoff) val maxRetries = config.as[Option[Int]]("max-retries").getOrElse(defaultMaxRetries) val uri = config.as[Option[String]]("endpoint").flatMap(createAndValidateUri) @@ -79,14 +89,13 @@ object WorkflowCallbackConfig extends LazyLogging { ) } - def createAndValidateUri(uriString: String): Option[URI] = { + def createAndValidateUri(uriString: String): Option[URI] = Try(new URI(uriString)) match { case Success(uri) => Option(uri) case Failure(err) => logger.warn(s"Failed to parse provided workflow callback URI (${uriString}): $err") None } - } } /** @@ -100,23 +109,22 @@ object WorkflowCallbackActor { uri: Option[String], terminalState: WorkflowState, workflowOutputs: CallOutputs, - failureMessage: List[String]) + failureMessage: List[String] + ) def props(serviceRegistryActor: ActorRef, callbackConfig: WorkflowCallbackConfig, httpClient: CallbackHttpHandler = CallbackHttpHandlerImpl - ) = Props( - new WorkflowCallbackActor( - serviceRegistryActor, - callbackConfig, - httpClient) + ) = Props( + new WorkflowCallbackActor(serviceRegistryActor, callbackConfig, httpClient) ).withDispatcher(IoDispatcher) } class WorkflowCallbackActor(serviceRegistryActor: ActorRef, config: WorkflowCallbackConfig, - httpClient: CallbackHttpHandler) - extends Actor with ActorLogging { + httpClient: CallbackHttpHandler +) extends Actor + with ActorLogging { // Create a dedicated thread pool for this actor so its damage is limited if we end up with // too many threads all taking a long time to do callbacks. If we're frequently saturating @@ -130,32 +138,50 @@ class WorkflowCallbackActor(serviceRegistryActor: ActorRef, case PerformCallbackCommand(workflowId, requestedCallbackUri, terminalState, outputs, failures) => // If no uri was provided to us here, fall back to the one in config. If there isn't // one there, do not perform a callback. - val callbackUri: Option[URI] = requestedCallbackUri.map(WorkflowCallbackConfig.createAndValidateUri).getOrElse(config.defaultUri) - callbackUri.map { uri => - performCallback(workflowId, uri, terminalState, outputs, failures) onComplete { - case Success(_) => - log.info(s"Successfully sent callback for workflow for workflow $workflowId in state $terminalState to $uri") - sendMetadata(workflowId, successful = true, uri) - case Failure(t) => - log.warning(s"Permanently failed to send callback for workflow $workflowId in state $terminalState to $uri: ${t.getMessage}") - sendMetadata(workflowId, successful = false, uri) + val callbackUri: Option[URI] = + requestedCallbackUri.map(WorkflowCallbackConfig.createAndValidateUri).getOrElse(config.defaultUri) + callbackUri + .map { uri => + performCallback(workflowId, uri, terminalState, outputs, failures) onComplete { + case Success(_) => + log.info( + s"Successfully sent callback for workflow for workflow $workflowId in state $terminalState to $uri" + ) + sendMetadata(workflowId, successful = true, uri) + case Failure(t) => + log.warning( + s"Permanently failed to send callback for workflow $workflowId in state $terminalState to $uri: ${t.getMessage}" + ) + sendMetadata(workflowId, successful = false, uri) + } } - }.getOrElse(()) + .getOrElse(()) case Broadcast(ShutdownCommand) | ShutdownCommand => context stop self case other => log.warning(s"WorkflowCallbackActor received an unexpected message: $other") } - private def makeHeaders: Future[List[HttpHeader]] = { - config.authMethod.toList.map(_.getAccessToken).map { - case Valid(header) => Future.successful(header) - case Invalid(err) => Future.failed(new RuntimeException(err.toString)) - } + private def makeHeaders: Future[List[HttpHeader]] = + config.authMethod.toList + .map(_.getAccessToken) + .map { + case Valid(header) => Future.successful(header) + case Invalid(err) => Future.failed(new RuntimeException(err.toString)) + } .map(t => t.map(t => RawHeader("Authorization", s"Bearer $t"))) .traverse(identity) - } - private def performCallback(workflowId: WorkflowId, callbackUri: URI, terminalState: WorkflowState, outputs: CallOutputs, failures: List[String]): Future[Done] = { - val callbackPostBody = CallbackMessage(workflowId.toString, terminalState.toString, outputs.outputs.map(entry => (entry._1.name, entry._2)), failures) + private def performCallback(workflowId: WorkflowId, + callbackUri: URI, + terminalState: WorkflowState, + outputs: CallOutputs, + failures: List[String] + ): Future[Done] = { + val callbackPostBody = CallbackMessage( + workflowId.toString, + terminalState.toString, + outputs.outputs.map(entry => (entry._1.identifier.fullyQualifiedName.value, entry._2)), + failures + ) for { entity <- Marshal(callbackPostBody).to[RequestEntity] headers <- makeHeaders @@ -164,7 +190,10 @@ class WorkflowCallbackActor(serviceRegistryActor: ActorRef, () => sendRequestOrFail(request), backoff = config.retryBackoff, maxRetries = Option(config.maxRetries), - onRetry = err => log.warning(s"Will retry after failure to send workflow callback for workflow $workflowId in state $terminalState to $callbackUri : $err") + onRetry = err => + log.warning( + s"Will retry after failure to send workflow callback for workflow $workflowId in state $terminalState to $callbackUri : $err" + ) ) result <- // Akka will get upset if we have a response body and leave it totally unread. @@ -174,15 +203,17 @@ class WorkflowCallbackActor(serviceRegistryActor: ActorRef, } private def sendRequestOrFail(request: HttpRequest): Future[HttpResponse] = - httpClient.sendRequest(request).flatMap(response => - if (response.status.isFailure()) { - response.entity.dataBytes.runFold(ByteString(""))(_ ++ _).map(_.utf8String) flatMap { errorBody => - Future.failed( - new RuntimeException(s"HTTP ${response.status.value}: $errorBody") - ) - } - } else Future.successful(response) - ) + httpClient + .sendRequest(request) + .flatMap(response => + if (response.status.isFailure()) { + response.entity.dataBytes.runFold(ByteString(""))(_ ++ _).map(_.utf8String) flatMap { errorBody => + Future.failed( + new RuntimeException(s"HTTP ${response.status.value}: $errorBody") + ) + } + } else Future.successful(response) + ) private def sendMetadata(workflowId: WorkflowId, successful: Boolean, uri: URI): Unit = { val events = List( @@ -210,7 +241,6 @@ trait CallbackHttpHandler { } object CallbackHttpHandlerImpl extends CallbackHttpHandler { - override def sendRequest(httpRequest: HttpRequest)(implicit actorSystem: ActorSystem): Future[HttpResponse] = { + override def sendRequest(httpRequest: HttpRequest)(implicit actorSystem: ActorSystem): Future[HttpResponse] = Http().singleRequest(httpRequest) - } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackJsonSupport.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackJsonSupport.scala index eb9b1c7ea9e..42eb4991c47 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackJsonSupport.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackJsonSupport.scala @@ -7,7 +7,8 @@ import wom.values.WomValue final case class CallbackMessage(workflowId: String, state: String, outputs: Map[String, WomValue], - failures: List[String]) + failures: List[String] +) object WorkflowCallbackJsonSupport extends DefaultJsonProtocol { implicit val callbackMessageFormat = jsonFormat4(CallbackMessage) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowFinalizationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowFinalizationActor.scala index 8731367cfb7..d94c8033584 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowFinalizationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowFinalizationActor.scala @@ -46,16 +46,18 @@ object WorkflowFinalizationActor { jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, initializationData: AllBackendInitializationData, - copyWorkflowOutputsActor: Option[Props]): Props = { - Props(new WorkflowFinalizationActor( - workflowDescriptor, - ioActor, - jobExecutionMap, - workflowOutputs, - initializationData, - copyWorkflowOutputsActor - )).withDispatcher(EngineDispatcher) - } + copyWorkflowOutputsActor: Option[Props] + ): Props = + Props( + new WorkflowFinalizationActor( + workflowDescriptor, + ioActor, + jobExecutionMap, + workflowOutputs, + initializationData, + copyWorkflowOutputsActor + ) + ).withDispatcher(EngineDispatcher) } case class WorkflowFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, @@ -63,8 +65,8 @@ case class WorkflowFinalizationActor(workflowDescriptor: EngineWorkflowDescripto jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, initializationData: AllBackendInitializationData, - copyWorkflowOutputsActorProps: Option[Props]) - extends WorkflowLifecycleActor[WorkflowFinalizationActorState] { + copyWorkflowOutputsActorProps: Option[Props] +) extends WorkflowLifecycleActor[WorkflowFinalizationActorState] { override lazy val workflowIdForLogging = workflowDescriptor.possiblyNotRootWorkflowId override lazy val rootWorkflowIdForLogging = workflowDescriptor.rootWorkflowId @@ -79,63 +81,72 @@ case class WorkflowFinalizationActor(workflowDescriptor: EngineWorkflowDescripto override def failureResponse(reasons: Seq[Throwable]) = WorkflowFinalizationFailedResponse(reasons) // If an engine or backend finalization actor (children of this actor) dies, send ourselves the failure and stop the child actor - override def supervisorStrategy = OneForOneStrategy() { - case failure => - self.tell(FinalizationFailed(failure), sender()) - Stop + override def supervisorStrategy = OneForOneStrategy() { case failure => + self.tell(FinalizationFailed(failure), sender()) + Stop } startWith(FinalizationPendingState, WorkflowLifecycleActorData.empty) - when(FinalizationPendingState) { - case Event(StartFinalizationCommand, _) => - val backendFinalizationActors = Try { - for { - (backend, calls) <- workflowDescriptor.backendAssignments.groupBy(_._2).safeMapValues(_.keySet) - props <- CromwellBackends.backendLifecycleFactoryActorByName(backend).map( - _.workflowFinalizationActorProps(workflowDescriptor.backendDescriptor, ioActor, calls, filterJobExecutionsForBackend(calls), workflowOutputs, initializationData.get(backend)) - ).valueOr(errors => throw AggregatedMessageException("Cannot validate backend factories", errors.toList)) - actor = context.actorOf(props, backend) - } yield actor - } - - val engineFinalizationActor = Try { copyWorkflowOutputsActorProps.map(context.actorOf(_, "CopyWorkflowOutputsActor")).toList } - - val allActors = for { - backendFinalizationActorsFromTry <- backendFinalizationActors - engineFinalizationActorFromTry <- engineFinalizationActor - } yield backendFinalizationActorsFromTry.toList ++ engineFinalizationActorFromTry - - allActors match { - case Failure(ex) => - sender() ! WorkflowFinalizationFailedResponse(Seq(ex)) - goto(WorkflowFinalizationFailedState) - case Success(actors) if actors.isEmpty => - sender() ! WorkflowFinalizationSucceededResponse - goto(FinalizationSucceededState) - case Success(actors) => - val actorSet = actors.toSet - actorSet.foreach(_ ! Finalize) - goto(FinalizationInProgressState) using stateData.withActors(actorSet) - case _ => - goto(WorkflowFinalizationFailedState) - } + when(FinalizationPendingState) { case Event(StartFinalizationCommand, _) => + val backendFinalizationActors = Try { + for { + (backend, calls) <- workflowDescriptor.backendAssignments.groupBy(_._2).safeMapValues(_.keySet) + props <- CromwellBackends + .backendLifecycleFactoryActorByName(backend) + .map( + _.workflowFinalizationActorProps(workflowDescriptor.backendDescriptor, + ioActor, + calls, + filterJobExecutionsForBackend(calls), + workflowOutputs, + initializationData.get(backend) + ) + ) + .valueOr(errors => throw AggregatedMessageException("Cannot validate backend factories", errors.toList)) + actor = context.actorOf(props, backend) + } yield actor + } + + val engineFinalizationActor = Try { + copyWorkflowOutputsActorProps.map(context.actorOf(_, "CopyWorkflowOutputsActor")).toList + } + + val allActors = for { + backendFinalizationActorsFromTry <- backendFinalizationActors + engineFinalizationActorFromTry <- engineFinalizationActor + } yield backendFinalizationActorsFromTry.toList ++ engineFinalizationActorFromTry + + allActors match { + case Failure(ex) => + sender() ! WorkflowFinalizationFailedResponse(Seq(ex)) + goto(WorkflowFinalizationFailedState) + case Success(actors) if actors.isEmpty => + sender() ! WorkflowFinalizationSucceededResponse + goto(FinalizationSucceededState) + case Success(actors) => + val actorSet = actors.toSet + actorSet.foreach(_ ! Finalize) + goto(FinalizationInProgressState) using stateData.withActors(actorSet) + case _ => + goto(WorkflowFinalizationFailedState) + } } // Only send to each backend the jobs that it executed - private def filterJobExecutionsForBackend(calls: Set[CommandCallNode]): JobExecutionMap = { - jobExecutionMap map { - case (wd, executedKeys) => wd -> (executedKeys filter { jobKey => calls.contains(jobKey.call) }) - } filter { - case (_, keys) => keys.nonEmpty + private def filterJobExecutionsForBackend(calls: Set[CommandCallNode]): JobExecutionMap = + jobExecutionMap map { case (wd, executedKeys) => + wd -> (executedKeys filter { jobKey => calls.contains(jobKey.call) }) + } filter { case (_, keys) => + keys.nonEmpty } - } when(FinalizationInProgressState) { case Event(FinalizationSuccess, stateData) => checkForDoneAndTransition(stateData.withSuccess(sender())) - case Event(FinalizationFailed(reason), stateData) => checkForDoneAndTransition(stateData.withFailure(sender(), reason)) + case Event(FinalizationFailed(reason), stateData) => + checkForDoneAndTransition(stateData.withFailure(sender(), reason)) } - when(FinalizationSucceededState) { FSM.NullFunction } - when(WorkflowFinalizationFailedState) { FSM.NullFunction } + when(FinalizationSucceededState)(FSM.NullFunction) + when(WorkflowFinalizationFailedState)(FSM.NullFunction) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/initialization/WorkflowInitializationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/initialization/WorkflowInitializationActor.scala index 2c0e8cb5fd3..6af8d7e6116 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/initialization/WorkflowInitializationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/initialization/WorkflowInitializationActor.scala @@ -12,7 +12,11 @@ import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.backend.CromwellBackends import cromwell.engine.workflow.lifecycle.WorkflowLifecycleActor._ import cromwell.engine.workflow.lifecycle.initialization.WorkflowInitializationActor._ -import cromwell.engine.workflow.lifecycle.{AbortableWorkflowLifecycleActor, EngineLifecycleActorAbortCommand, EngineLifecycleActorAbortedResponse} +import cromwell.engine.workflow.lifecycle.{ + AbortableWorkflowLifecycleActor, + EngineLifecycleActorAbortCommand, + EngineLifecycleActorAbortedResponse +} import scala.util.{Failure, Success, Try} @@ -22,7 +26,9 @@ object WorkflowInitializationActor { * States */ sealed trait WorkflowInitializationActorState extends WorkflowLifecycleActorState - sealed trait WorkflowInitializationActorTerminalState extends WorkflowInitializationActorState with WorkflowLifecycleActorTerminalState + sealed trait WorkflowInitializationActorTerminalState + extends WorkflowInitializationActorState + with WorkflowLifecycleActorTerminalState case object InitializationPendingState extends WorkflowInitializationActorState case object InitializationInProgressState extends WorkflowInitializationActorState @@ -41,25 +47,33 @@ object WorkflowInitializationActor { * Responses */ sealed trait WorkflowInitializationResponse - final case class WorkflowInitializationSucceededResponse(initializationData: AllBackendInitializationData) extends WorkflowLifecycleSuccessResponse with WorkflowInitializationResponse - case object WorkflowInitializationAbortedResponse extends EngineLifecycleActorAbortedResponse with WorkflowInitializationResponse - final case class WorkflowInitializationFailedResponse(reasons: Seq[Throwable]) extends WorkflowLifecycleFailureResponse with WorkflowInitializationResponse + final case class WorkflowInitializationSucceededResponse(initializationData: AllBackendInitializationData) + extends WorkflowLifecycleSuccessResponse + with WorkflowInitializationResponse + case object WorkflowInitializationAbortedResponse + extends EngineLifecycleActorAbortedResponse + with WorkflowInitializationResponse + final case class WorkflowInitializationFailedResponse(reasons: Seq[Throwable]) + extends WorkflowLifecycleFailureResponse + with WorkflowInitializationResponse def props(workflowIdForLogging: PossiblyNotRootWorkflowId, rootWorkflowIdForLogging: RootWorkflowId, workflowDescriptor: EngineWorkflowDescriptor, ioActor: ActorRef, serviceRegistryActor: ActorRef, - restarting: Boolean): Props = { - Props(new WorkflowInitializationActor( - workflowIdForLogging = workflowIdForLogging, - rootWorkflowIdForLogging = rootWorkflowIdForLogging, - workflowDescriptor = workflowDescriptor, - ioActor = ioActor, - serviceRegistryActor = serviceRegistryActor, - restarting = restarting - )).withDispatcher(EngineDispatcher) - } + restarting: Boolean + ): Props = + Props( + new WorkflowInitializationActor( + workflowIdForLogging = workflowIdForLogging, + rootWorkflowIdForLogging = rootWorkflowIdForLogging, + workflowDescriptor = workflowDescriptor, + ioActor = ioActor, + serviceRegistryActor = serviceRegistryActor, + restarting = restarting + ) + ).withDispatcher(EngineDispatcher) case class BackendActorAndBackend(actor: ActorRef, backend: String) } @@ -69,8 +83,8 @@ case class WorkflowInitializationActor(workflowIdForLogging: PossiblyNotRootWork workflowDescriptor: EngineWorkflowDescriptor, ioActor: ActorRef, serviceRegistryActor: ActorRef, - restarting: Boolean) - extends AbortableWorkflowLifecycleActor[WorkflowInitializationActorState] { + restarting: Boolean +) extends AbortableWorkflowLifecycleActor[WorkflowInitializationActorState] { startWith(InitializationPendingState, WorkflowLifecycleActorData.empty) val tag = self.path.name @@ -83,7 +97,9 @@ case class WorkflowInitializationActor(workflowIdForLogging: PossiblyNotRootWork override def successResponse(data: WorkflowLifecycleActorData) = { val actorsToBackends = backendActorsAndBackends.map(ab => ab.actor -> ab.backend).toMap val actorsToData = data.successes.map(ad => ad.actor -> ad.data).toMap - val allBackendInitializationData = AllBackendInitializationData(actorsToBackends collect { case (a, b) => b -> actorsToData(a) }) + val allBackendInitializationData = AllBackendInitializationData(actorsToBackends collect { case (a, b) => + b -> actorsToData(a) + }) WorkflowInitializationSucceededResponse(allBackendInitializationData) } override def failureResponse(reasons: Seq[Throwable]) = WorkflowInitializationFailedResponse(reasons) @@ -96,9 +112,17 @@ case class WorkflowInitializationActor(workflowIdForLogging: PossiblyNotRootWork val backendInitializationActors = Try { for { (backend, calls) <- workflowDescriptor.backendAssignments.groupBy(_._2).safeMapValues(_.keySet) - props <- CromwellBackends.backendLifecycleFactoryActorByName(backend).map(factory => - factory.workflowInitializationActorProps(workflowDescriptor.backendDescriptor, ioActor, calls, serviceRegistryActor, restarting) - ).valueOr(errors => throw AggregatedMessageException("Cannot validate backend factories", errors.toList)) + props <- CromwellBackends + .backendLifecycleFactoryActorByName(backend) + .map(factory => + factory.workflowInitializationActorProps(workflowDescriptor.backendDescriptor, + ioActor, + calls, + serviceRegistryActor, + restarting + ) + ) + .valueOr(errors => throw AggregatedMessageException("Cannot validate backend factories", errors.toList)) actor = context.actorOf(props, backend) } yield BackendActorAndBackend(actor, backend) } @@ -124,20 +148,24 @@ case class WorkflowInitializationActor(workflowIdForLogging: PossiblyNotRootWork } when(InitializationInProgressState) { - case Event(InitializationSuccess(initData), stateData) => checkForDoneAndTransition(stateData.withSuccess(sender(), initData)) - case Event(InitializationFailed(reason), stateData) => checkForDoneAndTransition(stateData.withFailure(sender(), reason)) + case Event(InitializationSuccess(initData), stateData) => + checkForDoneAndTransition(stateData.withSuccess(sender(), initData)) + case Event(InitializationFailed(reason), stateData) => + checkForDoneAndTransition(stateData.withFailure(sender(), reason)) case Event(EngineLifecycleActorAbortCommand, stateData) => stateData.actors foreach { _ ! BackendWorkflowInitializationActor.Abort } goto(InitializationAbortingState) } when(InitializationAbortingState) { - case Event(InitializationSuccess(initData), stateData) => checkForDoneAndTransition(stateData.withSuccess(sender(), initData)) - case Event(InitializationFailed(reason), stateData) => checkForDoneAndTransition(stateData.withFailure(sender(), reason)) + case Event(InitializationSuccess(initData), stateData) => + checkForDoneAndTransition(stateData.withSuccess(sender(), initData)) + case Event(InitializationFailed(reason), stateData) => + checkForDoneAndTransition(stateData.withFailure(sender(), reason)) case Event(BackendActorAbortedResponse, stateData) => checkForDoneAndTransition(stateData.withAborted(sender())) } - when(InitializationSucceededState) { FSM.NullFunction } - when(InitializationFailedState) { FSM.NullFunction } - when(InitializationsAbortedState) { FSM.NullFunction } + when(InitializationSucceededState)(FSM.NullFunction) + when(InitializationFailedState)(FSM.NullFunction) + when(InitializationsAbortedState)(FSM.NullFunction) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/materialization/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/materialization/MaterializeWorkflowDescriptorActor.scala index 70365819fe5..8e57c77f2de 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/materialization/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/materialization/MaterializeWorkflowDescriptorActor.scala @@ -2,6 +2,7 @@ package cromwell.engine.workflow.lifecycle.materialization import akka.actor.{ActorRef, FSM, LoggingFSM, Props, Status} import akka.pattern.pipe +import akka.util.Timeout import cats.data.EitherT._ import cats.data.NonEmptyList import cats.data.Validated.{Invalid, Valid} @@ -36,6 +37,7 @@ import cromwell.filesystems.gcs.batch.GcsBatchCommandBuilder import cromwell.languages.util.ImportResolver._ import cromwell.languages.util.LanguageFactoryUtil import cromwell.languages.{LanguageFactory, ValidatedWomNamespace} +import cromwell.services.auth.GithubAuthVendingSupport import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import eu.timepit.refined.refineV @@ -50,6 +52,7 @@ import wom.runtime.WomOutputRuntimeExtractor import wom.values.{WomString, WomValue} import scala.concurrent.Future +import scala.concurrent.duration.DurationInt import scala.language.postfixOps import scala.util.{Failure, Success, Try} @@ -63,10 +66,22 @@ object MaterializeWorkflowDescriptorActor { // exception if not initialized yet. private def cromwellBackends = CromwellBackends.instance.get - def props(serviceRegistryActor: ActorRef, workflowId: WorkflowId, cromwellBackends: => CromwellBackends = cromwellBackends, - importLocalFilesystem: Boolean, ioActorProxy: ActorRef, hogGroup: HogGroup): Props = { - Props(new MaterializeWorkflowDescriptorActor(serviceRegistryActor, workflowId, cromwellBackends, importLocalFilesystem, ioActorProxy, hogGroup)).withDispatcher(EngineDispatcher) - } + def props(serviceRegistryActor: ActorRef, + workflowId: WorkflowId, + cromwellBackends: => CromwellBackends = cromwellBackends, + importLocalFilesystem: Boolean, + ioActorProxy: ActorRef, + hogGroup: HogGroup + ): Props = + Props( + new MaterializeWorkflowDescriptorActor(serviceRegistryActor, + workflowId, + cromwellBackends, + importLocalFilesystem, + ioActorProxy, + hogGroup + ) + ).withDispatcher(EngineDispatcher) /* Commands @@ -75,15 +90,19 @@ object MaterializeWorkflowDescriptorActor { case class MaterializeWorkflowDescriptorCommand(workflowSourceFiles: WorkflowSourceFilesCollection, conf: Config, callCachingEnabled: Boolean, - invalidateBadCacheResults: Boolean) extends MaterializeWorkflowDescriptorActorMessage + invalidateBadCacheResults: Boolean + ) extends MaterializeWorkflowDescriptorActorMessage case object MaterializeWorkflowDescriptorAbortCommand /* Responses */ sealed trait WorkflowDescriptorMaterializationResult extends MaterializeWorkflowDescriptorActorMessage - case class MaterializeWorkflowDescriptorSuccessResponse(workflowDescriptor: EngineWorkflowDescriptor) extends WorkflowDescriptorMaterializationResult - case class MaterializeWorkflowDescriptorFailureResponse(reason: Throwable) extends Exception with WorkflowDescriptorMaterializationResult + case class MaterializeWorkflowDescriptorSuccessResponse(workflowDescriptor: EngineWorkflowDescriptor) + extends WorkflowDescriptorMaterializationResult + case class MaterializeWorkflowDescriptorFailureResponse(reason: Throwable) + extends Exception + with WorkflowDescriptorMaterializationResult /* States @@ -102,30 +121,30 @@ object MaterializeWorkflowDescriptorActor { private[lifecycle] def validateCallCachingMode(workflowOptions: WorkflowOptions, callCachingEnabled: Boolean, - invalidateBadCacheResults: Boolean): ErrorOr[CallCachingMode] = { + invalidateBadCacheResults: Boolean + ): ErrorOr[CallCachingMode] = { - def readOptionalOption(option: WorkflowOption): ErrorOr[Boolean] = { + def readOptionalOption(option: WorkflowOption): ErrorOr[Boolean] = workflowOptions.getBoolean(option.name) match { case Success(x) => x.validNel case Failure(_: OptionNotFoundException) => true.validNel case Failure(t) => t.getMessage.invalidNel } - } if (callCachingEnabled) { val readFromCache = readOptionalOption(ReadFromCache) val writeToCache = readOptionalOption(WriteToCache) - def errorOrCallCachingMode(callCachingOptions: CallCachingOptions): ErrorOr[CallCachingMode] = { + def errorOrCallCachingMode(callCachingOptions: CallCachingOptions): ErrorOr[CallCachingMode] = (readFromCache, writeToCache) mapN { case (false, false) => CallCachingOff case (true, false) => CallCachingActivity(ReadCache, callCachingOptions) case (false, true) => CallCachingActivity(WriteCache, callCachingOptions) case (true, true) => CallCachingActivity(ReadAndWriteCache, callCachingOptions) } - } - val errorOrMaybePrefixes: ErrorOr[Option[Vector[String]]] = workflowOptions.getVectorOfStrings("call_cache_hit_path_prefixes") + val errorOrMaybePrefixes: ErrorOr[Option[Vector[String]]] = + workflowOptions.getVectorOfStrings("call_cache_hit_path_prefixes") val errorOrCallCachingOptions: ErrorOr[CallCachingOptions] = errorOrMaybePrefixes.map { maybePrefixes => CallCachingOptions(invalidateBadCacheResults, maybePrefixes) @@ -134,8 +153,7 @@ object MaterializeWorkflowDescriptorActor { options <- errorOrCallCachingOptions mode <- errorOrCallCachingMode(options) } yield mode - } - else { + } else { CallCachingOff.validNel } } @@ -144,34 +162,40 @@ object MaterializeWorkflowDescriptorActor { def validateMemoryRetryMultiplier(workflowOptions: WorkflowOptions): ErrorOr[Unit] = { val optionName = WorkflowOptions.MemoryRetryMultiplier.name - def refineMultiplier(value: Double): ErrorOr[Unit] = { + def refineMultiplier(value: Double): ErrorOr[Unit] = refineV[MemoryRetryMultiplier](value.toDouble) match { case Left(_) => s"Workflow option '$optionName' is invalid. It should be in the range 1.0 ≤ n ≤ 99.0".invalidNel case Right(_) => ().validNel } - } workflowOptions.get(optionName) match { - case Success(value) => Try(value.toDouble) match { - case Success(v) => refineMultiplier(v) - case Failure(e) => (s"Workflow option '$optionName' is invalid. It should be of type Double and in the range " + - s"1.0 ≤ n ≤ 99.0. Error: ${ExceptionUtils.getMessage(e)}").invalidNel - } + case Success(value) => + Try(value.toDouble) match { + case Success(v) => refineMultiplier(v) + case Failure(e) => + (s"Workflow option '$optionName' is invalid. It should be of type Double and in the range " + + s"1.0 ≤ n ≤ 99.0. Error: ${ExceptionUtils.getMessage(e)}").invalidNel + } case Failure(OptionNotFoundException(_)) => // This is an optional... option, so "not found" is fine ().validNel - case Failure(e) => s"'$optionName' is specified in workflow options but value is not of expected Double type: ${e.getMessage}".invalidNel + case Failure(e) => + s"'$optionName' is specified in workflow options but value is not of expected Double type: ${e.getMessage}".invalidNel } } } // TODO WOM: need to decide where to draw the line between language specific initialization and WOM -class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, +class MaterializeWorkflowDescriptorActor(override val serviceRegistryActor: ActorRef, workflowId: WorkflowId, cromwellBackends: => CromwellBackends, importLocalFilesystem: Boolean, ioActorProxy: ActorRef, - hogGroup: HogGroup) extends LoggingFSM[MaterializeWorkflowDescriptorActorState, Unit] with StrictLogging with WorkflowLogging { + hogGroup: HogGroup +) extends LoggingFSM[MaterializeWorkflowDescriptorActorState, Unit] + with StrictLogging + with WorkflowLogging + with GithubAuthVendingSupport { import MaterializeWorkflowDescriptorActor._ val tag = self.path.name @@ -184,29 +208,36 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, protected val pathBuilderFactories: List[PathBuilderFactory] = EngineFilesystems.configuredPathBuilderFactories - startWith(ReadyToMaterializeState, ()) + final private val githubAuthVendingTimeout = Timeout(60.seconds) + startWith(ReadyToMaterializeState, ()) when(ReadyToMaterializeState) { - case Event(MaterializeWorkflowDescriptorCommand(workflowSourceFiles, conf, callCachingEnabled, invalidateBadCacheResults), _) => + case Event(MaterializeWorkflowDescriptorCommand(workflowSourceFiles, + conf, + callCachingEnabled, + invalidateBadCacheResults + ), + _ + ) => val replyTo = sender() workflowOptionsAndPathBuilders(workflowSourceFiles) match { case (workflowOptions, pathBuilders) => val futureDescriptor: Future[ErrorOr[EngineWorkflowDescriptor]] = pathBuilders flatMap { pb => - val engineIoFunctions = new EngineIoFunctions(pb, new AsyncIo(ioActorProxy, GcsBatchCommandBuilder), iOExecutionContext) - buildWorkflowDescriptor( - workflowId, - workflowSourceFiles, - conf, - callCachingEnabled, - invalidateBadCacheResults, - workflowOptions, - pb, - engineIoFunctions) - .value - .unsafeToFuture(). - map(_.toValidated) + val engineIoFunctions = + new EngineIoFunctions(pb, new AsyncIo(ioActorProxy, GcsBatchCommandBuilder), iOExecutionContext) + buildWorkflowDescriptor(workflowId, + workflowSourceFiles, + conf, + callCachingEnabled, + invalidateBadCacheResults, + workflowOptions, + pb, + engineIoFunctions + ).value + .unsafeToFuture() + .map(_.toValidated) } // Pipe the response to self, but make it look like it comes from the sender of the command @@ -223,18 +254,21 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, case Event(Valid(descriptor: EngineWorkflowDescriptor), _) => sender() ! MaterializeWorkflowDescriptorSuccessResponse(descriptor) goto(MaterializationSuccessfulState) - case Event(Invalid(error: NonEmptyList[String]@unchecked), _) => + case Event(Invalid(error: NonEmptyList[String] @unchecked), _) => workflowInitializationFailed(error, sender()) goto(MaterializationFailedState) case Event(Status.Failure(failure), _) => - workflowInitializationFailed(NonEmptyList.of(failure.getMessage, failure.getStackTrace.toList.map(_.toString):_*), sender()) + workflowInitializationFailed( + NonEmptyList.of(failure.getMessage, failure.getStackTrace.toList.map(_.toString): _*), + sender() + ) goto(MaterializationFailedState) } // Let these fall through to the whenUnhandled handler: - when(MaterializationSuccessfulState) { FSM.NullFunction } - when(MaterializationFailedState) { FSM.NullFunction } - when(MaterializationAbortedState) { FSM.NullFunction } + when(MaterializationSuccessfulState)(FSM.NullFunction) + when(MaterializationFailedState)(FSM.NullFunction) + when(MaterializationAbortedState)(FSM.NullFunction) onTransition { case oldState -> terminalState if terminalState.terminal => @@ -252,17 +286,18 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, stay() } - private def workflowInitializationFailed(errors: NonEmptyList[String], replyTo: ActorRef) = { - sender() ! MaterializeWorkflowDescriptorFailureResponse( - new IllegalArgumentException with MessageAggregation { - val exceptionContext = "Workflow input processing failed" - val errorMessages = errors.toList - }) - } + private def workflowInitializationFailed(errors: NonEmptyList[String], replyTo: ActorRef) = + sender() ! MaterializeWorkflowDescriptorFailureResponse(new IllegalArgumentException with MessageAggregation { + val exceptionContext = "Workflow input processing failed" + val errorMessages = errors.toList + }) - private def workflowOptionsAndPathBuilders(sourceFiles: WorkflowSourceFilesCollection): (WorkflowOptions, Future[List[PathBuilder]]) = { + private def workflowOptionsAndPathBuilders( + sourceFiles: WorkflowSourceFilesCollection + ): (WorkflowOptions, Future[List[PathBuilder]]) = { sourceFiles.workflowOptions - val pathBuilders = EngineFilesystems.pathBuildersForWorkflow(sourceFiles.workflowOptions, pathBuilderFactories)(context.system) + val pathBuilders = + EngineFilesystems.pathBuildersForWorkflow(sourceFiles.workflowOptions, pathBuilderFactories)(context.system) (sourceFiles.workflowOptions, pathBuilders) } @@ -273,7 +308,8 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, invalidateBadCacheResults: Boolean, workflowOptions: WorkflowOptions, pathBuilders: List[PathBuilder], - engineIoFunctions: EngineIoFunctions): IOChecked[EngineWorkflowDescriptor] = { + engineIoFunctions: EngineIoFunctions + ): IOChecked[EngineWorkflowDescriptor] = { def findFactory(workflowSource: WorkflowSource): ErrorOr[LanguageFactory] = { @@ -287,7 +323,10 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, factory } - def buildValidatedNamespace(factory: LanguageFactory, workflowSource: WorkflowSource, importResolvers: List[ImportResolver]): IOChecked[ValidatedWomNamespace] = { + def buildValidatedNamespace(factory: LanguageFactory, + workflowSource: WorkflowSource, + importResolvers: List[ImportResolver] + ): IOChecked[ValidatedWomNamespace] = factory.validateNamespace( sourceFiles, workflowSource, @@ -297,32 +336,47 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, engineIoFunctions, importResolvers ) - } val localFilesystemResolvers = if (importLocalFilesystem) DirectoryResolver.localFilesystemResolvers(None) else List.empty - val zippedResolverCheck: IOChecked[Option[DirectoryResolver]] = fromEither[IO](sourceFiles.importsZipFileOption match { - case None => None.validNelCheck - case Some(zipContent) => zippedImportResolver(zipContent, workflowId).toEither.map(Option.apply) - }) + val zippedResolverCheck: IOChecked[Option[DirectoryResolver]] = + fromEither[IO](sourceFiles.importsZipFileOption match { + case None => None.validNelCheck + case Some(zipContent) => zippedImportResolver(zipContent, workflowId).toEither.map(Option.apply) + }) val labels = convertJsonToLabels(sourceFiles.labelsJson) for { _ <- publishLabelsToMetadata(id, labels.asMap, serviceRegistryActor) zippedImportResolver <- zippedResolverCheck - importResolvers = zippedImportResolver.toList ++ localFilesystemResolvers :+ HttpResolver(None, Map.empty) - sourceAndResolvers <- fromEither[IO](LanguageFactoryUtil.findWorkflowSource(sourceFiles.workflowSource, sourceFiles.workflowUrl, importResolvers)) - _ = if(sourceFiles.workflowUrl.isDefined) publishWorkflowSourceToMetadata(id, sourceAndResolvers._1) + importAuthProviderOpt <- importAuthProvider(conf)(githubAuthVendingTimeout).toIOChecked + importResolvers = zippedImportResolver.toList ++ localFilesystemResolvers :+ HttpResolver( + None, + Map.empty, + importAuthProviderOpt.toList + ) + sourceAndResolvers <- fromEither[IO]( + LanguageFactoryUtil.findWorkflowSource(sourceFiles.workflowSource, sourceFiles.workflowUrl, importResolvers) + ) + _ = if (sourceFiles.workflowUrl.isDefined) publishWorkflowSourceToMetadata(id, sourceAndResolvers._1) factory <- findFactory(sourceAndResolvers._1).toIOChecked outputRuntimeExtractor <- factory.womOutputRuntimeExtractor.toValidated.toIOChecked validatedNamespace <- buildValidatedNamespace(factory, sourceAndResolvers._1, sourceAndResolvers._2) closeResult = sourceAndResolvers._2.traverse(_.cleanupIfNecessary()) _ = pushNamespaceMetadata(validatedNamespace) - ewd <- buildWorkflowDescriptor(id, validatedNamespace, workflowOptions, labels, conf, callCachingEnabled, - invalidateBadCacheResults, pathBuilders, outputRuntimeExtractor).toIOChecked + ewd <- buildWorkflowDescriptor(id, + validatedNamespace, + workflowOptions, + labels, + conf, + callCachingEnabled, + invalidateBadCacheResults, + pathBuilders, + outputRuntimeExtractor + ).toIOChecked } yield { closeResult match { case Valid(_) => () @@ -335,7 +389,10 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, } private def publishWorkflowSourceToMetadata(id: WorkflowId, workflowSource: WorkflowSource): Unit = { - val event = MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Workflow), MetadataValue(workflowSource)) + val event = MetadataEvent( + MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Workflow), + MetadataValue(workflowSource) + ) serviceRegistryActor ! PutMetadataAction(event) } @@ -347,7 +404,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, } private def pushLanguageToMetadata(languageName: String, languageVersion: String): Unit = { - val events = List ( + val events = List( MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.LanguageName), MetadataValue(languageName)), MetadataEvent( MetadataKey(workflowId, None, WorkflowMetadataKeys.LanguageVersionName), @@ -366,7 +423,9 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, case inputs => inputs flatMap { case (outputPort, womValue) => val inputName = outputPort.fullyQualifiedName - womValueToMetadataEvents(MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Inputs}:$inputName"), womValue) + womValueToMetadataEvents(MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Inputs}:$inputName"), + womValue + ) } } } @@ -385,28 +444,27 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, imported map { case (uri, value) => metadataEventForImportedFile(uri, value) } } - private def wfNameMetadata(name: String): MetadataEvent = { + private def wfNameMetadata(name: String): MetadataEvent = // Workflow name: MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.Name), MetadataValue(name)) - } - private def convertJsonToLabels(json: String): Labels = { + private def convertJsonToLabels(json: String): Labels = json.parseJson match { - case JsObject(inputs) => Labels(inputs.toVector.collect({ - case (key, JsString(value)) => Label(key, value) - })) + case JsObject(inputs) => + Labels(inputs.toVector.collect { case (key, JsString(value)) => + Label(key, value) + }) case _ => Labels(Vector.empty) } - } // Perform a fail-fast validation that the `use_reference_disks` workflow option is boolean if present. - private def validateUseReferenceDisks(workflowOptions: WorkflowOptions) : ErrorOr[Unit] = { + private def validateUseReferenceDisks(workflowOptions: WorkflowOptions): ErrorOr[Unit] = { val optionName = WorkflowOptions.UseReferenceDisks.name workflowOptions.getBoolean(optionName) match { case Success(_) => // If present must be boolean ().validNel - case Failure (OptionNotFoundException(_)) => + case Failure(OptionNotFoundException(_)) => // This is an optional... option, so "not found" is fine ().validNel case Failure(e) => @@ -423,29 +481,51 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, callCachingEnabled: Boolean, invalidateBadCacheResults: Boolean, pathBuilders: List[PathBuilder], - outputRuntimeExtractor: Option[WomOutputRuntimeExtractor]): ErrorOr[EngineWorkflowDescriptor] = { + outputRuntimeExtractor: Option[WomOutputRuntimeExtractor] + ): ErrorOr[EngineWorkflowDescriptor] = { val taskCalls = womNamespace.executable.graph.allNodes collect { case taskNode: CommandCallNode => taskNode } val defaultBackendName = conf.as[Option[String]]("backend.default") val failureModeValidation = validateWorkflowFailureMode(workflowOptions, conf) val backendAssignmentsValidation = validateBackendAssignments(taskCalls, workflowOptions, defaultBackendName) - val callCachingModeValidation = validateCallCachingMode(workflowOptions, callCachingEnabled, - invalidateBadCacheResults) + val callCachingModeValidation = + validateCallCachingMode(workflowOptions, callCachingEnabled, invalidateBadCacheResults) val useReferenceDisksValidation: ErrorOr[Unit] = validateUseReferenceDisks(workflowOptions) val memoryRetryMultiplierValidation: ErrorOr[Unit] = validateMemoryRetryMultiplier(workflowOptions) - (failureModeValidation, backendAssignmentsValidation, callCachingModeValidation, useReferenceDisksValidation, memoryRetryMultiplierValidation) mapN { - case (failureMode, backendAssignments, callCachingMode, _, _) => - val callable = womNamespace.executable.entryPoint - val backendDescriptor = BackendWorkflowDescriptor(id, callable, womNamespace.womValueInputs, workflowOptions, labels, hogGroup, List.empty, outputRuntimeExtractor) - EngineWorkflowDescriptor(callable, backendDescriptor, backendAssignments, failureMode, pathBuilders, callCachingMode) + (failureModeValidation, + backendAssignmentsValidation, + callCachingModeValidation, + useReferenceDisksValidation, + memoryRetryMultiplierValidation + ) mapN { case (failureMode, backendAssignments, callCachingMode, _, _) => + val callable = womNamespace.executable.entryPoint + val backendDescriptor = BackendWorkflowDescriptor(id, + callable, + womNamespace.womValueInputs, + workflowOptions, + labels, + hogGroup, + List.empty, + outputRuntimeExtractor + ) + EngineWorkflowDescriptor(callable, + backendDescriptor, + backendAssignments, + failureMode, + pathBuilders, + callCachingMode + ) } } - private def validateBackendAssignments(calls: Set[CommandCallNode], workflowOptions: WorkflowOptions, defaultBackendName: Option[String]): ErrorOr[Map[CommandCallNode, String]] = { + private def validateBackendAssignments(calls: Set[CommandCallNode], + workflowOptions: WorkflowOptions, + defaultBackendName: Option[String] + ): ErrorOr[Map[CommandCallNode, String]] = { val callToBackendMap = Try { calls map { call => val backendPriorities = Seq( @@ -456,7 +536,10 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, backendPriorities.flatten.headOption match { case Some(backendName) if cromwellBackends.isValidBackendName(backendName) => call -> backendName - case Some(backendName) => throw new Exception(s"Backend for call ${call.fullyQualifiedName} ('$backendName') not registered in configuration file") + case Some(backendName) => + throw new Exception( + s"Backend for call ${call.fullyQualifiedName} ('$backendName') not registered in configuration file" + ) case None => throw new Exception(s"No backend could be found for call ${call.fullyQualifiedName}") } } toMap @@ -464,7 +547,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, callToBackendMap match { case Success(backendMap) => - val backendMapAsString = backendMap.map({case (k, v) => s"${k.fullyQualifiedName} -> $v"}).mkString(", ") + val backendMapAsString = backendMap.map { case (k, v) => s"${k.fullyQualifiedName} -> $v" }.mkString(", ") workflowLogger.info(s"Call-to-Backend assignments: $backendMapAsString") backendMap.validNel case Failure(t) => t.getMessage.invalidNel @@ -476,25 +559,34 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, */ private def assignBackendUsingRuntimeAttrs(call: CommandCallNode): Option[String] = { val runtimeAttributesMap = call.callable.runtimeAttributes.attributes - runtimeAttributesMap.get(RuntimeBackendKey) map { wdlExpr => evaluateBackendNameExpression(call.fullyQualifiedName, wdlExpr) } + runtimeAttributesMap.get(RuntimeBackendKey) map { wdlExpr => + evaluateBackendNameExpression(call.fullyQualifiedName, wdlExpr) + } } - private def evaluateBackendNameExpression(callName: String, backendNameAsExp: WomExpression): String = { + private def evaluateBackendNameExpression(callName: String, backendNameAsExp: WomExpression): String = backendNameAsExp.evaluateValue(Map.empty, NoIoFunctionSet) match { case Valid(runtimeString: WomString) => runtimeString.valueString case Valid(x: WomValue) => - throw new Exception(s"Non-string values are not currently supported for backends! Cannot use backend '${x.valueString}' to backend to Call: $callName") + throw new Exception( + s"Non-string values are not currently supported for backends! Cannot use backend '${x.valueString}' to backend to Call: $callName" + ) case Invalid(errors) => // TODO WOM: need access to a "source string" for WomExpressions // TODO WOM: ErrorOrify this ? - throw AggregatedMessageException(s"Dynamic backends are not currently supported! Cannot assign backend '$backendNameAsExp' for Call: $callName", errors.toList) + throw AggregatedMessageException( + s"Dynamic backends are not currently supported! Cannot assign backend '$backendNameAsExp' for Call: $callName", + errors.toList + ) } - } - private def validateWorkflowFailureMode(workflowOptions: WorkflowOptions, conf: Config): ErrorOr[WorkflowFailureMode] = { + private def validateWorkflowFailureMode(workflowOptions: WorkflowOptions, + conf: Config + ): ErrorOr[WorkflowFailureMode] = { val modeString: Try[String] = workflowOptions.get(WorkflowOptions.WorkflowFailureMode) match { case Success(x) => Success(x) - case Failure(_: OptionNotFoundException) => Success(conf.as[Option[String]]("workflow-options.workflow-failure-mode") getOrElse DefaultWorkflowFailureMode) + case Failure(_: OptionNotFoundException) => + Success(conf.as[Option[String]]("workflow-options.workflow-failure-mode") getOrElse DefaultWorkflowFailureMode) case Failure(t) => Failure(t) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/DynamicRateLimiter.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/DynamicRateLimiter.scala index 6f1c068af5a..16b3026fc16 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/tokens/DynamicRateLimiter.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/DynamicRateLimiter.scala @@ -28,9 +28,8 @@ trait DynamicRateLimiter { this: Actor with Timers with ActorLogging => timers.startPeriodicTimer(ResetKey, ResetAction, dispensingRate.per) } - private def releaseTokens() = { + private def releaseTokens() = self ! TokensAvailable(dispensingRate.n) - } // When load is high, freeze token distribution private def highLoad(doLogging: Boolean = true) = { diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobTokenDispenserActor.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobTokenDispenserActor.scala index e97ae9974f4..e201d0917ba 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobTokenDispenserActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobTokenDispenserActor.scala @@ -29,23 +29,25 @@ class JobTokenDispenserActor(override val serviceRegistryActor: ActorRef, override val dispensingRate: DynamicRateLimiter.Rate, logInterval: Option[FiniteDuration], dispenserType: String, - tokenAllocatedDescription: String) - extends Actor + tokenAllocatedDescription: String +) extends Actor with ActorLogging with JobInstrumentation with CromwellInstrumentationScheduler with Timers with DynamicRateLimiter - with CromwellInstrumentation -{ + with CromwellInstrumentation { // Metrics paths are based on the dispenser type private val tokenDispenserMetricsBasePath: NonEmptyList[String] = NonEmptyList.of("token_dispenser", dispenserType) - private val tokenLeaseDurationMetricPath: NonEmptyList[String] = tokenDispenserMetricsBasePath :+ "token_hold_duration" + private val tokenLeaseDurationMetricPath: NonEmptyList[String] = + tokenDispenserMetricsBasePath :+ "token_hold_duration" - private val tokenDispenserMetricsActivityRates: NonEmptyList[String] = tokenDispenserMetricsBasePath :+ "activity_rate" - private val requestsEnqueuedMetricPath: NonEmptyList[String] = tokenDispenserMetricsActivityRates :+ "requests_enqueued" + private val tokenDispenserMetricsActivityRates: NonEmptyList[String] = + tokenDispenserMetricsBasePath :+ "activity_rate" + private val requestsEnqueuedMetricPath: NonEmptyList[String] = + tokenDispenserMetricsActivityRates :+ "requests_enqueued" private val tokensLeasedMetricPath: NonEmptyList[String] = tokenDispenserMetricsActivityRates :+ "tokens_dispensed" private val tokensReturnedMetricPath: NonEmptyList[String] = tokenDispenserMetricsActivityRates :+ "tokens_returned" @@ -90,7 +92,8 @@ class JobTokenDispenserActor(override val serviceRegistryActor: ActorRef, super.preStart() } - override def receive: Actor.Receive = tokenDispensingReceive.orElse(rateReceive).orElse(instrumentationReceive(instrumentationAction)) + override def receive: Actor.Receive = + tokenDispensingReceive.orElse(rateReceive).orElse(instrumentationReceive(instrumentationAction)) private def tokenDispensingReceive: Receive = { case JobTokenRequest(hogGroup, tokenType) => enqueue(sender(), hogGroup.value, tokenType) @@ -111,7 +114,7 @@ class JobTokenDispenserActor(override val serviceRegistryActor: ActorRef, count(tokensReturnedMetricPath, 0L, ServicesPrefix) } - private def enqueue(sndr: ActorRef, hogGroup: String, tokenType: JobTokenType): Unit = { + private def enqueue(sndr: ActorRef, hogGroup: String, tokenType: JobTokenType): Unit = if (tokenAssignments.contains(sndr)) { sndr ! JobTokenDispensed } else { @@ -121,12 +124,12 @@ class JobTokenDispenserActor(override val serviceRegistryActor: ActorRef, increment(requestsEnqueuedMetricPath, ServicesPrefix) () } - } private def dispense(n: Int) = if (tokenQueues.nonEmpty) { // Sort by backend name to avoid re-ordering across iterations: - val iterator = new RoundRobinQueueIterator(tokenQueues.toList.sortBy(_._1.backend).map(_._2), currentTokenQueuePointer) + val iterator = + new RoundRobinQueueIterator(tokenQueues.toList.sortBy(_._1.backend).map(_._2), currentTokenQueuePointer) // In rare cases, an abort might empty an inner queue between "available" and "dequeue", which could cause an // exception. @@ -139,11 +142,12 @@ class JobTokenDispenserActor(override val serviceRegistryActor: ActorRef, } if (nextTokens.nonEmpty) { - val hogGroupCounts = nextTokens.groupBy(t => t.queuePlaceholder.hogGroup).map { case (hogGroup, list) => s"$hogGroup: ${list.size}" } + val hogGroupCounts = + nextTokens.groupBy(t => t.queuePlaceholder.hogGroup).map { case (hogGroup, list) => s"$hogGroup: ${list.size}" } log.info(s"Assigned new job $dispenserType tokens to the following groups: ${hogGroupCounts.mkString(", ")}") } - nextTokens.foreach({ + nextTokens.foreach { case LeasedActor(queuePlaceholder, lease) if !tokenAssignments.contains(queuePlaceholder.actor) => tokenAssignments = tokenAssignments + (queuePlaceholder.actor -> TokenLeaseRecord(lease, OffsetDateTime.now())) incrementJob("Started") @@ -151,19 +155,21 @@ class JobTokenDispenserActor(override val serviceRegistryActor: ActorRef, queuePlaceholder.actor ! JobTokenDispensed // Only one token per actor, so if you've already got one, we don't need to use this new one: case LeasedActor(queuePlaceholder, lease) => - log.error(s"Programmer Error: Actor ${queuePlaceholder.actor.path} requested a job $dispenserType token more than once.") + log.error( + s"Programmer Error: Actor ${queuePlaceholder.actor.path} requested a job $dispenserType token more than once." + ) // Because this actor already has a lease assigned to it: // a) tell the actor that it has a lease // b) don't hold onto this new lease - release it and let some other actor take it instead queuePlaceholder.actor ! JobTokenDispensed lease.release() - }) + } tokenQueues = iterator.updatedQueues.map(queue => queue.tokenType -> queue).toMap currentTokenQueuePointer = iterator.updatedPointer } - private def release(actor: ActorRef): Unit = { + private def release(actor: ActorRef): Unit = tokenAssignments.get(actor) match { case Some(TokenLeaseRecord(leasedToken, timestamp)) => tokenAssignments -= actor @@ -175,7 +181,6 @@ class JobTokenDispenserActor(override val serviceRegistryActor: ActorRef, case None => log.error(s"Job {} token returned from incorrect actor: {}", dispenserType, actor.path.name) } - } private def onTerminate(terminee: ActorRef): Unit = { tokenAssignments.get(terminee) match { @@ -185,8 +190,8 @@ class JobTokenDispenserActor(override val serviceRegistryActor: ActorRef, case None => log.debug("Actor {} stopped before receiving a token, removing it from any queues if necessary", terminee) // This is a very inefficient way to remove the actor from the queue and can lead to very poor performance for a large queue and a large number of actors to remove - tokenQueues = tokenQueues map { - case (tokenType, tokenQueue) => tokenType -> tokenQueue.removeTokenlessActor(terminee) + tokenQueues = tokenQueues map { case (tokenType, tokenQueue) => + tokenType -> tokenQueue.removeTokenlessActor(terminee) } } context.unwatch(terminee) @@ -214,7 +219,9 @@ class JobTokenDispenserActor(override val serviceRegistryActor: ActorRef, } // Schedule the next log event: - context.system.scheduler.scheduleOnce(someInterval) { self ! LogJobTokenAllocation(someInterval) }(context.dispatcher) + context.system.scheduler.scheduleOnce(someInterval)(self ! LogJobTokenAllocation(someInterval))( + context.dispatcher + ) () } @@ -225,19 +232,23 @@ class JobTokenDispenserActor(override val serviceRegistryActor: ActorRef, so it's desirable that a group submitting to two or more backends pause workflow pickup globally when it exhausts tokens in one of the backends */ - private def tokenExhaustedGroups: ReplyLimitedGroups = { + private def tokenExhaustedGroups: ReplyLimitedGroups = ReplyLimitedGroups( tokenQueues.values.flatMap(_.eventLogger.tokenExhaustedGroups).toSet ) - } } object JobTokenDispenserActor { case object TokensTimerKey - def props(serviceRegistryActor: ActorRef, rate: DynamicRateLimiter.Rate, logInterval: Option[FiniteDuration], - dispenserType: String, tokenAllocatedDescription: String): Props = - Props(new JobTokenDispenserActor(serviceRegistryActor, rate, logInterval, dispenserType, tokenAllocatedDescription)).withDispatcher(EngineDispatcher) + def props(serviceRegistryActor: ActorRef, + rate: DynamicRateLimiter.Rate, + logInterval: Option[FiniteDuration], + dispenserType: String, + tokenAllocatedDescription: String + ): Props = + Props(new JobTokenDispenserActor(serviceRegistryActor, rate, logInterval, dispenserType, tokenAllocatedDescription)) + .withDispatcher(EngineDispatcher) case class JobTokenRequest(hogGroup: HogGroup, jobTokenType: JobTokenType) @@ -250,7 +261,11 @@ object JobTokenDispenserActor { implicit val tokenEncoder = deriveEncoder[JobTokenType] @JsonCodec(encodeOnly = true) - final case class TokenDispenserState(dispenserType: String, tokenTypes: Vector[TokenTypeState], pointer: Int, leased: Int) + final case class TokenDispenserState(dispenserType: String, + tokenTypes: Vector[TokenTypeState], + pointer: Int, + leased: Int + ) @JsonCodec(encodeOnly = true) final case class TokenTypeState(tokenType: JobTokenType, queue: TokenQueueState) diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/RoundRobinQueueIterator.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/RoundRobinQueueIterator.scala index 36e555be98b..472ca190912 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/tokens/RoundRobinQueueIterator.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/RoundRobinQueueIterator.scala @@ -7,7 +7,8 @@ import cromwell.engine.workflow.tokens.TokenQueue.{DequeueResult, LeasedActor} * It will keep rotating the list until it finds a queue with an element that can be dequeued. * If no queue can be dequeued, the iterator is empty. */ -final class RoundRobinQueueIterator(initialTokenQueue: List[TokenQueue], initialPointer: Int) extends Iterator[LeasedActor] { +final class RoundRobinQueueIterator(initialTokenQueue: List[TokenQueue], initialPointer: Int) + extends Iterator[LeasedActor] { // Assumes the number of queues won't change during iteration (it really shouldn't !) private val numberOfQueues = initialTokenQueue.size // Indicate the index of next queue to try to dequeue from. @@ -45,14 +46,14 @@ final class RoundRobinQueueIterator(initialTokenQueue: List[TokenQueue], initial val indexStream = ((pointer until numberOfQueues) ++ (0 until pointer)).to(LazyList) val dequeuedTokenStream = indexStream.map(index => tokenQueues(index).dequeue -> index) - val firstLeasedActor = dequeuedTokenStream.collectFirst({ + val firstLeasedActor = dequeuedTokenStream.collectFirst { case (DequeueResult(Some(dequeuedActor), newTokenQueue), index) => // Update the tokenQueues with the new queue tokenQueues = tokenQueues.updated(index, newTokenQueue) // Update the index. Add 1 to force trying all the queues as we call next, even if the first one is available pointer = (index + 1) % numberOfQueues dequeuedActor - }) + } firstLeasedActor } diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenEventLogger.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenEventLogger.scala index 7ee35f86cd5..e61c17c1949 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenEventLogger.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenEventLogger.scala @@ -22,29 +22,28 @@ case object NullTokenEventLogger extends TokenEventLogger { class CachingTokenEventLogger(cacheEntryTTL: FiniteDuration) extends TokenEventLogger { - private val groupCache = CacheBuilder.newBuilder() + private val groupCache = CacheBuilder + .newBuilder() .expireAfterWrite(cacheEntryTTL._1, cacheEntryTTL._2) .maximumSize(10000) .build[String, Object]() - override def flagTokenHog(hogGroup: String): Unit = { + override def flagTokenHog(hogGroup: String): Unit = groupCache.put(hogGroup, new Object()) - } override def tokenExhaustedGroups: Set[String] = { import scala.jdk.CollectionConverters._ groupCache.asMap().keySet().asScala.toSet } - - private val backendCache = CacheBuilder.newBuilder() + private val backendCache = CacheBuilder + .newBuilder() .expireAfterWrite(cacheEntryTTL._1, cacheEntryTTL._2) .maximumSize(10000) .build[String, Object]() - override def outOfTokens(backend: String): Unit = { + override def outOfTokens(backend: String): Unit = backendCache.put(backend, new Object()) - } override def tokenExhaustedBackends: Set[String] = { import scala.jdk.CollectionConverters._ diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenQueue.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenQueue.scala index af31384f415..c16ae9dad91 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenQueue.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenQueue.scala @@ -18,7 +18,8 @@ import scala.collection.immutable.Queue final case class TokenQueue(queues: Map[String, Queue[TokenQueuePlaceholder]], queueOrder: Vector[String], eventLogger: TokenEventLogger, - private [tokens] val pool: UnhoggableTokenPool) extends StrictLogging { + private[tokens] val pool: UnhoggableTokenPool +) extends StrictLogging { val tokenType = pool.tokenType /** @@ -31,7 +32,7 @@ final case class TokenQueue(queues: Map[String, Queue[TokenQueuePlaceholder]], * * @return the new token queue */ - def enqueue(placeholder: TokenQueuePlaceholder): TokenQueue = { + def enqueue(placeholder: TokenQueuePlaceholder): TokenQueue = if (queues.contains(placeholder.hogGroup)) { this.copy( queues = queues + (placeholder.hogGroup -> queues(placeholder.hogGroup).enqueue(placeholder)) @@ -42,7 +43,6 @@ final case class TokenQueue(queues: Map[String, Queue[TokenQueuePlaceholder]], queueOrder = queueOrder :+ placeholder.hogGroup ) } - } /** * Returns a dequeue'd actor if one exists and there's a token available for it @@ -57,7 +57,10 @@ final case class TokenQueue(queues: Map[String, Queue[TokenQueuePlaceholder]], recursingDequeue(guaranteedNonEmptyQueues, Vector.empty, queueOrder) } - private def recursingDequeue(queues: Map[String, Queue[TokenQueuePlaceholder]], queuesTried: Vector[String], queuesRemaining: Vector[String]): DequeueResult = { + private def recursingDequeue(queues: Map[String, Queue[TokenQueuePlaceholder]], + queuesTried: Vector[String], + queuesRemaining: Vector[String] + ): DequeueResult = if (queuesRemaining.isEmpty) { DequeueResult(None, this) } else { @@ -69,7 +72,9 @@ final case class TokenQueue(queues: Map[String, Queue[TokenQueuePlaceholder]], if (oldQueue.isEmpty) { // We should have caught this above. But just in case: - logger.warn(s"Programmer error: Empty token queue value still present in TokenQueue: $hogGroup *and* made it through into recursiveDequeue(!): $hogGroup") + logger.warn( + s"Programmer error: Empty token queue value still present in TokenQueue: $hogGroup *and* made it through into recursiveDequeue(!): $hogGroup" + ) recursingDequeue(queues, queuesTried :+ hogGroup, remainingHogGroups) } else { leaseTry match { @@ -80,7 +85,9 @@ final case class TokenQueue(queues: Map[String, Queue[TokenQueuePlaceholder]], } else { (queues + (hogGroup -> newQueue), remainingHogGroups ++ queuesTried :+ hogGroup) } - DequeueResult(Option(LeasedActor(placeholder, thl)), TokenQueue(newQueues, newQueueOrder, eventLogger, pool)) + DequeueResult(Option(LeasedActor(placeholder, thl)), + TokenQueue(newQueues, newQueueOrder, eventLogger, pool) + ) case TokenTypeExhausted => // The pool is completely full right now, so there's no benefit trying the other hog groups: eventLogger.outOfTokens(tokenType.backend) @@ -91,7 +98,6 @@ final case class TokenQueue(queues: Map[String, Queue[TokenQueuePlaceholder]], } } } - } def removeTokenlessActor(actor: ActorRef): TokenQueue = { val actorRemovedQueues = queues.map { case (hogGroup, queue) => @@ -139,7 +145,8 @@ object TokenQueue { case class LeasedActor(queuePlaceholder: TokenQueuePlaceholder, lease: Lease[JobToken]) { def actor: ActorRef = queuePlaceholder.actor } - def apply(tokenType: JobTokenType, logger: TokenEventLogger) = new TokenQueue(Map.empty, Vector.empty, logger, new UnhoggableTokenPool(tokenType)) + def apply(tokenType: JobTokenType, logger: TokenEventLogger) = + new TokenQueue(Map.empty, Vector.empty, logger, new UnhoggableTokenPool(tokenType)) final case class TokenQueuePlaceholder(actor: ActorRef, hogGroup: String) @JsonCodec diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/UnhoggableTokenPool.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/UnhoggableTokenPool.scala index 761748abb49..e4c3885b60c 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/tokens/UnhoggableTokenPool.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/UnhoggableTokenPool.scala @@ -12,13 +12,15 @@ import io.github.andrebeat.pool._ import scala.collection.immutable.HashSet import scala.collection.mutable -final class UnhoggableTokenPool(val tokenType: JobTokenType) extends SimplePool[JobToken]( - capacity = tokenType.maxPoolSize.getOrElse(UnhoggableTokenPool.MaxCapacity), - referenceType = ReferenceType.Strong, - _factory = () => JobToken(tokenType, UUID.randomUUID()), - _reset = Function.const(()), - _dispose = Function.const(()), - _healthCheck = Function.const(true)) { +final class UnhoggableTokenPool(val tokenType: JobTokenType) + extends SimplePool[JobToken]( + capacity = tokenType.maxPoolSize.getOrElse(UnhoggableTokenPool.MaxCapacity), + referenceType = ReferenceType.Strong, + _factory = () => JobToken(tokenType, UUID.randomUUID()), + _reset = Function.const(()), + _dispose = Function.const(()), + _healthCheck = Function.const(true) + ) { lazy val hogLimitOption: Option[Int] = tokenType match { case JobTokenType(_, Some(limit), hogFactor) if hogFactor > 1 => @@ -28,10 +30,11 @@ final class UnhoggableTokenPool(val tokenType: JobTokenType) extends SimplePool[ private[this] val hogGroupAssignments: mutable.Map[String, HashSet[JobToken]] = mutable.Map.empty - override def tryAcquire(): Option[Lease[JobToken]] = throw new UnsupportedOperationException("Use tryAcquire(hogGroup)") - - def available(hogGroup: String): UnhoggableTokenPoolAvailability = { + override def tryAcquire(): Option[Lease[JobToken]] = throw new UnsupportedOperationException( + "Use tryAcquire(hogGroup)" + ) + def available(hogGroup: String): UnhoggableTokenPoolAvailability = hogLimitOption match { case None if leased() < capacity => TokensAvailable case None => TokenTypeExhausted @@ -46,10 +49,8 @@ final class UnhoggableTokenPool(val tokenType: JobTokenType) extends SimplePool[ } } else TokenTypeExhausted } - } - - def tryAcquire(hogGroup: String): UnhoggableTokenPoolResult = { + def tryAcquire(hogGroup: String): UnhoggableTokenPoolResult = hogLimitOption match { case Some(hogLimit) => synchronized { @@ -75,9 +76,8 @@ final class UnhoggableTokenPool(val tokenType: JobTokenType) extends SimplePool[ case None => TokenTypeExhausted } } - } - def unhog(hogGroup: String, lease: Lease[JobToken]): Unit = { + def unhog(hogGroup: String, lease: Lease[JobToken]): Unit = hogLimitOption foreach { _ => synchronized { val newAssignment = hogGroupAssignments.getOrElse(hogGroup, HashSet.empty) - lease.get() @@ -89,15 +89,15 @@ final class UnhoggableTokenPool(val tokenType: JobTokenType) extends SimplePool[ } } } - } def poolState: TokenPoolState = { val (hogGroupUsages, hogLimitValue): (Option[Set[HogGroupState]], Option[Int]) = hogLimitOption match { case Some(hogLimit) => synchronized { - val entries: Set[HogGroupState] = hogGroupAssignments.toSet[(String, HashSet[JobToken])].map { case (hogGroup, set) => - HogGroupState(hogGroup, set.size, !hogGroupAssignments.get(hogGroup).forall(_.size < hogLimit)) - } + val entries: Set[HogGroupState] = + hogGroupAssignments.toSet[(String, HashSet[JobToken])].map { case (hogGroup, set) => + HogGroupState(hogGroup, set.size, !hogGroupAssignments.get(hogGroup).forall(_.size < hogLimit)) + } (Option(entries), Option(hogLimit)) } case None => (None, None) @@ -113,7 +113,9 @@ object UnhoggableTokenPool { sealed trait UnhoggableTokenPoolResult - final class TokenHoggingLease(lease: Lease[JobToken], hogGroup: String, pool: UnhoggableTokenPool) extends Lease[JobToken] with UnhoggableTokenPoolResult { + final class TokenHoggingLease(lease: Lease[JobToken], hogGroup: String, pool: UnhoggableTokenPool) + extends Lease[JobToken] + with UnhoggableTokenPoolResult { private[this] val dirty = new AtomicBoolean(false) override protected[this] def a: JobToken = lease.get() @@ -153,6 +155,11 @@ object UnhoggableTokenPool { final case class HogGroupState(hogGroup: String, used: Int, atLimit: Boolean) @JsonCodec - final case class TokenPoolState(hogGroups: Option[Set[HogGroupState]], hogLimit: Option[Int], capacity: Int, leased: Int, available: Boolean) + final case class TokenPoolState(hogGroups: Option[Set[HogGroupState]], + hogLimit: Option[Int], + capacity: Int, + leased: Int, + available: Boolean + ) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/AbortRequestScanningActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/AbortRequestScanningActor.scala index 1e565886941..6e8a8f96ea3 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/AbortRequestScanningActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/AbortRequestScanningActor.scala @@ -5,19 +5,26 @@ import com.google.common.cache.CacheBuilder import cromwell.core.{CacheConfig, WorkflowId} import cromwell.engine.workflow.WorkflowManagerActor import cromwell.engine.workflow.workflowstore.AbortRequestScanningActor.{AbortConfig, RunScan} -import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.{FindWorkflowsWithAbortRequested, FindWorkflowsWithAbortRequestedFailure, FindWorkflowsWithAbortRequestedSuccess} +import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.{ + FindWorkflowsWithAbortRequested, + FindWorkflowsWithAbortRequestedFailure, + FindWorkflowsWithAbortRequestedSuccess +} import scala.jdk.CollectionConverters._ import scala.concurrent.duration._ - class AbortRequestScanningActor(workflowStoreActor: ActorRef, workflowManagerActor: ActorRef, abortConfig: AbortConfig, - workflowHeartbeatConfig: WorkflowHeartbeatConfig) extends Actor with Timers with ActorLogging { + workflowHeartbeatConfig: WorkflowHeartbeatConfig +) extends Actor + with Timers + with ActorLogging { private val cache = { val cacheConfig = abortConfig.cacheConfig - CacheBuilder.newBuilder() + CacheBuilder + .newBuilder() .concurrencyLevel(cacheConfig.concurrency) .expireAfterWrite(cacheConfig.ttl.length, cacheConfig.ttl.unit) .build[WorkflowId, java.lang.Boolean]() @@ -52,6 +59,10 @@ object AbortRequestScanningActor { case object RunScan - def props(workflowStoreActor: ActorRef, workflowManagerActor: ActorRef, abortConfig: AbortConfig, workflowHeartbeatConfig: WorkflowHeartbeatConfig): Props = + def props(workflowStoreActor: ActorRef, + workflowManagerActor: ActorRef, + abortConfig: AbortConfig, + workflowHeartbeatConfig: WorkflowHeartbeatConfig + ): Props = Props(new AbortRequestScanningActor(workflowStoreActor, workflowManagerActor, abortConfig, workflowHeartbeatConfig)) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemorySubWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemorySubWorkflowStore.scala index c17a38b36fc..16557207b1b 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemorySubWorkflowStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemorySubWorkflowStore.scala @@ -14,7 +14,8 @@ class InMemorySubWorkflowStore(workflowStore: InMemoryWorkflowStore) extends Sub callFullyQualifiedName: String, jobIndex: Int, jobAttempt: Int, - subWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Unit] = { + subWorkflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Unit] = if (workflowStore.workflowStore.exists { case (wf, _) => wf.id.toString == rootWorkflowExecutionUuid }) { subWorkflowStore = subWorkflowStore + SubWorkflowStoreEntry( @@ -27,24 +28,24 @@ class InMemorySubWorkflowStore(workflowStore: InMemoryWorkflowStore) extends Sub ) Future.successful(()) } else Future.failed(new Throwable(s"No such root workflow: $rootWorkflowExecutionUuid")) - } override def querySubWorkflowStore(parentWorkflowExecutionUuid: String, callFqn: String, jobIndex: Int, - jobAttempt: Int)(implicit ec: ExecutionContext): Future[Option[SubWorkflowStoreEntry]] = { + jobAttempt: Int + )(implicit ec: ExecutionContext): Future[Option[SubWorkflowStoreEntry]] = Future.successful( - subWorkflowStore.find( - k => - k.parentWorkflowExecutionUuid == parentWorkflowExecutionUuid && - k.callFullyQualifiedName == callFqn && - k.callIndex == jobIndex && - k.callAttempt == jobAttempt + subWorkflowStore.find(k => + k.parentWorkflowExecutionUuid == parentWorkflowExecutionUuid && + k.callFullyQualifiedName == callFqn && + k.callIndex == jobIndex && + k.callAttempt == jobAttempt ) ) - } - override def removeSubWorkflowStoreEntries(parentWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Int] = { + override def removeSubWorkflowStoreEntries( + parentWorkflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Int] = { val toRemove = subWorkflowStore.filter(k => k.parentWorkflowExecutionUuid == parentWorkflowExecutionUuid) subWorkflowStore = subWorkflowStore -- toRemove Future.successful(toRemove.size) diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala index b6913f6172c..ebce8c92720 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala @@ -5,7 +5,11 @@ import cats.data.NonEmptyList import cromwell.core.{HogGroup, WorkflowId, WorkflowSourceFilesCollection} import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.WorkflowStoreAbortResponse.WorkflowStoreAbortResponse import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.WorkflowStoreState.WorkflowStoreState -import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.{WorkflowStoreAbortResponse, WorkflowStoreState, WorkflowSubmissionResponse} +import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.{ + WorkflowStoreAbortResponse, + WorkflowStoreState, + WorkflowSubmissionResponse +} import scala.concurrent.duration.FiniteDuration import scala.concurrent.{ExecutionContext, Future} @@ -18,12 +22,15 @@ class InMemoryWorkflowStore extends WorkflowStore { * Adds the requested WorkflowSourceFiles to the store and returns a WorkflowId for each one (in order) * for tracking purposes. */ - override def add(sources: NonEmptyList[WorkflowSourceFilesCollection])(implicit ec: ExecutionContext): Future[NonEmptyList[WorkflowSubmissionResponse]] = { - val actualWorkflowState = if (sources.head.workflowOnHold) WorkflowStoreState.OnHold else WorkflowStoreState.Submitted + override def add( + sources: NonEmptyList[WorkflowSourceFilesCollection] + )(implicit ec: ExecutionContext): Future[NonEmptyList[WorkflowSubmissionResponse]] = { + val actualWorkflowState = + if (sources.head.workflowOnHold) WorkflowStoreState.OnHold else WorkflowStoreState.Submitted val addedWorkflows = sources map { WorkflowIdAndSources(WorkflowId.randomId(), _) -> actualWorkflowState } workflowStore ++= addedWorkflows.toList.toMap - Future.successful(addedWorkflows map { - case (WorkflowIdAndSources(id, _), _) => WorkflowSubmissionResponse(actualWorkflowState, id) + Future.successful(addedWorkflows map { case (WorkflowIdAndSources(id, _), _) => + WorkflowSubmissionResponse(actualWorkflowState, id) }) } @@ -31,9 +38,15 @@ class InMemoryWorkflowStore extends WorkflowStore { * Retrieves up to n workflows which have not already been pulled into the engine and sets their pickedUp * flag to true */ - override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = { + override def fetchStartableWorkflows(n: Int, + cromwellId: String, + heartbeatTtl: FiniteDuration, + excludedGroups: Set[String] + )(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = { if (excludedGroups.nonEmpty) - throw new UnsupportedOperationException("Programmer Error: group filtering not supported for single-tenant/in-memory workflow store") + throw new UnsupportedOperationException( + "Programmer Error: group filtering not supported for single-tenant/in-memory workflow store" + ) val startableWorkflows = workflowStore filter { _._2 == WorkflowStoreState.Submitted } take n val updatedWorkflows = startableWorkflows map { _._1 -> WorkflowStoreState.Running } @@ -51,17 +64,18 @@ class InMemoryWorkflowStore extends WorkflowStore { override def initialize(implicit ec: ExecutionContext): Future[Unit] = Future.successful(()) - override def stats(implicit ec: ExecutionContext): Future[Map[WorkflowStoreState, Int]] = Future.successful(Map(WorkflowStoreState.Submitted -> workflowStore.size)) + override def stats(implicit ec: ExecutionContext): Future[Map[WorkflowStoreState, Int]] = + Future.successful(Map(WorkflowStoreState.Submitted -> workflowStore.size)) override def abortAllRunning()(implicit ec: ExecutionContext): Future[Unit] = { - workflowStore = workflowStore.map({ + workflowStore = workflowStore.map { case (workflow, WorkflowStoreState.Running) => workflow -> WorkflowStoreState.Aborting case (workflow, state) => workflow -> state - }) + } Future.successful(()) } - override def abort(id: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowStoreAbortResponse] = { + override def abort(id: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowStoreAbortResponse] = workflowStore collectFirst { case (workflowIdAndSources, workflowStoreState) if workflowIdAndSources.id == id => (workflowIdAndSources, workflowStoreState) @@ -76,21 +90,24 @@ class InMemoryWorkflowStore extends WorkflowStore { case None => Future.successful(WorkflowStoreAbortResponse.NotFound) } - } override def writeWorkflowHeartbeats(workflowIds: Set[(WorkflowId, OffsetDateTime)], - heartbeatDateTime: OffsetDateTime) - (implicit ec: ExecutionContext): Future[Int] = { + heartbeatDateTime: OffsetDateTime + )(implicit ec: ExecutionContext): Future[Int] = Future.successful(workflowIds.size) - } - override def switchOnHoldToSubmitted(id: WorkflowId)(implicit ec: ExecutionContext): Future[Unit] = Future.successful(()) + override def switchOnHoldToSubmitted(id: WorkflowId)(implicit ec: ExecutionContext): Future[Unit] = + Future.successful(()) - override def findWorkflowsWithAbortRequested(cromwellId: String)(implicit ec: ExecutionContext): Future[Iterable[WorkflowId]] = Future.successful(List.empty) + override def findWorkflowsWithAbortRequested(cromwellId: String)(implicit + ec: ExecutionContext + ): Future[Iterable[WorkflowId]] = Future.successful(List.empty) - override def findWorkflows(cromwellId: String)(implicit ec: ExecutionContext): Future[Iterable[WorkflowId]] = Future.successful(workflowStore.keys.map(_.id)) + override def findWorkflows(cromwellId: String)(implicit ec: ExecutionContext): Future[Iterable[WorkflowId]] = + Future.successful(workflowStore.keys.map(_.id)) - override def deleteFromStore(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[Int] = Future.successful(0) + override def deleteFromStore(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[Int] = + Future.successful(0) } final case class WorkflowIdAndSources(id: WorkflowId, sources: WorkflowSourceFilesCollection) diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala index b36cb008933..2243ff32573 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala @@ -13,7 +13,14 @@ import cromwell.database.sql.{MetadataSqlDatabase, WorkflowStoreSqlDatabase} import cromwell.database.sql.tables.WorkflowStoreEntry import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.WorkflowStoreAbortResponse.WorkflowStoreAbortResponse import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.WorkflowStoreState.WorkflowStoreState -import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.{DuplicateWorkflowIdsRequested, NotInOnHoldStateException, WorkflowIdsAlreadyInUseException, WorkflowStoreAbortResponse, WorkflowStoreState, WorkflowSubmissionResponse} +import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.{ + DuplicateWorkflowIdsRequested, + NotInOnHoldStateException, + WorkflowIdsAlreadyInUseException, + WorkflowStoreAbortResponse, + WorkflowStoreState, + WorkflowSubmissionResponse +} import eu.timepit.refined.api.Refined import eu.timepit.refined.collection._ @@ -23,17 +30,17 @@ import scala.concurrent.{ExecutionContext, Future} object SqlWorkflowStore { case class WorkflowSubmissionResponse(state: WorkflowStoreState, id: WorkflowId) - case class DuplicateWorkflowIdsRequested(workflowIds: Seq[WorkflowId]) extends - Exception (s"Requested workflow IDs are duplicated: ${workflowIds.mkString(", ")}") + case class DuplicateWorkflowIdsRequested(workflowIds: Seq[WorkflowId]) + extends Exception(s"Requested workflow IDs are duplicated: ${workflowIds.mkString(", ")}") - case class WorkflowIdsAlreadyInUseException(workflowIds: Seq[WorkflowId]) extends - Exception (s"Requested workflow IDs are already in use: ${workflowIds.mkString(", ")}") + case class WorkflowIdsAlreadyInUseException(workflowIds: Seq[WorkflowId]) + extends Exception(s"Requested workflow IDs are already in use: ${workflowIds.mkString(", ")}") - case class NotInOnHoldStateException(workflowId: WorkflowId) extends - Exception( - s"Couldn't change status of workflow $workflowId to " + - "'Submitted' because the workflow is not in 'On Hold' state" - ) + case class NotInOnHoldStateException(workflowId: WorkflowId) + extends Exception( + s"Couldn't change status of workflow $workflowId to " + + "'Submitted' because the workflow is not in 'On Hold' state" + ) object WorkflowStoreState extends Enumeration { type WorkflowStoreState = Value @@ -51,12 +58,14 @@ object SqlWorkflowStore { } } -case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase, metadataSqlDatabase: MetadataSqlDatabase) extends WorkflowStore { +case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase, metadataSqlDatabase: MetadataSqlDatabase) + extends WorkflowStore { + /** This is currently hardcoded to success but used to do stuff, left in place for now as a useful * startup initialization hook. */ override def initialize(implicit ec: ExecutionContext): Future[Unit] = Future.successful(()) - override def abort(id: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowStoreAbortResponse] = { + override def abort(id: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowStoreAbortResponse] = sqlDatabase.deleteOrUpdateWorkflowToState( workflowExecutionUuid = id.toString, workflowStateToDelete1 = WorkflowStoreState.OnHold.toString, @@ -70,33 +79,34 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase, metadataSqlDa case None => WorkflowStoreAbortResponse.NotFound } - } - override def findWorkflows(cromwellId: String)(implicit ec: ExecutionContext): Future[Iterable[WorkflowId]] = { + override def findWorkflows(cromwellId: String)(implicit ec: ExecutionContext): Future[Iterable[WorkflowId]] = sqlDatabase.findWorkflows(cromwellId) map { _ map WorkflowId.fromString } - } - override def findWorkflowsWithAbortRequested(cromwellId: String)(implicit ec: ExecutionContext): Future[Iterable[WorkflowId]] = { + override def findWorkflowsWithAbortRequested(cromwellId: String)(implicit + ec: ExecutionContext + ): Future[Iterable[WorkflowId]] = sqlDatabase.findWorkflowsWithAbortRequested(cromwellId) map { _ map WorkflowId.fromString } - } - override def abortAllRunning()(implicit ec: ExecutionContext): Future[Unit] = { + override def abortAllRunning()(implicit ec: ExecutionContext): Future[Unit] = sqlDatabase.setStateToState(WorkflowStoreState.Running.toString, WorkflowStoreState.Aborting.toString) - } - override def stats(implicit ec: ExecutionContext): Future[Map[WorkflowStoreState, Int]] = { + override def stats(implicit ec: ExecutionContext): Future[Map[WorkflowStoreState, Int]] = sqlDatabase.workflowStateCounts.map { - _ map { - case (key, value) => WorkflowStoreState.withName(key) -> value + _ map { case (key, value) => + WorkflowStoreState.withName(key) -> value } } - } /** * Retrieves up to n workflows which have not already been pulled into the engine and sets their pickedUp * flag to true */ - override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = { + override def fetchStartableWorkflows(n: Int, + cromwellId: String, + heartbeatTtl: FiniteDuration, + excludedGroups: Set[String] + )(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = { import cats.syntax.traverse._ import common.validation.Validation._ sqlDatabase.fetchWorkflowsInState( @@ -115,38 +125,44 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase, metadataSqlDa } override def writeWorkflowHeartbeats(workflowIds: Set[(WorkflowId, OffsetDateTime)], - heartbeatDateTime: OffsetDateTime) - (implicit ec: ExecutionContext): Future[Int] = { - val sortedWorkflowIds = workflowIds.toList sortBy(_._2) map (_._1.toString) + heartbeatDateTime: OffsetDateTime + )(implicit ec: ExecutionContext): Future[Int] = { + val sortedWorkflowIds = workflowIds.toList sortBy (_._2) map (_._1.toString) sqlDatabase.writeWorkflowHeartbeats(sortedWorkflowIds, heartbeatDateTime.toSystemTimestamp) } - def workflowAlreadyExists(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[Boolean] = { - Future.sequence(Seq( - sqlDatabase.checkWhetherWorkflowExists(workflowId.id.toString), - metadataSqlDatabase.getWorkflowStatus(workflowId.id.toString).map(_.nonEmpty) - )).map(_.exists(_ == true)) - } - - def findPreexistingWorkflowIds(workflowIds: Seq[WorkflowId])(implicit ec: ExecutionContext): Future[Seq[WorkflowId]] = { - Future.sequence(workflowIds.map(wfid => { - workflowAlreadyExists(wfid).map { - case true => Option(wfid) - case false => None - } - })).map { _.collect { case Some(existingId) => existingId } } - } + def workflowAlreadyExists(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[Boolean] = + Future + .sequence( + Seq( + sqlDatabase.checkWhetherWorkflowExists(workflowId.id.toString), + metadataSqlDatabase.getWorkflowStatus(workflowId.id.toString).map(_.nonEmpty) + ) + ) + .map(_.exists(_ == true)) + + def findPreexistingWorkflowIds(workflowIds: Seq[WorkflowId])(implicit ec: ExecutionContext): Future[Seq[WorkflowId]] = + Future + .sequence(workflowIds.map { wfid => + workflowAlreadyExists(wfid).map { + case true => Option(wfid) + case false => None + } + }) + .map(_.collect { case Some(existingId) => existingId }) /** * Adds the requested WorkflowSourceFiles to the store and returns a WorkflowId for each one (in order) * for tracking purposes. */ - override def add(sources: NonEmptyList[WorkflowSourceFilesCollection])(implicit ec: ExecutionContext): Future[NonEmptyList[WorkflowSubmissionResponse]] = { + override def add( + sources: NonEmptyList[WorkflowSourceFilesCollection] + )(implicit ec: ExecutionContext): Future[NonEmptyList[WorkflowSubmissionResponse]] = { val requestedWorkflowIds = sources.map(_.requestedWorkflowId).collect { case Some(id) => id } val duplicatedIds = requestedWorkflowIds.diff(requestedWorkflowIds.toSet.toSeq) - if(duplicatedIds.nonEmpty) { + if (duplicatedIds.nonEmpty) { Future.failed(DuplicateWorkflowIdsRequested(duplicatedIds)) } else { findPreexistingWorkflowIds(requestedWorkflowIds) flatMap { preexistingIds => @@ -168,7 +184,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase, metadataSqlDa } } - override def switchOnHoldToSubmitted(id: WorkflowId)(implicit ec: ExecutionContext): Future[Unit] = { + override def switchOnHoldToSubmitted(id: WorkflowId)(implicit ec: ExecutionContext): Future[Unit] = for { updated <- sqlDatabase.updateWorkflowState( id.toString, @@ -177,19 +193,17 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase, metadataSqlDa ) _ <- if (updated == 0) Future.failed(NotInOnHoldStateException(id)) else Future.successful(()) } yield () - } - override def deleteFromStore(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[Int] = { + override def deleteFromStore(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[Int] = sqlDatabase.removeWorkflowStoreEntry(workflowId.toString) - } private def fromWorkflowStoreEntry(workflowStoreEntry: WorkflowStoreEntry): ErrorOr[WorkflowToStart] = { - val workflowOptionsValidation: ErrorOr[WorkflowOptions] = WorkflowOptions.fromJsonString(workflowStoreEntry.workflowOptions.toRawString).toErrorOr + val workflowOptionsValidation: ErrorOr[WorkflowOptions] = + WorkflowOptions.fromJsonString(workflowStoreEntry.workflowOptions.toRawString).toErrorOr val startableStateValidation = workflowStoreStateToStartableState(workflowStoreEntry) (startableStateValidation, workflowOptionsValidation) mapN { (startableState, workflowOptions) => - val id = WorkflowId.fromString(workflowStoreEntry.workflowExecutionUuid) val sources = WorkflowSourceFilesCollection( @@ -207,7 +221,8 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase, metadataSqlDa requestedWorkflowId = Option(id) ) - val hogGroup: HogGroup = workflowStoreEntry.hogGroup.map(HogGroup(_)).getOrElse(HogGroup.decide(workflowOptions, id)) + val hogGroup: HogGroup = + workflowStoreEntry.hogGroup.map(HogGroup(_)).getOrElse(HogGroup.decide(workflowOptions, id)) WorkflowToStart( id = id, @@ -219,16 +234,15 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase, metadataSqlDa } } - private def workflowSubmissionState(workflowSourceFiles: WorkflowSourceFilesCollection) = { + private def workflowSubmissionState(workflowSourceFiles: WorkflowSourceFilesCollection) = if (workflowSourceFiles.workflowOnHold) WorkflowStoreState.OnHold else WorkflowStoreState.Submitted - } private def toWorkflowStoreEntry(workflowSourceFiles: WorkflowSourceFilesCollection): WorkflowStoreEntry = { import eu.timepit.refined._ - val nonEmptyJsonString: String Refined NonEmpty = refineMV[NonEmpty]("{}") + val nonEmptyJsonString: String Refined NonEmpty = refineMV[NonEmpty]("{}") val actualWorkflowState = workflowSubmissionState(workflowSourceFiles) diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowHeartbeatConfig.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowHeartbeatConfig.scala index d27d6d5b1f6..99d35c8a02d 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowHeartbeatConfig.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowHeartbeatConfig.scala @@ -25,13 +25,13 @@ import scala.concurrent.duration._ * @param writeBatchSize The maximum size of a write batch. * @param writeThreshold The threshold of heartbeat writes above which load is considered high. */ -case class WorkflowHeartbeatConfig( - cromwellId: String, - heartbeatInterval: FiniteDuration, - ttl: FiniteDuration, - failureShutdownDuration: FiniteDuration, - writeBatchSize: Int, - writeThreshold: Int) { +case class WorkflowHeartbeatConfig(cromwellId: String, + heartbeatInterval: FiniteDuration, + ttl: FiniteDuration, + failureShutdownDuration: FiniteDuration, + writeBatchSize: Int, + writeThreshold: Int +) { override def toString: String = this.asInstanceOf[WorkflowHeartbeatConfig].asJson.spaces2 } @@ -41,14 +41,12 @@ object WorkflowHeartbeatConfig { // compiler flag settings then promote to an error. // NOTE: This is a different encoding than circe's finiteDurationEncoder: https://github.com/circe/circe/pull/978 - private[engine] implicit lazy val encodeFiniteDuration: Encoder[FiniteDuration] = { + implicit private[engine] lazy val encodeFiniteDuration: Encoder[FiniteDuration] = Encoder.encodeString.contramap(_.toString) - } - private[engine] implicit lazy val encodeWorkflowHeartbeatConfig: Encoder[WorkflowHeartbeatConfig] = deriveEncoder + implicit private[engine] lazy val encodeWorkflowHeartbeatConfig: Encoder[WorkflowHeartbeatConfig] = deriveEncoder - def apply(config: Config): WorkflowHeartbeatConfig = { + def apply(config: Config): WorkflowHeartbeatConfig = validate(config).toTry("Errors parsing WorkflowHeartbeatConfig").get - } private def validate(config: Config): ErrorOr[WorkflowHeartbeatConfig] = { val randomSuffix = config diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala index 375a24db8b3..8ca4d5314ef 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala @@ -32,17 +32,21 @@ trait WorkflowStore { * Adds the requested WorkflowSourceFiles to the store and returns a WorkflowId for each one (in order) * for tracking purposes. */ - def add(sources: NonEmptyList[WorkflowSourceFilesCollection])(implicit ec: ExecutionContext): Future[NonEmptyList[WorkflowSubmissionResponse]] + def add(sources: NonEmptyList[WorkflowSourceFilesCollection])(implicit + ec: ExecutionContext + ): Future[NonEmptyList[WorkflowSubmissionResponse]] /** * Retrieves up to n workflows which have not already been pulled into the engine and sets their pickedUp * flag to true */ - def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] + def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String])( + implicit ec: ExecutionContext + ): Future[List[WorkflowToStart]] - def writeWorkflowHeartbeats(workflowIds: Set[(WorkflowId, OffsetDateTime)], - heartbeatDateTime: OffsetDateTime) - (implicit ec: ExecutionContext): Future[Int] + def writeWorkflowHeartbeats(workflowIds: Set[(WorkflowId, OffsetDateTime)], heartbeatDateTime: OffsetDateTime)( + implicit ec: ExecutionContext + ): Future[Int] def switchOnHoldToSubmitted(id: WorkflowId)(implicit ec: ExecutionContext): Future[Unit] diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreAccess.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreAccess.scala index 2086cd34256..16ad4ee669c 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreAccess.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreAccess.scala @@ -24,17 +24,20 @@ import scala.concurrent.{ExecutionContext, Future} */ sealed trait WorkflowStoreAccess { def writeWorkflowHeartbeats(workflowIds: NonEmptyVector[(WorkflowId, OffsetDateTime)], - heartbeatDateTime: OffsetDateTime) - (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] + heartbeatDateTime: OffsetDateTime + )(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] - def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String]) - (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[List[WorkflowToStart]] + def fetchStartableWorkflows(maxWorkflows: Int, + cromwellId: String, + heartbeatTtl: FiniteDuration, + excludedGroups: Set[String] + )(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[List[WorkflowToStart]] - def abort(workflowId: WorkflowId) - (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[WorkflowStoreAbortResponse] + def abort( + workflowId: WorkflowId + )(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[WorkflowStoreAbortResponse] - def deleteFromStore(workflowId: WorkflowId) - (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] + def deleteFromStore(workflowId: WorkflowId)(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] } @@ -45,23 +48,26 @@ sealed trait WorkflowStoreAccess { case class UncoordinatedWorkflowStoreAccess(store: WorkflowStore) extends WorkflowStoreAccess { override def writeWorkflowHeartbeats(workflowIds: NonEmptyVector[(WorkflowId, OffsetDateTime)], - heartbeatDateTime: OffsetDateTime) - (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] = { + heartbeatDateTime: OffsetDateTime + )(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] = store.writeWorkflowHeartbeats(workflowIds.toVector.toSet, heartbeatDateTime) - } - override def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String]) - (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[List[WorkflowToStart]] = { + override def fetchStartableWorkflows(maxWorkflows: Int, + cromwellId: String, + heartbeatTtl: FiniteDuration, + excludedGroups: Set[String] + )(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[List[WorkflowToStart]] = store.fetchStartableWorkflows(maxWorkflows, cromwellId, heartbeatTtl, excludedGroups) - } - override def deleteFromStore(workflowId: WorkflowId)(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] = { + override def deleteFromStore( + workflowId: WorkflowId + )(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] = store.deleteFromStore(workflowId) - } - override def abort(workflowId: WorkflowId)(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[WorkflowStoreAbortResponse] = { + override def abort( + workflowId: WorkflowId + )(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[WorkflowStoreAbortResponse] = store.abort(workflowId) - } } /** @@ -72,31 +78,42 @@ case class CoordinatedWorkflowStoreAccess(coordinatedWorkflowStoreAccessActor: A implicit val timeout = Timeout(WorkflowStoreCoordinatedAccessActor.Timeout) override def writeWorkflowHeartbeats(workflowIds: NonEmptyVector[(WorkflowId, OffsetDateTime)], - heartbeatDateTime: OffsetDateTime) - (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] = { - withRetryForTransactionRollback( - () => coordinatedWorkflowStoreAccessActor.ask(WorkflowStoreCoordinatedAccessActor.WriteHeartbeats(workflowIds, heartbeatDateTime)).mapTo[Int] + heartbeatDateTime: OffsetDateTime + )(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] = + withRetryForTransactionRollback(() => + coordinatedWorkflowStoreAccessActor + .ask(WorkflowStoreCoordinatedAccessActor.WriteHeartbeats(workflowIds, heartbeatDateTime)) + .mapTo[Int] ) - } - override def fetchStartableWorkflows(maxWorkflows: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String]) - (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[List[WorkflowToStart]] = { - val message = WorkflowStoreCoordinatedAccessActor.FetchStartableWorkflows(maxWorkflows, cromwellId, heartbeatTtl, excludedGroups) - withRetryForTransactionRollback( - () => coordinatedWorkflowStoreAccessActor.ask(message).mapTo[List[WorkflowToStart]] + override def fetchStartableWorkflows(maxWorkflows: Int, + cromwellId: String, + heartbeatTtl: FiniteDuration, + excludedGroups: Set[String] + )(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[List[WorkflowToStart]] = { + val message = WorkflowStoreCoordinatedAccessActor.FetchStartableWorkflows(maxWorkflows, + cromwellId, + heartbeatTtl, + excludedGroups ) + withRetryForTransactionRollback(() => coordinatedWorkflowStoreAccessActor.ask(message).mapTo[List[WorkflowToStart]]) } - override def deleteFromStore(workflowId: WorkflowId)(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] = { - withRetryForTransactionRollback( - () => coordinatedWorkflowStoreAccessActor.ask(WorkflowStoreCoordinatedAccessActor.DeleteFromStore(workflowId)).mapTo[Int] + override def deleteFromStore( + workflowId: WorkflowId + )(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[Int] = + withRetryForTransactionRollback(() => + coordinatedWorkflowStoreAccessActor + .ask(WorkflowStoreCoordinatedAccessActor.DeleteFromStore(workflowId)) + .mapTo[Int] ) - } - override def abort(workflowId: WorkflowId)(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[WorkflowStoreAbortResponse] = { - withRetryForTransactionRollback( - () => coordinatedWorkflowStoreAccessActor.ask(WorkflowStoreCoordinatedAccessActor.Abort(workflowId)).mapTo[WorkflowStoreAbortResponse] + override def abort( + workflowId: WorkflowId + )(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[WorkflowStoreAbortResponse] = + withRetryForTransactionRollback(() => + coordinatedWorkflowStoreAccessActor + .ask(WorkflowStoreCoordinatedAccessActor.Abort(workflowId)) + .mapTo[WorkflowStoreAbortResponse] ) - } } - diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala index 4bb31db4a6d..2d309f369c2 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala @@ -11,23 +11,23 @@ import cromwell.engine.CromwellTerminator import cromwell.util.GracefulShutdownHelper import cromwell.util.GracefulShutdownHelper.ShutdownCommand -final case class WorkflowStoreActor private( - workflowStore: WorkflowStore, +final case class WorkflowStoreActor private (workflowStore: WorkflowStore, workflowStoreAccess: WorkflowStoreAccess, serviceRegistryActor: ActorRef, terminator: CromwellTerminator, abortAllJobsOnTerminate: Boolean, - workflowHeartbeatConfig: WorkflowHeartbeatConfig) - extends Actor with ActorLogging with GracefulShutdownHelper { + workflowHeartbeatConfig: WorkflowHeartbeatConfig +) extends Actor + with ActorLogging + with GracefulShutdownHelper { import WorkflowStoreActor._ implicit val ec = context.dispatcher lazy val workflowStoreSubmitActor: ActorRef = context.actorOf( - WorkflowStoreSubmitActor.props( - workflowStoreDatabase = workflowStore, - serviceRegistryActor = serviceRegistryActor), - "WorkflowStoreSubmitActor") + WorkflowStoreSubmitActor.props(workflowStoreDatabase = workflowStore, serviceRegistryActor = serviceRegistryActor), + "WorkflowStoreSubmitActor" + ) lazy val workflowStoreEngineActor: ActorRef = context.actorOf( WorkflowStoreEngineActor.props( @@ -35,19 +35,24 @@ final case class WorkflowStoreActor private( workflowStoreAccess = workflowStoreAccess, serviceRegistryActor = serviceRegistryActor, abortAllJobsOnTerminate = abortAllJobsOnTerminate, - workflowHeartbeatConfig = workflowHeartbeatConfig), - "WorkflowStoreEngineActor") + workflowHeartbeatConfig = workflowHeartbeatConfig + ), + "WorkflowStoreEngineActor" + ) lazy val workflowStoreHeartbeatWriteActor: ActorRef = context.actorOf( WorkflowStoreHeartbeatWriteActor.props( workflowStoreAccess = workflowStoreAccess, workflowHeartbeatConfig = workflowHeartbeatConfig, terminator = terminator, - serviceRegistryActor = serviceRegistryActor), - "WorkflowStoreHeartbeatWriteActor") + serviceRegistryActor = serviceRegistryActor + ), + "WorkflowStoreHeartbeatWriteActor" + ) override def receive = { - case ShutdownCommand => waitForActorsAndShutdown(NonEmptyList.of(workflowStoreSubmitActor, workflowStoreEngineActor)) + case ShutdownCommand => + waitForActorsAndShutdown(NonEmptyList.of(workflowStoreSubmitActor, workflowStoreEngineActor)) case cmd: WorkflowStoreActorSubmitCommand => workflowStoreSubmitActor forward cmd case cmd: WorkflowStoreActorEngineCommand => workflowStoreEngineActor forward cmd case cmd: WorkflowStoreWriteHeartbeatCommand => workflowStoreHeartbeatWriteActor forward cmd @@ -75,26 +80,32 @@ object WorkflowStoreActor { sealed trait WorkflowStoreActorSubmitCommand final case class SubmitWorkflow(source: WorkflowSourceFilesCollection) extends WorkflowStoreActorSubmitCommand - final case class BatchSubmitWorkflows(sources: NonEmptyList[WorkflowSourceFilesCollection]) extends WorkflowStoreActorSubmitCommand + final case class BatchSubmitWorkflows(sources: NonEmptyList[WorkflowSourceFilesCollection]) + extends WorkflowStoreActorSubmitCommand final case object GetWorkflowStoreStats - case class WorkflowStoreWriteHeartbeatCommand(workflowId: WorkflowId, submissionTime: OffsetDateTime, heartbeatTime: OffsetDateTime = OffsetDateTime.now()) + case class WorkflowStoreWriteHeartbeatCommand(workflowId: WorkflowId, + submissionTime: OffsetDateTime, + heartbeatTime: OffsetDateTime = OffsetDateTime.now() + ) def props( - workflowStoreDatabase: WorkflowStore, - workflowStoreAccess: WorkflowStoreAccess, - serviceRegistryActor: ActorRef, - terminator: CromwellTerminator, - abortAllJobsOnTerminate: Boolean, - workflowHeartbeatConfig: WorkflowHeartbeatConfig - ) = { - Props(WorkflowStoreActor( - workflowStore = workflowStoreDatabase, - workflowStoreAccess = workflowStoreAccess, - serviceRegistryActor = serviceRegistryActor, - terminator = terminator, - abortAllJobsOnTerminate = abortAllJobsOnTerminate, - workflowHeartbeatConfig = workflowHeartbeatConfig)).withDispatcher(EngineDispatcher) - } + workflowStoreDatabase: WorkflowStore, + workflowStoreAccess: WorkflowStoreAccess, + serviceRegistryActor: ActorRef, + terminator: CromwellTerminator, + abortAllJobsOnTerminate: Boolean, + workflowHeartbeatConfig: WorkflowHeartbeatConfig + ) = + Props( + WorkflowStoreActor( + workflowStore = workflowStoreDatabase, + workflowStoreAccess = workflowStoreAccess, + serviceRegistryActor = serviceRegistryActor, + terminator = terminator, + abortAllJobsOnTerminate = abortAllJobsOnTerminate, + workflowHeartbeatConfig = workflowHeartbeatConfig + ) + ).withDispatcher(EngineDispatcher) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActor.scala index 3d36a5a8062..2adbfe25da5 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActor.scala @@ -13,7 +13,6 @@ import scala.concurrent.{Await, ExecutionContext, Future} import scala.language.postfixOps import scala.util.{Failure, Success, Try} - /** * Serializes access to the workflow store for workflow store writers that acquire locks to multiple rows inside a single * transaction and otherwise are prone to deadlock. @@ -44,12 +43,18 @@ class WorkflowStoreCoordinatedAccessActor(workflowStore: WorkflowStore) extends object WorkflowStoreCoordinatedAccessActor { final case class WriteHeartbeats(workflowIds: NonEmptyVector[(WorkflowId, OffsetDateTime)], - heartbeatDateTime: OffsetDateTime) - final case class FetchStartableWorkflows(count: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String]) + heartbeatDateTime: OffsetDateTime + ) + final case class FetchStartableWorkflows(count: Int, + cromwellId: String, + heartbeatTtl: FiniteDuration, + excludedGroups: Set[String] + ) final case class DeleteFromStore(workflowId: WorkflowId) final case class Abort(workflowId: WorkflowId) val Timeout = 1 minute - def props(workflowStore: WorkflowStore): Props = Props(new WorkflowStoreCoordinatedAccessActor(workflowStore)).withDispatcher(Dispatcher.IoDispatcher) + def props(workflowStore: WorkflowStore): Props = + Props(new WorkflowStoreCoordinatedAccessActor(workflowStore)).withDispatcher(Dispatcher.IoDispatcher) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala index 44ecc09f1c8..a2fec62eda9 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala @@ -5,7 +5,7 @@ import cats.data.NonEmptyList import cromwell.core.Dispatcher._ import cromwell.core.WorkflowProcessingEvents.DescriptionEventValue.PickedUp import cromwell.core._ -import cromwell.core.abort.{WorkflowAbortFailureResponse, WorkflowAbortRequestedResponse, WorkflowAbortedResponse} +import cromwell.core.abort.{WorkflowAbortedResponse, WorkflowAbortFailureResponse, WorkflowAbortRequestedResponse} import cromwell.engine.instrumentation.WorkflowInstrumentation import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException import cromwell.engine.workflow.{WorkflowMetadataHelper, WorkflowProcessingEventPublishing} @@ -20,12 +20,17 @@ import org.apache.commons.lang3.exception.ExceptionUtils import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} -final case class WorkflowStoreEngineActor private(store: WorkflowStore, - workflowStoreAccess: WorkflowStoreAccess, - serviceRegistryActor: ActorRef, - abortAllJobsOnTerminate: Boolean, - workflowHeartbeatConfig: WorkflowHeartbeatConfig) - extends LoggingFSM[WorkflowStoreActorState, WorkflowStoreActorData] with ActorLogging with WorkflowInstrumentation with CromwellInstrumentationScheduler with WorkflowMetadataHelper with Timers { +final case class WorkflowStoreEngineActor private (store: WorkflowStore, + workflowStoreAccess: WorkflowStoreAccess, + serviceRegistryActor: ActorRef, + abortAllJobsOnTerminate: Boolean, + workflowHeartbeatConfig: WorkflowHeartbeatConfig +) extends LoggingFSM[WorkflowStoreActorState, WorkflowStoreActorData] + with ActorLogging + with WorkflowInstrumentation + with CromwellInstrumentationScheduler + with WorkflowMetadataHelper + with Timers { implicit val actorSystem: ActorSystem = context.system implicit val ec: ExecutionContext = context.dispatcher @@ -63,12 +68,11 @@ final case class WorkflowStoreEngineActor private(store: WorkflowStore, stay() using stateData.withPendingCommand(x, sender()) } - when(Idle) { - case Event(cmd: WorkflowStoreActorEngineCommand, _) => - if (stateData.currentOperation.nonEmpty || stateData.pendingOperations.nonEmpty) { - log.error("Non-empty WorkflowStoreActorData when in Idle state: {}", stateData) - } - startNewWork(cmd, sender(), stateData.withCurrentCommand(cmd, sender())) + when(Idle) { case Event(cmd: WorkflowStoreActorEngineCommand, _) => + if (stateData.currentOperation.nonEmpty || stateData.pendingOperations.nonEmpty) { + log.error("Non-empty WorkflowStoreActorData when in Idle state: {}", stateData) + } + startNewWork(cmd, sender(), stateData.withCurrentCommand(cmd, sender())) } when(Working) { @@ -93,12 +97,14 @@ final case class WorkflowStoreEngineActor private(store: WorkflowStore, stay() } - onTransition { - case fromState -> toState => - log.debug("WorkflowStore moving from {} (using {}) to {} (using {})", fromState, stateData, toState, nextStateData) + onTransition { case fromState -> toState => + log.debug("WorkflowStore moving from {} (using {}) to {} (using {})", fromState, stateData, toState, nextStateData) } - private def startNewWork(command: WorkflowStoreActorEngineCommand, sndr: ActorRef, nextData: WorkflowStoreActorData) = { + private def startNewWork(command: WorkflowStoreActorEngineCommand, + sndr: ActorRef, + nextData: WorkflowStoreActorData + ) = { val work: Future[Any] = command match { case FetchRunnableWorkflows(count, excludedGroups) => newWorkflowMessage(count, excludedGroups) map { response => @@ -113,7 +119,11 @@ final case class WorkflowStoreEngineActor private(store: WorkflowStore, ) workflowsIds foreach { w => - WorkflowProcessingEventPublishing.publish(w, workflowHeartbeatConfig.cromwellId, PickedUp, serviceRegistryActor) + WorkflowProcessingEventPublishing.publish(w, + workflowHeartbeatConfig.cromwellId, + PickedUp, + serviceRegistryActor + ) } case NoNewWorkflowsToStart => log.debug("No workflows fetched by {}", workflowHeartbeatConfig.cromwellId) @@ -122,10 +132,10 @@ final case class WorkflowStoreEngineActor private(store: WorkflowStore, sndr ! response } case FindWorkflowsWithAbortRequested(cromwellId) => - store.findWorkflowsWithAbortRequested(cromwellId) map { - ids => sndr ! FindWorkflowsWithAbortRequestedSuccess(ids) - } recover { - case t => sndr ! FindWorkflowsWithAbortRequestedFailure(t) + store.findWorkflowsWithAbortRequested(cromwellId) map { ids => + sndr ! FindWorkflowsWithAbortRequestedSuccess(ids) + } recover { case t => + sndr ! FindWorkflowsWithAbortRequestedFailure(t) } case AbortWorkflowCommand(id) => workflowStoreAccess.abort(id) map { workflowStoreAbortResponse => @@ -139,14 +149,16 @@ final case class WorkflowStoreEngineActor private(store: WorkflowStore, pushCurrentStateToMetadataService(id, WorkflowAborting) sndr ! WorkflowAbortRequestedResponse(id) case WorkflowStoreAbortResponse.NotFound => - sndr ! WorkflowAbortFailureResponse(id, new WorkflowNotFoundException(s"Couldn't abort $id because no workflow with that ID is in progress")) - } recover { - case t => - val message = s"Unable to update workflow store to abort $id" - log.error(t, message) - // A generic exception type like RuntimeException will produce a 500 at the API layer, which seems appropriate - // given we don't know much about what went wrong here. `t.getMessage` so the cause propagates to the client. - sndr ! WorkflowAbortFailureResponse(id, new RuntimeException(s"$message: ${t.getMessage}", t)) + sndr ! WorkflowAbortFailureResponse( + id, + new WorkflowNotFoundException(s"Couldn't abort $id because no workflow with that ID is in progress") + ) + } recover { case t => + val message = s"Unable to update workflow store to abort $id" + log.error(t, message) + // A generic exception type like RuntimeException will produce a 500 at the API layer, which seems appropriate + // given we don't know much about what went wrong here. `t.getMessage` so the cause propagates to the client. + sndr ! WorkflowAbortFailureResponse(id, new RuntimeException(s"$message: ${t.getMessage}", t)) } case AbortAllRunningWorkflowsCommandAndStop => store.abortAllRunning() map { _ => @@ -158,11 +170,10 @@ final case class WorkflowStoreEngineActor private(store: WorkflowStore, sndr ! WorkflowOnHoldToSubmittedSuccess(id) pushCurrentStateToMetadataService(id, WorkflowSubmitted) log.info(s"Status changed to 'Submitted' for $id") - } recover { - case t => - val message = s"Couldn't change the status to 'Submitted' from 'On Hold' for workflow $id" - log.error(message) - sndr ! WorkflowOnHoldToSubmittedFailure(id, t) + } recover { case t => + val message = s"Couldn't change the status to 'Submitted' from 'On Hold' for workflow $id" + log.error(message) + sndr ! WorkflowOnHoldToSubmittedFailure(id, t) } case oops => log.error("Unexpected type of start work command: {}", oops.getClass.getSimpleName) @@ -172,64 +183,81 @@ final case class WorkflowStoreEngineActor private(store: WorkflowStore, goto(Working) using nextData } - private def addWorkCompletionHooks[A](command: WorkflowStoreActorEngineCommand, work: Future[A]) = { + private def addWorkCompletionHooks[A](command: WorkflowStoreActorEngineCommand, work: Future[A]) = work.onComplete { case Success(_) => self ! WorkDone case Failure(t) => - log.error("Error occurred during {}: {} because {}", command.getClass.getSimpleName, t.toString, ExceptionUtils.getStackTrace(t)) + log.error("Error occurred during {}: {} because {}", + command.getClass.getSimpleName, + t.toString, + ExceptionUtils.getStackTrace(t) + ) self ! WorkDone } - } /** * Fetches at most n workflows, and builds the correct response message based on if there were any workflows or not */ - private def newWorkflowMessage(maxWorkflows: Int, excludedGroups: Set[String]): Future[WorkflowStoreEngineActorResponse] = { - def fetchStartableWorkflowsIfNeeded = { + private def newWorkflowMessage(maxWorkflows: Int, + excludedGroups: Set[String] + ): Future[WorkflowStoreEngineActorResponse] = { + def fetchStartableWorkflowsIfNeeded = if (maxWorkflows > 0) { - workflowStoreAccess.fetchStartableWorkflows(maxWorkflows, workflowHeartbeatConfig.cromwellId, workflowHeartbeatConfig.ttl, excludedGroups) + workflowStoreAccess.fetchStartableWorkflows(maxWorkflows, + workflowHeartbeatConfig.cromwellId, + workflowHeartbeatConfig.ttl, + excludedGroups + ) } else { Future.successful(List.empty[WorkflowToStart]) } - } fetchStartableWorkflowsIfNeeded map { case x :: xs => NewWorkflowsToStart(NonEmptyList.of(x, xs: _*)) case _ => NoNewWorkflowsToStart - } recover { - case e => - // Log the error but return a successful Future so as not to hang future workflow store polls. - log.error(e, "Error trying to fetch new workflows") - NoNewWorkflowsToStart + } recover { case e => + // Log the error but return a successful Future so as not to hang future workflow store polls. + log.error(e, "Error trying to fetch new workflows") + NoNewWorkflowsToStart } } } object WorkflowStoreEngineActor { def props( - workflowStore: WorkflowStore, - workflowStoreAccess: WorkflowStoreAccess, - serviceRegistryActor: ActorRef, - abortAllJobsOnTerminate: Boolean, - workflowHeartbeatConfig: WorkflowHeartbeatConfig - ) = { - Props(WorkflowStoreEngineActor(workflowStore, workflowStoreAccess, serviceRegistryActor, abortAllJobsOnTerminate, workflowHeartbeatConfig)).withDispatcher(EngineDispatcher) - } + workflowStore: WorkflowStore, + workflowStoreAccess: WorkflowStoreAccess, + serviceRegistryActor: ActorRef, + abortAllJobsOnTerminate: Boolean, + workflowHeartbeatConfig: WorkflowHeartbeatConfig + ) = + Props( + WorkflowStoreEngineActor(workflowStore, + workflowStoreAccess, + serviceRegistryActor, + abortAllJobsOnTerminate, + workflowHeartbeatConfig + ) + ).withDispatcher(EngineDispatcher) sealed trait WorkflowStoreEngineActorResponse case object NoNewWorkflowsToStart extends WorkflowStoreEngineActorResponse - final case class NewWorkflowsToStart(workflows: NonEmptyList[WorkflowToStart]) extends WorkflowStoreEngineActorResponse + final case class NewWorkflowsToStart(workflows: NonEmptyList[WorkflowToStart]) + extends WorkflowStoreEngineActorResponse final case class WorkflowStoreActorCommandWithSender(command: WorkflowStoreActorEngineCommand, sender: ActorRef) - final case class WorkflowStoreActorData(currentOperation: Option[WorkflowStoreActorCommandWithSender], pendingOperations: List[WorkflowStoreActorCommandWithSender]) { - def withCurrentCommand(command: WorkflowStoreActorEngineCommand, sender: ActorRef) = this.copy(currentOperation = Option(WorkflowStoreActorCommandWithSender(command, sender))) - def withPendingCommand(newCommand: WorkflowStoreActorEngineCommand, sender: ActorRef) = this.copy(pendingOperations = this.pendingOperations :+ WorkflowStoreActorCommandWithSender(newCommand, sender)) - def pop = { + final case class WorkflowStoreActorData(currentOperation: Option[WorkflowStoreActorCommandWithSender], + pendingOperations: List[WorkflowStoreActorCommandWithSender] + ) { + def withCurrentCommand(command: WorkflowStoreActorEngineCommand, sender: ActorRef) = + this.copy(currentOperation = Option(WorkflowStoreActorCommandWithSender(command, sender))) + def withPendingCommand(newCommand: WorkflowStoreActorEngineCommand, sender: ActorRef) = + this.copy(pendingOperations = this.pendingOperations :+ WorkflowStoreActorCommandWithSender(newCommand, sender)) + def pop = if (pendingOperations.isEmpty) { WorkflowStoreActorData(None, List.empty) } else { WorkflowStoreActorData(Option(pendingOperations.head), pendingOperations.tail) } - } } sealed trait WorkflowStoreActorState @@ -239,5 +267,6 @@ object WorkflowStoreEngineActor { sealed trait WorkflowOnHoldToSubmittedResponse case class WorkflowOnHoldToSubmittedSuccess(workflowId: WorkflowId) extends WorkflowOnHoldToSubmittedResponse - case class WorkflowOnHoldToSubmittedFailure(workflowId: WorkflowId, failure: Throwable) extends WorkflowOnHoldToSubmittedResponse + case class WorkflowOnHoldToSubmittedFailure(workflowId: WorkflowId, failure: Throwable) + extends WorkflowOnHoldToSubmittedResponse } diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreHeartbeatWriteActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreHeartbeatWriteActor.scala index ee88627cdfe..2cefd06ad7f 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreHeartbeatWriteActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreHeartbeatWriteActor.scala @@ -1,6 +1,6 @@ package cromwell.engine.workflow.workflowstore -import java.time.{OffsetDateTime, Duration => JDuration} +import java.time.{Duration => JDuration, OffsetDateTime} import java.util.concurrent.TimeUnit import akka.actor.{ActorRef, ActorSystem, CoordinatedShutdown, Props} @@ -20,11 +20,10 @@ import scala.util.{Failure, Success, Try} case class WorkflowStoreHeartbeatWriteActor(workflowStoreAccess: WorkflowStoreAccess, workflowHeartbeatConfig: WorkflowHeartbeatConfig, terminator: CromwellTerminator, - override val serviceRegistryActor: ActorRef) - - extends EnhancedBatchActor[WorkflowStoreWriteHeartbeatCommand]( - flushRate = workflowHeartbeatConfig.heartbeatInterval, - batchSize = workflowHeartbeatConfig.writeBatchSize) { + override val serviceRegistryActor: ActorRef +) extends EnhancedBatchActor[WorkflowStoreWriteHeartbeatCommand](flushRate = workflowHeartbeatConfig.heartbeatInterval, + batchSize = workflowHeartbeatConfig.writeBatchSize + ) { implicit val actorSystem: ActorSystem = context.system @@ -32,7 +31,7 @@ case class WorkflowStoreHeartbeatWriteActor(workflowStoreAccess: WorkflowStoreAc private val failureShutdownDuration = workflowHeartbeatConfig.failureShutdownDuration - //noinspection ActorMutableStateInspection + // noinspection ActorMutableStateInspection private var lastSuccessOption: Option[OffsetDateTime] = None /** @@ -40,50 +39,52 @@ case class WorkflowStoreHeartbeatWriteActor(workflowStoreAccess: WorkflowStoreAc * * @return the number of elements processed */ - override protected def process(data: NonEmptyVector[WorkflowStoreWriteHeartbeatCommand]): Future[Int] = instrumentedProcess { - val now = OffsetDateTime.now() - - val warnDuration = (failureShutdownDuration + workflowHeartbeatConfig.heartbeatInterval) / 2 - val warnThreshold = warnDuration.toNanos - val errorThreshold = failureShutdownDuration.toNanos - - // Traverse these heartbeats looking for staleness of warning or error severity. - val (warningIds, errorIds) = data.foldLeft((Seq.empty[WorkflowId], Seq.empty[WorkflowId])) { case ((w, e), h) => - val staleness = JDuration.between(h.heartbeatTime, now).toNanos - - // w = warning ids, e = error ids, h = heartbeat datum - if (staleness > errorThreshold) (w, e :+ h.workflowId) - else if (staleness > warnThreshold) (w :+ h.workflowId, e) - else (w, e) - } + override protected def process(data: NonEmptyVector[WorkflowStoreWriteHeartbeatCommand]): Future[Int] = + instrumentedProcess { + val now = OffsetDateTime.now() + + val warnDuration = (failureShutdownDuration + workflowHeartbeatConfig.heartbeatInterval) / 2 + val warnThreshold = warnDuration.toNanos + val errorThreshold = failureShutdownDuration.toNanos + + // Traverse these heartbeats looking for staleness of warning or error severity. + val (warningIds, errorIds) = data.foldLeft((Seq.empty[WorkflowId], Seq.empty[WorkflowId])) { case ((w, e), h) => + val staleness = JDuration.between(h.heartbeatTime, now).toNanos + + // w = warning ids, e = error ids, h = heartbeat datum + if (staleness > errorThreshold) (w, e :+ h.workflowId) + else if (staleness > warnThreshold) (w :+ h.workflowId, e) + else (w, e) + } - if (warningIds.nonEmpty) { - log.warning( - "Found {} stale workflow heartbeats (more than {} old): {}", - warningIds.size.toString, - warnDuration.toString(), - warningIds.mkString(", ") - ) - } + if (warningIds.nonEmpty) { + log.warning( + "Found {} stale workflow heartbeats (more than {} old): {}", + warningIds.size.toString, + warnDuration.toString(), + warningIds.mkString(", ") + ) + } - if (errorIds.isEmpty) { - val processFuture = workflowStoreAccess.writeWorkflowHeartbeats(data.map { h => (h.workflowId, h.submissionTime) }, now) - processFuture transform { - // Track the `Try`, and then return the original `Try`. Similar to `andThen` but doesn't swallow exceptions. - _ <| trackRepeatedFailures(now, data.length) + if (errorIds.isEmpty) { + val processFuture = + workflowStoreAccess.writeWorkflowHeartbeats(data.map(h => (h.workflowId, h.submissionTime)), now) + processFuture transform { + // Track the `Try`, and then return the original `Try`. Similar to `andThen` but doesn't swallow exceptions. + _ <| trackRepeatedFailures(now, data.length) + } + } else { + log.error( + "Shutting down Cromwell instance {} as {} stale workflow heartbeats (more than {} old) were found: {}", + workflowHeartbeatConfig.cromwellId, + errorIds.size.toString, + failureShutdownDuration.toString(), + errorIds.mkString(", ") + ) + terminator.beginCromwellShutdown(WorkflowStoreHeartbeatWriteActor.Shutdown) + Future.successful(0) } - } else { - log.error( - "Shutting down Cromwell instance {} as {} stale workflow heartbeats (more than {} old) were found: {}", - workflowHeartbeatConfig.cromwellId, - errorIds.size.toString, - failureShutdownDuration.toString(), - errorIds.mkString(", ") - ) - terminator.beginCromwellShutdown(WorkflowStoreHeartbeatWriteActor.Shutdown) - Future.successful(0) } - } override def receive: Receive = enhancedReceive.orElse(super.receive) override protected def weightFunction(command: WorkflowStoreWriteHeartbeatCommand) = 1 @@ -100,7 +101,7 @@ case class WorkflowStoreHeartbeatWriteActor(workflowStoreAccess: WorkflowStoreAc We are expecting the underlying FSM to ensure that the call to this method does NOT occur in parallel, waiting for the call to `process` to complete. */ - private def trackRepeatedFailures(heartbeatDateTime: OffsetDateTime, workflowCount: Int)(processTry: Try[Int]): Unit = { + private def trackRepeatedFailures(heartbeatDateTime: OffsetDateTime, workflowCount: Int)(processTry: Try[Int]): Unit = processTry match { case Success(_) => lastSuccessOption = Option(heartbeatDateTime) @@ -118,15 +119,17 @@ case class WorkflowStoreHeartbeatWriteActor(workflowStoreAccess: WorkflowStoreAc if (failureJDuration.toNanos >= failureShutdownDuration.toNanos) { val failureUnits = failureShutdownDuration.unit val failureLength = FiniteDuration(failureJDuration.toNanos, TimeUnit.NANOSECONDS).toUnit(failureUnits) - log.error(String.format( - "Shutting down %s as at least %d heartbeat write errors have occurred between %s and %s (%s %s)", - workflowHeartbeatConfig.cromwellId, - Integer.valueOf(workflowCount), - lastSuccess, - now, - failureLength.toString, - failureUnits.toString.toLowerCase - )) + log.error( + String.format( + "Shutting down %s as at least %d heartbeat write errors have occurred between %s and %s (%s %s)", + workflowHeartbeatConfig.cromwellId, + Integer.valueOf(workflowCount), + lastSuccess, + now, + failureLength.toString, + failureUnits.toString.toLowerCase + ) + ) terminator.beginCromwellShutdown(WorkflowStoreHeartbeatWriteActor.Shutdown) } () @@ -135,7 +138,6 @@ case class WorkflowStoreHeartbeatWriteActor(workflowStoreAccess: WorkflowStoreAc terminator.beginCromwellShutdown(WorkflowStoreHeartbeatWriteActor.Shutdown) () } - } } @@ -143,16 +145,17 @@ object WorkflowStoreHeartbeatWriteActor { object Shutdown extends CoordinatedShutdown.Reason def props( - workflowStoreAccess: WorkflowStoreAccess, - workflowHeartbeatConfig: WorkflowHeartbeatConfig, - terminator: CromwellTerminator, - serviceRegistryActor: ActorRef - ): Props = + workflowStoreAccess: WorkflowStoreAccess, + workflowHeartbeatConfig: WorkflowHeartbeatConfig, + terminator: CromwellTerminator, + serviceRegistryActor: ActorRef + ): Props = Props( WorkflowStoreHeartbeatWriteActor( workflowStoreAccess = workflowStoreAccess, workflowHeartbeatConfig = workflowHeartbeatConfig, terminator = terminator, serviceRegistryActor = serviceRegistryActor - )).withDispatcher(EngineDispatcher) + ) + ).withDispatcher(EngineDispatcher) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala index 5ba36f8befe..6920e1a40be 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala @@ -14,7 +14,11 @@ import cromwell.engine.workflow.WorkflowProcessingEventPublishing._ import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.WorkflowStoreState.WorkflowStoreState import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.{WorkflowStoreState, WorkflowSubmissionResponse} import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._ -import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.{WorkflowSubmitFailed, WorkflowSubmittedToStore, WorkflowsBatchSubmittedToStore} +import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.{ + WorkflowsBatchSubmittedToStore, + WorkflowSubmitFailed, + WorkflowSubmittedToStore +} import cromwell.services.metadata.MetadataService.PutMetadataAction import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import spray.json._ @@ -22,8 +26,12 @@ import spray.json._ import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} -final case class WorkflowStoreSubmitActor(store: WorkflowStore, serviceRegistryActor: ActorRef) extends Actor - with ActorLogging with WorkflowMetadataHelper with MonitoringCompanionHelper with WorkflowInstrumentation { +final case class WorkflowStoreSubmitActor(store: WorkflowStore, serviceRegistryActor: ActorRef) + extends Actor + with ActorLogging + with WorkflowMetadataHelper + with MonitoringCompanionHelper + with WorkflowInstrumentation { implicit val ec: ExecutionContext = context.dispatcher val workflowStoreReceive: Receive = { @@ -80,32 +88,32 @@ final case class WorkflowStoreSubmitActor(store: WorkflowStore, serviceRegistryA removeWork() } } - + override def receive = workflowStoreReceive.orElse(monitoringReceive) - private def convertDatabaseStateToApiState(workflowStoreState: WorkflowStoreState): WorkflowState ={ + private def convertDatabaseStateToApiState(workflowStoreState: WorkflowStoreState): WorkflowState = workflowStoreState match { case WorkflowStoreState.Submitted => WorkflowSubmitted case WorkflowStoreState.OnHold => WorkflowOnHold case WorkflowStoreState.Aborting => WorkflowAborting case WorkflowStoreState.Running => WorkflowRunning } - } - private def storeWorkflowSources(sources: NonEmptyList[WorkflowSourceFilesCollection]): Future[NonEmptyList[WorkflowSubmissionResponse]] = { + private def storeWorkflowSources( + sources: NonEmptyList[WorkflowSourceFilesCollection] + ): Future[NonEmptyList[WorkflowSubmissionResponse]] = for { workflowSubmissionResponses <- store.add(sources) } yield workflowSubmissionResponses - } - private def convertJsonToLabelsMap(json: String): Map[String, String] = { + private def convertJsonToLabelsMap(json: String): Map[String, String] = json.parseJson match { - case JsObject(inputs) => inputs.collect({ - case (key, JsString(value)) => key -> value - }) + case JsObject(inputs) => + inputs.collect { case (key, JsString(value)) => + key -> value + } case _ => Map.empty } - } /** * Runs processing on workflow source files before they are stored. @@ -114,61 +122,97 @@ final case class WorkflowStoreSubmitActor(store: WorkflowStore, serviceRegistryA * @param source Original workflow source * @return Attempted updated workflow source */ - private def processSource(processOptions: WorkflowOptions => WorkflowOptions) - (source: WorkflowSourceFilesCollection): WorkflowSourceFilesCollection = { - + private def processSource(processOptions: WorkflowOptions => WorkflowOptions)( + source: WorkflowSourceFilesCollection + ): WorkflowSourceFilesCollection = source.setOptions(processOptions(source.workflowOptions)) - } /** * Takes the workflow id and sends it over to the metadata service w/ default empty values for inputs/outputs */ - private def registerSubmission( - id: WorkflowId, - originalSourceFiles: WorkflowSourceFilesCollection): Unit = { + private def registerSubmission(id: WorkflowId, originalSourceFiles: WorkflowSourceFilesCollection): Unit = { // Increment the workflow submitted count incrementWorkflowState(WorkflowSubmitted) - val actualWorkflowState = if(originalSourceFiles.workflowOnHold) - WorkflowOnHold - else - WorkflowSubmitted + val actualWorkflowState = + if (originalSourceFiles.workflowOnHold) + WorkflowOnHold + else + WorkflowSubmitted val sourceFiles = processSource(_.clearEncryptedValues)(originalSourceFiles) - val submissionEvents: List[MetadataEvent] = List( - MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionTime), MetadataValue(OffsetDateTime.now)), - MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Inputs)), - MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Outputs)), - MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.Status), MetadataValue(actualWorkflowState)), - - MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Workflow), MetadataValue(sourceFiles.workflowSource.orNull)), - MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_WorkflowUrl), MetadataValue(sourceFiles.workflowUrl.orNull)), - MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Root), MetadataValue(sourceFiles.workflowRoot.orNull)), - MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Inputs), MetadataValue(sourceFiles.inputsJson)), - MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Options), MetadataValue(sourceFiles.workflowOptions.asPrettyJson)), - MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Labels), MetadataValue(sourceFiles.labelsJson)) - ) - - // Don't publish metadata for either workflow type or workflow type version if not defined. - val workflowTypeAndVersionEvents: List[Option[MetadataEvent]] = List( - sourceFiles.workflowType map { wt => MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_WorkflowType), MetadataValue(wt)) }, - sourceFiles.workflowTypeVersion map { wtv => MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_WorkflowTypeVersion), MetadataValue(wtv)) } + val submissionEvents: List[MetadataEvent] = List( + MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionTime), MetadataValue(OffsetDateTime.now)), + MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Inputs)), + MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Outputs)), + MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.Status), MetadataValue(actualWorkflowState)), + MetadataEvent( + MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Workflow), + MetadataValue(sourceFiles.workflowSource.orNull) + ), + MetadataEvent( + MetadataKey(id, + None, + WorkflowMetadataKeys.SubmissionSection, + WorkflowMetadataKeys.SubmissionSection_WorkflowUrl + ), + MetadataValue(sourceFiles.workflowUrl.orNull) + ), + MetadataEvent( + MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Root), + MetadataValue(sourceFiles.workflowRoot.orNull) + ), + MetadataEvent( + MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Inputs), + MetadataValue(sourceFiles.inputsJson) + ), + MetadataEvent( + MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Options), + MetadataValue(sourceFiles.workflowOptions.asPrettyJson) + ), + MetadataEvent( + MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Labels), + MetadataValue(sourceFiles.labelsJson) ) + ) + + // Don't publish metadata for either workflow type or workflow type version if not defined. + val workflowTypeAndVersionEvents: List[Option[MetadataEvent]] = List( + sourceFiles.workflowType map { wt => + MetadataEvent(MetadataKey(id, + None, + WorkflowMetadataKeys.SubmissionSection, + WorkflowMetadataKeys.SubmissionSection_WorkflowType + ), + MetadataValue(wt) + ) + }, + sourceFiles.workflowTypeVersion map { wtv => + MetadataEvent(MetadataKey(id, + None, + WorkflowMetadataKeys.SubmissionSection, + WorkflowMetadataKeys.SubmissionSection_WorkflowTypeVersion + ), + MetadataValue(wtv) + ) + } + ) - serviceRegistryActor ! PutMetadataAction(submissionEvents ++ workflowTypeAndVersionEvents.flatten) - () + serviceRegistryActor ! PutMetadataAction(submissionEvents ++ workflowTypeAndVersionEvents.flatten) + () } } object WorkflowStoreSubmitActor { - def props(workflowStoreDatabase: WorkflowStore, serviceRegistryActor: ActorRef) = { + def props(workflowStoreDatabase: WorkflowStore, serviceRegistryActor: ActorRef) = Props(WorkflowStoreSubmitActor(workflowStoreDatabase, serviceRegistryActor)).withDispatcher(ApiDispatcher) - } sealed trait WorkflowStoreSubmitActorResponse - final case class WorkflowSubmittedToStore(workflowId: WorkflowId, state: WorkflowState) extends WorkflowStoreSubmitActorResponse - final case class WorkflowsBatchSubmittedToStore(workflowIds: NonEmptyList[WorkflowId], state: WorkflowState) extends WorkflowStoreSubmitActorResponse + final case class WorkflowSubmittedToStore(workflowId: WorkflowId, state: WorkflowState) + extends WorkflowStoreSubmitActorResponse + final case class WorkflowsBatchSubmittedToStore(workflowIds: NonEmptyList[WorkflowId], state: WorkflowState) + extends WorkflowStoreSubmitActorResponse final case class WorkflowSubmitFailed(throwable: Throwable) extends WorkflowStoreSubmitActorResponse } diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala index 382b963670d..c8e9a19ac78 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala @@ -27,4 +27,5 @@ final case class WorkflowToStart(id: WorkflowId, submissionTime: OffsetDateTime, sources: WorkflowSourceFilesCollection, state: StartableState, - hogGroup: HogGroup) extends HasWorkflowIdAndSources + hogGroup: HogGroup +) extends HasWorkflowIdAndSources diff --git a/engine/src/main/scala/cromwell/jobstore/JobResultJsonFormatter.scala b/engine/src/main/scala/cromwell/jobstore/JobResultJsonFormatter.scala index 204998d115d..92c8dd407cb 100644 --- a/engine/src/main/scala/cromwell/jobstore/JobResultJsonFormatter.scala +++ b/engine/src/main/scala/cromwell/jobstore/JobResultJsonFormatter.scala @@ -17,8 +17,12 @@ object JobResultJsonFormatter extends DefaultJsonProtocol { } implicit object CallOutputsFormat extends RootJsonFormat[CallOutputs] { - def write(value: CallOutputs) = value.outputs.map({case (port, v) => port.identifier.fullyQualifiedName.value -> v }).toJson - def read(value: JsValue): CallOutputs = throw new UnsupportedOperationException("Cannot deserialize outputs to output ports") + def write(value: CallOutputs) = value.outputs.map { case (port, v) => + port.identifier.fullyQualifiedName.value -> v + }.toJson + def read(value: JsValue): CallOutputs = throw new UnsupportedOperationException( + "Cannot deserialize outputs to output ports" + ) } implicit val JobResultSuccessFormat = jsonFormat2(JobResultSuccess) diff --git a/engine/src/main/scala/cromwell/jobstore/JobStore.scala b/engine/src/main/scala/cromwell/jobstore/JobStore.scala index abc5e8765b4..a98928686fe 100644 --- a/engine/src/main/scala/cromwell/jobstore/JobStore.scala +++ b/engine/src/main/scala/cromwell/jobstore/JobStore.scala @@ -7,8 +7,12 @@ import wom.graph.GraphNodePort.OutputPort import scala.concurrent.{ExecutionContext, Future} trait JobStore { - def writeToDatabase(workflowCompletions: Seq[WorkflowCompletion], jobCompletions: Seq[JobCompletion], batchSize: Int)(implicit ec: ExecutionContext): Future[Unit] - def readJobResult(jobStoreKey: JobStoreKey, taskOutputs: Seq[OutputPort])(implicit ec: ExecutionContext): Future[Option[JobResult]] + def writeToDatabase(workflowCompletions: Seq[WorkflowCompletion], jobCompletions: Seq[JobCompletion], batchSize: Int)( + implicit ec: ExecutionContext + ): Future[Unit] + def readJobResult(jobStoreKey: JobStoreKey, taskOutputs: Seq[OutputPort])(implicit + ec: ExecutionContext + ): Future[Option[JobResult]] } object JobStore { diff --git a/engine/src/main/scala/cromwell/jobstore/JobStoreActor.scala b/engine/src/main/scala/cromwell/jobstore/JobStoreActor.scala index 6e14f6b8682..fd86080bc0d 100644 --- a/engine/src/main/scala/cromwell/jobstore/JobStoreActor.scala +++ b/engine/src/main/scala/cromwell/jobstore/JobStoreActor.scala @@ -12,14 +12,25 @@ import wom.graph.GraphNodePort.OutputPort import scala.concurrent.duration._ import scala.language.postfixOps + /** * Joins the service registry API to the JobStoreReaderActor and JobStoreWriterActor. * * This level of indirection is a tiny bit awkward but allows the database to be injected. */ -class JobStoreActor(jobStore: JobStore, dbBatchSize: Int, dbFlushRate: FiniteDuration, registryActor: ActorRef, workflowStoreAccess: WorkflowStoreAccess) extends Actor with ActorLogging with GracefulShutdownHelper { +class JobStoreActor(jobStore: JobStore, + dbBatchSize: Int, + dbFlushRate: FiniteDuration, + registryActor: ActorRef, + workflowStoreAccess: WorkflowStoreAccess +) extends Actor + with ActorLogging + with GracefulShutdownHelper { import JobStoreActor._ - val jobStoreWriterActor = context.actorOf(JobStoreWriterActor.props(jobStore, dbBatchSize, dbFlushRate, registryActor, workflowStoreAccess), "JobStoreWriterActor") + val jobStoreWriterActor = context.actorOf( + JobStoreWriterActor.props(jobStore, dbBatchSize, dbFlushRate, registryActor, workflowStoreAccess), + "JobStoreWriterActor" + ) val jobStoreReaderActor = context.actorOf(JobStoreReaderActor.props(jobStore, registryActor), "JobStoreReaderActor") override def receive: Receive = { @@ -47,16 +58,19 @@ object JobStoreActor { case class JobStoreWriteFailure(reason: Throwable) extends JobStoreWriterResponse sealed trait JobStoreReaderCommand extends JobStoreCommand + /** * Message to query the JobStoreReaderActor, asks whether the specified job has already been completed. */ case class QueryJobCompletion(jobKey: JobStoreKey, taskOutputs: Seq[OutputPort]) extends JobStoreReaderCommand sealed trait JobStoreReaderResponse + /** * Message which indicates that a job has already completed, and contains the results of the job */ case class JobComplete(jobResult: JobResult) extends JobStoreReaderResponse + /** * Indicates that the job has not been completed yet. Makes no statement about whether the job is * running versus unstarted or (maybe?) doesn't even exist! @@ -65,7 +79,9 @@ object JobStoreActor { case class JobStoreReadFailure(reason: Throwable) extends JobStoreReaderResponse - def props(database: JobStore, registryActor: ActorRef, workflowStoreAccess: WorkflowStoreAccess) = Props(new JobStoreActor(database, dbBatchSize, dbFlushRate, registryActor, workflowStoreAccess)).withDispatcher(EngineDispatcher) + def props(database: JobStore, registryActor: ActorRef, workflowStoreAccess: WorkflowStoreAccess) = Props( + new JobStoreActor(database, dbBatchSize, dbFlushRate, registryActor, workflowStoreAccess) + ).withDispatcher(EngineDispatcher) val dbFlushRate = 1 second diff --git a/engine/src/main/scala/cromwell/jobstore/JobStoreReaderActor.scala b/engine/src/main/scala/cromwell/jobstore/JobStoreReaderActor.scala index ba8d1665aa3..341c757fdb8 100644 --- a/engine/src/main/scala/cromwell/jobstore/JobStoreReaderActor.scala +++ b/engine/src/main/scala/cromwell/jobstore/JobStoreReaderActor.scala @@ -12,11 +12,13 @@ import cromwell.services.EnhancedThrottlerActor import scala.util.{Failure, Success} object JobStoreReaderActor { - def props(database: JobStore, registryActor: ActorRef) = Props(new JobStoreReaderActor(database, registryActor, LoadConfig.JobStoreReadThreshold)).withDispatcher(EngineDispatcher) + def props(database: JobStore, registryActor: ActorRef) = Props( + new JobStoreReaderActor(database, registryActor, LoadConfig.JobStoreReadThreshold) + ).withDispatcher(EngineDispatcher) } class JobStoreReaderActor(database: JobStore, override val serviceRegistryActor: ActorRef, override val threshold: Int) - extends EnhancedThrottlerActor[CommandAndReplyTo[QueryJobCompletion]] + extends EnhancedThrottlerActor[CommandAndReplyTo[QueryJobCompletion]] with ActorLogging { override def processHead(head: CommandAndReplyTo[QueryJobCompletion]) = instrumentedProcess { @@ -35,7 +37,7 @@ class JobStoreReaderActor(database: JobStore, override val serviceRegistryActor: override def receive = enhancedReceive.orElse(super.receive) override protected def instrumentationPath = NonEmptyList.of("store", "read") override protected def instrumentationPrefix = InstrumentationPrefixes.JobPrefix - override def commandToData(snd: ActorRef) = { - case query: QueryJobCompletion => CommandAndReplyTo(query, sender()) + override def commandToData(snd: ActorRef) = { case query: QueryJobCompletion => + CommandAndReplyTo(query, sender()) } } diff --git a/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala b/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala index f00693d1daf..ea9113bd101 100644 --- a/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala +++ b/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala @@ -16,50 +16,51 @@ import scala.concurrent.duration._ import scala.language.postfixOps import scala.util.{Failure, Success} - case class JobStoreWriterActor(jsd: JobStore, override val batchSize: Int, override val flushRate: FiniteDuration, serviceRegistryActor: ActorRef, threshold: Int, workflowStoreAccess: WorkflowStoreAccess - ) - extends EnhancedBatchActor[CommandAndReplyTo[JobStoreWriterCommand]](flushRate, batchSize) { +) extends EnhancedBatchActor[CommandAndReplyTo[JobStoreWriterCommand]](flushRate, batchSize) { - override protected def process(nonEmptyData: NonEmptyVector[CommandAndReplyTo[JobStoreWriterCommand]]): Future[Int] = instrumentedProcess { - implicit val actorSystem: ActorSystem = context.system + override protected def process(nonEmptyData: NonEmptyVector[CommandAndReplyTo[JobStoreWriterCommand]]): Future[Int] = + instrumentedProcess { + implicit val actorSystem: ActorSystem = context.system - val data = nonEmptyData.toVector - log.debug("Flushing {} job store commands to the DB", data.length) - val completions = data.collect({ case CommandAndReplyTo(c: JobStoreWriterCommand, _) => c.completion }) + val data = nonEmptyData.toVector + log.debug("Flushing {} job store commands to the DB", data.length) + val completions = data.collect { case CommandAndReplyTo(c: JobStoreWriterCommand, _) => c.completion } - if (completions.nonEmpty) { - val workflowCompletions = completions collect { case w: WorkflowCompletion => w } - val completedWorkflowIds = workflowCompletions map { _.workflowId } toSet - // Filter job completions that also have a corresponding workflow completion; these would just be - // immediately deleted anyway. - val jobCompletions = completions.toList collect { case j: JobCompletion if !completedWorkflowIds.contains(j.key.workflowId) => j } - val jobStoreAction: Future[Unit] = jsd.writeToDatabase(workflowCompletions, jobCompletions, batchSize) - val workflowStoreAction: Future[List[Int]] = Future.sequence { - completedWorkflowIds.map(workflowStoreAccess.deleteFromStore(_)).toList - } + if (completions.nonEmpty) { + val workflowCompletions = completions collect { case w: WorkflowCompletion => w } + val completedWorkflowIds = workflowCompletions map { _.workflowId } toSet + // Filter job completions that also have a corresponding workflow completion; these would just be + // immediately deleted anyway. + val jobCompletions = completions.toList collect { + case j: JobCompletion if !completedWorkflowIds.contains(j.key.workflowId) => j + } + val jobStoreAction: Future[Unit] = jsd.writeToDatabase(workflowCompletions, jobCompletions, batchSize) + val workflowStoreAction: Future[List[Int]] = Future.sequence { + completedWorkflowIds.map(workflowStoreAccess.deleteFromStore(_)).toList + } - val combinedAction: Future[Unit] = for { - _ <- jobStoreAction - _ <- workflowStoreAction - } yield () + val combinedAction: Future[Unit] = for { + _ <- jobStoreAction + _ <- workflowStoreAction + } yield () - combinedAction onComplete { - case Success(_) => - data foreach { case CommandAndReplyTo(c: JobStoreWriterCommand, r) => r ! JobStoreWriteSuccess(c) } - case Failure(error) => - log.error(error, "Failed to write job store entries to database") - data foreach { case CommandAndReplyTo(_, r) => r ! JobStoreWriteFailure(error) } - } + combinedAction onComplete { + case Success(_) => + data foreach { case CommandAndReplyTo(c: JobStoreWriterCommand, r) => r ! JobStoreWriteSuccess(c) } + case Failure(error) => + log.error(error, "Failed to write job store entries to database") + data foreach { case CommandAndReplyTo(_, r) => r ! JobStoreWriteFailure(error) } + } - combinedAction.map(_ => 1) - } else Future.successful(0) - } + combinedAction.map(_ => 1) + } else Future.successful(0) + } // EnhancedBatchActor overrides override def receive = enhancedReceive.orElse(super.receive) @@ -76,7 +77,15 @@ object JobStoreWriterActor { dbBatchSize: Int, dbFlushRate: FiniteDuration, registryActor: ActorRef, - workflowStoreAccess: WorkflowStoreAccess): Props = { - Props(new JobStoreWriterActor(jobStoreDatabase, dbBatchSize, dbFlushRate, registryActor, LoadConfig.JobStoreWriteThreshold, workflowStoreAccess)).withDispatcher(EngineDispatcher) - } + workflowStoreAccess: WorkflowStoreAccess + ): Props = + Props( + new JobStoreWriterActor(jobStoreDatabase, + dbBatchSize, + dbFlushRate, + registryActor, + LoadConfig.JobStoreWriteThreshold, + workflowStoreAccess + ) + ).withDispatcher(EngineDispatcher) } diff --git a/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala b/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala index baeaa79221d..b6f28a8a24d 100644 --- a/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala +++ b/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala @@ -19,7 +19,10 @@ import scala.concurrent.{ExecutionContext, Future} class SqlJobStore(sqlDatabase: EngineSqlDatabase) extends JobStore { val log = LoggerFactory.getLogger(classOf[SqlJobStore]) - override def writeToDatabase(workflowCompletions: Seq[WorkflowCompletion], jobCompletions: Seq[JobCompletion], batchSize: Int)(implicit ec: ExecutionContext): Future[Unit] = { + override def writeToDatabase(workflowCompletions: Seq[WorkflowCompletion], + jobCompletions: Seq[JobCompletion], + batchSize: Int + )(implicit ec: ExecutionContext): Future[Unit] = { val completedWorkflowIds = workflowCompletions.toList.map(_.workflowId.toString) for { _ <- sqlDatabase.addJobStores(jobCompletions map toDatabase, batchSize) @@ -28,23 +31,24 @@ class SqlJobStore(sqlDatabase: EngineSqlDatabase) extends JobStore { } yield () } - private def toDatabase(jobCompletion: JobCompletion): JobStoreJoin = { + private def toDatabase(jobCompletion: JobCompletion): JobStoreJoin = jobCompletion match { case JobCompletion(key, JobResultSuccess(returnCode, jobOutputs)) => - val entry = JobStoreEntry( - key.workflowId.toString, - key.callFqn, - key.index.fromIndex, - key.attempt, - jobSuccessful = true, - returnCode, - None, - None) + val entry = JobStoreEntry(key.workflowId.toString, + key.callFqn, + key.index.fromIndex, + key.attempt, + jobSuccessful = true, + returnCode, + None, + None + ) val jobStoreResultSimpletons = - jobOutputs.outputs.simplify.map { - womValueSimpleton => JobStoreSimpletonEntry( - womValueSimpleton.simpletonKey, womValueSimpleton.simpletonValue.valueString.toClobOption, - womValueSimpleton.simpletonValue.womType.stableName) + jobOutputs.outputs.simplify.map { womValueSimpleton => + JobStoreSimpletonEntry(womValueSimpleton.simpletonKey, + womValueSimpleton.simpletonValue.valueString.toClobOption, + womValueSimpleton.simpletonValue.womType.stableName + ) } JobStoreJoin(entry, jobStoreResultSimpletons.toSeq) case JobCompletion(key, JobResultFailure(returnCode, throwable, retryable)) => @@ -56,14 +60,19 @@ class SqlJobStore(sqlDatabase: EngineSqlDatabase) extends JobStore { jobSuccessful = false, returnCode, Option(throwable.getMessage).toClobOption, - Option(retryable)) + Option(retryable) + ) JobStoreJoin(entry, Seq.empty) } - } - override def readJobResult(jobStoreKey: JobStoreKey, taskOutputs: Seq[OutputPort])(implicit ec: ExecutionContext): Future[Option[JobResult]] = { - sqlDatabase.queryJobStores(jobStoreKey.workflowId.toString, jobStoreKey.callFqn, jobStoreKey.index.fromIndex, - jobStoreKey.attempt) map { + override def readJobResult(jobStoreKey: JobStoreKey, taskOutputs: Seq[OutputPort])(implicit + ec: ExecutionContext + ): Future[Option[JobResult]] = + sqlDatabase.queryJobStores(jobStoreKey.workflowId.toString, + jobStoreKey.callFqn, + jobStoreKey.index.fromIndex, + jobStoreKey.attempt + ) map { _ map { case JobStoreJoin(entry, simpletonEntries) => entry match { case JobStoreEntry(_, _, _, _, true, returnCode, None, None, _) => @@ -72,12 +81,12 @@ class SqlJobStore(sqlDatabase: EngineSqlDatabase) extends JobStore { JobResultSuccess(returnCode, jobOutputs) case JobStoreEntry(_, _, _, _, false, returnCode, Some(_), Some(retryable), _) => JobResultFailure(returnCode, - JobAlreadyFailedInJobStore(jobStoreKey.tag, entry.exceptionMessage.toRawString), - retryable) + JobAlreadyFailedInJobStore(jobStoreKey.tag, entry.exceptionMessage.toRawString), + retryable + ) case bad => throw new Exception(s"Invalid contents of JobStore table: $bad") } } } - } } diff --git a/engine/src/main/scala/cromwell/jobstore/jobstore_.scala b/engine/src/main/scala/cromwell/jobstore/jobstore_.scala index 886de33ebdb..9bcd641330e 100644 --- a/engine/src/main/scala/cromwell/jobstore/jobstore_.scala +++ b/engine/src/main/scala/cromwell/jobstore/jobstore_.scala @@ -1,7 +1,12 @@ package cromwell.jobstore import cromwell.backend.BackendJobDescriptorKey -import cromwell.backend.BackendJobExecutionActor.{FetchedFromJobStore, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{ + FetchedFromJobStore, + JobFailedNonRetryableResponse, + JobFailedRetryableResponse, + JobSucceededResponse +} import cromwell.core.{CallOutputs, WorkflowId} case class JobStoreKey(workflowId: WorkflowId, callFqn: String, index: Option[Int], attempt: Int) { @@ -13,7 +18,15 @@ sealed trait JobResult { def toBackendJobResponse(key: BackendJobDescriptorKey) = this match { // Always puts `None` for `dockerImageUsed` for a successfully completed job on restart. This shouldn't be a // problem since `saveJobCompletionToJobStore` in EJEA will already have sent this to metadata. - case JobResultSuccess(returnCode, jobOutputs) => JobSucceededResponse(key, returnCode, jobOutputs, None, Seq.empty, None, resultGenerationMode = FetchedFromJobStore) + case JobResultSuccess(returnCode, jobOutputs) => + JobSucceededResponse(key, + returnCode, + jobOutputs, + None, + Seq.empty, + None, + resultGenerationMode = FetchedFromJobStore + ) case JobResultFailure(returnCode, reason, false) => JobFailedNonRetryableResponse(key, reason, returnCode) case JobResultFailure(returnCode, reason, true) => JobFailedRetryableResponse(key, reason, returnCode) } diff --git a/engine/src/main/scala/cromwell/jobstore/package.scala b/engine/src/main/scala/cromwell/jobstore/package.scala index 1e4dbaecbbb..6d5bdb834e1 100644 --- a/engine/src/main/scala/cromwell/jobstore/package.scala +++ b/engine/src/main/scala/cromwell/jobstore/package.scala @@ -4,6 +4,7 @@ import cromwell.core.{JobKey, WorkflowId} package object jobstore { implicit class EnhancedJobKey(val jobKey: JobKey) extends AnyVal { - def toJobStoreKey(workflowId: WorkflowId): JobStoreKey = JobStoreKey(workflowId, jobKey.node.fullyQualifiedName, jobKey.index, jobKey.attempt) + def toJobStoreKey(workflowId: WorkflowId): JobStoreKey = + JobStoreKey(workflowId, jobKey.node.fullyQualifiedName, jobKey.index, jobKey.attempt) } } diff --git a/engine/src/main/scala/cromwell/logging/TerminalLayout.scala b/engine/src/main/scala/cromwell/logging/TerminalLayout.scala index 8871341f976..6bb67e5a1af 100644 --- a/engine/src/main/scala/cromwell/logging/TerminalLayout.scala +++ b/engine/src/main/scala/cromwell/logging/TerminalLayout.scala @@ -20,15 +20,14 @@ object TerminalLayout { } implicit class ColorString(msg: String) { - def colorizeUuids: String = { - "UUID\\((.*?)\\)".r.findAllMatchIn(msg).foldLeft(msg) { - case (l, r) => - val color = if (Option(System.getProperty("RAINBOW_UUID")).isDefined) - Math.abs(17 * r.group(1).substring(0,8).map(_.toInt).product) % 209 + 22 + def colorizeUuids: String = + "UUID\\((.*?)\\)".r.findAllMatchIn(msg).foldLeft(msg) { case (l, r) => + val color = + if (Option(System.getProperty("RAINBOW_UUID")).isDefined) + Math.abs(17 * r.group(1).substring(0, 8).map(_.toInt).product) % 209 + 22 else 2 - l.replace(r.group(0), TerminalUtil.highlight(color, r.group(1))) + l.replace(r.group(0), TerminalUtil.highlight(color, r.group(1))) } - } def colorizeCommand: String = msg.replaceAll("`([^`]*?)`", TerminalUtil.highlight(5, "$1")) } } diff --git a/engine/src/main/scala/cromwell/server/CromwellAkkaLogFilter.scala b/engine/src/main/scala/cromwell/server/CromwellAkkaLogFilter.scala index 31dc6408b77..9789b354f37 100644 --- a/engine/src/main/scala/cromwell/server/CromwellAkkaLogFilter.scala +++ b/engine/src/main/scala/cromwell/server/CromwellAkkaLogFilter.scala @@ -4,17 +4,19 @@ import akka.actor.ActorSystem import akka.event.EventStream import akka.event.slf4j.Slf4jLoggingFilter -class CromwellAkkaLogFilter(settings: ActorSystem.Settings, eventStream: EventStream) extends Slf4jLoggingFilter(settings, eventStream) { - override def isErrorEnabled(logClass: Class[_], logSource: String) = { +class CromwellAkkaLogFilter(settings: ActorSystem.Settings, eventStream: EventStream) + extends Slf4jLoggingFilter(settings, eventStream) { + override def isErrorEnabled(logClass: Class[_], logSource: String) = /* * This might filter out too much but it's the finest granularity we have here - * The goal is to not log the - * "Outgoing request stream error akka.stream.AbruptTerminationException: + * The goal is to not log the + * "Outgoing request stream error akka.stream.AbruptTerminationException: * Processor actor [Actor[akka://cromwell-system/user/StreamSupervisor-1/flow-6-0-mergePreferred#1200284127]] terminated abruptly" * type of message - * + * * See https://github.com/akka/akka-http/issues/907 and https://github.com/akka/akka/issues/18747 */ - super.isErrorEnabled(logClass, logSource) && !(logSource.startsWith("akka.actor.ActorSystemImpl") && CromwellShutdown.shutdownInProgress()) - } + super.isErrorEnabled(logClass, logSource) && !(logSource.startsWith( + "akka.actor.ActorSystemImpl" + ) && CromwellShutdown.shutdownInProgress()) } diff --git a/engine/src/main/scala/cromwell/server/CromwellDeadLetterListener.scala b/engine/src/main/scala/cromwell/server/CromwellDeadLetterListener.scala index 5cea81c37d2..c7af7991b6c 100644 --- a/engine/src/main/scala/cromwell/server/CromwellDeadLetterListener.scala +++ b/engine/src/main/scala/cromwell/server/CromwellDeadLetterListener.scala @@ -6,14 +6,15 @@ import cats.Show import cats.syntax.show._ class CromwellDeadLetterListener extends DeadLetterListener with ActorLogging { - implicit val showActor: Show[ActorRef] = Show.show(actor => - s"Actor of path ${actor.path} toString ${actor.toString()}" - ) - + implicit val showActor: Show[ActorRef] = + Show.show(actor => s"Actor of path ${actor.path} toString ${actor.toString()}") + def shutdownReceive: Receive = { // This silences the dead letter messages when Cromwell is shutting down case DeadLetter(msg, from, to) if CromwellShutdown.shutdownInProgress() => - log.debug(s"Got a dead letter during Cromwell shutdown. Sent by\n${from.show}\nto ${to.show}\n consisting of message: $msg\n ") + log.debug( + s"Got a dead letter during Cromwell shutdown. Sent by\n${from.show}\nto ${to.show}\n consisting of message: $msg\n " + ) } override def receive = shutdownReceive.orElse(super.receive) } diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala index c37d6466fcc..b4cdbef735b 100644 --- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala +++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala @@ -20,7 +20,11 @@ import cromwell.engine.io.{IoActor, IoActorProxy} import cromwell.engine.workflow.WorkflowManagerActor import cromwell.engine.workflow.WorkflowManagerActor.AbortAllWorkflowsCommand import cromwell.engine.workflow.lifecycle.execution.callcaching.{CallCache, CallCacheReadActor, CallCacheWriteActor} -import cromwell.engine.workflow.lifecycle.finalization.{CopyWorkflowLogsActor, WorkflowCallbackActor, WorkflowCallbackConfig} +import cromwell.engine.workflow.lifecycle.finalization.{ + CopyWorkflowLogsActor, + WorkflowCallbackActor, + WorkflowCallbackConfig +} import cromwell.engine.workflow.tokens.{DynamicRateLimiter, JobTokenDispenserActor} import cromwell.engine.workflow.workflowstore.AbortRequestScanningActor.AbortConfig import cromwell.engine.workflow.workflowstore._ @@ -54,9 +58,11 @@ abstract class CromwellRootActor(terminator: CromwellTerminator, gracefulShutdown: Boolean, abortJobsOnTerminate: Boolean, val serverMode: Boolean, - protected val config: Config) - (implicit materializer: ActorMaterializer) - extends Actor with ActorLogging with GracefulShutdownHelper { + protected val config: Config +)(implicit materializer: ActorMaterializer) + extends Actor + with ActorLogging + with GracefulShutdownHelper { import CromwellRootActor._ @@ -72,9 +78,11 @@ abstract class CromwellRootActor(terminator: CromwellTerminator, lazy val systemConfig = config.getConfig("system") lazy val serviceRegistryActor: ActorRef = context.actorOf(ServiceRegistryActor.props(config), "ServiceRegistryActor") - lazy val numberOfWorkflowLogCopyWorkers = systemConfig.as[Option[Int]]("number-of-workflow-log-copy-workers").getOrElse(DefaultNumberOfWorkflowLogCopyWorkers) + lazy val numberOfWorkflowLogCopyWorkers = + systemConfig.as[Option[Int]]("number-of-workflow-log-copy-workers").getOrElse(DefaultNumberOfWorkflowLogCopyWorkers) - lazy val workflowStore: WorkflowStore = SqlWorkflowStore(EngineServicesStore.engineDatabaseInterface, MetadataServicesStore.metadataDatabaseInterface) + lazy val workflowStore: WorkflowStore = + SqlWorkflowStore(EngineServicesStore.engineDatabaseInterface, MetadataServicesStore.metadataDatabaseInterface) val workflowStoreAccess: WorkflowStoreAccess = { val coordinatedWorkflowStoreAccess = config.as[Option[Boolean]]("system.coordinated-workflow-store-access") @@ -90,41 +98,49 @@ abstract class CromwellRootActor(terminator: CromwellTerminator, } lazy val workflowStoreActor = - context.actorOf(WorkflowStoreActor.props( - workflowStoreDatabase = workflowStore, - workflowStoreAccess = workflowStoreAccess, - serviceRegistryActor = serviceRegistryActor, - terminator = terminator, - abortAllJobsOnTerminate = abortJobsOnTerminate, - workflowHeartbeatConfig = workflowHeartbeatConfig), - "WorkflowStoreActor") + context.actorOf( + WorkflowStoreActor.props( + workflowStoreDatabase = workflowStore, + workflowStoreAccess = workflowStoreAccess, + serviceRegistryActor = serviceRegistryActor, + terminator = terminator, + abortAllJobsOnTerminate = abortJobsOnTerminate, + workflowHeartbeatConfig = workflowHeartbeatConfig + ), + "WorkflowStoreActor" + ) lazy val jobStore: JobStore = new SqlJobStore(EngineServicesStore.engineDatabaseInterface) - lazy val jobStoreActor: ActorRef = context.actorOf(JobStoreActor.props(jobStore, serviceRegistryActor, workflowStoreAccess), "JobStoreActor") + lazy val jobStoreActor: ActorRef = + context.actorOf(JobStoreActor.props(jobStore, serviceRegistryActor, workflowStoreAccess), "JobStoreActor") lazy val subWorkflowStore: SubWorkflowStore = new SqlSubWorkflowStore(EngineServicesStore.engineDatabaseInterface) - lazy val subWorkflowStoreActor: ActorRef = context.actorOf(SubWorkflowStoreActor.props(subWorkflowStore), "SubWorkflowStoreActor") + lazy val subWorkflowStoreActor: ActorRef = + context.actorOf(SubWorkflowStoreActor.props(subWorkflowStore), "SubWorkflowStoreActor") lazy val ioConfig: IoConfig = config.as[IoConfig] - lazy val ioActor: ActorRef = context.actorOf( - IoActor.props( - ioConfig = ioConfig, - serviceRegistryActor = serviceRegistryActor, - applicationName = GoogleConfiguration(config).applicationName), - "IoActor") + lazy val ioActor: ActorRef = context.actorOf(IoActor.props(ioConfig = ioConfig, + serviceRegistryActor = serviceRegistryActor, + applicationName = + GoogleConfiguration(config).applicationName + ), + "IoActor" + ) lazy val ioActorProxy: ActorRef = context.actorOf(IoActorProxy.props(ioActor), "IoProxy") // Register the IoActor with the service registry: serviceRegistryActor ! IoActorRef(ioActorProxy) - lazy val workflowLogCopyRouter: ActorRef = context.actorOf(RoundRobinPool(numberOfWorkflowLogCopyWorkers) - .withSupervisorStrategy(CopyWorkflowLogsActor.strategy) - .props(CopyWorkflowLogsActor.props(serviceRegistryActor, ioActor)), - "WorkflowLogCopyRouter") + lazy val workflowLogCopyRouter: ActorRef = context.actorOf( + RoundRobinPool(numberOfWorkflowLogCopyWorkers) + .withSupervisorStrategy(CopyWorkflowLogsActor.strategy) + .props(CopyWorkflowLogsActor.props(serviceRegistryActor, ioActor)), + "WorkflowLogCopyRouter" + ) private val workflowCallbackConfig = WorkflowCallbackConfig(config.getConfig("workflow-state-callback")) - lazy val workflowCallbackActor: Option[ActorRef] = { + lazy val workflowCallbackActor: Option[ActorRef] = if (workflowCallbackConfig.enabled) { val props = WorkflowCallbackActor.props( serviceRegistryActor, @@ -132,21 +148,25 @@ abstract class CromwellRootActor(terminator: CromwellTerminator, ) Option(context.actorOf(props, "WorkflowCallbackActor")) } else None - } - //Call-caching config validation + // Call-caching config validation lazy val callCachingConfig = config.getConfig("call-caching") lazy val callCachingEnabled = callCachingConfig.getBoolean("enabled") lazy val callInvalidateBadCacheResults = callCachingConfig.getBoolean("invalidate-bad-cache-results") lazy val callCache: CallCache = new CallCache(EngineServicesStore.engineDatabaseInterface) - lazy val numberOfCacheReadWorkers = config.getConfig("system").as[Option[Int]]("number-of-cache-read-workers").getOrElse(DefaultNumberOfCacheReadWorkers) + lazy val numberOfCacheReadWorkers = config + .getConfig("system") + .as[Option[Int]]("number-of-cache-read-workers") + .getOrElse(DefaultNumberOfCacheReadWorkers) lazy val callCacheReadActor = context.actorOf(RoundRobinPool(numberOfCacheReadWorkers) - .props(CallCacheReadActor.props(callCache, serviceRegistryActor)), - "CallCacheReadActor") + .props(CallCacheReadActor.props(callCache, serviceRegistryActor)), + "CallCacheReadActor" + ) - lazy val callCacheWriteActor = context.actorOf(CallCacheWriteActor.props(callCache, serviceRegistryActor), "CallCacheWriteActor") + lazy val callCacheWriteActor = + context.actorOf(CallCacheWriteActor.props(callCache, serviceRegistryActor), "CallCacheWriteActor") // Docker Actor lazy val ioEc = context.system.dispatchers.lookup(Dispatcher.IoDispatcher) @@ -160,22 +180,49 @@ abstract class CromwellRootActor(terminator: CromwellTerminator, case DockerRemoteLookup => DockerInfoActor.remoteRegistriesFromConfig(DockerConfiguration.dockerHashLookupConfig) } - lazy val dockerHashActor = context.actorOf(DockerInfoActor.props(dockerFlows, dockerActorQueueSize, - dockerConf.cacheEntryTtl, dockerConf.cacheSize), "DockerHashActor") + lazy val dockerHashActor = context.actorOf( + DockerInfoActor.props(dockerFlows, dockerActorQueueSize, dockerConf.cacheEntryTtl, dockerConf.cacheSize), + "DockerHashActor" + ) lazy val backendSingletons = CromwellBackends.instance.get.backendLifecycleActorFactories map { - case (name, factory) => name -> (factory.backendSingletonActorProps(serviceRegistryActor) map { context.actorOf(_, s"$name-Singleton") }) + case (name, factory) => + name -> (factory.backendSingletonActorProps(serviceRegistryActor) map { context.actorOf(_, s"$name-Singleton") }) } lazy val backendSingletonCollection = BackendSingletonCollection(backendSingletons) - lazy val jobRestartCheckRate: DynamicRateLimiter.Rate = DynamicRateLimiter.Rate(systemConfig.as[Int]("job-restart-check-rate-control.jobs"), systemConfig.as[FiniteDuration]("job-restart-check-rate-control.per")) - lazy val jobExecutionRate: DynamicRateLimiter.Rate = DynamicRateLimiter.Rate(systemConfig.as[Int]("job-rate-control.jobs"), systemConfig.as[FiniteDuration]("job-rate-control.per")) - - lazy val restartCheckTokenLogInterval: Option[FiniteDuration] = systemConfig.as[Option[Int]]("job-restart-check-rate-control.token-log-interval-seconds").map(_.seconds) - lazy val executionTokenLogInterval: Option[FiniteDuration] = systemConfig.as[Option[Int]]("hog-safety.token-log-interval-seconds").map(_.seconds) - - lazy val jobRestartCheckTokenDispenserActor: ActorRef = context.actorOf(JobTokenDispenserActor.props(serviceRegistryActor, jobRestartCheckRate, restartCheckTokenLogInterval, "restart checking", "CheckingRestart"), "JobRestartCheckTokenDispenser") - lazy val jobExecutionTokenDispenserActor: ActorRef = context.actorOf(JobTokenDispenserActor.props(serviceRegistryActor, jobExecutionRate, executionTokenLogInterval, "execution", ExecutionStatus.Running.toString), "JobExecutionTokenDispenser") + lazy val jobRestartCheckRate: DynamicRateLimiter.Rate = DynamicRateLimiter.Rate( + systemConfig.as[Int]("job-restart-check-rate-control.jobs"), + systemConfig.as[FiniteDuration]("job-restart-check-rate-control.per") + ) + lazy val jobExecutionRate: DynamicRateLimiter.Rate = DynamicRateLimiter.Rate( + systemConfig.as[Int]("job-rate-control.jobs"), + systemConfig.as[FiniteDuration]("job-rate-control.per") + ) + + lazy val restartCheckTokenLogInterval: Option[FiniteDuration] = + systemConfig.as[Option[Int]]("job-restart-check-rate-control.token-log-interval-seconds").map(_.seconds) + lazy val executionTokenLogInterval: Option[FiniteDuration] = + systemConfig.as[Option[Int]]("hog-safety.token-log-interval-seconds").map(_.seconds) + + lazy val jobRestartCheckTokenDispenserActor: ActorRef = context.actorOf( + JobTokenDispenserActor.props(serviceRegistryActor, + jobRestartCheckRate, + restartCheckTokenLogInterval, + "restart checking", + "CheckingRestart" + ), + "JobRestartCheckTokenDispenser" + ) + lazy val jobExecutionTokenDispenserActor: ActorRef = context.actorOf( + JobTokenDispenserActor.props(serviceRegistryActor, + jobExecutionRate, + executionTokenLogInterval, + "execution", + ExecutionStatus.Running.toString + ), + "JobExecutionTokenDispenser" + ) lazy val workflowManagerActor = context.actorOf( WorkflowManagerActor.props( @@ -196,19 +243,24 @@ abstract class CromwellRootActor(terminator: CromwellTerminator, jobExecutionTokenDispenserActor = jobExecutionTokenDispenserActor, backendSingletonCollection = backendSingletonCollection, serverMode = serverMode, - workflowHeartbeatConfig = workflowHeartbeatConfig), - "WorkflowManagerActor") + workflowHeartbeatConfig = workflowHeartbeatConfig + ), + "WorkflowManagerActor" + ) val abortRequestScanningActor = { val abortConfigBlock = config.as[Option[Config]]("system.abort") - val abortCacheConfig = CacheConfig.config(caching = abortConfigBlock.flatMap { _.as[Option[Config]]("cache") }, + val abortCacheConfig = CacheConfig.config(caching = abortConfigBlock.flatMap(_.as[Option[Config]]("cache")), defaultConcurrency = 1, defaultSize = 100000L, - defaultTtl = 20 minutes) + defaultTtl = 20 minutes + ) val abortConfig = AbortConfig( - scanFrequency = abortConfigBlock.flatMap { _.as[Option[FiniteDuration]]("scan-frequency") } getOrElse (30 seconds), + scanFrequency = abortConfigBlock.flatMap { + _.as[Option[FiniteDuration]]("scan-frequency") + } getOrElse (30 seconds), cacheConfig = abortCacheConfig ) @@ -262,8 +314,8 @@ abstract class CromwellRootActor(terminator: CromwellTerminator, } } - override def receive = { - case message => logger.error(s"Unknown message received by CromwellRootActor: $message") + override def receive = { case message => + logger.error(s"Unknown message received by CromwellRootActor: $message") } /** diff --git a/engine/src/main/scala/cromwell/server/CromwellServer.scala b/engine/src/main/scala/cromwell/server/CromwellServer.scala index 76f784875fc..cc2a1d409e9 100644 --- a/engine/src/main/scala/cromwell/server/CromwellServer.scala +++ b/engine/src/main/scala/cromwell/server/CromwellServer.scala @@ -20,26 +20,29 @@ object CromwellServer { def run(gracefulShutdown: Boolean, abortJobsOnTerminate: Boolean)(cromwellSystem: CromwellSystem): Future[Any] = { implicit val actorSystem = cromwellSystem.actorSystem implicit val materializer = cromwellSystem.materializer - actorSystem.actorOf(CromwellServerActor.props(cromwellSystem, gracefulShutdown, abortJobsOnTerminate), "cromwell-service") + actorSystem.actorOf(CromwellServerActor.props(cromwellSystem, gracefulShutdown, abortJobsOnTerminate), + "cromwell-service" + ) actorSystem.whenTerminated } } -class CromwellServerActor(cromwellSystem: CromwellSystem, gracefulShutdown: Boolean, abortJobsOnTerminate: Boolean)(override implicit val materializer: ActorMaterializer) - extends CromwellRootActor( - terminator = cromwellSystem, - gracefulShutdown = gracefulShutdown, - abortJobsOnTerminate = abortJobsOnTerminate, - serverMode = true, - config = cromwellSystem.config - ) +class CromwellServerActor(cromwellSystem: CromwellSystem, gracefulShutdown: Boolean, abortJobsOnTerminate: Boolean)( + implicit override val materializer: ActorMaterializer +) extends CromwellRootActor( + terminator = cromwellSystem, + gracefulShutdown = gracefulShutdown, + abortJobsOnTerminate = abortJobsOnTerminate, + serverMode = true, + config = cromwellSystem.config + ) with CromwellApiService with CromwellInstrumentationActor with WesRouteSupport with SwaggerService with ActorLogging { implicit val actorSystem = context.system - override implicit val ec = context.dispatcher + implicit override val ec = context.dispatcher override def actorRefFactory: ActorContext = context val webserviceConf = cromwellSystem.config.getConfig("webservice") @@ -71,7 +74,7 @@ class CromwellServerActor(cromwellSystem: CromwellSystem, gracefulShutdown: Bool If/when CromwellServer behaves like a better async citizen, we may be less paranoid about our async log messages not appearing due to the actor system shutdown. For now, synchronously print to the stderr so that the user has some idea of why the server failed to start up. - */ + */ Console.err.println(s"Binding failed interface $interface port $port") e.printStackTrace(Console.err) cromwellSystem.shutdownActorSystem() @@ -85,7 +88,9 @@ class CromwellServerActor(cromwellSystem: CromwellSystem, gracefulShutdown: Bool } object CromwellServerActor { - def props(cromwellSystem: CromwellSystem, gracefulShutdown: Boolean, abortJobsOnTerminate: Boolean)(implicit materializer: ActorMaterializer): Props = { - Props(new CromwellServerActor(cromwellSystem, gracefulShutdown, abortJobsOnTerminate)).withDispatcher(EngineDispatcher) - } + def props(cromwellSystem: CromwellSystem, gracefulShutdown: Boolean, abortJobsOnTerminate: Boolean)(implicit + materializer: ActorMaterializer + ): Props = + Props(new CromwellServerActor(cromwellSystem, gracefulShutdown, abortJobsOnTerminate)) + .withDispatcher(EngineDispatcher) } diff --git a/engine/src/main/scala/cromwell/server/CromwellShutdown.scala b/engine/src/main/scala/cromwell/server/CromwellShutdown.scala index ce204249b1e..991d8dfabb6 100644 --- a/engine/src/main/scala/cromwell/server/CromwellShutdown.scala +++ b/engine/src/main/scala/cromwell/server/CromwellShutdown.scala @@ -59,17 +59,16 @@ object CromwellShutdown extends GracefulStopSupport { /** * Register a task to unbind from the port during the ServiceUnbind phase. */ - def registerUnbindTask(actorSystem: ActorSystem, serverBinding: Future[Http.ServerBinding]) = { + def registerUnbindTask(actorSystem: ActorSystem, serverBinding: Future[Http.ServerBinding]) = instance(actorSystem).addTask(CoordinatedShutdown.PhaseServiceUnbind, "UnbindingServerPort") { () => // At this point it's still safe to schedule work on the actor system's dispatcher implicit val ec = actorSystem.dispatcher for { binding <- serverBinding _ <- binding.unbind() - _ = logger.info("Http server unbound.") + _ = logger.info("Http server unbound.") } yield Done } - } /** * Register tasks on the coordinated shutdown instance allowing a controlled, ordered shutdown process @@ -77,29 +76,28 @@ object CromwellShutdown extends GracefulStopSupport { * Calling this method will add a JVM shutdown hook. */ def registerShutdownTasks( - cromwellId: String, - abortJobsOnTerminate: Boolean, - actorSystem: ActorSystem, - workflowManagerActor: ActorRef, - logCopyRouter: ActorRef, - workflowCallbackActor: Option[ActorRef], - jobTokenDispenser: ActorRef, - jobStoreActor: ActorRef, - workflowStoreActor: ActorRef, - subWorkflowStoreActor: ActorRef, - callCacheWriteActor: ActorRef, - ioActor: ActorRef, - dockerHashActor: ActorRef, - serviceRegistryActor: ActorRef, - materializer: ActorMaterializer - ): Unit = { + cromwellId: String, + abortJobsOnTerminate: Boolean, + actorSystem: ActorSystem, + workflowManagerActor: ActorRef, + logCopyRouter: ActorRef, + workflowCallbackActor: Option[ActorRef], + jobTokenDispenser: ActorRef, + jobStoreActor: ActorRef, + workflowStoreActor: ActorRef, + subWorkflowStoreActor: ActorRef, + callCacheWriteActor: ActorRef, + ioActor: ActorRef, + dockerHashActor: ActorRef, + serviceRegistryActor: ActorRef, + materializer: ActorMaterializer + ): Unit = { val coordinatedShutdown = this.instance(actorSystem) - def shutdownActor(actor: ActorRef, - phase: String, - message: AnyRef, - customTimeout: Option[FiniteDuration] = None)(implicit executionContext: ExecutionContext) = { + def shutdownActor(actor: ActorRef, phase: String, message: AnyRef, customTimeout: Option[FiniteDuration] = None)( + implicit executionContext: ExecutionContext + ) = coordinatedShutdown.addTask(phase, s"stop${actor.path.name.capitalize}") { () => val timeout = coordinatedShutdown.timeout(phase) logger.info(s"Shutting down ${actor.path.name} - Timeout = ${timeout.toSeconds} seconds") @@ -115,7 +113,6 @@ object CromwellShutdown extends GracefulStopSupport { action map { _ => Done } } - } implicit val ec = actorSystem.dispatcher @@ -131,61 +128,70 @@ object CromwellShutdown extends GracefulStopSupport { } /* 2) The socket is unbound from the port in the CoordinatedShutdown.PhaseServiceUnbind - * See cromwell.engine.server.CromwellServer + * See cromwell.engine.server.CromwellServer */ /* 3) Finish processing all requests: - * - Release any WorkflowStore entries held by this Cromwell instance. - * - Publish workflow processing event metadata for the released workflows. - * - Stop the WorkflowStore: The port is not bound anymore so we can't have new submissions. - * Process what's left in the message queue and stop. - * Note that it's possible that some submissions are still asynchronously being prepared at the - * akka http API layer (CromwellApiService) to be sent to the WorkflowStore. - * Those submissions might be lost if the WorkflowStore shuts itself down when it's finished processing its current work. - * In that case the "ask" over in the CromwellAPIService will fail with a AskTimeoutException and should be handled appropriately. - * This process still ensures that no submission can make it to the database without a response being sent back to the client. - * - * - Stop WorkflowManagerActor: We've already stopped starting new workflows but the Running workflows are still - * going. This is tricky because all the actor hierarchy under the WMA can be in a variety of state combinations. - * Specifically there is an asynchronous gap in several cases between emission of messages towards engine level - * actors (job store, cache store, etc...) and emission of messages towards the metadata service for the same logical - * event (e.g: job complete). - * The current behavior upon restart however is to re-play the graph, skipping execution of completed jobs (determined by - * engine job store) but still re-submitting all related metadata events. This is likely sub-optimal, but is used here - * to simply stop the WMA (which will trigger all its descendants to be stopped recursively) without more coordination. - * Indeed even if the actor is stopped in between the above mentioned gap, metadata will be re-submitted anyway on restart, - * even for completed jobs. - * - * - Stop the LogCopyRouter: it can generate metadata events and must therefore be stopped before the service registry. - * Wrap the ShutdownCommand in a Broadcast message so the router forwards it to all its routees - * Use the ShutdownCommand because a PoisonPill could stop the routees in the middle of "transaction" - * with the IoActor. The routees handle the ShutdownCommand properly and shutdown only when they have - * no outstanding requests to the IoActor. When all routees are dead the router automatically stops itself. - * - * - Stop the job token dispenser: stop it before stopping WMA and its EJEA descendants because - * the dispenser is watching all EJEAs and would be flooded by Terminated messages otherwise - */ + * - Release any WorkflowStore entries held by this Cromwell instance. + * - Publish workflow processing event metadata for the released workflows. + * - Stop the WorkflowStore: The port is not bound anymore so we can't have new submissions. + * Process what's left in the message queue and stop. + * Note that it's possible that some submissions are still asynchronously being prepared at the + * akka http API layer (CromwellApiService) to be sent to the WorkflowStore. + * Those submissions might be lost if the WorkflowStore shuts itself down when it's finished processing its current work. + * In that case the "ask" over in the CromwellAPIService will fail with a AskTimeoutException and should be handled appropriately. + * This process still ensures that no submission can make it to the database without a response being sent back to the client. + * + * - Stop WorkflowManagerActor: We've already stopped starting new workflows but the Running workflows are still + * going. This is tricky because all the actor hierarchy under the WMA can be in a variety of state combinations. + * Specifically there is an asynchronous gap in several cases between emission of messages towards engine level + * actors (job store, cache store, etc...) and emission of messages towards the metadata service for the same logical + * event (e.g: job complete). + * The current behavior upon restart however is to re-play the graph, skipping execution of completed jobs (determined by + * engine job store) but still re-submitting all related metadata events. This is likely sub-optimal, but is used here + * to simply stop the WMA (which will trigger all its descendants to be stopped recursively) without more coordination. + * Indeed even if the actor is stopped in between the above mentioned gap, metadata will be re-submitted anyway on restart, + * even for completed jobs. + * + * - Stop the LogCopyRouter: it can generate metadata events and must therefore be stopped before the service registry. + * Wrap the ShutdownCommand in a Broadcast message so the router forwards it to all its routees + * Use the ShutdownCommand because a PoisonPill could stop the routees in the middle of "transaction" + * with the IoActor. The routees handle the ShutdownCommand properly and shutdown only when they have + * no outstanding requests to the IoActor. When all routees are dead the router automatically stops itself. + * + * - Stop the job token dispenser: stop it before stopping WMA and its EJEA descendants because + * the dispenser is watching all EJEAs and would be flooded by Terminated messages otherwise + */ coordinatedShutdown.addTask(CoordinatedShutdown.PhaseServiceRequestsDone, "releaseWorkflowStoreEntries") { () => - EngineServicesStore.engineDatabaseInterface.releaseWorkflowStoreEntries(cromwellId).map(count => { - logger.info("{} workflows released by {}", count, cromwellId) - }).as(Done) + EngineServicesStore.engineDatabaseInterface + .releaseWorkflowStoreEntries(cromwellId) + .map(count => logger.info("{} workflows released by {}", count, cromwellId)) + .as(Done) } - coordinatedShutdown.addTask(CoordinatedShutdown.PhaseServiceRequestsDone, "publishMetadataForReleasedWorkflowStoreEntries") { () => - EngineServicesStore.engineDatabaseInterface.findWorkflows(cromwellId).map(ids => - ids foreach { id => - WorkflowProcessingEventPublishing.publish(WorkflowId.fromString(id), cromwellId, Released, serviceRegistryActor) - } - ).as(Done) + coordinatedShutdown.addTask(CoordinatedShutdown.PhaseServiceRequestsDone, + "publishMetadataForReleasedWorkflowStoreEntries" + ) { () => + EngineServicesStore.engineDatabaseInterface + .findWorkflows(cromwellId) + .map(ids => + ids foreach { id => + WorkflowProcessingEventPublishing + .publish(WorkflowId.fromString(id), cromwellId, Released, serviceRegistryActor) + } + ) + .as(Done) } shutdownActor(workflowStoreActor, CoordinatedShutdown.PhaseServiceRequestsDone, ShutdownCommand) shutdownActor(logCopyRouter, CoordinatedShutdown.PhaseServiceRequestsDone, Broadcast(ShutdownCommand)) - workflowCallbackActor.foreach(wca => shutdownActor(wca, CoordinatedShutdown.PhaseServiceRequestsDone, Broadcast(ShutdownCommand))) + workflowCallbackActor.foreach(wca => + shutdownActor(wca, CoordinatedShutdown.PhaseServiceRequestsDone, Broadcast(ShutdownCommand)) + ) shutdownActor(jobTokenDispenser, CoordinatedShutdown.PhaseServiceRequestsDone, ShutdownCommand) - + /* - * Aborting is only a special case of shutdown. Instead of sending a PoisonPill, send a AbortAllWorkflowsCommand - * Also attach this task to a special shutdown phase allowing for a longer timeout. + * Aborting is only a special case of shutdown. Instead of sending a PoisonPill, send a AbortAllWorkflowsCommand + * Also attach this task to a special shutdown phase allowing for a longer timeout. */ if (abortJobsOnTerminate) { val abortTimeout = coordinatedShutdown.timeout(PhaseAbortAllWorkflows) @@ -197,10 +203,10 @@ object CromwellShutdown extends GracefulStopSupport { } /* 4) Shutdown connection pools - * This will close all akka http opened connection pools tied to the actor system. - * The pools stop accepting new work but are given a chance to execute the work submitted prior to the shutdown call. - * When this future returns, all outstanding connections to client will be terminated. - * Note that this also includes connection pools like the one used to lookup docker hashes. + * This will close all akka http opened connection pools tied to the actor system. + * The pools stop accepting new work but are given a chance to execute the work submitted prior to the shutdown call. + * When this future returns, all outstanding connections to client will be terminated. + * Note that this also includes connection pools like the one used to lookup docker hashes. */ coordinatedShutdown.addTask(CoordinatedShutdown.PhaseServiceStop, "TerminatingConnections") { () => Http(actorSystem).shutdownAllConnectionPools() as { @@ -210,14 +216,20 @@ object CromwellShutdown extends GracefulStopSupport { } /* 5) Stop system level actors that require writing to the database or I/O - * - SubWorkflowStoreActor - * - JobStoreActor - * - CallCacheWriteActor - * - ServiceRegistryActor - * - DockerHashActor - * - IoActor - */ - List(subWorkflowStoreActor, jobStoreActor, callCacheWriteActor, serviceRegistryActor, dockerHashActor, ioActor) foreach { + * - SubWorkflowStoreActor + * - JobStoreActor + * - CallCacheWriteActor + * - ServiceRegistryActor + * - DockerHashActor + * - IoActor + */ + List(subWorkflowStoreActor, + jobStoreActor, + callCacheWriteActor, + serviceRegistryActor, + dockerHashActor, + ioActor + ) foreach { shutdownActor(_, PhaseStopIoActivity, ShutdownCommand) } @@ -236,12 +248,13 @@ object CromwellShutdown extends GracefulStopSupport { // 7) Close out the backend used for WDL HTTP import resolution // http://sttp.readthedocs.io/en/latest/backends/start_stop.html - coordinatedShutdown.addTask(CoordinatedShutdown.PhaseBeforeActorSystemTerminate, "wdlHttpImportResolverBackend") { () => - Future { - HttpResolver.closeBackendIfNecessary() - logger.info("WDL HTTP import resolver closed") - Done - } + coordinatedShutdown.addTask(CoordinatedShutdown.PhaseBeforeActorSystemTerminate, "wdlHttpImportResolverBackend") { + () => + Future { + HttpResolver.closeBackendIfNecessary() + logger.info("WDL HTTP import resolver closed") + Done + } } } } diff --git a/engine/src/main/scala/cromwell/server/CromwellSystem.scala b/engine/src/main/scala/cromwell/server/CromwellSystem.scala index 1a6a709c8f3..ec9b95999d0 100644 --- a/engine/src/main/scala/cromwell/server/CromwellSystem.scala +++ b/engine/src/main/scala/cromwell/server/CromwellSystem.scala @@ -40,26 +40,23 @@ trait CromwellSystem extends CromwellTerminator { implicit final lazy val actorSystem = newActorSystem() implicit final lazy val materializer = ActorMaterializer() - implicit private final lazy val ec = actorSystem.dispatcher + implicit final private lazy val ec = actorSystem.dispatcher - override def beginCromwellShutdown(reason: CoordinatedShutdown.Reason): Future[Done] = { + override def beginCromwellShutdown(reason: CoordinatedShutdown.Reason): Future[Done] = CromwellShutdown.instance(actorSystem).run(reason) - } - def shutdownActorSystem(): Future[Terminated] = { + def shutdownActorSystem(): Future[Terminated] = // If the actor system is already terminated it's already too late for a clean shutdown // Note: This does not protect again starting 2 shutdowns concurrently if (!actorSystem.whenTerminated.isCompleted) { Http().shutdownAllConnectionPools() flatMap { _ => shutdownMaterializerAndActorSystem() - } recoverWith { - case _ => - // we still want to shutdown the materializer and actor system if shutdownAllConnectionPools failed - shutdownMaterializerAndActorSystem() + } recoverWith { case _ => + // we still want to shutdown the materializer and actor system if shutdownAllConnectionPools failed + shutdownMaterializerAndActorSystem() } } else actorSystem.whenTerminated - } - + private def shutdownMaterializerAndActorSystem() = { materializer.shutdown() actorSystem.terminate() diff --git a/engine/src/main/scala/cromwell/subworkflowstore/EmptySubWorkflowStoreActor.scala b/engine/src/main/scala/cromwell/subworkflowstore/EmptySubWorkflowStoreActor.scala index cbbae2410c3..82e7a733817 100644 --- a/engine/src/main/scala/cromwell/subworkflowstore/EmptySubWorkflowStoreActor.scala +++ b/engine/src/main/scala/cromwell/subworkflowstore/EmptySubWorkflowStoreActor.scala @@ -7,7 +7,7 @@ import cromwell.util.GracefulShutdownHelper.ShutdownCommand class EmptySubWorkflowStoreActor extends Actor with ActorLogging { override def receive: Receive = { - case register: RegisterSubWorkflow => sender() ! SubWorkflowStoreRegisterSuccess(register) + case register: RegisterSubWorkflow => sender() ! SubWorkflowStoreRegisterSuccess(register) case query: QuerySubWorkflow => sender() ! SubWorkflowNotFound(query) case _: WorkflowComplete => // No-op! case ShutdownCommand => context stop self diff --git a/engine/src/main/scala/cromwell/subworkflowstore/SqlSubWorkflowStore.scala b/engine/src/main/scala/cromwell/subworkflowstore/SqlSubWorkflowStore.scala index 64f21275ff4..acf760c8143 100644 --- a/engine/src/main/scala/cromwell/subworkflowstore/SqlSubWorkflowStore.scala +++ b/engine/src/main/scala/cromwell/subworkflowstore/SqlSubWorkflowStore.scala @@ -10,7 +10,8 @@ class SqlSubWorkflowStore(subWorkflowStoreSqlDatabase: SubWorkflowStoreSqlDataba callFullyQualifiedName: String, jobIndex: Int, jobAttempt: Int, - subWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Unit] = { + subWorkflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Unit] = subWorkflowStoreSqlDatabase.addSubWorkflowStoreEntry( rootWorkflowExecutionUuid, parentWorkflowExecutionUuid, @@ -19,13 +20,16 @@ class SqlSubWorkflowStore(subWorkflowStoreSqlDatabase: SubWorkflowStoreSqlDataba jobAttempt, subWorkflowExecutionUuid ) - } - override def querySubWorkflowStore(parentWorkflowExecutionUuid: String, callFqn: String, jobIndex: Int, jobAttempt: Int)(implicit ec: ExecutionContext): Future[Option[SubWorkflowStoreEntry]] = { + override def querySubWorkflowStore(parentWorkflowExecutionUuid: String, + callFqn: String, + jobIndex: Int, + jobAttempt: Int + )(implicit ec: ExecutionContext): Future[Option[SubWorkflowStoreEntry]] = subWorkflowStoreSqlDatabase.querySubWorkflowStore(parentWorkflowExecutionUuid, callFqn, jobIndex, jobAttempt) - } - override def removeSubWorkflowStoreEntries(parentWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Int] = { + override def removeSubWorkflowStoreEntries(parentWorkflowExecutionUuid: String)(implicit + ec: ExecutionContext + ): Future[Int] = subWorkflowStoreSqlDatabase.removeSubWorkflowStoreEntries(parentWorkflowExecutionUuid) - } } diff --git a/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStore.scala b/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStore.scala index 8ad92fa9bae..ef303be8e0e 100644 --- a/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStore.scala +++ b/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStore.scala @@ -10,10 +10,12 @@ trait SubWorkflowStore { callFullyQualifiedName: String, jobIndex: Int, jobAttempt: Int, - subWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Unit] + subWorkflowExecutionUuid: String + )(implicit ec: ExecutionContext): Future[Unit] - def querySubWorkflowStore(parentWorkflowExecutionUuid: String, callFqn: String, jobIndex: Int, jobAttempt: Int) - (implicit ec: ExecutionContext): Future[Option[SubWorkflowStoreEntry]] + def querySubWorkflowStore(parentWorkflowExecutionUuid: String, callFqn: String, jobIndex: Int, jobAttempt: Int)( + implicit ec: ExecutionContext + ): Future[Option[SubWorkflowStoreEntry]] def removeSubWorkflowStoreEntries(parentWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Int] } diff --git a/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala b/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala index 22ec0476220..a3540c3497a 100644 --- a/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala +++ b/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala @@ -11,7 +11,7 @@ import scala.concurrent.ExecutionContext import scala.util.{Failure, Success} class SubWorkflowStoreActor(database: SubWorkflowStore) extends Actor with ActorLogging with MonitoringCompanionHelper { - + implicit val ec: ExecutionContext = context.dispatcher val subWorkflowStoreReceive: Receive = { @@ -19,9 +19,9 @@ class SubWorkflowStoreActor(database: SubWorkflowStore) extends Actor with Actor case query: QuerySubWorkflow => querySubWorkflow(sender(), query) case complete: WorkflowComplete => workflowComplete(sender(), complete) } - + override def receive = subWorkflowStoreReceive.orElse(monitoringReceive) - + private def registerSubWorkflow(replyTo: ActorRef, command: RegisterSubWorkflow) = { addWork() database.addSubWorkflowStoreEntry( @@ -31,9 +31,9 @@ class SubWorkflowStoreActor(database: SubWorkflowStore) extends Actor with Actor command.jobKey.index.fromIndex, command.jobKey.attempt, command.subWorkflowExecutionUuid.toString - ) onComplete { + ) onComplete { case Success(_) => - replyTo ! SubWorkflowStoreRegisterSuccess(command) + replyTo ! SubWorkflowStoreRegisterSuccess(command) removeWork() case Failure(ex) => replyTo ! SubWorkflowStoreFailure(command, ex) @@ -43,7 +43,11 @@ class SubWorkflowStoreActor(database: SubWorkflowStore) extends Actor with Actor private def querySubWorkflow(replyTo: ActorRef, command: QuerySubWorkflow) = { val jobKey = command.jobKey - database.querySubWorkflowStore(command.parentWorkflowExecutionUuid.toString, jobKey.node.fullyQualifiedName, jobKey.index.fromIndex, jobKey.attempt) onComplete { + database.querySubWorkflowStore(command.parentWorkflowExecutionUuid.toString, + jobKey.node.fullyQualifiedName, + jobKey.index.fromIndex, + jobKey.attempt + ) onComplete { case Success(Some(result)) => replyTo ! SubWorkflowFound(result) case Success(None) => replyTo ! SubWorkflowNotFound(command) case Failure(ex) => replyTo ! SubWorkflowStoreFailure(command, ex) @@ -54,27 +58,33 @@ class SubWorkflowStoreActor(database: SubWorkflowStore) extends Actor with Actor addWork() database.removeSubWorkflowStoreEntries(command.workflowExecutionUuid.toString) onComplete { case Success(_) => removeWork() - case Failure(ex) => + case Failure(ex) => replyTo ! SubWorkflowStoreFailure(command, ex) removeWork() } } - + } object SubWorkflowStoreActor { sealed trait SubWorkflowStoreActorCommand - case class RegisterSubWorkflow(rootWorkflowExecutionUuid: WorkflowId, parentWorkflowExecutionUuid: WorkflowId, jobKey: JobKey, subWorkflowExecutionUuid: WorkflowId) extends SubWorkflowStoreActorCommand - case class QuerySubWorkflow(parentWorkflowExecutionUuid: WorkflowId, jobKey: JobKey) extends SubWorkflowStoreActorCommand + case class RegisterSubWorkflow(rootWorkflowExecutionUuid: WorkflowId, + parentWorkflowExecutionUuid: WorkflowId, + jobKey: JobKey, + subWorkflowExecutionUuid: WorkflowId + ) extends SubWorkflowStoreActorCommand + case class QuerySubWorkflow(parentWorkflowExecutionUuid: WorkflowId, jobKey: JobKey) + extends SubWorkflowStoreActorCommand case class WorkflowComplete(workflowExecutionUuid: WorkflowId) extends SubWorkflowStoreActorCommand sealed trait SubWorkflowStoreActorResponse case class SubWorkflowStoreRegisterSuccess(command: RegisterSubWorkflow) extends SubWorkflowStoreActorResponse case class SubWorkflowFound(subWorkflowStoreEntry: SubWorkflowStoreEntry) extends SubWorkflowStoreActorResponse case class SubWorkflowNotFound(command: QuerySubWorkflow) extends SubWorkflowStoreActorResponse - - case class SubWorkflowStoreFailure(command: SubWorkflowStoreActorCommand, failure: Throwable) extends SubWorkflowStoreActorResponse - + + case class SubWorkflowStoreFailure(command: SubWorkflowStoreActorCommand, failure: Throwable) + extends SubWorkflowStoreActorResponse + def props(database: SubWorkflowStore) = Props( new SubWorkflowStoreActor(database) ).withDispatcher(EngineDispatcher) diff --git a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala index 61bd2ee7d01..ceb4c1b67e7 100644 --- a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala +++ b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala @@ -6,7 +6,6 @@ import spray.json._ import wdl.draft2.model.FullyQualifiedName import wom.values.WomValue - case class WorkflowStatusResponse(id: String, status: String) case class WorkflowSubmitResponse(id: String, status: String) @@ -21,16 +20,16 @@ case class WorkflowMetadataQueryParameters(outputs: Boolean = true, timings: Boo object APIResponse { - private def constructFailureResponse(status: String, ex: Throwable) = { + private def constructFailureResponse(status: String, ex: Throwable) = ex match { case exceptionWithErrors: MessageAggregation => - FailureResponse( - status, - exceptionWithErrors.exceptionContext, - Option(exceptionWithErrors.errorMessages.toVector)) - case e: Throwable => FailureResponse(status, e.getMessage, Option(e.getCause).map(c => Vector(ExceptionUtils.getMessage(c)))) + FailureResponse(status, + exceptionWithErrors.exceptionContext, + Option(exceptionWithErrors.errorMessages.toVector) + ) + case e: Throwable => + FailureResponse(status, e.getMessage, Option(e.getCause).map(c => Vector(ExceptionUtils.getMessage(c)))) } - } /** When the data submitted in the request is incorrect. */ def fail(ex: Throwable) = constructFailureResponse("fail", ex) diff --git a/engine/src/main/scala/cromwell/webservice/EngineStatsActor.scala b/engine/src/main/scala/cromwell/webservice/EngineStatsActor.scala index ce123db4d20..6aa5edc94fb 100644 --- a/engine/src/main/scala/cromwell/webservice/EngineStatsActor.scala +++ b/engine/src/main/scala/cromwell/webservice/EngineStatsActor.scala @@ -13,7 +13,9 @@ import scala.concurrent.duration._ * Because of the vagaries of timing, etc this is intended to give a rough idea of what's going on instead of * being a ground truth. */ -final case class EngineStatsActor(workflowActors: List[ActorRef], replyTo: ActorRef, timeout: FiniteDuration) extends Actor with ActorLogging { +final case class EngineStatsActor(workflowActors: List[ActorRef], replyTo: ActorRef, timeout: FiniteDuration) + extends Actor + with ActorLogging { implicit val ec = context.dispatcher private var jobCounts = Map.empty[ActorRef, Int] @@ -23,7 +25,7 @@ final case class EngineStatsActor(workflowActors: List[ActorRef], replyTo: Actor * Because of sub workflows there is currently no reliable way to know if we received responses from all running WEAs. * For now, we always wait for the timeout duration before responding to give a chance to all WEAs to respond (even nested ones). * This could be improved by having WEAs wait for their sub WEAs before sending back the response. - */ + */ val scheduledMsg = context.system.scheduler.scheduleOnce(timeout, self, ShutItDown) if (workflowActors.isEmpty) reportStats() @@ -47,9 +49,8 @@ final case class EngineStatsActor(workflowActors: List[ActorRef], replyTo: Actor object EngineStatsActor { import scala.language.postfixOps - def props(workflowActors: List[ActorRef], replyTo: ActorRef, timeout: FiniteDuration = MaxTimeToWait) = { + def props(workflowActors: List[ActorRef], replyTo: ActorRef, timeout: FiniteDuration = MaxTimeToWait) = Props(EngineStatsActor(workflowActors, replyTo, timeout)).withDispatcher(Dispatcher.ApiDispatcher) - } sealed abstract class EngineStatsActorMessage private case object ShutItDown extends EngineStatsActorMessage diff --git a/engine/src/main/scala/cromwell/webservice/LabelsManagerActor.scala b/engine/src/main/scala/cromwell/webservice/LabelsManagerActor.scala index ca85c8de694..9d317152ba9 100644 --- a/engine/src/main/scala/cromwell/webservice/LabelsManagerActor.scala +++ b/engine/src/main/scala/cromwell/webservice/LabelsManagerActor.scala @@ -11,7 +11,8 @@ import spray.json.{DefaultJsonProtocol, JsObject, JsString} object LabelsManagerActor { - def props(serviceRegistryActor: ActorRef): Props = Props(new LabelsManagerActor(serviceRegistryActor)).withDispatcher(Dispatcher.ApiDispatcher) + def props(serviceRegistryActor: ActorRef): Props = + Props(new LabelsManagerActor(serviceRegistryActor)).withDispatcher(Dispatcher.ApiDispatcher) final case class LabelsData(workflowId: WorkflowId, labels: Labels) @@ -49,7 +50,7 @@ class LabelsManagerActor(serviceRegistryActor: ActorRef) extends Actor with Acto /* Ask the metadata store for the current set of labels, so we can return the full label set to the user. At this point in the actor lifecycle, wfId has already been filled out so the .get is safe - */ + */ serviceRegistryActor ! GetLabels(wfId.get) case SuccessfulMetadataJsonResponse(_, jsObject) => /* @@ -64,7 +65,7 @@ class LabelsManagerActor(serviceRegistryActor: ActorRef) extends Actor with Acto the return packet, this is a likely cause. At this point in the actor lifecycle, newLabels will have been filled in so the .get is safe - */ + */ def replaceOrAddLabel(originalJson: JsObject, label: Label): JsObject = { val labels = originalJson.fields.get("labels").map(_.asJsObject.fields).getOrElse(Map.empty) @@ -83,7 +84,9 @@ class LabelsManagerActor(serviceRegistryActor: ActorRef) extends Actor with Acto At this point in the actor lifecycle, wfId has already been filled out so the .get is safe */ - target ! FailedLabelsManagerResponse(new RuntimeException(s"Unable to update labels for ${wfId.get} due to ${f.reason.getMessage}")) + target ! FailedLabelsManagerResponse( + new RuntimeException(s"Unable to update labels for ${wfId.get} due to ${f.reason.getMessage}") + ) context stop self } } diff --git a/engine/src/main/scala/cromwell/webservice/PartialWorkflowSources.scala b/engine/src/main/scala/cromwell/webservice/PartialWorkflowSources.scala index 9862dc01151..22adbe6e0f8 100644 --- a/engine/src/main/scala/cromwell/webservice/PartialWorkflowSources.scala +++ b/engine/src/main/scala/cromwell/webservice/PartialWorkflowSources.scala @@ -36,7 +36,8 @@ final case class PartialWorkflowSources(workflowSource: Option[WorkflowSource] = zippedImports: Option[Array[Byte]] = None, warnings: Seq[String] = List.empty, workflowOnHold: Boolean, - requestedWorkflowIds: Vector[WorkflowId]) + requestedWorkflowIds: Vector[WorkflowId] +) object PartialWorkflowSources { val log = LoggerFactory.getLogger(classOf[PartialWorkflowSources]) @@ -56,15 +57,29 @@ object PartialWorkflowSources { val workflowOnHoldKey = "workflowOnHold" val RequestedWorkflowIdKey = "requestedWorkflowId" - val allKeys = List(WdlSourceKey, WorkflowUrlKey, WorkflowRootKey, WorkflowSourceKey, WorkflowTypeKey, WorkflowTypeVersionKey, WorkflowInputsKey, - WorkflowOptionsKey, labelsKey, WdlDependenciesKey, WorkflowDependenciesKey, workflowOnHoldKey, RequestedWorkflowIdKey) + val allKeys = List( + WdlSourceKey, + WorkflowUrlKey, + WorkflowRootKey, + WorkflowSourceKey, + WorkflowTypeKey, + WorkflowTypeVersionKey, + WorkflowInputsKey, + WorkflowOptionsKey, + labelsKey, + WdlDependenciesKey, + WorkflowDependenciesKey, + workflowOnHoldKey, + RequestedWorkflowIdKey + ) val allPrefixes = List(WorkflowInputsAuxPrefix) val MaxWorkflowUrlLength = 2000 def fromSubmitRoute(formData: Map[String, ByteString], - allowNoInputs: Boolean): Try[Seq[WorkflowSourceFilesCollection]] = { + allowNoInputs: Boolean + ): Try[Seq[WorkflowSourceFilesCollection]] = { import cats.syntax.apply._ import cats.syntax.traverse._ import cats.syntax.validated._ @@ -78,7 +93,7 @@ object PartialWorkflowSources { val unrecognized: ErrorOr[Unit] = formData.keySet .filterNot(name => allKeys.contains(name) || allPrefixes.exists(name.startsWith)) .toList - .map(name => s"Unexpected body part name: $name") match { + .map(name => s"Unexpected body part name: $name") match { case Nil => ().validNel case head :: tail => NonEmptyList.of(head, tail: _*).invalid } @@ -88,17 +103,17 @@ object PartialWorkflowSources { val workflowSource = getStringValue(WorkflowSourceKey) val workflowUrl = getStringValue(WorkflowUrlKey) - def deprecationWarning(out: String, in: String)(actual: String): String = { + def deprecationWarning(out: String, in: String)(actual: String): String = if (actual == out) { val warning = Array( s"The '$out' parameter name has been deprecated in favor of '$in'.", s"Support for '$out' will be removed from future versions of Cromwell.", - s"Please switch to using '$in' in future submissions.").mkString(" ") + s"Please switch to using '$in' in future submissions." + ).mkString(" ") log.warn(warning) warning } else "" - } val wdlSourceDeprecationWarning: String => String = deprecationWarning(out = WdlSourceKey, in = WorkflowSourceKey) val wdlSourceWarning = wdlSource.as(WdlSourceKey) map wdlSourceDeprecationWarning @@ -110,7 +125,8 @@ object PartialWorkflowSources { case (Some(_), Some(_), None) => s"$WdlSourceKey and $WorkflowSourceKey can't both be supplied".invalidNel case (None, Some(_), Some(_)) => s"$WorkflowSourceKey and $WorkflowUrlKey can't both be supplied".invalidNel case (Some(_), None, Some(_)) => s"$WdlSourceKey and $WorkflowUrlKey can't both be supplied".invalidNel - case (Some(_), Some(_), Some(_)) => s"$WdlSourceKey, $WorkflowSourceKey and $WorkflowUrlKey all 3 can't be supplied".invalidNel + case (Some(_), Some(_), Some(_)) => + s"$WdlSourceKey, $WorkflowSourceKey and $WorkflowUrlKey all 3 can't be supplied".invalidNel case (None, None, None) => s"$WorkflowSourceKey or $WorkflowUrlKey needs to be supplied".invalidNel } @@ -124,17 +140,20 @@ object PartialWorkflowSources { case None => Vector.empty.validNel } - val workflowInputsAux: ErrorOr[Map[Int, String]] = formData.toList.flatTraverse[ErrorOr, (Int, String)]({ - case (name, value) if name.startsWith(WorkflowInputsAuxPrefix) => - Try(name.stripPrefix(WorkflowInputsAuxPrefix).toInt).toErrorOr.map(index => List((index, value.utf8String))) - case _ => List.empty.validNel - }).map(_.toMap) + val workflowInputsAux: ErrorOr[Map[Int, String]] = formData.toList + .flatTraverse[ErrorOr, (Int, String)] { + case (name, value) if name.startsWith(WorkflowInputsAuxPrefix) => + Try(name.stripPrefix(WorkflowInputsAuxPrefix).toInt).toErrorOr.map(index => List((index, value.utf8String))) + case _ => List.empty.validNel + } + .map(_.toMap) // dependencies val wdlDependencies = getArrayValue(WdlDependenciesKey) val workflowDependencies = getArrayValue(WorkflowDependenciesKey) - val wdlDependenciesDeprecationWarning: String => String = deprecationWarning(out = "wdlDependencies", in = "workflowDependencies") + val wdlDependenciesDeprecationWarning: String => String = + deprecationWarning(out = "wdlDependencies", in = "workflowDependencies") val wdlDependenciesWarning = wdlDependencies.as(WdlDependenciesKey) map wdlDependenciesDeprecationWarning val workflowDependenciesFinal: ErrorOr[Option[Array[Byte]]] = (wdlDependencies, workflowDependencies) match { @@ -146,15 +165,22 @@ object PartialWorkflowSources { val onHold: ErrorOr[Boolean] = getBooleanValue(workflowOnHoldKey).getOrElse(false.validNel) - (unrecognized, workflowSourceFinal, requestedIds, workflowInputs, workflowInputsAux, workflowDependenciesFinal, onHold) mapN { - case (_, source, ids, inputs, aux, dep, onHoldActual) => PartialWorkflowSources( + (unrecognized, + workflowSourceFinal, + requestedIds, + workflowInputs, + workflowInputsAux, + workflowDependenciesFinal, + onHold + ) mapN { case (_, source, ids, inputs, aux, dep, onHoldActual) => + PartialWorkflowSources( workflowSource = source, workflowUrl = workflowUrl, workflowRoot = getStringValue(WorkflowRootKey), workflowType = getStringValue(WorkflowTypeKey), workflowTypeVersion = getStringValue(WorkflowTypeVersionKey), workflowInputs = inputs, - workflowInputsAux= aux, + workflowInputsAux = aux, workflowOptions = getStringValue(WorkflowOptionsKey), customLabels = getStringValue(labelsKey), zippedImports = dep, @@ -171,20 +197,28 @@ object PartialWorkflowSources { } } - private def arrayTypeElementValidation[A](data: String, interpretElementFunction: _root_.io.circe.Json => ErrorOr[A]) = { + private def arrayTypeElementValidation[A](data: String, + interpretElementFunction: _root_.io.circe.Json => ErrorOr[A] + ) = { import cats.syntax.validated._ val parseInputsTry = Try { YamlUtils.parse(data) match { // If it's an array, treat each element as an individual input object, otherwise simply toString the whole thing - case Right(json) => json.asArray.map(_.traverse(interpretElementFunction)).getOrElse(interpretElementFunction(json).map(Vector(_))).validNel - case Left(error) => s"Input file is not a valid yaml or json. Inputs data: '$data'. Error: ${ExceptionUtils.getMessage(error)}.".invalidNel + case Right(json) => + json.asArray + .map(_.traverse(interpretElementFunction)) + .getOrElse(interpretElementFunction(json).map(Vector(_))) + .validNel + case Left(error) => + s"Input file is not a valid yaml or json. Inputs data: '$data'. Error: ${ExceptionUtils.getMessage(error)}.".invalidNel } } parseInputsTry match { case Success(v) => v.flatten - case Failure(error) => s"Input file is not a valid yaml or json. Inputs data: '$data'. Error: ${ExceptionUtils.getMessage(error)}.".invalidNel + case Failure(error) => + s"Input file is not a valid yaml or json. Inputs data: '$data'. Error: ${ExceptionUtils.getMessage(error)}.".invalidNel } } @@ -203,17 +237,17 @@ object PartialWorkflowSources { import _root_.io.circe.Printer import cats.syntax.validated._ - def interpretEachElement(json: Json): ErrorOr[WorkflowId] = { + def interpretEachElement(json: Json): ErrorOr[WorkflowId] = if (json.isString) { Try(WorkflowId.fromString(json.asString.get)).toErrorOrWithContext("parse requested workflow ID as UUID") } else s"Requested workflow IDs must be strings but got: '${json.printWith(Printer.noSpaces)}'".invalidNel - } arrayTypeElementValidation(data, interpretEachElement) } private def partialSourcesToSourceCollections(partialSources: ErrorOr[PartialWorkflowSources], - allowNoInputs: Boolean): ErrorOr[Seq[WorkflowSourceFilesCollection]] = { + allowNoInputs: Boolean + ): ErrorOr[Seq[WorkflowSourceFilesCollection]] = { case class RequestedIdAndInputs(requestedId: Option[WorkflowId], inputs: WorkflowJson) def validateInputsAndRequestedIds(pws: PartialWorkflowSources): ErrorOr[Seq[RequestedIdAndInputs]] = { @@ -221,7 +255,9 @@ object PartialWorkflowSources { case (true, true) => Vector("{}").validNel case (true, false) => "No inputs were provided".invalidNel case _ => - val sortedInputAuxes = pws.workflowInputsAux.toSeq.sortBy { case (index, _) => index } map { case(_, inputJson) => Option(inputJson) } + val sortedInputAuxes = pws.workflowInputsAux.toSeq.sortBy { case (index, _) => index } map { + case (_, inputJson) => Option(inputJson) + } pws.workflowInputs.toList.traverse[ErrorOr, String] { workflowInputSet: WorkflowJson => mergeMaps(Seq(Option(workflowInputSet)) ++ sortedInputAuxes).map(_.toString) } @@ -231,7 +267,10 @@ object PartialWorkflowSources { if (pws.requestedWorkflowIds.isEmpty) { (workflowInputs map { i => RequestedIdAndInputs(None, i) }).validNel } else if (pws.requestedWorkflowIds.size == workflowInputs.size) { - (pws.requestedWorkflowIds.zip(workflowInputs).map { case (id, inputs) => RequestedIdAndInputs(Option(id), inputs) }).validNel + pws.requestedWorkflowIds + .zip(workflowInputs) + .map { case (id, inputs) => RequestedIdAndInputs(Option(id), inputs) } + .validNel } else { s"Mismatch between requested IDs count (${pws.requestedWorkflowIds.size}) and workflow inputs counts (${workflowInputs.size})".invalidNel } @@ -239,18 +278,23 @@ object PartialWorkflowSources { } def validateOptions(options: Option[WorkflowOptionsJson]): ErrorOr[WorkflowOptions] = - WorkflowOptions.fromJsonString(options.getOrElse("{}")).toErrorOr leftMap { _ map { i => s"Invalid workflow options provided: $i" } } + WorkflowOptions.fromJsonString(options.getOrElse("{}")).toErrorOr leftMap { + _ map { i => s"Invalid workflow options provided: $i" } + } - def validateLabels(labels: WorkflowJson) : ErrorOr[WorkflowJson] = { + def validateLabels(labels: WorkflowJson): ErrorOr[WorkflowJson] = { - def validateKeyValuePair(key: String, value: String): ErrorOr[Unit] = (Label.validateLabelKey(key), Label.validateLabelValue(value)).tupled.void + def validateKeyValuePair(key: String, value: String): ErrorOr[Unit] = + (Label.validateLabelKey(key), Label.validateLabelValue(value)).tupled.void - def validateLabelRestrictions(inputs: Map[String, JsValue]): ErrorOr[Unit] = { - inputs.toList.traverse[ErrorOr, Unit]({ - case (key, JsString(s)) => validateKeyValuePair(key, s) - case (key, other) => s"Invalid label $key: $other : Labels must be strings. ${Label.LabelExpectationsMessage}".invalidNel - }).void - } + def validateLabelRestrictions(inputs: Map[String, JsValue]): ErrorOr[Unit] = + inputs.toList + .traverse[ErrorOr, Unit] { + case (key, JsString(s)) => validateKeyValuePair(key, s) + case (key, other) => + s"Invalid label $key: $other : Labels must be strings. ${Label.LabelExpectationsMessage}".invalidNel + } + .void Try(labels.parseJson) match { case Success(JsObject(inputs)) => validateLabelRestrictions(inputs).map(_ => labels) @@ -262,11 +306,12 @@ object PartialWorkflowSources { partialSources match { case Valid(partialSource) => (validateInputsAndRequestedIds(partialSource), - validateOptions(partialSource.workflowOptions), - validateLabels(partialSource.customLabels.getOrElse("{}")), - partialSource.workflowUrl.traverse(validateWorkflowUrl)) mapN { - case (wfInputsAndIds, wfOptions, workflowLabels, wfUrl) => - wfInputsAndIds.map { case RequestedIdAndInputs(id, inputsJson) => WorkflowSourceFilesCollection( + validateOptions(partialSource.workflowOptions), + validateLabels(partialSource.customLabels.getOrElse("{}")), + partialSource.workflowUrl.traverse(validateWorkflowUrl) + ) mapN { case (wfInputsAndIds, wfOptions, workflowLabels, wfUrl) => + wfInputsAndIds.map { case RequestedIdAndInputs(id, inputsJson) => + WorkflowSourceFilesCollection( workflowSource = partialSource.workflowSource, workflowUrl = wfUrl, workflowRoot = partialSource.workflowRoot, @@ -278,7 +323,9 @@ object PartialWorkflowSources { importsFile = partialSource.zippedImports, warnings = partialSource.warnings, workflowOnHold = partialSource.workflowOnHold, - requestedWorkflowId = id) } + requestedWorkflowId = id + ) + } } case Invalid(err) => err.invalid } @@ -294,27 +341,28 @@ object PartialWorkflowSources { } def validateWorkflowUrl(workflowUrl: String): ErrorOr[WorkflowUrl] = { - def convertStringToUrl(workflowUrl: String): ErrorOr[WorkflowUrl] = { + def convertStringToUrl(workflowUrl: String): ErrorOr[WorkflowUrl] = Try(new URL(workflowUrl)) match { case Success(_) => workflowUrl.validNel case Failure(e) => s"Error while validating workflow url: ${e.getMessage}".invalidNel } - } val len = workflowUrl.length - if (len > MaxWorkflowUrlLength) s"Invalid workflow url: url has length $len, longer than the maximum allowed $MaxWorkflowUrlLength characters".invalidNel + if (len > MaxWorkflowUrlLength) + s"Invalid workflow url: url has length $len, longer than the maximum allowed $MaxWorkflowUrlLength characters".invalidNel else convertStringToUrl(workflowUrl) } - private def toMap(someInput: Option[String]): ErrorOr[Map[String, JsValue]] = { + private def toMap(someInput: Option[String]): ErrorOr[Map[String, JsValue]] = someInput match { case Some(input: String) => - Try(input.parseJson).toErrorOrWithContext(s"parse input: '$input', which is not a valid json. Please check for syntactical errors.") flatMap { + Try(input.parseJson).toErrorOrWithContext( + s"parse input: '$input', which is not a valid json. Please check for syntactical errors." + ) flatMap { case JsObject(inputMap) => inputMap.validNel - case j: JsValue => s"Submitted input '$input' of type ${j.getClass.getSimpleName} is not a valid JSON object.".invalidNel + case j: JsValue => + s"Submitted input '$input' of type ${j.getClass.getSimpleName} is not a valid JSON object.".invalidNel } case None => Map.empty[String, JsValue].validNel } - } } - diff --git a/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala b/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala index d63ab3bcb51..dd1d88d5f5d 100644 --- a/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala +++ b/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala @@ -41,7 +41,7 @@ trait SwaggerUiHttpService { * * @return Route serving the swagger UI. */ - final def swaggerUiRoute: Route = { + final def swaggerUiRoute: Route = pathEndOrSingleSlash { get { serveIndex @@ -61,13 +61,13 @@ trait SwaggerUiHttpService { // the subject of the CVE linked below while preserving any fragment identifiers to scroll to the right spot in // the Swagger UI. // https://github.com/swagger-api/swagger-ui/security/advisories/GHSA-qrmm-w75w-3wpx - (path("swagger" / "index.html") | path ("swagger")) { + (path("swagger" / "index.html") | path("swagger")) { get { redirect("/", StatusCodes.MovedPermanently) } } - } } + /** * An extension of HttpService to serve up a resource containing the swagger api as yaml or json. The resource * directory and path on the classpath must match the path for route. The resource can be any file type supported by the @@ -75,9 +75,8 @@ trait SwaggerUiHttpService { */ trait SwaggerResourceHttpService { - def getBasePathOverride(): Option[String] = { + def getBasePathOverride(): Option[String] = Option(System.getenv("SWAGGER_BASE_PATH")) - } /** * @return The directory for the resource under the classpath, and in the url @@ -104,18 +103,21 @@ trait SwaggerResourceHttpService { */ final def swaggerResourceRoute: Route = { // Serve Cromwell API docs from either `/swagger/cromwell.yaml` or just `cromwell.yaml`. - val swaggerDocsDirective = path(separateOnSlashes(swaggerDocsPath)) | path(s"$swaggerServiceName.$swaggerResourceType") + val swaggerDocsDirective = + path(separateOnSlashes(swaggerDocsPath)) | path(s"$swaggerServiceName.$swaggerResourceType") - def injectBasePath(basePath: Option[String])(response: HttpResponse): HttpResponse = { + def injectBasePath(basePath: Option[String])(response: HttpResponse): HttpResponse = basePath match { case _ if response.status != StatusCodes.OK => response case None => response - case Some(base_path) => response.mapEntity { entity => - val swapperFlow: Flow[ByteString, ByteString, Any] = Flow[ByteString].map(byteString => ByteString.apply(byteString.utf8String.replace("#basePath: ...", "basePath: " + base_path))) - entity.transformDataBytes(swapperFlow) - } + case Some(base_path) => + response.mapEntity { entity => + val swapperFlow: Flow[ByteString, ByteString, Any] = Flow[ByteString].map(byteString => + ByteString.apply(byteString.utf8String.replace("#basePath: ...", "basePath: " + base_path)) + ) + entity.transformDataBytes(swapperFlow) + } } - } val route = get { swaggerDocsDirective { @@ -135,6 +137,7 @@ trait SwaggerResourceHttpService { * Extends the SwaggerUiHttpService and SwaggerResourceHttpService to serve up both. */ trait SwaggerUiResourceHttpService extends SwaggerUiHttpService with SwaggerResourceHttpService { + /** * @return A route that redirects to the swagger UI and returns the swagger resource. */ diff --git a/engine/src/main/scala/cromwell/webservice/WebServiceUtils.scala b/engine/src/main/scala/cromwell/webservice/WebServiceUtils.scala index 7483b300f2b..ab5f26caa37 100644 --- a/engine/src/main/scala/cromwell/webservice/WebServiceUtils.scala +++ b/engine/src/main/scala/cromwell/webservice/WebServiceUtils.scala @@ -18,28 +18,35 @@ trait WebServiceUtils { type MaterializedFormData = Map[String, ByteString] - def materializeFormData(formData: Multipart.FormData)(implicit timeout: Timeout, materializer: Materializer, executionContext: ExecutionContext): Future[MaterializedFormData] = { - formData.parts.mapAsync[(String, ByteString)](1) { - bodyPart => bodyPart.toStrict(timeout.duration)(materializer).map(strict => bodyPart.name -> strict.entity.data)(executionContext) - }.runFold(Map.empty[String, ByteString])((map, tuple) => map + tuple)(materializer) - } + def materializeFormData(formData: Multipart.FormData)(implicit + timeout: Timeout, + materializer: Materializer, + executionContext: ExecutionContext + ): Future[MaterializedFormData] = + formData.parts + .mapAsync[(String, ByteString)](1) { bodyPart => + bodyPart + .toStrict(timeout.duration)(materializer) + .map(strict => bodyPart.name -> strict.entity.data)(executionContext) + } + .runFold(Map.empty[String, ByteString])((map, tuple) => map + tuple)(materializer) /** * Completes a response of a Product, probably a case class, using an implicit marshaller, probably a json encoder. */ - def completeResponse[A <: Product](statusCode: StatusCode, value: A, warnings: Seq[String]) - (implicit mt: ToEntityMarshaller[A]): Route = { + def completeResponse[A <: Product](statusCode: StatusCode, value: A, warnings: Seq[String])(implicit + mt: ToEntityMarshaller[A] + ): Route = complete((statusCode, warningHeaders(warnings), value)) - } // 2.13 added this, not sure why the baseline version was compiling actually. /** * Completes a response of a List of Product (probably a case class), using an implicit marshaller, probably a json encoder. */ - def completeResponse[A <: Product](statusCode: StatusCode, values: List[A], warnings: Seq[String]) - (implicit mt: ToEntityMarshaller[List[A]]): Route = { + def completeResponse[A <: Product](statusCode: StatusCode, values: List[A], warnings: Seq[String])(implicit + mt: ToEntityMarshaller[List[A]] + ): Route = complete((statusCode, warningHeaders(warnings), values)) - } /** * Completes a response of string with the supplied content type. @@ -52,11 +59,11 @@ trait WebServiceUtils { def completeResponse(statusCode: StatusCode, contentType: ContentType.NonBinary, value: String, - warnings: Seq[String]): Route = { + warnings: Seq[String] + ): Route = complete((statusCode, warningHeaders(warnings), HttpEntity(contentType, value))) - } - def warningHeaders(warnings: Seq[String]): List[HttpHeader] = { + def warningHeaders(warnings: Seq[String]): List[HttpHeader] = warnings.toList map { warning => /* Need a quoted string. @@ -65,12 +72,11 @@ trait WebServiceUtils { Using a poor version of ~~#! https://github.com/akka/akka-http/blob/v10.0.9/akka-http-core/src/main/scala/akka/http/impl/util/Rendering.scala#L206 */ - val quotedString = "\"" + warning.replaceAll("\"","\\\\\"").replaceAll("[\\r\\n]+", " ").trim + "\"" + val quotedString = "\"" + warning.replaceAll("\"", "\\\\\"").replaceAll("[\\r\\n]+", " ").trim + "\"" // https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.46 RawHeader("Warning", s"299 cromwell/$cromwellVersion $quotedString") } - } } object WebServiceUtils extends WebServiceUtils { @@ -79,17 +85,14 @@ object WebServiceUtils extends WebServiceUtils { // A: There are customers who rely on the pretty printing to display the error directly in a terminal or GUI. // AEN 2018-12-05 implicit class EnhancedThrowable(val e: Throwable) extends AnyVal { - def failRequest(statusCode: StatusCode, warnings: Seq[String] = Vector.empty): Route = { + def failRequest(statusCode: StatusCode, warnings: Seq[String] = Vector.empty): Route = completeResponse(statusCode, ContentTypes.`application/json`, prettyPrint(APIResponse.fail(e)), warnings) - } - def errorRequest(statusCode: StatusCode, warnings: Seq[String] = Vector.empty): Route = { + def errorRequest(statusCode: StatusCode, warnings: Seq[String] = Vector.empty): Route = completeResponse(statusCode, ContentTypes.`application/json`, prettyPrint(APIResponse.error(e)), warnings) - } } - private def prettyPrint(failureResponse: FailureResponse): String ={ + private def prettyPrint(failureResponse: FailureResponse): String = // .asJson cannot live inside a value class like `EnhancedThrowable`, hence the object method failureResponse.asJson.printWith(Printer.spaces2.copy(dropNullValues = true, colonLeft = "")) - } } diff --git a/engine/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala b/engine/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala index 0bb398f67af..b504818c1a4 100644 --- a/engine/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala +++ b/engine/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala @@ -13,7 +13,7 @@ import cromwell.services.metadata.MetadataArchiveStatus import cromwell.services.metadata.MetadataService._ import cromwell.util.JsonFormatting.WomValueJsonFormatter._ import cromwell.webservice.routes.CromwellApiService.BackendResponse -import spray.json.{DefaultJsonProtocol, JsString, JsValue, JsonFormat, RootJsonFormat} +import spray.json.{DefaultJsonProtocol, JsonFormat, JsString, JsValue, RootJsonFormat} object WorkflowJsonSupport extends DefaultJsonProtocol { implicit val workflowStatusResponseProtocol = jsonFormat2(WorkflowStatusResponse) @@ -25,18 +25,24 @@ object WorkflowJsonSupport extends DefaultJsonProtocol { implicit val BackendResponseFormat = jsonFormat2(BackendResponse) implicit val callAttempt = jsonFormat2(CallAttempt) - implicit val workflowOptionsFormatter: JsonFormat[WorkflowOptions] = new JsonFormat[WorkflowOptions] { + implicit val workflowOptionsFormatter: JsonFormat[WorkflowOptions] = new JsonFormat[WorkflowOptions] { override def read(json: JsValue): WorkflowOptions = json match { case str: JsString => WorkflowOptions.fromJsonString(str.value).get - case other => throw new UnsupportedOperationException(s"Cannot use ${other.getClass.getSimpleName} value. Expected a workflow options String") + case other => + throw new UnsupportedOperationException( + s"Cannot use ${other.getClass.getSimpleName} value. Expected a workflow options String" + ) } override def write(obj: WorkflowOptions): JsValue = JsString(obj.asPrettyJson) } - implicit val workflowIdFormatter: JsonFormat[WorkflowId] = new JsonFormat[WorkflowId] { + implicit val workflowIdFormatter: JsonFormat[WorkflowId] = new JsonFormat[WorkflowId] { override def read(json: JsValue): WorkflowId = json match { case str: JsString => WorkflowId.fromString(str.value) - case other => throw new UnsupportedOperationException(s"Cannot use ${other.getClass.getSimpleName} value. Expected a workflow ID String") + case other => + throw new UnsupportedOperationException( + s"Cannot use ${other.getClass.getSimpleName} value. Expected a workflow ID String" + ) } override def write(obj: WorkflowId): JsValue = JsString(obj.id.toString) } @@ -58,7 +64,7 @@ object WorkflowJsonSupport extends DefaultJsonProtocol { // By default the formatter for JsValues prints them out ADT-style. // In the case of SuccessResponses, we just want raw JsValues to be included in our output verbatim. - private implicit val identityJsValueFormatter = new RootJsonFormat[JsValue] { + implicit private val identityJsValueFormatter = new RootJsonFormat[JsValue] { override def read(json: JsValue): JsValue = json override def write(obj: JsValue): JsValue = obj } diff --git a/engine/src/main/scala/cromwell/webservice/routes/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/routes/CromwellApiService.scala index a1c4f023135..d75eed4ccd8 100644 --- a/engine/src/main/scala/cromwell/webservice/routes/CromwellApiService.scala +++ b/engine/src/main/scala/cromwell/webservice/routes/CromwellApiService.scala @@ -10,7 +10,7 @@ import akka.http.scaladsl.model._ import akka.http.scaladsl.model.ContentTypes._ import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.{ExceptionHandler, Route} -import akka.pattern.{AskTimeoutException, ask} +import akka.pattern.{ask, AskTimeoutException} import akka.stream.ActorMaterializer import akka.util.Timeout import cats.data.NonEmptyList @@ -23,7 +23,12 @@ import cromwell.core.{path => _, _} import cromwell.engine.backend.BackendConfiguration import cromwell.engine.instrumentation.HttpInstrumentation import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException -import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheDiffActor.{CachedCallNotFoundException, CallCacheDiffActorResponse, FailedCallCacheDiffResponse, SuccessfulCallCacheDiffResponse} +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheDiffActor.{ + CachedCallNotFoundException, + CallCacheDiffActorResponse, + FailedCallCacheDiffResponse, + SuccessfulCallCacheDiffResponse +} import cromwell.engine.workflow.lifecycle.execution.callcaching.{CallCacheDiffActor, CallCacheDiffQueryParameter} import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.NotInOnHoldStateException import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStoreEngineActor, WorkflowStoreSubmitActor} @@ -42,7 +47,11 @@ import scala.concurrent.{ExecutionContext, Future, TimeoutException} import scala.io.Source import scala.util.{Failure, Success, Try} -trait CromwellApiService extends HttpInstrumentation with MetadataRouteSupport with WomtoolRouteSupport with WebServiceUtils { +trait CromwellApiService + extends HttpInstrumentation + with MetadataRouteSupport + with WomtoolRouteSupport + with WebServiceUtils { import CromwellApiService._ implicit def actorRefFactory: ActorRefFactory @@ -60,11 +69,14 @@ trait CromwellApiService extends HttpInstrumentation with MetadataRouteSupport w val engineRoutes = concat( path("engine" / Segment / "stats") { _ => get { - completeResponse(StatusCodes.Forbidden, APIResponse.fail(new RuntimeException("The /stats endpoint is currently disabled.")), warnings = Seq.empty) + completeResponse(StatusCodes.Forbidden, + APIResponse.fail(new RuntimeException("The /stats endpoint is currently disabled.")), + warnings = Seq.empty + ) } }, path("engine" / Segment / "version") { _ => - get { complete(versionResponse) } + get(complete(versionResponse)) }, path("engine" / Segment / "status") { _ => onComplete(serviceRegistryActor.ask(GetCurrentStatus).mapTo[StatusCheckResponse]) { @@ -72,14 +84,15 @@ trait CromwellApiService extends HttpInstrumentation with MetadataRouteSupport w val httpCode = if (status.ok) StatusCodes.OK else StatusCodes.InternalServerError complete(ToResponseMarshallable((httpCode, status.systems))) case Failure(e: TimeoutException) => e.failRequest(StatusCodes.ServiceUnavailable) - case Failure(_) => new RuntimeException("Unable to gather engine status").failRequest(StatusCodes.InternalServerError) + case Failure(_) => + new RuntimeException("Unable to gather engine status").failRequest(StatusCodes.InternalServerError) } } ) val workflowRoutes = path("workflows" / Segment / "backends") { _ => - get { instrumentRequest { complete(ToResponseMarshallable(backendResponse)) } } + get(instrumentRequest(complete(ToResponseMarshallable(backendResponse)))) } ~ path("workflows" / Segment / "callcaching" / "diff") { _ => parameterSeq { parameters => @@ -87,11 +100,15 @@ trait CromwellApiService extends HttpInstrumentation with MetadataRouteSupport w instrumentRequest { CallCacheDiffQueryParameter.fromParameters(parameters) match { case Valid(queryParameter) => - val diffActor = actorRefFactory.actorOf(CallCacheDiffActor.props(serviceRegistryActor), "CallCacheDiffActor-" + UUID.randomUUID()) + val diffActor = actorRefFactory.actorOf(CallCacheDiffActor.props(serviceRegistryActor), + "CallCacheDiffActor-" + UUID.randomUUID() + ) onComplete(diffActor.ask(queryParameter).mapTo[CallCacheDiffActorResponse]) { case Success(r: SuccessfulCallCacheDiffResponse) => complete(r) - case Success(r: FailedCallCacheDiffResponse) => r.reason.errorRequest(StatusCodes.InternalServerError) - case Failure(_: AskTimeoutException) if CromwellShutdown.shutdownInProgress() => serviceShuttingDownResponse + case Success(r: FailedCallCacheDiffResponse) => + r.reason.errorRequest(StatusCodes.InternalServerError) + case Failure(_: AskTimeoutException) if CromwellShutdown.shutdownInProgress() => + serviceShuttingDownResponse case Failure(e: CachedCallNotFoundException) => e.errorRequest(StatusCodes.NotFound) case Failure(e: TimeoutException) => e.failRequest(StatusCodes.ServiceUnavailable) case Failure(e) => e.errorRequest(StatusCodes.InternalServerError) @@ -142,13 +159,21 @@ trait CromwellApiService extends HttpInstrumentation with MetadataRouteSupport w path("workflows" / Segment / Segment / "releaseHold") { (_, possibleWorkflowId) => post { instrumentRequest { - val response = validateWorkflowIdInMetadata(possibleWorkflowId, serviceRegistryActor) flatMap { workflowId => - workflowStoreActor.ask(WorkflowStoreActor.WorkflowOnHoldToSubmittedCommand(workflowId)).mapTo[WorkflowStoreEngineActor.WorkflowOnHoldToSubmittedResponse] + val response = validateWorkflowIdInMetadata(possibleWorkflowId, serviceRegistryActor) flatMap { + workflowId => + workflowStoreActor + .ask(WorkflowStoreActor.WorkflowOnHoldToSubmittedCommand(workflowId)) + .mapTo[WorkflowStoreEngineActor.WorkflowOnHoldToSubmittedResponse] } - onComplete(response){ - case Success(WorkflowStoreEngineActor.WorkflowOnHoldToSubmittedFailure(_, e: NotInOnHoldStateException)) => e.errorRequest(StatusCodes.Forbidden) - case Success(WorkflowStoreEngineActor.WorkflowOnHoldToSubmittedFailure(_, e)) => e.errorRequest(StatusCodes.InternalServerError) - case Success(r: WorkflowStoreEngineActor.WorkflowOnHoldToSubmittedSuccess) => completeResponse(StatusCodes.OK, toResponse(r.workflowId, WorkflowSubmitted), Seq.empty) + onComplete(response) { + case Success( + WorkflowStoreEngineActor.WorkflowOnHoldToSubmittedFailure(_, e: NotInOnHoldStateException) + ) => + e.errorRequest(StatusCodes.Forbidden) + case Success(WorkflowStoreEngineActor.WorkflowOnHoldToSubmittedFailure(_, e)) => + e.errorRequest(StatusCodes.InternalServerError) + case Success(r: WorkflowStoreEngineActor.WorkflowOnHoldToSubmittedSuccess) => + completeResponse(StatusCodes.OK, toResponse(r.workflowId, WorkflowSubmitted), Seq.empty) case Failure(e: UnrecognizedWorkflowException) => e.failRequest(StatusCodes.NotFound) case Failure(e: InvalidWorkflowException) => e.failRequest(StatusCodes.BadRequest) case Failure(e) => e.errorRequest(StatusCodes.InternalServerError) @@ -157,44 +182,49 @@ trait CromwellApiService extends HttpInstrumentation with MetadataRouteSupport w } } ~ metadataRoutes - private def metadataLookupForTimingRoute(workflowId: WorkflowId): Future[MetadataJsonResponse] = { val includeKeys = NonEmptyList.of("start", "end", "executionStatus", "executionEvents", "subWorkflowMetadata") - val readMetadataRequest = (w: WorkflowId) => GetSingleWorkflowMetadataAction(w, Option(includeKeys), None, expandSubWorkflows = true) + val readMetadataRequest = (w: WorkflowId) => + GetSingleWorkflowMetadataAction(w, Option(includeKeys), None, expandSubWorkflows = true) serviceRegistryActor.ask(readMetadataRequest(workflowId)).mapTo[MetadataJsonResponse] } - private def completeTimingRouteResponse(metadataResponse: Future[MetadataJsonResponse]) = { + private def completeTimingRouteResponse(metadataResponse: Future[MetadataJsonResponse]) = onComplete(metadataResponse) { case Success(r: SuccessfulMetadataJsonResponse) => - Try(Source.fromResource("workflowTimings/workflowTimings.html").mkString) match { case Success(wfTimingsContent) => - val response = HttpResponse(entity = wfTimingsContent.replace("\"{{REPLACE_THIS_WITH_METADATA}}\"", r.responseJson.toString)) + val response = HttpResponse(entity = + wfTimingsContent.replace("\"{{REPLACE_THIS_WITH_METADATA}}\"", r.responseJson.toString) + ) complete(response.withEntity(response.entity.withContentType(`text/html(UTF-8)`))) - case Failure(e) => completeResponse(StatusCodes.InternalServerError, APIResponse.fail(new RuntimeException("Error while loading workflowTimings.html", e)), Seq.empty) + case Failure(e) => + completeResponse(StatusCodes.InternalServerError, + APIResponse.fail(new RuntimeException("Error while loading workflowTimings.html", e)), + Seq.empty + ) } case Success(r: FailedMetadataJsonResponse) => r.reason.errorRequest(StatusCodes.InternalServerError) case Failure(_: AskTimeoutException) if CromwellShutdown.shutdownInProgress() => serviceShuttingDownResponse case Failure(e: TimeoutException) => e.failRequest(StatusCodes.ServiceUnavailable) case Failure(e) => e.failRequest(StatusCodes.InternalServerError) } - } - private def toResponse(workflowId: WorkflowId, workflowState: WorkflowState): WorkflowSubmitResponse = { + private def toResponse(workflowId: WorkflowId, workflowState: WorkflowState): WorkflowSubmitResponse = WorkflowSubmitResponse(workflowId.toString, workflowState.toString) - } private def submitRequest(formData: Multipart.FormData, isSingleSubmission: Boolean): Route = { - def getWorkflowState(workflowOnHold: Boolean): WorkflowState = { + def getWorkflowState(workflowOnHold: Boolean): WorkflowState = if (workflowOnHold) WorkflowOnHold else WorkflowSubmitted - } - def askSubmit(command: WorkflowStoreActor.WorkflowStoreActorSubmitCommand, warnings: Seq[String], workflowState: WorkflowState): Route = { + def askSubmit(command: WorkflowStoreActor.WorkflowStoreActorSubmitCommand, + warnings: Seq[String], + workflowState: WorkflowState + ): Route = // NOTE: Do not blindly copy the akka-http -to- ask-actor pattern below without knowing the pros and cons. onComplete(workflowStoreActor.ask(command).mapTo[WorkflowStoreSubmitActor.WorkflowStoreSubmitActorResponse]) { case Success(w) => @@ -210,14 +240,16 @@ trait CromwellApiService extends HttpInstrumentation with MetadataRouteSupport w case Failure(e: TimeoutException) => e.failRequest(StatusCodes.ServiceUnavailable) case Failure(e) => e.failRequest(StatusCodes.InternalServerError, warnings) } - } onComplete(materializeFormData(formData)) { case Success(data) => PartialWorkflowSources.fromSubmitRoute(data, allowNoInputs = isSingleSubmission) match { case Success(workflowSourceFiles) if isSingleSubmission && workflowSourceFiles.size == 1 => val warnings = workflowSourceFiles.flatMap(_.warnings) - askSubmit(WorkflowStoreActor.SubmitWorkflow(workflowSourceFiles.head), warnings, getWorkflowState(workflowSourceFiles.head.workflowOnHold)) + askSubmit(WorkflowStoreActor.SubmitWorkflow(workflowSourceFiles.head), + warnings, + getWorkflowState(workflowSourceFiles.head.workflowOnHold) + ) // Catches the case where someone has gone through the single submission endpoint w/ more than one workflow case Success(workflowSourceFiles) if isSingleSubmission => val warnings = workflowSourceFiles.flatMap(_.warnings) @@ -227,7 +259,9 @@ trait CromwellApiService extends HttpInstrumentation with MetadataRouteSupport w val warnings = workflowSourceFiles.flatMap(_.warnings) askSubmit( WorkflowStoreActor.BatchSubmitWorkflows(NonEmptyList.fromListUnsafe(workflowSourceFiles.toList)), - warnings, getWorkflowState(workflowSourceFiles.head.workflowOnHold)) + warnings, + getWorkflowState(workflowSourceFiles.head.workflowOnHold) + ) case Failure(t) => t.failRequest(StatusCodes.BadRequest) } case Failure(e: TimeoutException) => e.failRequest(StatusCodes.ServiceUnavailable) @@ -248,12 +282,13 @@ object CromwellApiService { workflowStoreActor: ActorRef, workflowManagerActor: ActorRef, successHandler: PartialFunction[SuccessfulAbortResponse, Route] = standardAbortSuccessHandler, - errorHandler: PartialFunction[Throwable, Route] = standardAbortErrorHandler) - (implicit timeout: Timeout): Route = { + errorHandler: PartialFunction[Throwable, Route] = standardAbortErrorHandler + )(implicit timeout: Timeout): Route = handleExceptions(ExceptionHandler(errorHandler)) { Try(WorkflowId.fromString(possibleWorkflowId)) match { case Success(workflowId) => - val response = workflowStoreActor.ask(WorkflowStoreActor.AbortWorkflowCommand(workflowId)).mapTo[AbortResponse] + val response = + workflowStoreActor.ask(WorkflowStoreActor.AbortWorkflowCommand(workflowId)).mapTo[AbortResponse] onComplete(response) { case Success(x: SuccessfulAbortResponse) => successHandler(x) case Success(x: WorkflowAbortFailureResponse) => throw x.failure @@ -262,14 +297,15 @@ object CromwellApiService { case Failure(_) => throw InvalidWorkflowException(possibleWorkflowId) } } - } /** * The abort success handler for typical cases, i.e. cromwell's API. */ private def standardAbortSuccessHandler: PartialFunction[SuccessfulAbortResponse, Route] = { - case WorkflowAbortedResponse(id) => complete(ToResponseMarshallable(WorkflowAbortResponse(id.toString, WorkflowAborted.toString))) - case WorkflowAbortRequestedResponse(id) => complete(ToResponseMarshallable(WorkflowAbortResponse(id.toString, WorkflowAborting.toString))) + case WorkflowAbortedResponse(id) => + complete(ToResponseMarshallable(WorkflowAbortResponse(id.toString, WorkflowAborted.toString))) + case WorkflowAbortRequestedResponse(id) => + complete(ToResponseMarshallable(WorkflowAbortResponse(id.toString, WorkflowAborting.toString))) } /** @@ -283,9 +319,10 @@ object CromwellApiService { case e: Exception => e.errorRequest(StatusCodes.InternalServerError) } - def validateWorkflowIdInMetadata(possibleWorkflowId: String, - serviceRegistryActor: ActorRef) - (implicit timeout: Timeout, executor: ExecutionContext): Future[WorkflowId] = { + def validateWorkflowIdInMetadata(possibleWorkflowId: String, serviceRegistryActor: ActorRef)(implicit + timeout: Timeout, + executor: ExecutionContext + ): Future[WorkflowId] = Try(WorkflowId.fromString(possibleWorkflowId)) match { case Success(w) => serviceRegistryActor.ask(ValidateWorkflowIdInMetadata(w)).mapTo[WorkflowValidationResponse] flatMap { @@ -295,11 +332,11 @@ object CromwellApiService { } case Failure(_) => Future.failed(InvalidWorkflowException(possibleWorkflowId)) } - } - def validateWorkflowIdInMetadataSummaries(possibleWorkflowId: String, - serviceRegistryActor: ActorRef) - (implicit timeout: Timeout, executor: ExecutionContext): Future[WorkflowId] = { + def validateWorkflowIdInMetadataSummaries(possibleWorkflowId: String, serviceRegistryActor: ActorRef)(implicit + timeout: Timeout, + executor: ExecutionContext + ): Future[WorkflowId] = Try(WorkflowId.fromString(possibleWorkflowId)) match { case Success(w) => serviceRegistryActor.ask(ValidateWorkflowIdInMetadataSummaries(w)).mapTo[WorkflowValidationResponse] map { @@ -309,17 +346,20 @@ object CromwellApiService { } case Failure(_) => Future.failed(InvalidWorkflowException(possibleWorkflowId)) } - } final case class BackendResponse(supportedBackends: List[String], defaultBackend: String) final case class UnrecognizedWorkflowException(id: WorkflowId) extends Exception(s"Unrecognized workflow ID: $id") - final case class InvalidWorkflowException(possibleWorkflowId: String) extends Exception(s"Invalid workflow ID: '$possibleWorkflowId'.") + final case class InvalidWorkflowException(possibleWorkflowId: String) + extends Exception(s"Invalid workflow ID: '$possibleWorkflowId'.") val cromwellVersion = VersionUtil.getVersion("cromwell-engine") val swaggerUiVersion = VersionUtil.getVersion("swagger-ui", VersionUtil.sbtDependencyVersion("swaggerUi")) - val backendResponse = BackendResponse(BackendConfiguration.AllBackendEntries.map(_.name).sorted, BackendConfiguration.DefaultBackendEntry.name) + val backendResponse = BackendResponse(BackendConfiguration.AllBackendEntries.map(_.name).sorted, + BackendConfiguration.DefaultBackendEntry.name + ) val versionResponse = JsObject(Map("cromwell" -> cromwellVersion.toJson)) - val serviceShuttingDownResponse = new Exception("Cromwell service is shutting down.").failRequest(StatusCodes.ServiceUnavailable) + val serviceShuttingDownResponse = + new Exception("Cromwell service is shutting down.").failRequest(StatusCodes.ServiceUnavailable) } diff --git a/engine/src/main/scala/cromwell/webservice/routes/MetadataRouteSupport.scala b/engine/src/main/scala/cromwell/webservice/routes/MetadataRouteSupport.scala index b84ff59d5a3..4ec1d0babc7 100644 --- a/engine/src/main/scala/cromwell/webservice/routes/MetadataRouteSupport.scala +++ b/engine/src/main/scala/cromwell/webservice/routes/MetadataRouteSupport.scala @@ -1,6 +1,6 @@ package cromwell.webservice.routes -import java.time.{OffsetDateTime, Duration => JDuration} +import java.time.{Duration => JDuration, OffsetDateTime} import java.util.concurrent.TimeUnit import akka.actor.{ActorRef, ActorRefFactory} @@ -9,14 +9,14 @@ import akka.http.scaladsl.marshalling.ToResponseMarshallable import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route -import akka.pattern.{AskTimeoutException, ask} +import akka.pattern.{ask, AskTimeoutException} import akka.util.Timeout import cats.data.NonEmptyList import cats.data.Validated.{Invalid, Valid} import cromwell.core.Dispatcher.ApiDispatcher import cromwell.core.instrumentation.InstrumentationPrefixes.ServicesPrefix import cromwell.core.labels.Labels -import cromwell.core.{WorkflowId, WorkflowMetadataKeys, path => _} +import cromwell.core.{path => _, WorkflowId, WorkflowMetadataKeys} import cromwell.engine.instrumentation.HttpInstrumentation import cromwell.server.CromwellShutdown import cromwell.services._ @@ -38,7 +38,6 @@ import scala.concurrent.duration._ import scala.concurrent.{ExecutionContext, Future, TimeoutException} import scala.util.{Failure, Success} - trait MetadataRouteSupport extends HttpInstrumentation { implicit def actorRefFactory: ActorRefFactory implicit val ec: ExecutionContext @@ -83,15 +82,21 @@ trait MetadataRouteSupport extends HttpInstrumentation { encodeResponse { path("workflows" / Segment / Segment / "metadata") { (_, possibleWorkflowId) => instrumentRequest { - parameters((Symbol("includeKey").*, Symbol("excludeKey").*, Symbol("expandSubWorkflows").as[Boolean].?)) { (includeKeys, excludeKeys, expandSubWorkflowsOption) => - val includeKeysOption = NonEmptyList.fromList(includeKeys.toList) - val excludeKeysOption = NonEmptyList.fromList(excludeKeys.toList) - val expandSubWorkflows = expandSubWorkflowsOption.getOrElse(false) + parameters((Symbol("includeKey").*, Symbol("excludeKey").*, Symbol("expandSubWorkflows").as[Boolean].?)) { + (includeKeys, excludeKeys, expandSubWorkflowsOption) => + val includeKeysOption = NonEmptyList.fromList(includeKeys.toList) + val excludeKeysOption = NonEmptyList.fromList(excludeKeys.toList) + val expandSubWorkflows = expandSubWorkflowsOption.getOrElse(false) - metadataLookup( - possibleWorkflowId, - (w: WorkflowId) => GetSingleWorkflowMetadataAction(w, includeKeysOption, excludeKeysOption, expandSubWorkflows), - serviceRegistryActor) + metadataLookup(possibleWorkflowId, + (w: WorkflowId) => + GetSingleWorkflowMetadataAction(w, + includeKeysOption, + excludeKeysOption, + expandSubWorkflows + ), + serviceRegistryActor + ) } } } @@ -107,7 +112,8 @@ trait MetadataRouteSupport extends HttpInstrumentation { entity(as[Map[String, String]]) { parameterMap => instrumentRequest { Labels.validateMapOfLabels(parameterMap) match { - case Valid(labels) => patchLabelsRequest(possibleWorkflowId, labels, serviceRegistryActor, actorRefFactory) + case Valid(labels) => + patchLabelsRequest(possibleWorkflowId, labels, serviceRegistryActor, actorRefFactory) case Invalid(e) => val iae = new IllegalArgumentException(e.toList.mkString(",")) iae.failRequest(StatusCodes.BadRequest) @@ -141,41 +147,42 @@ object MetadataRouteSupport { private def processMetadataArchivedResponse(workflowId: WorkflowId, archiveStatus: MetadataArchiveStatus, endTime: Option[OffsetDateTime], - additionalMsg: String = ""): JsObject = { + additionalMsg: String = "" + ): JsObject = { val baseMessage = "Cromwell has archived this workflow's metadata according to the lifecycle policy." val timeSinceMessage = endTime map { timestamp => val duration = FiniteDuration(JDuration.between(timestamp, OffsetDateTime.now()).toMillis, TimeUnit.MILLISECONDS) s" The workflow completed at $timestamp, which was ${duration} ago." - } getOrElse("") - val additionalDetails = if (archiveStatus == MetadataArchiveStatus.ArchivedAndDeleted) - " It is available in the archive bucket, or via a support request in the case of a managed instance." - else "" + } getOrElse "" + val additionalDetails = + if (archiveStatus == MetadataArchiveStatus.ArchivedAndDeleted) + " It is available in the archive bucket, or via a support request in the case of a managed instance." + else "" - JsObject(Map( - WorkflowMetadataKeys.Id -> JsString(workflowId.toString), - WorkflowMetadataKeys.MetadataArchiveStatus -> JsString(archiveStatus.toString), - WorkflowMetadataKeys.Message -> JsString(baseMessage + timeSinceMessage + additionalDetails + additionalMsg) - )) + JsObject( + Map( + WorkflowMetadataKeys.Id -> JsString(workflowId.toString), + WorkflowMetadataKeys.MetadataArchiveStatus -> JsString(archiveStatus.toString), + WorkflowMetadataKeys.Message -> JsString(baseMessage + timeSinceMessage + additionalDetails + additionalMsg) + ) + ) } def metadataLookup(possibleWorkflowId: String, request: WorkflowId => BuildMetadataJsonAction, - serviceRegistryActor: ActorRef) - (implicit timeout: Timeout, - ec: ExecutionContext): Route = { + serviceRegistryActor: ActorRef + )(implicit timeout: Timeout, ec: ExecutionContext): Route = completeMetadataBuilderResponse(metadataBuilderActorRequest(possibleWorkflowId, request, serviceRegistryActor)) - } - def queryMetadata(parameters: Seq[(String, String)], - serviceRegistryActor: ActorRef)(implicit timeout: Timeout): Route = { + def queryMetadata(parameters: Seq[(String, String)], serviceRegistryActor: ActorRef)(implicit + timeout: Timeout + ): Route = completeMetadataQueryResponse(metadataQueryRequest(parameters, serviceRegistryActor)) - } def metadataBuilderActorRequest(possibleWorkflowId: String, request: WorkflowId => BuildMetadataJsonAction, - serviceRegistryActor: ActorRef) - (implicit timeout: Timeout, - ec: ExecutionContext): Future[MetadataJsonResponse] = { + serviceRegistryActor: ActorRef + )(implicit timeout: Timeout, ec: ExecutionContext): Future[MetadataJsonResponse] = { def recordHistoricalMetadataLookupMetrics(endTime: Option[OffsetDateTime]): Unit = { val timeSinceEndTime = endTime match { @@ -183,8 +190,11 @@ object MetadataRouteSupport { case None => 0.seconds } - val lagInstrumentationPath = MetadataServiceActor.MetadataInstrumentationPrefix :+ "archiver" :+ "historical_metadata_lookup" :+ "lag" - val lagMessage = InstrumentationServiceMessage(CromwellTiming(CromwellBucket(ServicesPrefix.toList, lagInstrumentationPath), timeSinceEndTime)) + val lagInstrumentationPath = + MetadataServiceActor.MetadataInstrumentationPrefix :+ "archiver" :+ "historical_metadata_lookup" :+ "lag" + val lagMessage = InstrumentationServiceMessage( + CromwellTiming(CromwellBucket(ServicesPrefix.toList, lagInstrumentationPath), timeSinceEndTime) + ) serviceRegistryActor ! lagMessage val interestingDayMarks = (5.to(55, step = 5) ++ 0.to(4)).map(d => (d.days, s"${d}_day_old")) @@ -192,29 +202,39 @@ object MetadataRouteSupport { (interestingDayMarks ++ interestingMonthMarks) foreach { case (timeSpan, metricName) if timeSinceEndTime >= timeSpan => - val oldMetadataCounterPath = MetadataServiceActor.MetadataInstrumentationPrefix :+ "archiver" :+ "historical_metadata_lookup" :+ "lookup_age_counts" :+ metricName - val oldMetadataCounterMessage = InstrumentationServiceMessage(CromwellIncrement(CromwellBucket(ServicesPrefix.toList, oldMetadataCounterPath))) + val oldMetadataCounterPath = + MetadataServiceActor.MetadataInstrumentationPrefix :+ "archiver" :+ "historical_metadata_lookup" :+ "lookup_age_counts" :+ metricName + val oldMetadataCounterMessage = InstrumentationServiceMessage( + CromwellIncrement(CromwellBucket(ServicesPrefix.toList, oldMetadataCounterPath)) + ) serviceRegistryActor ! oldMetadataCounterMessage case _ => // Do nothing } } def checkIfMetadataDeletedAndRespond(id: WorkflowId, - metadataRequest: BuildWorkflowMetadataJsonWithOverridableSourceAction): Future[MetadataJsonResponse] = { - serviceRegistryActor.ask(FetchWorkflowMetadataArchiveStatusAndEndTime(id)).mapTo[FetchWorkflowArchiveStatusAndEndTimeResponse] flatMap { + metadataRequest: BuildWorkflowMetadataJsonWithOverridableSourceAction + ): Future[MetadataJsonResponse] = + serviceRegistryActor + .ask(FetchWorkflowMetadataArchiveStatusAndEndTime(id)) + .mapTo[FetchWorkflowArchiveStatusAndEndTimeResponse] flatMap { case WorkflowMetadataArchivedStatusAndEndTime(archiveStatus, endTime) => recordHistoricalMetadataLookupMetrics(endTime) - if (archiveStatus.isDeleted) Future.successful(SuccessfulMetadataJsonResponse(metadataRequest, processMetadataArchivedResponse(id, archiveStatus, endTime))) + if (archiveStatus.isDeleted) + Future.successful( + SuccessfulMetadataJsonResponse(metadataRequest, + processMetadataArchivedResponse(id, archiveStatus, endTime) + ) + ) else serviceRegistryActor.ask(request(id)).mapTo[MetadataJsonResponse] case FailedToGetArchiveStatusAndEndTime(e) => Future.failed(e) } - } validateWorkflowIdInMetadata(possibleWorkflowId, serviceRegistryActor) flatMap { id => /* for requests made to one of /metadata, /logs or /outputs endpoints, perform an additional check to see if metadata for the workflow has been archived and deleted or not (as they interact with metadata table) - */ + */ request(id) match { case m: BuildWorkflowMetadataJsonWithOverridableSourceAction => checkIfMetadataDeletedAndRespond(id, m) case _ => serviceRegistryActor.ask(request(id)).mapTo[MetadataJsonResponse] @@ -222,8 +242,9 @@ object MetadataRouteSupport { } } - def completeMetadataBuilderResponse(response: Future[MetadataJsonResponse]): Route = { - onComplete(response) { case Success(r: SuccessfulMetadataJsonResponse) => complete(r.responseJson) + def completeMetadataBuilderResponse(response: Future[MetadataJsonResponse]): Route = + onComplete(response) { + case Success(r: SuccessfulMetadataJsonResponse) => complete(r.responseJson) case Success(r: FailedMetadataJsonResponse) => r.reason.errorRequest(StatusCodes.InternalServerError) case Failure(_: AskTimeoutException) if CromwellShutdown.shutdownInProgress() => serviceShuttingDownResponse case Failure(e: UnrecognizedWorkflowException) => e.failRequest(StatusCodes.NotFound) @@ -231,12 +252,11 @@ object MetadataRouteSupport { case Failure(e: TimeoutException) => e.failRequest(StatusCodes.ServiceUnavailable) case Failure(e) => e.errorRequest(StatusCodes.InternalServerError) } - } - def metadataQueryRequest(parameters: Seq[(String, String)], - serviceRegistryActor: ActorRef)(implicit timeout: Timeout): Future[MetadataQueryResponse] = { + def metadataQueryRequest(parameters: Seq[(String, String)], serviceRegistryActor: ActorRef)(implicit + timeout: Timeout + ): Future[MetadataQueryResponse] = serviceRegistryActor.ask(QueryForWorkflowsMatchingParameters(parameters)).mapTo[MetadataQueryResponse] - } def completeMetadataQueryResponse(response: Future[MetadataQueryResponse]): Route = { import cromwell.webservice.WorkflowJsonSupport.workflowQueryResponse @@ -250,41 +270,49 @@ object MetadataRouteSupport { } } - def completePatchLabelsResponse(response: Future[LabelsManagerActorResponse]): Route = { + def completePatchLabelsResponse(response: Future[LabelsManagerActorResponse]): Route = onComplete(response) { case Success(r: BuiltLabelsManagerResponse) => complete(r.response) - case Success(r: WorkflowArchivedLabelsManagerResponse) => completeResponse(StatusCodes.BadRequest, r.response, Seq.empty) + case Success(r: WorkflowArchivedLabelsManagerResponse) => + completeResponse(StatusCodes.BadRequest, r.response, Seq.empty) case Success(e: FailedLabelsManagerResponse) => e.reason.failRequest(StatusCodes.InternalServerError) case Failure(e: UnrecognizedWorkflowException) => e.failRequest(StatusCodes.NotFound) case Failure(e: TimeoutException) => e.failRequest(StatusCodes.ServiceUnavailable) case Failure(e) => e.errorRequest(StatusCodes.InternalServerError) } - } def patchLabelsRequest(possibleWorkflowId: String, labels: Labels, serviceRegistryActor: ActorRef, - actorRefFactory: ActorRefFactory) - (implicit timeout: Timeout, ec: ExecutionContext): Route = { + actorRefFactory: ActorRefFactory + )(implicit timeout: Timeout, ec: ExecutionContext): Route = { - def checkIfMetadataArchivedAndRespond(id: WorkflowId, archiveStatusResponse: FetchWorkflowArchiveStatusAndEndTimeResponse): Future[LabelsManagerActorResponse] = { + def checkIfMetadataArchivedAndRespond(id: WorkflowId, + archiveStatusResponse: FetchWorkflowArchiveStatusAndEndTimeResponse + ): Future[LabelsManagerActorResponse] = archiveStatusResponse match { case WorkflowMetadataArchivedStatusAndEndTime(archiveStatus, endTime) => if (archiveStatus.isArchived) { - val message = " As a result, new labels can't be added or existing labels can't be updated for this workflow." - Future.successful(WorkflowArchivedLabelsManagerResponse(processMetadataArchivedResponse(id, archiveStatus, endTime, message))) - } - else { - val lma = actorRefFactory.actorOf(LabelsManagerActor.props(serviceRegistryActor).withDispatcher(ApiDispatcher)) + val message = + " As a result, new labels can't be added or existing labels can't be updated for this workflow." + Future.successful( + WorkflowArchivedLabelsManagerResponse( + processMetadataArchivedResponse(id, archiveStatus, endTime, message) + ) + ) + } else { + val lma = + actorRefFactory.actorOf(LabelsManagerActor.props(serviceRegistryActor).withDispatcher(ApiDispatcher)) lma.ask(LabelsAddition(LabelsData(id, labels))).mapTo[LabelsManagerActorResponse] } case FailedToGetArchiveStatusAndEndTime(e) => Future.failed(e) } - } val response = for { id <- validateWorkflowIdInMetadataSummaries(possibleWorkflowId, serviceRegistryActor) - archiveStatusResponse <- serviceRegistryActor.ask(FetchWorkflowMetadataArchiveStatusAndEndTime(id)).mapTo[FetchWorkflowArchiveStatusAndEndTimeResponse] + archiveStatusResponse <- serviceRegistryActor + .ask(FetchWorkflowMetadataArchiveStatusAndEndTime(id)) + .mapTo[FetchWorkflowArchiveStatusAndEndTimeResponse] response <- checkIfMetadataArchivedAndRespond(id, archiveStatusResponse) } yield response diff --git a/engine/src/main/scala/cromwell/webservice/routes/WomtoolRouteSupport.scala b/engine/src/main/scala/cromwell/webservice/routes/WomtoolRouteSupport.scala index 0690c11dcb5..b073c3dae8d 100644 --- a/engine/src/main/scala/cromwell/webservice/routes/WomtoolRouteSupport.scala +++ b/engine/src/main/scala/cromwell/webservice/routes/WomtoolRouteSupport.scala @@ -2,20 +2,27 @@ package cromwell.webservice.routes import akka.actor.{ActorRef, ActorRefFactory} import akka.http.scaladsl.model._ +import akka.http.scaladsl.model.headers.OAuth2BearerToken import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route -import akka.pattern.ask import akka.stream.ActorMaterializer import akka.util.Timeout import cromwell.core.{WorkflowOptions, WorkflowSourceFilesCollection} -import cromwell.services.womtool.WomtoolServiceMessages.{DescribeFailure, DescribeRequest, DescribeResult, DescribeSuccess} +import cromwell.languages.util.ImportResolver.ImportAuthProvider +import cromwell.services.auth.GithubAuthVendingSupport +import cromwell.services.womtool.WomtoolServiceMessages.{ + DescribeFailure, + DescribeRequest, + DescribeResult, + DescribeSuccess +} import cromwell.webservice.WebServiceUtils import cromwell.webservice.WebServiceUtils.EnhancedThrowable import scala.concurrent.ExecutionContext import scala.util.{Failure, Success} -trait WomtoolRouteSupport extends WebServiceUtils { +trait WomtoolRouteSupport extends WebServiceUtils with GithubAuthVendingSupport { implicit def actorRefFactory: ActorRefFactory implicit val ec: ExecutionContext @@ -28,17 +35,25 @@ trait WomtoolRouteSupport extends WebServiceUtils { path("womtool" / Segment / "describe") { _ => post { entity(as[Multipart.FormData]) { formData: Multipart.FormData => - onComplete(materializeFormData(formData)) { - case Success(data) => - validateAndSubmitRequest(data) - case Failure(e) => - e.failRequest(StatusCodes.InternalServerError) + extractCredentials { creds => + val authProviders: List[ImportAuthProvider] = creds match { + case Some(OAuth2BearerToken(token)) => List(importAuthProvider(token)) + case _ => List.empty + } + onComplete(materializeFormData(formData)(timeout, materializer, ec)) { + case Success(data) => + validateAndSubmitRequest(data, authProviders) + case Failure(e) => + e.failRequest(StatusCodes.InternalServerError) + } } } } } - private def validateAndSubmitRequest(data: MaterializedFormData): Route = { + private def validateAndSubmitRequest(data: MaterializedFormData, + importAuthProviders: List[ImportAuthProvider] + ): Route = { // TODO: move constants to WebServiceUtils, adopt in PartialWorkflowSources val workflowSource = data.get("workflowSource").map(_.utf8String) val workflowUrl = data.get("workflowUrl").map(_.utf8String) @@ -61,7 +76,7 @@ trait WomtoolRouteSupport extends WebServiceUtils { requestedWorkflowId = None ) - onComplete(serviceRegistryActor.ask(DescribeRequest(wsfc)).mapTo[DescribeResult]) { + onComplete(serviceRegistryActor.ask(DescribeRequest(wsfc, importAuthProviders)).mapTo[DescribeResult]) { case Success(response: DescribeSuccess) => import cromwell.services.womtool.models.WorkflowDescription.workflowDescriptionEncoder import de.heikoseeberger.akkahttpcirce.ErrorAccumulatingCirceSupport._ diff --git a/engine/src/main/scala/cromwell/webservice/routes/wes/CromwellMetadata.scala b/engine/src/main/scala/cromwell/webservice/routes/wes/CromwellMetadata.scala index d8c38de8dc9..0ff96489169 100644 --- a/engine/src/main/scala/cromwell/webservice/routes/wes/CromwellMetadata.scala +++ b/engine/src/main/scala/cromwell/webservice/routes/wes/CromwellMetadata.scala @@ -8,7 +8,7 @@ final case class CromwellSubmittedFiles(workflow: Option[String], options: Option[String], inputs: Option[String], labels: Option[String] - ) +) final case class CromwellCallsMetadata(shardIndex: Option[Int], commandLine: Option[String], @@ -17,7 +17,7 @@ final case class CromwellCallsMetadata(shardIndex: Option[Int], end: Option[String], stdout: Option[String], stderr: Option[String] - ) +) final case class CromwellMetadata(workflowName: Option[String], id: String, @@ -27,7 +27,7 @@ final case class CromwellMetadata(workflowName: Option[String], submittedFiles: CromwellSubmittedFiles, outputs: Option[JsObject], calls: Option[Map[String, Seq[CromwellCallsMetadata]]] - ) { +) { import CromwellMetadata._ def wesRunLog: WesRunLog = { @@ -35,7 +35,8 @@ final case class CromwellMetadata(workflowName: Option[String], val workflowTags = submittedFiles.labels.map(JsonParser(_).asJsObject) val workflowEngineParams = submittedFiles.options.map(JsonParser(_).asJsObject) - val workflowRequest = WesRunRequest(workflow_params = workflowParams, + val workflowRequest = WesRunRequest( + workflow_params = workflowParams, workflow_type = submittedFiles.workflowType.getOrElse("None supplied"), workflow_type_version = submittedFiles.workflowTypeVersion.getOrElse("None supplied"), tags = workflowTags, @@ -44,12 +45,12 @@ final case class CromwellMetadata(workflowName: Option[String], ) val workflowLogData = WesLog(name = workflowName, - cmd = None, - start_time = start, - end_time = end, - stdout = None, - stderr = None, - exit_code = None + cmd = None, + start_time = start, + end_time = end, + stdout = None, + stderr = None, + exit_code = None ) val taskLogs = for { @@ -74,7 +75,9 @@ object CromwellMetadata { import spray.json.DefaultJsonProtocol._ implicit val cromwellCallsMetadataFormat: JsonFormat[CromwellCallsMetadata] = jsonFormat7(CromwellCallsMetadata.apply) - implicit val cromwellSubmittedFilesFormat: JsonFormat[CromwellSubmittedFiles] = jsonFormat6(CromwellSubmittedFiles.apply) + implicit val cromwellSubmittedFilesFormat: JsonFormat[CromwellSubmittedFiles] = jsonFormat6( + CromwellSubmittedFiles.apply + ) implicit val cromwellMetadataFormat: JsonFormat[CromwellMetadata] = jsonFormat8(CromwellMetadata.apply) def fromJson(json: String): CromwellMetadata = { diff --git a/engine/src/main/scala/cromwell/webservice/routes/wes/RunListResponse.scala b/engine/src/main/scala/cromwell/webservice/routes/wes/RunListResponse.scala index b9bea188061..11ca85082ac 100644 --- a/engine/src/main/scala/cromwell/webservice/routes/wes/RunListResponse.scala +++ b/engine/src/main/scala/cromwell/webservice/routes/wes/RunListResponse.scala @@ -8,13 +8,11 @@ import cromwell.webservice.routes.wes.WesState.fromStatusString case class RunListResponse(runs: List[WesRunStatus], next_page_token: String) object RunListResponse { - def fromMetadataQueryResponse(response: MetadataService.MetadataQueryResponse): WesResponse = { - - response match { + def fromMetadataQueryResponse(response: MetadataService.MetadataQueryResponse): WesResponse = + response match { case w: WorkflowQuerySuccess => val runs = w.response.results.toList.map(x => WesRunStatus(x.id, fromStatusString(x.status))) WesResponseRunList(runs) case f: WorkflowQueryFailure => WesErrorResponse(f.reason.getMessage, StatusCodes.BadRequest.intValue) } - } } diff --git a/engine/src/main/scala/cromwell/webservice/routes/wes/ServiceInfo.scala b/engine/src/main/scala/cromwell/webservice/routes/wes/ServiceInfo.scala index 02710a5c7e1..c6a3a8401cd 100644 --- a/engine/src/main/scala/cromwell/webservice/routes/wes/ServiceInfo.scala +++ b/engine/src/main/scala/cromwell/webservice/routes/wes/ServiceInfo.scala @@ -49,9 +49,12 @@ object ServiceInfo { /** * Generate any runtime level information and create a response to the client */ - def toWesResponse(workflowStoreActor: ActorRef)(implicit ec: ExecutionContext, timeout: Timeout): Future[WesStatusInfoResponse] = { + def toWesResponse( + workflowStoreActor: ActorRef + )(implicit ec: ExecutionContext, timeout: Timeout): Future[WesStatusInfoResponse] = workflowStats(workflowStoreActor).map(stats => - WesStatusInfoResponse(WorkflowTypeVersion, + WesStatusInfoResponse( + WorkflowTypeVersion, SupportedWesVersions, SupportedFilesystemProtocols, WorkflowEngineVerisons, @@ -59,18 +62,20 @@ object ServiceInfo { stats, AuthInstructionsUrl.toString, ContactInfoUrl.toString, - Tags) + Tags + ) ) - } /** * Retrieve a map from state to count for all represented non-terminal workflow states */ - private def workflowStats(workflowStoreActor: ActorRef)(implicit ec: ExecutionContext, timeout: Timeout): Future[Map[WesState, Int]] = { - workflowStoreActor.ask(GetWorkflowStoreStats) + private def workflowStats( + workflowStoreActor: ActorRef + )(implicit ec: ExecutionContext, timeout: Timeout): Future[Map[WesState, Int]] = + workflowStoreActor + .ask(GetWorkflowStoreStats) .mapTo[Map[WorkflowState, Int]] .map(m => m.map(e => WesState.fromCromwellStatus(e._1) -> e._2)) // Convert WorkflowState -> WesState - } } /* diff --git a/engine/src/main/scala/cromwell/webservice/routes/wes/WesResponse.scala b/engine/src/main/scala/cromwell/webservice/routes/wes/WesResponse.scala index 58313cb5946..9d2ca2a5e82 100644 --- a/engine/src/main/scala/cromwell/webservice/routes/wes/WesResponse.scala +++ b/engine/src/main/scala/cromwell/webservice/routes/wes/WesResponse.scala @@ -17,22 +17,22 @@ final case class WesRunLog(run_id: String, run_log: Option[WesLog], task_logs: Option[List[WesLog]], outputs: Option[JsObject] - ) extends WesResponse +) extends WesResponse object WesRunLog { def fromJson(json: String): WesRunLog = CromwellMetadata.fromJson(json).wesRunLog } - final case class WesStatusInfoResponse(workflow_type_version: Map[String, Iterable[String]], - supported_wes_versions: Iterable[String], - supported_filesystem_protocols: Iterable[String], - workflow_engine_versions: Map[String, String], - default_workflow_engine_parameters: Iterable[DefaultWorkflowEngineParameter], - system_state_counts: Map[WesState, Int], - auth_instructions_url: String, - contact_info_url: String, - tags: Map[String, String]) extends WesResponse + supported_wes_versions: Iterable[String], + supported_filesystem_protocols: Iterable[String], + workflow_engine_versions: Map[String, String], + default_workflow_engine_parameters: Iterable[DefaultWorkflowEngineParameter], + system_state_counts: Map[WesState, Int], + auth_instructions_url: String, + contact_info_url: String, + tags: Map[String, String] +) extends WesResponse object WesResponseJsonSupport extends SprayJsonSupport with DefaultJsonProtocol { import WesStateJsonSupport._ @@ -51,7 +51,7 @@ object WesResponseJsonSupport extends SprayJsonSupport with DefaultJsonProtocol implicit object WesResponseFormat extends RootJsonFormat[WesResponse] { import spray.json._ - def write(r: WesResponse) = { + def write(r: WesResponse) = r match { case r: WesRunId => r.toJson case s: WesRunStatus => s.toJson @@ -61,8 +61,9 @@ object WesResponseJsonSupport extends SprayJsonSupport with DefaultJsonProtocol case m: WesResponseWorkflowMetadata => m.toJson case w: WesRunLog => w.toJson } - } - def read(value: JsValue) = throw new UnsupportedOperationException("Reading WesResponse objects from JSON is not supported") + def read(value: JsValue) = throw new UnsupportedOperationException( + "Reading WesResponse objects from JSON is not supported" + ) } } diff --git a/engine/src/main/scala/cromwell/webservice/routes/wes/WesRouteSupport.scala b/engine/src/main/scala/cromwell/webservice/routes/wes/WesRouteSupport.scala index 41973bb8981..bd5977ae853 100644 --- a/engine/src/main/scala/cromwell/webservice/routes/wes/WesRouteSupport.scala +++ b/engine/src/main/scala/cromwell/webservice/routes/wes/WesRouteSupport.scala @@ -5,7 +5,7 @@ import akka.http.scaladsl.model.{Multipart, StatusCode, StatusCodes} import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.directives.RouteDirectives.complete import akka.http.scaladsl.server.{Directive1, Route} -import akka.pattern.{AskTimeoutException, ask} +import akka.pattern.{ask, AskTimeoutException} import akka.stream.ActorMaterializer import akka.util.Timeout import cats.data.NonEmptyList @@ -16,12 +16,18 @@ import cromwell.engine.instrumentation.HttpInstrumentation import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStoreSubmitActor} import cromwell.server.CromwellShutdown -import cromwell.services.metadata.MetadataService.{BuildMetadataJsonAction, GetSingleWorkflowMetadataAction, GetStatus, MetadataServiceResponse, StatusLookupFailed} +import cromwell.services.metadata.MetadataService.{ + BuildMetadataJsonAction, + GetSingleWorkflowMetadataAction, + GetStatus, + MetadataServiceResponse, + StatusLookupFailed +} import cromwell.services.{FailedMetadataJsonResponse, SuccessfulMetadataJsonResponse} import cromwell.webservice.PartialWorkflowSources -import cromwell.webservice.WebServiceUtils.{EnhancedThrowable, completeResponse, materializeFormData} +import cromwell.webservice.WebServiceUtils.{completeResponse, materializeFormData, EnhancedThrowable} import cromwell.webservice.routes.CromwellApiService -import cromwell.webservice.routes.CromwellApiService.{UnrecognizedWorkflowException, validateWorkflowIdInMetadata} +import cromwell.webservice.routes.CromwellApiService.{validateWorkflowIdInMetadata, UnrecognizedWorkflowException} import cromwell.webservice.routes.MetadataRouteSupport.{metadataBuilderActorRequest, metadataQueryRequest} import cromwell.webservice.routes.wes.WesResponseJsonSupport._ import cromwell.webservice.routes.wes.WesRouteSupport.{respondWithWesError, _} @@ -31,8 +37,6 @@ import scala.concurrent.duration.FiniteDuration import scala.concurrent.{ExecutionContext, Future, TimeoutException} import scala.util.{Failure, Success} - - trait WesRouteSupport extends HttpInstrumentation { val serviceRegistryActor: ActorRef @@ -55,7 +59,7 @@ trait WesRouteSupport extends HttpInstrumentation { - It'd require a fairly substantial refactor of the MetadataBuilderActor to be more general - It's expected that for now the usage of these endpoints will not be extensive, so the protections of the regulator should not be necessary - */ + */ val wesRoutes: Route = instrumentRequest { concat( @@ -74,19 +78,25 @@ trait WesRouteSupport extends HttpInstrumentation { } ~ post { extractSubmission() { submission => - wesSubmitRequest(submission.entity, - isSingleSubmission = true) + wesSubmitRequest(submission.entity, isSingleSubmission = true) } } }, path("runs" / Segment) { workflowId => get { // this is what it was like in code found in the project… it perhaps isn’t ideal but doesn’t seem to hurt, so leaving it like this for now. - completeCromwellResponse(runLog(workflowId, (w: WorkflowId) => GetSingleWorkflowMetadataAction(w, None, None, expandSubWorkflows = false), serviceRegistryActor)) + completeCromwellResponse( + runLog(workflowId, + (w: WorkflowId) => GetSingleWorkflowMetadataAction(w, None, None, expandSubWorkflows = false), + serviceRegistryActor + ) + ) } }, path("runs" / Segment / "status") { possibleWorkflowId => - val response = validateWorkflowIdInMetadata(possibleWorkflowId, serviceRegistryActor).flatMap(w => serviceRegistryActor.ask(GetStatus(w)).mapTo[MetadataServiceResponse]) + val response = validateWorkflowIdInMetadata(possibleWorkflowId, serviceRegistryActor).flatMap(w => + serviceRegistryActor.ask(GetStatus(w)).mapTo[MetadataServiceResponse] + ) // WES can also return a 401 or a 403 but that requires user auth knowledge which Cromwell doesn't currently have onComplete(response) { case Success(SuccessfulMetadataJsonResponse(_, jsObject)) => @@ -104,10 +114,11 @@ trait WesRouteSupport extends HttpInstrumentation { path("runs" / Segment / "cancel") { possibleWorkflowId => post { CromwellApiService.abortWorkflow(possibleWorkflowId, - workflowStoreActor, - workflowManagerActor, - successHandler = WesAbortSuccessHandler, - errorHandler = WesAbortErrorHandler) + workflowStoreActor, + workflowManagerActor, + successHandler = WesAbortSuccessHandler, + errorHandler = WesAbortErrorHandler + ) } } ) @@ -115,22 +126,22 @@ trait WesRouteSupport extends HttpInstrumentation { ) } - def toWesResponse(workflowId: WorkflowId, workflowState: WorkflowState): WesRunStatus = { + def toWesResponse(workflowId: WorkflowId, workflowState: WorkflowState): WesRunStatus = WesRunStatus(workflowId.toString, WesState.fromCromwellStatus(workflowState)) - } - def toWesResponseId(workflowId: WorkflowId): WesRunId ={ + def toWesResponseId(workflowId: WorkflowId): WesRunId = WesRunId(workflowId.toString) - } def wesSubmitRequest(formData: Multipart.FormData, isSingleSubmission: Boolean): Route = { - def getWorkflowState(workflowOnHold: Boolean): WorkflowState = { + def getWorkflowState(workflowOnHold: Boolean): WorkflowState = if (workflowOnHold) WorkflowOnHold else WorkflowSubmitted - } - def sendToWorkflowStore(command: WorkflowStoreActor.WorkflowStoreActorSubmitCommand, warnings: Seq[String], workflowState: WorkflowState): Route = { + def sendToWorkflowStore(command: WorkflowStoreActor.WorkflowStoreActorSubmitCommand, + warnings: Seq[String], + workflowState: WorkflowState + ): Route = // NOTE: Do not blindly copy the akka-http -to- ask-actor pattern below without knowing the pros and cons. onComplete(workflowStoreActor.ask(command).mapTo[WorkflowStoreSubmitActor.WorkflowStoreSubmitActorResponse]) { case Success(w) => @@ -142,18 +153,21 @@ trait WesRouteSupport extends HttpInstrumentation { case WorkflowStoreSubmitActor.WorkflowSubmitFailed(throwable) => respondWithWesError(throwable.getLocalizedMessage, StatusCodes.BadRequest) } - case Failure(_: AskTimeoutException) if CromwellShutdown.shutdownInProgress() => respondWithWesError("Cromwell service is shutting down", StatusCodes.InternalServerError) + case Failure(_: AskTimeoutException) if CromwellShutdown.shutdownInProgress() => + respondWithWesError("Cromwell service is shutting down", StatusCodes.InternalServerError) case Failure(e: TimeoutException) => e.failRequest(StatusCodes.ServiceUnavailable) case Failure(e) => e.failRequest(StatusCodes.InternalServerError, warnings) } - } onComplete(materializeFormData(formData)) { case Success(data) => PartialWorkflowSources.fromSubmitRoute(data, allowNoInputs = isSingleSubmission) match { case Success(workflowSourceFiles) if isSingleSubmission && workflowSourceFiles.size == 1 => val warnings = workflowSourceFiles.flatMap(_.warnings) - sendToWorkflowStore(WorkflowStoreActor.SubmitWorkflow(workflowSourceFiles.head), warnings, getWorkflowState(workflowSourceFiles.head.workflowOnHold)) + sendToWorkflowStore(WorkflowStoreActor.SubmitWorkflow(workflowSourceFiles.head), + warnings, + getWorkflowState(workflowSourceFiles.head.workflowOnHold) + ) // Catches the case where someone has gone through the single submission endpoint w/ more than one workflow case Success(workflowSourceFiles) if isSingleSubmission => val warnings = workflowSourceFiles.flatMap(_.warnings) @@ -163,7 +177,9 @@ trait WesRouteSupport extends HttpInstrumentation { val warnings = workflowSourceFiles.flatMap(_.warnings) sendToWorkflowStore( WorkflowStoreActor.BatchSubmitWorkflows(NonEmptyList.fromListUnsafe(workflowSourceFiles.toList)), - warnings, getWorkflowState(workflowSourceFiles.head.workflowOnHold)) + warnings, + getWorkflowState(workflowSourceFiles.head.workflowOnHold) + ) case Failure(t) => t.failRequest(StatusCodes.BadRequest) } case Failure(e: TimeoutException) => e.failRequest(StatusCodes.ServiceUnavailable) @@ -172,64 +188,67 @@ trait WesRouteSupport extends HttpInstrumentation { } } - - object WesRouteSupport { import WesResponseJsonSupport._ - implicit lazy val duration: FiniteDuration = ConfigFactory.load().as[FiniteDuration]("akka.http.server.request-timeout") + implicit lazy val duration: FiniteDuration = + ConfigFactory.load().as[FiniteDuration]("akka.http.server.request-timeout") implicit lazy val timeout: Timeout = duration import scala.concurrent.ExecutionContext.Implicits.global val NotFoundError = WesErrorResponse("The requested workflow run wasn't found", StatusCodes.NotFound.intValue) - def WesAbortSuccessHandler: PartialFunction[SuccessfulAbortResponse, Route] = { - case response => complete(WesRunId(response.workflowId.toString)) + def WesAbortSuccessHandler: PartialFunction[SuccessfulAbortResponse, Route] = { case response => + complete(WesRunId(response.workflowId.toString)) } def WesAbortErrorHandler: PartialFunction[Throwable, Route] = { // There are also some auth situations which should be handled, but at the moment Cromwell doesn't allow for those case e: IllegalStateException => respondWithWesError(e.getLocalizedMessage, StatusCodes.Forbidden) case e: WorkflowNotFoundException => respondWithWesError(e.getLocalizedMessage, StatusCodes.NotFound) - case _: AskTimeoutException if CromwellShutdown.shutdownInProgress() => respondWithWesError("Cromwell service is shutting down", StatusCodes.InternalServerError) + case _: AskTimeoutException if CromwellShutdown.shutdownInProgress() => + respondWithWesError("Cromwell service is shutting down", StatusCodes.InternalServerError) case e: Exception => respondWithWesError(e.getLocalizedMessage, StatusCodes.InternalServerError) } - private def respondWithWesError(errorMsg: String, status: StatusCode): Route = { + private def respondWithWesError(errorMsg: String, status: StatusCode): Route = complete((status, WesErrorResponse(errorMsg, status.intValue))) - } - def extractSubmission(): Directive1[WesSubmission] = { - formFields(( - "workflow_params".?, - "workflow_type".?, - "workflow_type_version".?, - "tags".?, - "workflow_engine_parameters".?, - "workflow_url".?, - "workflow_attachment".as[String].* - )).as(WesSubmission) - } + def extractSubmission(): Directive1[WesSubmission] = + formFields( + ( + "workflow_params".?, + "workflow_type".?, + "workflow_type_version".?, + "tags".?, + "workflow_engine_parameters".?, + "workflow_url".?, + "workflow_attachment".as[String].* + ) + ).as(WesSubmission) - def completeCromwellResponse(future: => Future[WesResponse]): Route = { + def completeCromwellResponse(future: => Future[WesResponse]): Route = onComplete(future) { case Success(response: WesResponse) => complete(response) case Failure(e) => complete(WesErrorResponse(e.getMessage, StatusCodes.InternalServerError.intValue)) } - } - def listRuns(pageSize: Option[Int], pageToken: Option[String], serviceRegistryActor: ActorRef): Future[WesResponse] = { + def listRuns(pageSize: Option[Int], pageToken: Option[String], serviceRegistryActor: ActorRef): Future[WesResponse] = // FIXME: to handle - page_size, page_token // FIXME: How to handle next_page_token in response? - metadataQueryRequest(Seq.empty[(String, String)], serviceRegistryActor).map(RunListResponse.fromMetadataQueryResponse) - } + metadataQueryRequest(Seq.empty[(String, String)], serviceRegistryActor) + .map(RunListResponse.fromMetadataQueryResponse) - def runLog(workflowId: String, request: WorkflowId => BuildMetadataJsonAction, serviceRegistryActor: ActorRef): Future[WesResponse] = { + def runLog(workflowId: String, + request: WorkflowId => BuildMetadataJsonAction, + serviceRegistryActor: ActorRef + ): Future[WesResponse] = { val metadataJsonResponse = metadataBuilderActorRequest(workflowId, request, serviceRegistryActor) metadataJsonResponse.map { case SuccessfulMetadataJsonResponse(_, responseJson) => WesRunLog.fromJson(responseJson.toString()) - case FailedMetadataJsonResponse(_, reason) => WesErrorResponse(reason.getMessage, StatusCodes.InternalServerError.intValue) + case FailedMetadataJsonResponse(_, reason) => + WesErrorResponse(reason.getMessage, StatusCodes.InternalServerError.intValue) } } -} \ No newline at end of file +} diff --git a/engine/src/main/scala/cromwell/webservice/routes/wes/WesRunLog.scala b/engine/src/main/scala/cromwell/webservice/routes/wes/WesRunLog.scala index c882ebf4f8b..240fce01341 100644 --- a/engine/src/main/scala/cromwell/webservice/routes/wes/WesRunLog.scala +++ b/engine/src/main/scala/cromwell/webservice/routes/wes/WesRunLog.scala @@ -2,7 +2,6 @@ package cromwell.webservice.routes.wes import spray.json.JsObject - final case class WesLog(name: Option[String], cmd: Option[Seq[String]], start_time: Option[String], @@ -10,7 +9,7 @@ final case class WesLog(name: Option[String], stdout: Option[String], stderr: Option[String], exit_code: Option[Int] - ) +) final case class WesRunRequest(workflow_params: Option[JsObject], workflow_type: String, @@ -18,4 +17,4 @@ final case class WesRunRequest(workflow_params: Option[JsObject], tags: Option[JsObject], workflow_engine_parameters: Option[JsObject], workflow_url: Option[String] - ) +) diff --git a/engine/src/main/scala/cromwell/webservice/routes/wes/WesState.scala b/engine/src/main/scala/cromwell/webservice/routes/wes/WesState.scala index f7feee727b8..58cf69e6a08 100644 --- a/engine/src/main/scala/cromwell/webservice/routes/wes/WesState.scala +++ b/engine/src/main/scala/cromwell/webservice/routes/wes/WesState.scala @@ -6,24 +6,23 @@ import spray.json.{DefaultJsonProtocol, JsObject, JsString, JsValue, RootJsonFor object WesState { sealed trait WesState extends Product with Serializable { val name: String } - case object Unknown extends WesState { override val name = "UNKNOWN"} - case object Queued extends WesState { override val name = "QUEUED"} - case object Initializing extends WesState { override val name = "INITIALIZING"} - case object Running extends WesState { override val name = "RUNNING"} - case object Paused extends WesState { override val name = "PAUSED"} - case object Complete extends WesState { override val name = "COMPLETE"} - case object ExecutorError extends WesState { override val name = "EXECUTOR_ERROR"} - case object SystemError extends WesState { override val name = "SYSTEM_ERROR"} - case object Canceled extends WesState { override val name = "CANCELED"} - case object Canceling extends WesState { override val name = "CANCELING"} + case object Unknown extends WesState { override val name = "UNKNOWN" } + case object Queued extends WesState { override val name = "QUEUED" } + case object Initializing extends WesState { override val name = "INITIALIZING" } + case object Running extends WesState { override val name = "RUNNING" } + case object Paused extends WesState { override val name = "PAUSED" } + case object Complete extends WesState { override val name = "COMPLETE" } + case object ExecutorError extends WesState { override val name = "EXECUTOR_ERROR" } + case object SystemError extends WesState { override val name = "SYSTEM_ERROR" } + case object Canceled extends WesState { override val name = "CANCELED" } + case object Canceling extends WesState { override val name = "CANCELING" } - def fromStatusString(status: Option[String]): WesState = { + def fromStatusString(status: Option[String]): WesState = status match { case Some(status) => fromCromwellStatus(WorkflowState.withName(status)) case None => Unknown } - } - def fromCromwellStatus(cromwellStatus: WorkflowState): WesState = { + def fromCromwellStatus(cromwellStatus: WorkflowState): WesState = cromwellStatus match { case WorkflowOnHold => Paused case WorkflowSubmitted => Queued @@ -34,18 +33,24 @@ object WesState { case WorkflowFailed => ExecutorError case _ => Unknown } - } def fromCromwellStatusJson(jsonResponse: JsObject): WesState = { - val statusString = jsonResponse.fields.get("status").collect { - case str: JsString => str.value - }.getOrElse(throw new IllegalArgumentException(s"Could not coerce Cromwell status response ${jsonResponse.compactPrint} into a valid WES status")) + val statusString = jsonResponse.fields + .get("status") + .collect { case str: JsString => + str.value + } + .getOrElse( + throw new IllegalArgumentException( + s"Could not coerce Cromwell status response ${jsonResponse.compactPrint} into a valid WES status" + ) + ) fromCromwellStatus(WorkflowState.withName(statusString)) } - def fromString(status: String): WesState = { + def fromString(status: String): WesState = status match { case Unknown.name => Unknown case Queued.name => Queued @@ -59,7 +64,6 @@ object WesState { case Canceling.name => Canceling case doh => throw new IllegalArgumentException(s"Invalid status attempting to be coerced to WesState: $doh") } - } } object WesStateJsonSupport extends SprayJsonSupport with DefaultJsonProtocol { @@ -70,8 +74,8 @@ object WesStateJsonSupport extends SprayJsonSupport with DefaultJsonProtocol { def read(json: JsValue): WesState = json match { - case JsString(string) => WesState.fromString(string) - case other => throw new UnsupportedOperationException(s"Cannot deserialize $other into a WesState") - } + case JsString(string) => WesState.fromString(string) + case other => throw new UnsupportedOperationException(s"Cannot deserialize $other into a WesState") + } } } diff --git a/engine/src/main/scala/cromwell/webservice/routes/wes/WesSubmission.scala b/engine/src/main/scala/cromwell/webservice/routes/wes/WesSubmission.scala index 1db2cd801b8..a3de8bbb24a 100644 --- a/engine/src/main/scala/cromwell/webservice/routes/wes/WesSubmission.scala +++ b/engine/src/main/scala/cromwell/webservice/routes/wes/WesSubmission.scala @@ -10,7 +10,7 @@ final case class WesSubmission(workflowParams: Option[String], workflowEngineParameters: Option[String], workflowUrl: Option[String], workflowAttachment: Iterable[String] - ) { +) { val entity: Multipart.FormData = { /* FIXME: @@ -27,15 +27,29 @@ final case class WesSubmission(workflowParams: Option[String], Content-Disposition headers on each of these files which can be used to describe directory structure and such for relative import resolution */ - val sourcePart = workflowAttachment.headOption map { a => Multipart.FormData.BodyPart(WorkflowSourceKey, HttpEntity(MediaTypes.`application/json`, a)) } - - val urlPart = workflowUrl map { u => Multipart.FormData.BodyPart(WorkflowUrlKey, HttpEntity(MediaTypes.`application/json`, u)) } - - val typePart = workflowType map { w => Multipart.FormData.BodyPart(WorkflowTypeKey, HttpEntity(MediaTypes.`application/json`, w)) } - val typeVersionPart = workflowTypeVersion map { v => Multipart.FormData.BodyPart(WorkflowTypeVersionKey, HttpEntity(MediaTypes.`application/json`, v)) } - val inputsPart = workflowParams map { p => Multipart.FormData.BodyPart(WorkflowInputsKey, HttpEntity(MediaTypes.`application/json`, p)) } - val optionsPart = workflowEngineParameters map { o => Multipart.FormData.BodyPart(WorkflowOptionsKey, HttpEntity(MediaTypes.`application/json`, o)) } - val labelsPart = tags map { t => Multipart.FormData.BodyPart(labelsKey, HttpEntity(MediaTypes.`application/json`, t)) } + val sourcePart = workflowAttachment.headOption map { a => + Multipart.FormData.BodyPart(WorkflowSourceKey, HttpEntity(MediaTypes.`application/json`, a)) + } + + val urlPart = workflowUrl map { u => + Multipart.FormData.BodyPart(WorkflowUrlKey, HttpEntity(MediaTypes.`application/json`, u)) + } + + val typePart = workflowType map { w => + Multipart.FormData.BodyPart(WorkflowTypeKey, HttpEntity(MediaTypes.`application/json`, w)) + } + val typeVersionPart = workflowTypeVersion map { v => + Multipart.FormData.BodyPart(WorkflowTypeVersionKey, HttpEntity(MediaTypes.`application/json`, v)) + } + val inputsPart = workflowParams map { p => + Multipart.FormData.BodyPart(WorkflowInputsKey, HttpEntity(MediaTypes.`application/json`, p)) + } + val optionsPart = workflowEngineParameters map { o => + Multipart.FormData.BodyPart(WorkflowOptionsKey, HttpEntity(MediaTypes.`application/json`, o)) + } + val labelsPart = tags map { t => + Multipart.FormData.BodyPart(labelsKey, HttpEntity(MediaTypes.`application/json`, t)) + } val parts = List(sourcePart, urlPart, typePart, typeVersionPart, inputsPart, optionsPart, labelsPart).flatten diff --git a/engine/src/main/scala/cromwell/webservice/webservice_.scala b/engine/src/main/scala/cromwell/webservice/webservice_.scala index d68ba0bdb1f..08b57cfc378 100644 --- a/engine/src/main/scala/cromwell/webservice/webservice_.scala +++ b/engine/src/main/scala/cromwell/webservice/webservice_.scala @@ -35,5 +35,5 @@ object Patterns { \. # Literal dot. (\d+) # Captured shard digits. )? # End outer optional noncapturing group for shard. - """.trim.r // The trim is necessary as (?x) must be at the beginning of the regex. + """.trim.r // The trim is necessary as (?x) must be at the beginning of the regex. } diff --git a/engine/src/test/scala/cromwell/MetadataWatchActor.scala b/engine/src/test/scala/cromwell/MetadataWatchActor.scala index 2c43ff74463..e9c7710d79f 100644 --- a/engine/src/test/scala/cromwell/MetadataWatchActor.scala +++ b/engine/src/test/scala/cromwell/MetadataWatchActor.scala @@ -18,7 +18,7 @@ final case class MetadataWatchActor(promise: Promise[Unit], matchers: Matcher*) var unsatisfiedMatchers = matchers def tryMatchingEvents(events: Iterable[MetadataEvent]) = { - unsatisfiedMatchers = unsatisfiedMatchers.filterNot { m => m.matches(events) } + unsatisfiedMatchers = unsatisfiedMatchers.filterNot(m => m.matches(events)) if (unsatisfiedMatchers.isEmpty) { promise.trySuccess(()) () @@ -41,7 +41,8 @@ final case class MetadataWatchActor(promise: Promise[Unit], matchers: Matcher*) object MetadataWatchActor { - def props(promise: Promise[Unit], matchers: Matcher*): Props = Props(MetadataWatchActor(promise, matchers: _*)).withDispatcher(EngineDispatcher) + def props(promise: Promise[Unit], matchers: Matcher*): Props = + Props(MetadataWatchActor(promise, matchers: _*)).withDispatcher(EngineDispatcher) trait Matcher { private var _fullEventList: List[MetadataEvent] = List.empty @@ -57,7 +58,8 @@ object MetadataWatchActor { def checkMetadataValueContains(key: String, actual: MetadataValue, expected: String): Boolean = { val result = actual.value.contains(expected) - if (!result) addNearMissInfo(s"Key $key had unexpected value.\nActual value: ${actual.value}\n\nDid not contain: $expected") + if (!result) + addNearMissInfo(s"Key $key had unexpected value.\nActual value: ${actual.value}\n\nDid not contain: $expected") result } } @@ -67,20 +69,28 @@ object MetadataWatchActor { case None => false } - final case class JobKeyMetadataKeyAndValueContainStringMatcher(jobKeyCheck: Option[MetadataJobKey] => Boolean, key: String, value: String) extends Matcher { - def _matches(events: Iterable[MetadataEvent]): Boolean = { - events.exists(e => e.key.key.contains(key) && jobKeyCheck(e.key.jobKey) && e.value.exists { v => v.valueType == MetadataString && checkMetadataValueContains(e.key.key, v, value) }) - } + final case class JobKeyMetadataKeyAndValueContainStringMatcher(jobKeyCheck: Option[MetadataJobKey] => Boolean, + key: String, + value: String + ) extends Matcher { + def _matches(events: Iterable[MetadataEvent]): Boolean = + events.exists(e => + e.key.key.contains(key) && jobKeyCheck(e.key.jobKey) && e.value.exists { v => + v.valueType == MetadataString && checkMetadataValueContains(e.key.key, v, value) + } + ) } abstract class KeyMatchesRegexAndValueContainsStringMatcher(keyTemplate: String, value: String) extends Matcher { val templateRegex = keyTemplate.r - def _matches(events: Iterable[MetadataEvent]): Boolean = { - events.exists(e => templateRegex.findFirstIn(e.key.key).isDefined && - e.value.exists { v => checkMetadataValueContains(e.key.key, v, value) }) - } + def _matches(events: Iterable[MetadataEvent]): Boolean = + events.exists(e => + templateRegex.findFirstIn(e.key.key).isDefined && + e.value.exists(v => checkMetadataValueContains(e.key.key, v, value)) + ) } val failurePattern = """failures\[\d*\].*\:message""" - final case class FailureMatcher(value: String) extends KeyMatchesRegexAndValueContainsStringMatcher(failurePattern, value) { } + final case class FailureMatcher(value: String) + extends KeyMatchesRegexAndValueContainsStringMatcher(failurePattern, value) {} } diff --git a/engine/src/test/scala/cromwell/engine/MockCromwellTerminator.scala b/engine/src/test/scala/cromwell/engine/MockCromwellTerminator.scala index 1ef165ce14b..24b21541a1a 100644 --- a/engine/src/test/scala/cromwell/engine/MockCromwellTerminator.scala +++ b/engine/src/test/scala/cromwell/engine/MockCromwellTerminator.scala @@ -6,7 +6,6 @@ import akka.actor.CoordinatedShutdown import scala.concurrent.Future object MockCromwellTerminator extends CromwellTerminator { - override def beginCromwellShutdown(notUsed: CoordinatedShutdown.Reason): Future[Done] = { + override def beginCromwellShutdown(notUsed: CoordinatedShutdown.Reason): Future[Done] = Future.successful(Done) - } } diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala index f31e4d72002..6fc7ed0fbd5 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala @@ -10,13 +10,26 @@ import wom.graph.CommandCallNode import scala.concurrent.{ExecutionContext, Future} object DefaultBackendJobExecutionActor { - def props(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor) = Props(DefaultBackendJobExecutionActor(jobDescriptor, configurationDescriptor)) + def props(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor) = Props( + DefaultBackendJobExecutionActor(jobDescriptor, configurationDescriptor) + ) } -case class DefaultBackendJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, override val configurationDescriptor: BackendConfigurationDescriptor) extends BackendJobExecutionActor { - override def execute: Future[BackendJobExecutionResponse] = { - Future.successful(JobSucceededResponse(jobDescriptor.key, Some(0), CallOutputs((jobDescriptor.taskCall.outputPorts map taskOutputToJobOutput).toMap), None, Seq.empty, dockerImageUsed = None, resultGenerationMode = RunOnBackend)) - } +case class DefaultBackendJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, + override val configurationDescriptor: BackendConfigurationDescriptor +) extends BackendJobExecutionActor { + override def execute: Future[BackendJobExecutionResponse] = + Future.successful( + JobSucceededResponse( + jobDescriptor.key, + Some(0), + CallOutputs((jobDescriptor.taskCall.outputPorts map taskOutputToJobOutput).toMap), + None, + Seq.empty, + dockerImageUsed = None, + resultGenerationMode = RunOnBackend + ) + ) override def recover = execute @@ -24,25 +37,26 @@ case class DefaultBackendJobExecutionActor(override val jobDescriptor: BackendJo } class DefaultBackendLifecycleActorFactory(val name: String, val configurationDescriptor: BackendConfigurationDescriptor) - extends BackendLifecycleActorFactory { + extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[CommandCallNode], serviceRegistryActor: ActorRef, - restarting: Boolean): Option[Props] = None + restarting: Boolean + ): Option[Props] = None override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef, - backendSingletonActor: Option[ActorRef]): Props = { + backendSingletonActor: Option[ActorRef] + ): Props = DefaultBackendJobExecutionActor.props(jobDescriptor, configurationDescriptor) - } override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor, jobKey: BackendJobDescriptorKey, initializationData: Option[BackendInitializationData], ioActorProxy: ActorRef, - ec: ExecutionContext): IoFunctionSet = NoIoFunctionSet + ec: ExecutionContext + ): IoFunctionSet = NoIoFunctionSet } - diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendJobExecutionActor.scala b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendJobExecutionActor.scala index 6bc7b04adce..69685d58176 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendJobExecutionActor.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendJobExecutionActor.scala @@ -2,26 +2,39 @@ package cromwell.engine.backend.mock import akka.actor.Props import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobExecutionActor} -import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse} +import cromwell.backend.BackendJobExecutionActor.{ + BackendJobExecutionResponse, + JobFailedNonRetryableResponse, + JobFailedRetryableResponse +} import scala.concurrent.Future object RetryableBackendJobExecutionActor { - def props(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor) = Props(RetryableBackendJobExecutionActor(jobDescriptor, configurationDescriptor)) + def props(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor) = Props( + RetryableBackendJobExecutionActor(jobDescriptor, configurationDescriptor) + ) } -final case class RetryableBackendJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, override val configurationDescriptor: BackendConfigurationDescriptor) extends BackendJobExecutionActor { +final case class RetryableBackendJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, + override val configurationDescriptor: BackendConfigurationDescriptor +) extends BackendJobExecutionActor { val attempts = 3 - override def execute: Future[BackendJobExecutionResponse] = { + override def execute: Future[BackendJobExecutionResponse] = if (jobDescriptor.key.attempt < attempts) { - Future.successful(JobFailedRetryableResponse(jobDescriptor.key, new RuntimeException("An apparent transient Exception!"), None)) - } - else { - Future.successful(JobFailedNonRetryableResponse(jobDescriptor.key, new RuntimeException("A permanent Exception! Yikes, what a pickle!"), None)) + Future.successful( + JobFailedRetryableResponse(jobDescriptor.key, new RuntimeException("An apparent transient Exception!"), None) + ) + } else { + Future.successful( + JobFailedNonRetryableResponse(jobDescriptor.key, + new RuntimeException("A permanent Exception! Yikes, what a pickle!"), + None + ) + ) } - } override def recover = execute diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala index 3ee7afc8767..23ea949f9d6 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala @@ -8,25 +8,27 @@ import wom.graph.CommandCallNode import scala.concurrent.ExecutionContext class RetryableBackendLifecycleActorFactory(val name: String, - val configurationDescriptor: BackendConfigurationDescriptor) - extends BackendLifecycleActorFactory { + val configurationDescriptor: BackendConfigurationDescriptor +) extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[CommandCallNode], serviceRegistryActor: ActorRef, - restarting: Boolean): Option[Props] = None + restarting: Boolean + ): Option[Props] = None override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef, - backendSingletonActor: Option[ActorRef]): Props = { + backendSingletonActor: Option[ActorRef] + ): Props = RetryableBackendJobExecutionActor.props(jobDescriptor, configurationDescriptor) - } override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor, jobKey: BackendJobDescriptorKey, initializationData: Option[BackendInitializationData], ioActorProxy: ActorRef, - ec: ExecutionContext): IoFunctionSet = NoIoFunctionSet + ec: ExecutionContext + ): IoFunctionSet = NoIoFunctionSet } diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/package.scala b/engine/src/test/scala/cromwell/engine/backend/mock/package.scala index 4de47444c64..ef75c0df703 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/package.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/package.scala @@ -17,6 +17,7 @@ package object mock { case WomSingleFileType => WomSingleFile("/root/of/all/evil") case WomArrayType(memberType) => WomArray(WomArrayType(memberType), List(sampleValue(memberType))) case WomObjectType => WomObject(Map("a" -> WomString("1"), "b" -> WomString("2"))) - case WomMapType(keyType, valueType) => WomMap(WomMapType(keyType, valueType), Map(sampleValue(keyType) -> sampleValue(valueType))) + case WomMapType(keyType, valueType) => + WomMap(WomMapType(keyType, valueType), Map(sampleValue(keyType) -> sampleValue(valueType))) } } diff --git a/engine/src/test/scala/cromwell/engine/io/IoActorProxyGcsBatchSpec.scala b/engine/src/test/scala/cromwell/engine/io/IoActorProxyGcsBatchSpec.scala index 6c69fe98af6..57b85dd2a3b 100644 --- a/engine/src/test/scala/cromwell/engine/io/IoActorProxyGcsBatchSpec.scala +++ b/engine/src/test/scala/cromwell/engine/io/IoActorProxyGcsBatchSpec.scala @@ -21,7 +21,12 @@ import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext} import scala.language.postfixOps -class IoActorProxyGcsBatchSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with ImplicitSender with Eventually { +class IoActorProxyGcsBatchSpec + extends TestKitSuite + with AnyFlatSpecLike + with Matchers + with ImplicitSender + with Eventually { behavior of "IoActor [GCS Batch]" implicit val ec: ExecutionContext = system.dispatcher @@ -74,10 +79,11 @@ class IoActorProxyGcsBatchSpec extends TestKitSuite with AnyFlatSpecLike with Ma dst: GcsPath, directory: GcsPath, testActorName: String, - serviceRegistryActorName: String) = { + serviceRegistryActorName: String + ) = { val testActor = TestActorRef( factory = new IoActor(IoActorConfig, TestProbe(serviceRegistryActorName).ref, "cromwell test"), - name = testActorName, + name = testActorName ) val copyCommand = GcsBatchCopyCommand.forPaths(src, dst).get @@ -102,20 +108,24 @@ class IoActorProxyGcsBatchSpec extends TestKitSuite with AnyFlatSpecLike with Ma received1.size shouldBe 5 received1 forall { _.isInstanceOf[IoSuccess[_]] } shouldBe true - received1 collect { - case IoSuccess(_: GcsBatchSizeCommand, fileSize: Long) => fileSize shouldBe 5 + received1 collect { case IoSuccess(_: GcsBatchSizeCommand, fileSize: Long) => + fileSize shouldBe 5 } - received1 collect { - case IoSuccess(_: GcsBatchCrc32Command, hash: String) => hash shouldBe "mnG7TA==" + received1 collect { case IoSuccess(_: GcsBatchCrc32Command, hash: String) => + hash shouldBe "mnG7TA==" } received1 collect { - case IoSuccess(command: GcsBatchIsDirectoryCommand, isDirectory: Boolean) if command.file.pathAsString == directory.pathAsString => isDirectory shouldBe true + case IoSuccess(command: GcsBatchIsDirectoryCommand, isDirectory: Boolean) + if command.file.pathAsString == directory.pathAsString => + isDirectory shouldBe true } received1 collect { - case IoSuccess(command: GcsBatchIsDirectoryCommand, isDirectory: Boolean) if command.file.pathAsString == src.pathAsString => isDirectory shouldBe false + case IoSuccess(command: GcsBatchIsDirectoryCommand, isDirectory: Boolean) + if command.file.pathAsString == src.pathAsString => + isDirectory shouldBe false } testActor ! deleteSrcCommand @@ -136,7 +146,7 @@ class IoActorProxyGcsBatchSpec extends TestKitSuite with AnyFlatSpecLike with Ma dst = dst, directory = directory, testActorName = "testActor-batch", - serviceRegistryActorName = "serviceRegistryActor-batch", + serviceRegistryActorName = "serviceRegistryActor-batch" ) } @@ -146,14 +156,14 @@ class IoActorProxyGcsBatchSpec extends TestKitSuite with AnyFlatSpecLike with Ma dst = dstRequesterPays, directory = directoryRequesterPays, testActorName = "testActor-batch-rp", - serviceRegistryActorName = "serviceRegistryActor-batch-rp", + serviceRegistryActorName = "serviceRegistryActor-batch-rp" ) } it should "copy files across GCS storage classes" taggedAs IntegrationTest in { val testActor = TestActorRef( factory = new IoActor(IoActorConfig, TestProbe("serviceRegistryActor").ref, "cromwell test"), - name = "testActor", + name = "testActor" ) val copyCommand = GcsBatchCopyCommand.forPaths(srcRegional, dstMultiRegional).get diff --git a/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala b/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala index e6209ff5958..880e9028282 100644 --- a/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala @@ -37,7 +37,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I it should "copy a file" in { val testActor = TestActorRef( factory = new IoActor(IoActorConfig, TestProbe("serviceRegistryActorCopy").ref, "cromwell test"), - name = "testActorCopy", + name = "testActorCopy" ) val src = DefaultPathBuilder.createTempFile() @@ -59,7 +59,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I it should "write to a file" in { val testActor = TestActorRef( factory = new IoActor(IoActorConfig, TestProbe("serviceRegistryActorWrite").ref, "cromwell test"), - name = "testActorWrite", + name = "testActorWrite" ) val src = DefaultPathBuilder.createTempFile() @@ -79,7 +79,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I it should "delete a file" in { val testActor = TestActorRef( factory = new IoActor(IoActorConfig, TestProbe("serviceRegistryActorDelete").ref, "cromwell test"), - name = "testActorDelete", + name = "testActorDelete" ) val src = DefaultPathBuilder.createTempFile() @@ -98,7 +98,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I it should "read a file" in { val testActor = TestActorRef( factory = new IoActor(IoActorConfig, TestProbe("serviceRegistryActorRead").ref, "cromwell test"), - name = "testActorRead", + name = "testActorRead" ) val src = DefaultPathBuilder.createTempFile() @@ -120,7 +120,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I it should "read only the first bytes of file" in { val testActor = TestActorRef( factory = new IoActor(IoActorConfig, TestProbe("serviceRegistryActorFirstBytes").ref, "cromwell test"), - name = "testActorFirstBytes", + name = "testActorFirstBytes" ) val src = DefaultPathBuilder.createTempFile() @@ -142,7 +142,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I it should "read the file if it's under the byte limit" in { val testActor = TestActorRef( factory = new IoActor(IoActorConfig, TestProbe("serviceRegistryActorByteLimit").ref, "cromwell test"), - name = "testActorByteLimit", + name = "testActorByteLimit" ) val src = DefaultPathBuilder.createTempFile() @@ -164,7 +164,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I it should "fail if the file is larger than the read limit" in { val testActor = TestActorRef( factory = new IoActor(IoActorConfig, TestProbe("serviceRegistryActorReadLimit").ref, "cromwell test"), - name = "testActorReadLimit", + name = "testActorReadLimit" ) val src = DefaultPathBuilder.createTempFile() @@ -174,8 +174,10 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I testActor ! readCommand expectMsgPF(5 seconds) { - case _: IoSuccess[_] => fail("Command should have failed because the read limit was < file size and failOnOverflow was true") - case response: IoFailure[_] => response.failure.getMessage shouldBe s"[Attempted 1 time(s)] - IOException: Could not read from ${src.pathAsString}: File ${src.pathAsString} is larger than requested maximum of 2 Bytes." + case _: IoSuccess[_] => + fail("Command should have failed because the read limit was < file size and failOnOverflow was true") + case response: IoFailure[_] => + response.failure.getMessage shouldBe s"[Attempted 1 time(s)] - IOException: Could not read from ${src.pathAsString}: File ${src.pathAsString} is larger than requested maximum of 2 Bytes." } src.delete() @@ -184,7 +186,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I it should "return a file size" in { val testActor = TestActorRef( factory = new IoActor(IoActorConfig, TestProbe("serviceRegistryActorSize").ref, "cromwell test"), - name = "testActorSize", + name = "testActorSize" ) val src = DefaultPathBuilder.createTempFile() @@ -206,7 +208,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I it should "return a file md5 hash (local)" in { val testActor = TestActorRef( factory = new IoActor(IoActorConfig, TestProbe("serviceRegistryActorHash").ref, "cromwell test"), - name = "testActorHash", + name = "testActorHash" ) val src = DefaultPathBuilder.createTempFile() @@ -228,7 +230,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I it should "touch a file (local)" in { val testActor = TestActorRef( factory = new IoActor(IoActorConfig, TestProbe("serviceRegistryActorTouch").ref, "cromwell test"), - name = "testActorTouch", + name = "testActorTouch" ) val src = DefaultPathBuilder.createTempFile() @@ -259,25 +261,45 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I new SocketException(), new SocketTimeoutException(), new IOException("text Error getting access token for service account some other text"), - - new IOException("Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 500 Internal Server Error\nBackend Error"), - new IOException("Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 500 Internal Server Error Backend Error"), - - new IOException("Could not read from gs://broad-epi-cromwell/workflows/ChipSeq/ce6a5671-baf6-4734-a32b-abf3d9138e9b/call-epitope_classifier/memory_retry_rc: 503 Service Unavailable\nBackend Error"), - new IOException("Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 503 Service Unavailable Backend Error"), - - new IOException("Could not read from gs://mccarroll-mocha/cromwell/cromwell-executions/mocha/86d47e9a-5745-4ec0-b4eb-0164f073e5f4/call-idat2gtc/shard-73/rc: 504 Gateway Timeout\nGET https://storage.googleapis.com/download/storage/v1/b/mccarroll-mocha/o/cromwell%2Fcromwell-executions%2Fmocha%2F86d47e9a-5745-4ec0-b4eb-0164f073e5f4%2Fcall-idat2gtc%2Fshard-73%2Frc?alt=media"), + new IOException( + "Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 500 Internal Server Error\nBackend Error" + ), + new IOException( + "Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 500 Internal Server Error Backend Error" + ), + new IOException( + "Could not read from gs://broad-epi-cromwell/workflows/ChipSeq/ce6a5671-baf6-4734-a32b-abf3d9138e9b/call-epitope_classifier/memory_retry_rc: 503 Service Unavailable\nBackend Error" + ), + new IOException( + "Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 503 Service Unavailable Backend Error" + ), + new IOException( + "Could not read from gs://mccarroll-mocha/cromwell/cromwell-executions/mocha/86d47e9a-5745-4ec0-b4eb-0164f073e5f4/call-idat2gtc/shard-73/rc: 504 Gateway Timeout\nGET https://storage.googleapis.com/download/storage/v1/b/mccarroll-mocha/o/cromwell%2Fcromwell-executions%2Fmocha%2F86d47e9a-5745-4ec0-b4eb-0164f073e5f4%2Fcall-idat2gtc%2Fshard-73%2Frc?alt=media" + ), // Prove that `isRetryable` successfully recurses to unwrap the lowest-level Throwable - new IOException(new Throwable("Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 500 Internal Server Error Backend Error")), - new IOException(new Throwable("Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 503 Service Unavailable Backend Error")), - - new IOException("Some other text. Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 503 Service Unavailable"), - new IOException("Some other text. Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 504 Gateway Timeout"), + new IOException( + new Throwable( + "Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 500 Internal Server Error Backend Error" + ) + ), + new IOException( + new Throwable( + "Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 503 Service Unavailable Backend Error" + ) + ), + new IOException( + "Some other text. Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 503 Service Unavailable" + ), + new IOException( + "Some other text. Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-4688/rc: 504 Gateway Timeout" + ) ) - retryables foreach { e => withClue(e) { - RetryableRequestSupport.isRetryable(e) shouldBe true } + retryables foreach { e => + withClue(e) { + RetryableRequestSupport.isRetryable(e) shouldBe true + } } } @@ -288,7 +310,9 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I new IOException("502 HTTP Status Code"), new Exception("502 HTTP Status Code"), new Exception("5xx HTTP Status Code"), - new IOException("Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-500/rc: 404 File Not Found") + new IOException( + "Could not read from gs://fc-secure-/JointGenotyping//call-HardFilterAndMakeSitesOnlyVcf/shard-500/rc: 404 File Not Found" + ) ) nonRetryables foreach { RetryableRequestSupport.isRetryable(_) shouldBe false } diff --git a/engine/src/test/scala/cromwell/engine/io/gcs/GcsBatchCommandContextSpec.scala b/engine/src/test/scala/cromwell/engine/io/gcs/GcsBatchCommandContextSpec.scala index a7bf613efa1..ac19b4d8ad1 100644 --- a/engine/src/test/scala/cromwell/engine/io/gcs/GcsBatchCommandContextSpec.scala +++ b/engine/src/test/scala/cromwell/engine/io/gcs/GcsBatchCommandContextSpec.scala @@ -13,7 +13,11 @@ import org.scalatest.matchers.should.Matchers import scala.util.{Failure, Success} class GcsBatchCommandContextSpec - extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Eventually with BeforeAndAfter { + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with Eventually + with BeforeAndAfter { behavior of "GcsBatchCommandContext" it should "handle exceptions in success handlers" in { @@ -34,7 +38,10 @@ class GcsBatchCommandContextSpec exceptionSpewingCommandContext.promise.future.value.get match { case Success(oops) => fail(s"Should not have produced a success: $oops") - case Failure(error) => error.getMessage should be("Error processing IO response in onSuccessCallback: Ill behaved code that throws in mapGoogleResponse") + case Failure(error) => + error.getMessage should be( + "Error processing IO response in onSuccessCallback: Ill behaved code that throws in mapGoogleResponse" + ) } } @@ -48,7 +55,7 @@ class GcsBatchCommandContextSpec exceptionSpewingCommandContext.promise.isCompleted should be(false) // Simulate a failure response from an underlying IO operation: - exceptionSpewingCommandContext.callback.onFailure(new GoogleJsonError { }, new HttpHeaders()) + exceptionSpewingCommandContext.callback.onFailure(new GoogleJsonError {}, new HttpHeaders()) eventually { exceptionSpewingCommandContext.promise.isCompleted should be(true) @@ -56,7 +63,10 @@ class GcsBatchCommandContextSpec exceptionSpewingCommandContext.promise.future.value.get match { case Success(oops) => fail(s"Should not have produced a success: $oops") - case Failure(error) => error.getMessage should be("Error processing IO response in onFailureCallback: Ill behaved code that throws in onFailure") + case Failure(error) => + error.getMessage should be( + "Error processing IO response in onFailureCallback: Ill behaved code that throws in onFailure" + ) } } @@ -78,7 +88,10 @@ class GcsBatchCommandContextSpec errorReturningCommandContext.promise.future.value.get match { case Success(oops) => fail(s"Should not have produced a success: $oops") - case Failure(error) => error.getMessage should be("Unexpected result in successful Google API call:\nWell behaved code that returns an error in mapGoogleResponse") + case Failure(error) => + error.getMessage should be( + "Unexpected result in successful Google API call:\nWell behaved code that returns an error in mapGoogleResponse" + ) } } } diff --git a/engine/src/test/scala/cromwell/engine/io/gcs/GcsBatchFlowSpec.scala b/engine/src/test/scala/cromwell/engine/io/gcs/GcsBatchFlowSpec.scala index c133e7618c0..9b0a99752fe 100644 --- a/engine/src/test/scala/cromwell/engine/io/gcs/GcsBatchFlowSpec.scala +++ b/engine/src/test/scala/cromwell/engine/io/gcs/GcsBatchFlowSpec.scala @@ -17,14 +17,20 @@ import scala.concurrent.duration._ import scala.concurrent.{ExecutionContextExecutor, Future} import scala.language.postfixOps -class GcsBatchFlowSpec extends TestKitSuite with AnyFlatSpecLike with CromwellTimeoutSpec with Matchers - with PrivateMethodTester with MockSugar { +class GcsBatchFlowSpec + extends TestKitSuite + with AnyFlatSpecLike + with CromwellTimeoutSpec + with Matchers + with PrivateMethodTester + with MockSugar { private val NoopOnRetry: IoCommandContext[_] => Throwable => Unit = _ => _ => () private val NoopOnBackpressure: Option[Double] => Unit = _ => () "GcsBatchFlow" should "know what read forbidden bucket failures look like" in { - val ErrorTemplate = "foo@bar.iam.gserviceaccount.com does not have storage.objects.%s access to %s/three_step/f0000000-baaa-f000-baaa-f00000000000/call-foo/foo.log" + val ErrorTemplate = + "foo@bar.iam.gserviceaccount.com does not have storage.objects.%s access to %s/three_step/f0000000-baaa-f000-baaa-f00000000000/call-foo/foo.log" val UnreadableBucketName = "unreadable-bucket" val objectReadOperationNames = Set( @@ -41,7 +47,9 @@ class GcsBatchFlowSpec extends TestKitSuite with AnyFlatSpecLike with CromwellTi } yield new StorageException(code, String.format(ErrorTemplate, op, UnreadableBucketName)) } // Can't flatMap this since any Nones would just be squashed out. - storageExceptions(objectReadOperationNames) map (_.getMessage) map GcsBatchFlow.getReadForbiddenBucket map { _.get } shouldBe Set(UnreadableBucketName) + storageExceptions(objectReadOperationNames) map (_.getMessage) map GcsBatchFlow.getReadForbiddenBucket map { + _.get + } shouldBe Set(UnreadableBucketName) // A sampling of write operations, not an exhaustive list. val objectWriteOperationNames = Set( @@ -49,9 +57,13 @@ class GcsBatchFlowSpec extends TestKitSuite with AnyFlatSpecLike with CromwellTi "delete", "insert" ) - storageExceptions(objectWriteOperationNames) map (_.getMessage) flatMap GcsBatchFlow.getReadForbiddenBucket shouldBe Set.empty + storageExceptions( + objectWriteOperationNames + ) map (_.getMessage) flatMap GcsBatchFlow.getReadForbiddenBucket shouldBe Set.empty - Set(new RuntimeException("random exception")) map (_.getMessage) flatMap GcsBatchFlow.getReadForbiddenBucket shouldBe Set.empty + Set( + new RuntimeException("random exception") + ) map (_.getMessage) flatMap GcsBatchFlow.getReadForbiddenBucket shouldBe Set.empty } "GcsBatchFlow" should "not throw unhandled exception and kill the thread when trying to recover from unretryable exception with null error message" in { @@ -63,19 +75,23 @@ class GcsBatchFlowSpec extends TestKitSuite with AnyFlatSpecLike with CromwellTi onRetry = NoopOnRetry, onBackpressure = NoopOnBackpressure, applicationName = "testAppName", - backpressureStaleness = 5 seconds) + backpressureStaleness = 5 seconds + ) val mockGcsPath = GcsPath( nioPath = CloudStorageFileSystem.forBucket("bucket").getPath("test"), apiStorage = mock[com.google.api.services.storage.Storage], cloudStorage = mock[com.google.cloud.storage.Storage], - projectId = "GcsBatchFlowSpec-project", + projectId = "GcsBatchFlowSpec-project" ) - val gcsBatchCommandContext = GcsBatchCommandContext(GcsBatchCrc32Command.forPath(mockGcsPath).get, TestProbe().ref, 5) - val recoverCommandPrivateMethod = PrivateMethod[PartialFunction[Throwable, Future[GcsBatchResponse[_]]]](Symbol("recoverCommand")) + val gcsBatchCommandContext = + GcsBatchCommandContext(GcsBatchCrc32Command.forPath(mockGcsPath).get, TestProbe().ref, 5) + val recoverCommandPrivateMethod = + PrivateMethod[PartialFunction[Throwable, Future[GcsBatchResponse[_]]]](Symbol("recoverCommand")) val partialFuncAcceptingThrowable = gcsBatchFlow invokePrivate recoverCommandPrivateMethod(gcsBatchCommandContext) - val futureRes = partialFuncAcceptingThrowable(new NullPointerException(null)) // no unhandled exceptions should be thrown here + val futureRes = + partialFuncAcceptingThrowable(new NullPointerException(null)) // no unhandled exceptions should be thrown here futureRes.isCompleted shouldBe true } } diff --git a/engine/src/test/scala/cromwell/engine/io/nio/NioFlowSpec.scala b/engine/src/test/scala/cromwell/engine/io/nio/NioFlowSpec.scala index 41c121da0cc..5859684e756 100644 --- a/engine/src/test/scala/cromwell/engine/io/nio/NioFlowSpec.scala +++ b/engine/src/test/scala/cromwell/engine/io/nio/NioFlowSpec.scala @@ -38,12 +38,12 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with private val NoopOnRetry: IoCommandContext[_] => Throwable => Unit = _ => _ => () private val NoopOnBackpressure: Option[Double] => Unit = _ => () - private val flow = new NioFlow( - parallelism = 1, - onRetryCallback = NoopOnRetry, - onBackpressure = NoopOnBackpressure, - numberOfAttempts = 3, - commandBackpressureStaleness = 5 seconds)(system).flow + private val flow = new NioFlow(parallelism = 1, + onRetryCallback = NoopOnRetry, + onBackpressure = NoopOnBackpressure, + numberOfAttempts = 3, + commandBackpressureStaleness = 5 seconds + )(system).flow implicit val materializer: ActorMaterializer = ActorMaterializer() private val replyTo = mock[ActorRef] @@ -106,7 +106,7 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with val stream = testSource.via(flow).toMat(readSink)(Keep.right) stream.run() map { case (IoFailure(_, EnhancedCromwellIoException(_, receivedException)), _) => - receivedException.getMessage should include ("UnknownHost") + receivedException.getMessage should include("UnknownHost") case (ack, _) => fail(s"size should have failed with UnknownHost but didn't:\n$ack\n\n") } } @@ -120,7 +120,7 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with val stream = testSource.via(flow).toMat(readSink)(Keep.right) stream.run() map { case (IoFailure(_, EnhancedCromwellIoException(_, receivedException)), _) => - receivedException.getMessage should include ("Couldn't fetch size") + receivedException.getMessage should include("Couldn't fetch size") case (ack, _) => fail(s"size should have failed but didn't:\n$ack\n\n") } } @@ -135,7 +135,8 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with val stream = testSource.via(flow).toMat(readSink)(Keep.right) stream.run() map { - case (success: IoSuccess[_], _) => assert(success.result.asInstanceOf[String] == "5d41402abc4b2a76b9719d911017c592") + case (success: IoSuccess[_], _) => + assert(success.result.asInstanceOf[String] == "5d41402abc4b2a76b9719d911017c592") case _ => fail("hash returned an unexpected message") } } @@ -177,16 +178,19 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with it should "fail if DrsPath hash doesn't match checksum" in { val testPath = mock[DrsPath] when(testPath.limitFileContent(any[Option[Int]], any[Boolean])(any[ExecutionContext])).thenReturn("hello".getBytes) - when(testPath.getFileHash).thenReturn(FileHash(HashType.Crc32c, "boom")) // correct Base64-encoded crc32c checksum is "9a71bb4c" + when(testPath.getFileHash).thenReturn( + FileHash(HashType.Crc32c, "boom") + ) // correct Base64-encoded crc32c checksum is "9a71bb4c" - val context = DefaultCommandContext(contentAsStringCommand(testPath, Option(100), failOnOverflow = true).get, replyTo) + val context = + DefaultCommandContext(contentAsStringCommand(testPath, Option(100), failOnOverflow = true).get, replyTo) val testSource = Source.single(context) val stream = testSource.via(flow).toMat(readSink)(Keep.right) stream.run() map { case (IoFailure(_, EnhancedCromwellIoException(_, receivedException)), _) => - receivedException.getMessage should include ("Failed checksum") + receivedException.getMessage should include("Failed checksum") case (ack, _) => fail(s"read returned an unexpected message:\n$ack\n\n") } } @@ -202,7 +206,8 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with .thenReturn(FileHash(HashType.Crc32c, "boom")) .thenReturn(FileHash(HashType.Crc32c, "9a71bb4c")) - val context = DefaultCommandContext(contentAsStringCommand(testPath, Option(100), failOnOverflow = true).get, replyTo) + val context = + DefaultCommandContext(contentAsStringCommand(testPath, Option(100), failOnOverflow = true).get, replyTo) val testSource = Source.single(context) val stream = testSource.via(flow).toMat(readSink)(Keep.right) @@ -225,7 +230,8 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with when(testPath.md5HexString) .thenReturn(Success(None)) - val context = DefaultCommandContext(contentAsStringCommand(testPath, Option(100), failOnOverflow = true).get, replyTo) + val context = + DefaultCommandContext(contentAsStringCommand(testPath, Option(100), failOnOverflow = true).get, replyTo) val testSource = Source.single(context) val stream = testSource.via(flow).toMat(readSink)(Keep.right) @@ -290,7 +296,7 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with it should "delete a Nio path with swallowIoExceptions true" in { val testPath = DefaultPathBuilder.build("/this/does/not/exist").get - //noinspection RedundantDefaultArgument + // noinspection RedundantDefaultArgument val context = DefaultCommandContext(deleteCommand(testPath, swallowIoExceptions = true).get, replyTo) val testSource = Source.single(context) @@ -328,21 +334,20 @@ class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with val testSource = Source.single(context) - val customFlow = new NioFlow( - parallelism = 1, - onRetryCallback = NoopOnRetry, - onBackpressure = NoopOnBackpressure, - numberOfAttempts = 3, - commandBackpressureStaleness = 5 seconds)(system) { + val customFlow = new NioFlow(parallelism = 1, + onRetryCallback = NoopOnRetry, + onBackpressure = NoopOnBackpressure, + numberOfAttempts = 3, + commandBackpressureStaleness = 5 seconds + )(system) { private var tries = 0 - override def handleSingleCommand(ioSingleCommand: IoCommand[_]): IO[IoSuccess[_]] = { + override def handleSingleCommand(ioSingleCommand: IoCommand[_]): IO[IoSuccess[_]] = IO { tries += 1 if (tries < 3) throw new StorageException(500, "message") else IoSuccess(ioSingleCommand, "content") } - } }.flow val stream = testSource.via(customFlow).toMat(readSink)(Keep.right) diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowDockerLookupActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowDockerLookupActorSpec.scala index 1d2652b83f1..667861801a4 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowDockerLookupActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowDockerLookupActorSpec.scala @@ -9,9 +9,14 @@ import cromwell.core.retry.SimpleExponentialBackoff import cromwell.core.{TestKitSuite, WorkflowId} import cromwell.database.slick.EngineSlickDatabase import cromwell.database.sql.tables.DockerHashStoreEntry -import cromwell.docker.DockerInfoActor.{DockerInfoFailedResponse, DockerInfoSuccessResponse, DockerInformation} +import cromwell.docker.DockerInfoActor.{DockerInfoFailedResponse, DockerInformation, DockerInfoSuccessResponse} import cromwell.docker.{DockerHashResult, DockerImageIdentifier, DockerImageIdentifierWithoutHash, DockerInfoRequest} -import cromwell.engine.workflow.WorkflowDockerLookupActor.{DockerHashActorTimeout, Running, WorkflowDockerLookupFailure, WorkflowDockerTerminalFailure} +import cromwell.engine.workflow.WorkflowDockerLookupActor.{ + DockerHashActorTimeout, + Running, + WorkflowDockerLookupFailure, + WorkflowDockerTerminalFailure +} import cromwell.engine.workflow.WorkflowDockerLookupActorSpec._ import cromwell.engine.workflow.workflowstore.{StartableState, Submitted} import cromwell.services.EngineServicesStore @@ -25,9 +30,8 @@ import scala.concurrent.{ExecutionContext, Future} import scala.language.postfixOps import scala.util.control.NoStackTrace - class WorkflowDockerLookupActorSpec - extends TestKitSuite + extends TestKitSuite with AnyFlatSpecLike with Matchers with ImplicitSender @@ -55,11 +59,11 @@ class WorkflowDockerLookupActorSpec } it should "wait and resubmit the docker request when it gets a backpressure message" in { - val backoff = SimpleExponentialBackoff(2.seconds, 10.minutes, 2D) + val backoff = SimpleExponentialBackoff(2.seconds, 10.minutes, 2d) val lookupActor = TestActorRef( Props(new TestWorkflowDockerLookupActor(workflowId, dockerHashingActor.ref, Submitted, backoff)), - dockerSendingActor.ref, + dockerSendingActor.ref ) lookupActor.tell(LatestRequest, dockerSendingActor.ref) @@ -75,7 +79,8 @@ class WorkflowDockerLookupActorSpec Future.successful(()) } - val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = false, db)) + val lookupActor = + TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = false, db)) lookupActor.tell(LatestRequest, dockerSendingActor.ref) // The WorkflowDockerLookupActor should not have the hash for this tag yet and will need to query the dockerHashingActor. @@ -94,7 +99,8 @@ class WorkflowDockerLookupActorSpec } it should "soldier on after docker hashing actor timeouts" in { - val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = false)) + val lookupActor = + TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = false)) lookupActor.tell(LatestRequest, dockerSendingActor.ref) lookupActor.tell(OlderRequest, dockerSendingActor.ref) @@ -131,7 +137,13 @@ class WorkflowDockerLookupActorSpec // BA-6495 it should "not fail and enter terminal state when response for certain image id from DockerHashingActor arrived after the self-imposed timeout" in { - val lookupActor = TestFSMRef(new WorkflowDockerLookupActor(workflowId, dockerHashingActor.ref, isRestart = false, EngineServicesStore.engineDatabaseInterface)) + val lookupActor = TestFSMRef( + new WorkflowDockerLookupActor(workflowId, + dockerHashingActor.ref, + isRestart = false, + EngineServicesStore.engineDatabaseInterface + ) + ) lookupActor.tell(LatestRequest, dockerSendingActor.ref) @@ -142,7 +154,8 @@ class WorkflowDockerLookupActorSpec // WorkflowDockerLookupActor actually sends DockerHashActorTimeout to itself lookupActor.tell(timeout, lookupActor) - val failedRequest: WorkflowDockerLookupFailure = dockerSendingActor.receiveOne(2 seconds).asInstanceOf[WorkflowDockerLookupFailure] + val failedRequest: WorkflowDockerLookupFailure = + dockerSendingActor.receiveOne(2 seconds).asInstanceOf[WorkflowDockerLookupFailure] failedRequest.request shouldBe LatestRequest lookupActor.tell(LatestRequest, dockerSendingActor.ref) @@ -160,7 +173,8 @@ class WorkflowDockerLookupActorSpec } it should "respond appropriately to docker hash lookup failures" in { - val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = false)) + val lookupActor = + TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = false)) lookupActor.tell(LatestRequest, dockerSendingActor.ref) lookupActor.tell(OlderRequest, dockerSendingActor.ref) @@ -176,7 +190,9 @@ class WorkflowDockerLookupActorSpec val mixedResponses = results collect { case msg: DockerInfoSuccessResponse => msg // Scoop out the request here since we can't match the exception on the whole message. - case msg: WorkflowDockerLookupFailure if msg.reason.getMessage == "Failed to get docker hash for ubuntu:older Lookup failed" => msg.request + case msg: WorkflowDockerLookupFailure + if msg.reason.getMessage == "Failed to get docker hash for ubuntu:older Lookup failed" => + msg.request } Set(LatestSuccessResponse, OlderRequest) should equal(mixedResponses) @@ -190,11 +206,11 @@ class WorkflowDockerLookupActorSpec it should "reuse previously looked up hashes following a restart" in { val db = dbWithQuery { - Future.successful( - Seq(LatestStoreEntry(workflowId), OlderStoreEntry(workflowId))) + Future.successful(Seq(LatestStoreEntry(workflowId), OlderStoreEntry(workflowId))) } - val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = true, db)) + val lookupActor = + TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = true, db)) lookupActor.tell(LatestRequest, dockerSendingActor.ref) lookupActor.tell(OlderRequest, dockerSendingActor.ref) @@ -209,7 +225,8 @@ class WorkflowDockerLookupActorSpec it should "not try to look up hashes if not restarting" in { val db = dbWithWrite(Future.successful(())) - val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = false, db)) + val lookupActor = + TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = false, db)) lookupActor.tell(LatestRequest, dockerSendingActor.ref) lookupActor.tell(OlderRequest, dockerSendingActor.ref) @@ -231,7 +248,8 @@ class WorkflowDockerLookupActorSpec if (numWrites == 1) Future.failed(new RuntimeException("Fake exception from a test.")) else Future.successful(()) } - val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = false, db)) + val lookupActor = + TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = false, db)) lookupActor.tell(LatestRequest, dockerSendingActor.ref) // The WorkflowDockerLookupActor should not have the hash for this tag yet and will need to query the dockerHashingActor. @@ -256,7 +274,8 @@ class WorkflowDockerLookupActorSpec Future.failed(new Exception("Don't worry this is just a dummy failure in a test") with NoStackTrace) } - val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = true, db)) + val lookupActor = + TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = true, db)) lookupActor.tell(LatestRequest, dockerSendingActor.ref) dockerHashingActor.expectNoMessage() @@ -267,14 +286,17 @@ class WorkflowDockerLookupActorSpec it should "emit a terminal failure message if unable to parse hashes read from the database on restart" in { val db = dbWithQuery { numReads = numReads + 1 - Future.successful(Seq( - DockerHashStoreEntry(workflowId.toString, Latest, "md5:AAAAA", None), - // missing the "algorithm:" preceding the hash value so this should fail parsing. - DockerHashStoreEntry(workflowId.toString, Older, "BBBBB", None) - )) + Future.successful( + Seq( + DockerHashStoreEntry(workflowId.toString, Latest, "md5:AAAAA", None), + // missing the "algorithm:" preceding the hash value so this should fail parsing. + DockerHashStoreEntry(workflowId.toString, Older, "BBBBB", None) + ) + ) } - val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = true, db)) + val lookupActor = + TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, isRestart = true, db)) lookupActor.tell(LatestRequest, dockerSendingActor.ref) dockerHashingActor.expectNoMessage() @@ -282,25 +304,26 @@ class WorkflowDockerLookupActorSpec numReads should equal(1) } - def dbWithWrite(writeFn: => Future[Unit]): EngineSlickDatabase = { + def dbWithWrite(writeFn: => Future[Unit]): EngineSlickDatabase = databaseInterface(write = _ => writeFn) - } - def dbWithQuery(queryFn: => Future[Seq[DockerHashStoreEntry]]): EngineSlickDatabase = { + def dbWithQuery(queryFn: => Future[Seq[DockerHashStoreEntry]]): EngineSlickDatabase = databaseInterface(query = _ => queryFn) - } def databaseInterface(query: String => Future[Seq[DockerHashStoreEntry]] = abjectFailure, - write: DockerHashStoreEntry => Future[Unit] = abjectFailure): EngineSlickDatabase = { + write: DockerHashStoreEntry => Future[Unit] = abjectFailure + ): EngineSlickDatabase = new EngineSlickDatabase(DatabaseConfig) { - override def queryDockerHashStoreEntries(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Seq[DockerHashStoreEntry]] = query(workflowExecutionUuid) + override def queryDockerHashStoreEntries(workflowExecutionUuid: String)(implicit + ec: ExecutionContext + ): Future[Seq[DockerHashStoreEntry]] = query(workflowExecutionUuid) - override def addDockerHashStoreEntry(dockerHashStoreEntry: DockerHashStoreEntry)(implicit ec: ExecutionContext): Future[Unit] = write(dockerHashStoreEntry) + override def addDockerHashStoreEntry(dockerHashStoreEntry: DockerHashStoreEntry)(implicit + ec: ExecutionContext + ): Future[Unit] = write(dockerHashStoreEntry) }.initialized(EngineServicesStore.EngineLiquibaseSettings) - } } - object WorkflowDockerLookupActorSpec { val Latest = "ubuntu:latest" val Older = "ubuntu:older" @@ -313,8 +336,10 @@ object WorkflowDockerLookupActorSpec { val LatestRequest: DockerInfoRequest = DockerInfoRequest(LatestImageId) val OlderRequest: DockerInfoRequest = DockerInfoRequest(OlderImageId) - def LatestStoreEntry(workflowId: WorkflowId): DockerHashStoreEntry = DockerHashStoreEntry(workflowId.toString, Latest, "md5:AAAAAAAA", None) - def OlderStoreEntry(workflowId: WorkflowId): DockerHashStoreEntry = DockerHashStoreEntry(workflowId.toString, Older, "md5:BBBBBBBB", None) + def LatestStoreEntry(workflowId: WorkflowId): DockerHashStoreEntry = + DockerHashStoreEntry(workflowId.toString, Latest, "md5:AAAAAAAA", None) + def OlderStoreEntry(workflowId: WorkflowId): DockerHashStoreEntry = + DockerHashStoreEntry(workflowId.toString, Older, "md5:BBBBBBBB", None) val LatestSuccessResponse: DockerInfoSuccessResponse = DockerInfoSuccessResponse(DockerInformation(DockerHashResult("md5", "AAAAAAAA"), None), LatestRequest) @@ -325,12 +350,15 @@ object WorkflowDockerLookupActorSpec { def abjectFailure[A, B]: A => Future[B] = _ => Future.failed(new RuntimeException("Should not be called!")) - class TestWorkflowDockerLookupActor(workflowId: WorkflowId, dockerHashingActor: ActorRef, startState: StartableState, backoff: Backoff) - extends WorkflowDockerLookupActor( - workflowId, - dockerHashingActor, - startState.restarted, - EngineServicesStore.engineDatabaseInterface) { + class TestWorkflowDockerLookupActor(workflowId: WorkflowId, + dockerHashingActor: ActorRef, + startState: StartableState, + backoff: Backoff + ) extends WorkflowDockerLookupActor(workflowId, + dockerHashingActor, + startState.restarted, + EngineServicesStore.engineDatabaseInterface + ) { override protected def initialBackoff(): Backoff = backoff } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/ValidatingCachingConfigSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/ValidatingCachingConfigSpec.scala index 095721569e7..b5a46729b24 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/ValidatingCachingConfigSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/ValidatingCachingConfigSpec.scala @@ -8,26 +8,28 @@ import org.scalatest.prop.TableDrivenPropertyChecks import scala.util.{Failure, Success, Try} -class ValidatingCachingConfigSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with TableDrivenPropertyChecks { +class ValidatingCachingConfigSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with TableDrivenPropertyChecks { - it should "run config tests" in { - val cases = Table[String, Any]( - ("config" , "exceptionMessage" ), - ("enabled = not-a-boolean", "String: 1: enabled has type STRING rather than BOOLEAN" ), - ("enabled = true" , true ), - ("enabled = false" , false ), - ("enabled = 1" , "String: 1: enabled has type NUMBER rather than BOOLEAN" ), - ("" , "String: 1: No configuration setting found for key 'enabled'" ) - ) + it should "run config tests" in { + val cases = Table[String, Any]( + ("config", "exceptionMessage"), + ("enabled = not-a-boolean", "String: 1: enabled has type STRING rather than BOOLEAN"), + ("enabled = true", true), + ("enabled = false", false), + ("enabled = 1", "String: 1: enabled has type NUMBER rather than BOOLEAN"), + ("", "String: 1: No configuration setting found for key 'enabled'") + ) - forEvery(cases) { (config, expected) => - val rootConfig = ConfigFactory.parseString(config) - Try(rootConfig.getBoolean("enabled")) match { - case Success(what) => what shouldBe a [java.lang.Boolean] - case Failure(exception) => exception.getMessage should be (expected) - } + forEvery(cases) { (config, expected) => + val rootConfig = ConfigFactory.parseString(config) + Try(rootConfig.getBoolean("enabled")) match { + case Success(what) => what shouldBe a[java.lang.Boolean] + case Failure(exception) => exception.getMessage should be(expected) } } } - - +} diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/ValidatingCallCachingModeSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/ValidatingCallCachingModeSpec.scala index e7dc1aeaa4f..f2a4aafa6c7 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/ValidatingCallCachingModeSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/ValidatingCallCachingModeSpec.scala @@ -12,7 +12,11 @@ import org.scalatest.prop.TableDrivenPropertyChecks import scala.util.{Success, Try} -class ValidatingCallCachingModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with TableDrivenPropertyChecks { +class ValidatingCallCachingModeSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with TableDrivenPropertyChecks { def makeOptions(writeOpt: Option[Boolean], readOpt: Option[Boolean]) = { val writeValue = writeOpt map { v => s""""write_to_cache": $v""" } @@ -29,54 +33,54 @@ class ValidatingCallCachingModeSpec extends AnyFlatSpec with CromwellTimeoutSpec val allCombinations = (for { writeOption <- options readOption <- options - } yield (makeOptions(writeOption, readOption))).toSet + } yield makeOptions(writeOption, readOption)).toSet // writeCache is ON when config is ON and write_to_cache is None or true val writeCacheOnCombinations = (for { writeOption <- options if writeOption.isEmpty || writeOption.get readOption <- options - } yield (makeOptions(writeOption, readOption))).toSet + } yield makeOptions(writeOption, readOption)).toSet // readCache is ON when config is ON and read_from_cache is None or true val readCacheOnCombinations = (for { writeOption <- options readOption <- options if readOption.isEmpty || readOption.get - } yield (makeOptions(writeOption, readOption))).toSet + } yield makeOptions(writeOption, readOption)).toSet val writeCacheOffCombinations = allCombinations -- writeCacheOnCombinations val readCacheOffCombinations = allCombinations -- readCacheOnCombinations - validateCallCachingMode( - "write cache on options", - writeCacheOnCombinations, - callCachingEnabled, - invalidBadCaсheResults) { _.writeToCache should be(true) } - validateCallCachingMode( - "read cache on options", - readCacheOnCombinations, - callCachingEnabled, - invalidBadCaсheResults) { _.readFromCache should be(true) } - validateCallCachingMode( - "write cache off options", - writeCacheOffCombinations, - callCachingEnabled, - invalidBadCaсheResults) { _.writeToCache should be(false) } - validateCallCachingMode( - "read cache off options", - readCacheOffCombinations, - callCachingEnabled, - invalidBadCaсheResults) { _.readFromCache should be(false) } + validateCallCachingMode("write cache on options", + writeCacheOnCombinations, + callCachingEnabled, + invalidBadCaсheResults + )(_.writeToCache should be(true)) + validateCallCachingMode("read cache on options", readCacheOnCombinations, callCachingEnabled, invalidBadCaсheResults)( + _.readFromCache should be(true) + ) + validateCallCachingMode("write cache off options", + writeCacheOffCombinations, + callCachingEnabled, + invalidBadCaсheResults + )(_.writeToCache should be(false)) + validateCallCachingMode("read cache off options", + readCacheOffCombinations, + callCachingEnabled, + invalidBadCaсheResults + )(_.readFromCache should be(false)) private def validateCallCachingMode(testName: String, wfOptions: Set[Try[WorkflowOptions]], callCachingEnabled: Boolean, - invalidBadCacheResults: Boolean) - (verificationFunction: CallCachingMode => Assertion): Unit = { + invalidBadCacheResults: Boolean + )(verificationFunction: CallCachingMode => Assertion): Unit = it should s"correctly identify $testName" in { - wfOptions foreach { + wfOptions foreach { case Success(wfOptions) => MaterializeWorkflowDescriptorActor.validateCallCachingMode(wfOptions, - callCachingEnabled, invalidBadCacheResults) match { + callCachingEnabled, + invalidBadCacheResults + ) match { case Valid(activity) => verificationFunction(activity) case Invalid(errors) => val errorsList = errors.toList.mkString(", ") @@ -85,5 +89,4 @@ class ValidatingCallCachingModeSpec extends AnyFlatSpec with CromwellTimeoutSpec case x => fail(s"Unexpected test tuple: $x") } } - } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/deletion/DeleteWorkflowFilesActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/deletion/DeleteWorkflowFilesActorSpec.scala index 1d117e0a8bc..d1759afef9d 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/deletion/DeleteWorkflowFilesActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/deletion/DeleteWorkflowFilesActorSpec.scala @@ -12,7 +12,10 @@ import cromwell.core.path.Path import cromwell.core.path.PathFactory.PathBuilders import cromwell.core.retry.SimpleExponentialBackoff import cromwell.engine.io.IoAttempts.EnhancedCromwellIoException -import cromwell.engine.workflow.lifecycle.deletion.DeleteWorkflowFilesActor.{StartWorkflowFilesDeletion, WaitingForIoResponses} +import cromwell.engine.workflow.lifecycle.deletion.DeleteWorkflowFilesActor.{ + StartWorkflowFilesDeletion, + WaitingForIoResponses +} import cromwell.filesystems.gcs.batch.{GcsBatchCommandBuilder, GcsBatchDeleteCommand} import cromwell.filesystems.gcs.{GcsPath, GcsPathBuilder, MockGcsPathBuilder} import cromwell.services.metadata.MetadataService.PutMetadataAction @@ -31,10 +34,7 @@ import scala.concurrent.duration._ import scala.util.control.NoStackTrace import scala.util.{Failure, Try} -class DeleteWorkflowFilesActorSpec extends TestKitSuite - with AnyFlatSpecLike - with Matchers - with BeforeAndAfter { +class DeleteWorkflowFilesActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with BeforeAndAfter { val mockPathBuilder: GcsPathBuilder = MockGcsPathBuilder.instance val mockPathBuilders = List(mockPathBuilder) @@ -50,7 +50,10 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite var rootWorkflowRoots: Set[Path] = _ var allOutputs: CallOutputs = _ var finalOutputs: CallOutputs = _ - var testDeleteWorkflowFilesActor: TestFSMRef[DeleteWorkflowFilesActor.DeleteWorkflowFilesActorState, DeleteWorkflowFilesActor.DeleteWorkflowFilesActorStateData, MockDeleteWorkflowFilesActor] = _ + var testDeleteWorkflowFilesActor: TestFSMRef[DeleteWorkflowFilesActor.DeleteWorkflowFilesActorState, + DeleteWorkflowFilesActor.DeleteWorkflowFilesActorStateData, + MockDeleteWorkflowFilesActor + ] = _ var gcsFilePath: GcsPath = _ before { @@ -60,44 +63,103 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite rootWorkflowRoots = Set[Path](mockPathBuilder.build(rootWorkflowExecutionDir).get) testProbe = TestProbe(s"test-probe-$rootWorkflowId") - allOutputs = CallOutputs(Map( - GraphNodeOutputPort(WomIdentifier(LocalName("main_output"),FullyQualifiedName("main_workflow.main_output")), WomSingleFileType, null) - -> WomSingleFile(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt"), - ExpressionBasedOutputPort(WomIdentifier(LocalName("first_task.first_task_output_2"),FullyQualifiedName("first_sub_workflow.first_task.first_task_output_2")), WomSingleFileType, null, null) - -> WomSingleFile(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file2.txt"), - GraphNodeOutputPort(WomIdentifier(LocalName("first_output_file"),FullyQualifiedName("first_sub_workflow.first_output_file")), WomSingleFileType, null) - -> WomSingleFile(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt"), - ExpressionBasedOutputPort(WomIdentifier(LocalName("first_task.first_task_output_1"),FullyQualifiedName("first_sub_workflow.first_task.first_task_output_1")), WomSingleFileType, null, null) - -> WomSingleFile(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt"), - GraphNodeOutputPort(WomIdentifier(LocalName("second_output_file"),FullyQualifiedName("first_sub_workflow.second_output_file")), WomSingleFileType, null) - -> WomSingleFile(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file2.txt") - )) - finalOutputs = CallOutputs(Map( - GraphNodeOutputPort(WomIdentifier(LocalName("main_output"),FullyQualifiedName("main_workflow.main_output")), WomSingleFileType, null) - -> WomSingleFile(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file2.txt") - )) - - testDeleteWorkflowFilesActor = TestFSMRef(new MockDeleteWorkflowFilesActor(rootWorkflowId, emptyWorkflowIdSet, rootWorkflowRoots, finalOutputs, allOutputs, mockPathBuilders, serviceRegistryActor.ref, ioActor.ref)) + allOutputs = CallOutputs( + Map( + GraphNodeOutputPort(WomIdentifier(LocalName("main_output"), FullyQualifiedName("main_workflow.main_output")), + WomSingleFileType, + null + ) + -> WomSingleFile( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt" + ), + ExpressionBasedOutputPort( + WomIdentifier(LocalName("first_task.first_task_output_2"), + FullyQualifiedName("first_sub_workflow.first_task.first_task_output_2") + ), + WomSingleFileType, + null, + null + ) + -> WomSingleFile( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file2.txt" + ), + GraphNodeOutputPort(WomIdentifier(LocalName("first_output_file"), + FullyQualifiedName("first_sub_workflow.first_output_file") + ), + WomSingleFileType, + null + ) + -> WomSingleFile( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt" + ), + ExpressionBasedOutputPort( + WomIdentifier(LocalName("first_task.first_task_output_1"), + FullyQualifiedName("first_sub_workflow.first_task.first_task_output_1") + ), + WomSingleFileType, + null, + null + ) + -> WomSingleFile( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt" + ), + GraphNodeOutputPort(WomIdentifier(LocalName("second_output_file"), + FullyQualifiedName("first_sub_workflow.second_output_file") + ), + WomSingleFileType, + null + ) + -> WomSingleFile( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file2.txt" + ) + ) + ) + finalOutputs = CallOutputs( + Map( + GraphNodeOutputPort(WomIdentifier(LocalName("main_output"), FullyQualifiedName("main_workflow.main_output")), + WomSingleFileType, + null + ) + -> WomSingleFile( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file2.txt" + ) + ) + ) + + testDeleteWorkflowFilesActor = TestFSMRef( + new MockDeleteWorkflowFilesActor(rootWorkflowId, + emptyWorkflowIdSet, + rootWorkflowRoots, + finalOutputs, + allOutputs, + mockPathBuilders, + serviceRegistryActor.ref, + ioActor.ref + ) + ) testProbe watch testDeleteWorkflowFilesActor - gcsFilePath = mockPathBuilder.build(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt").get + gcsFilePath = mockPathBuilder + .build( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt" + ) + .get } it should "follow the expected golden-path lifecycle" in { testDeleteWorkflowFilesActor ! StartWorkflowFilesDeletion - serviceRegistryActor.expectMsgPF(10.seconds) { - case m: PutMetadataAction => - val event = m.events.head - m.events.size shouldBe 1 - event.key.workflowId shouldBe rootWorkflowId - event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus - event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(InProgress) + serviceRegistryActor.expectMsgPF(10.seconds) { case m: PutMetadataAction => + val event = m.events.head + m.events.size shouldBe 1 + event.key.workflowId shouldBe rootWorkflowId + event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus + event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(InProgress) } - ioActor.expectMsgPF(10.seconds) { - case cmd: IoDeleteCommand => cmd.file shouldBe gcsFilePath + ioActor.expectMsgPF(10.seconds) { case cmd: IoDeleteCommand => + cmd.file shouldBe gcsFilePath } eventually { @@ -107,13 +169,12 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite testDeleteWorkflowFilesActor ! IoSuccess(GcsBatchDeleteCommand.forPath(gcsFilePath, swallowIOExceptions = false).get, ()) - serviceRegistryActor.expectMsgPF(10.seconds) { - case m: PutMetadataAction => - val event = m.events.head - m.events.size shouldBe 1 - event.key.workflowId shouldBe rootWorkflowId - event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus - event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(Succeeded) + serviceRegistryActor.expectMsgPF(10.seconds) { case m: PutMetadataAction => + val event = m.events.head + m.events.size shouldBe 1 + event.key.workflowId shouldBe rootWorkflowId + event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus + event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(Succeeded) } testProbe.expectTerminated(testDeleteWorkflowFilesActor, 10.seconds) @@ -123,19 +184,17 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite testDeleteWorkflowFilesActor ! StartWorkflowFilesDeletion - serviceRegistryActor.expectMsgPF(10.seconds) { - case m: PutMetadataAction => - val event = m.events.head - m.events.size shouldBe 1 - event.key.workflowId shouldBe rootWorkflowId - event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus - event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(InProgress) + serviceRegistryActor.expectMsgPF(10.seconds) { case m: PutMetadataAction => + val event = m.events.head + m.events.size shouldBe 1 + event.key.workflowId shouldBe rootWorkflowId + event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus + event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(InProgress) } val expectedDeleteCommand = GcsBatchDeleteCommand.forPath(gcsFilePath, swallowIOExceptions = false).get - ioActor.expectMsgPF(10.seconds) { - case `expectedDeleteCommand` => // woohoo! + ioActor.expectMsgPF(10.seconds) { case `expectedDeleteCommand` => // woohoo! } eventually { @@ -153,13 +212,12 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite ioActor.send(testDeleteWorkflowFilesActor, IoSuccess(expectedDeleteCommand, ())) - serviceRegistryActor.expectMsgPF(10.seconds) { - case m: PutMetadataAction => - val event = m.events.head - m.events.size shouldBe 1 - event.key.workflowId shouldBe rootWorkflowId - event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus - event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(Succeeded) + serviceRegistryActor.expectMsgPF(10.seconds) { case m: PutMetadataAction => + val event = m.events.head + m.events.size shouldBe 1 + event.key.workflowId shouldBe rootWorkflowId + event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus + event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(Succeeded) } testProbe.expectTerminated(testDeleteWorkflowFilesActor, 10.seconds) @@ -169,17 +227,16 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite testDeleteWorkflowFilesActor ! StartWorkflowFilesDeletion - serviceRegistryActor.expectMsgPF(10.seconds) { - case m: PutMetadataAction => - val event = m.events.head - m.events.size shouldBe 1 - event.key.workflowId shouldBe rootWorkflowId - event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus - event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(InProgress) + serviceRegistryActor.expectMsgPF(10.seconds) { case m: PutMetadataAction => + val event = m.events.head + m.events.size shouldBe 1 + event.key.workflowId shouldBe rootWorkflowId + event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus + event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(InProgress) } - ioActor.expectMsgPF(10.seconds) { - case cmd: IoDeleteCommand => cmd.file shouldBe gcsFilePath + ioActor.expectMsgPF(10.seconds) { case cmd: IoDeleteCommand => + cmd.file shouldBe gcsFilePath } eventually { @@ -189,63 +246,59 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite testDeleteWorkflowFilesActor ! IoFailure( command = GcsBatchDeleteCommand.forPath(gcsFilePath, swallowIOExceptions = false).get, - failure = new Exception(s"Something is fishy!"), + failure = new Exception(s"Something is fishy!") ) - serviceRegistryActor.expectMsgPF(10.seconds) { - case m: PutMetadataAction => - val event = m.events.head - m.events.size shouldBe 1 - event.key.workflowId shouldBe rootWorkflowId - event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus - event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(Failed) + serviceRegistryActor.expectMsgPF(10.seconds) { case m: PutMetadataAction => + val event = m.events.head + m.events.size shouldBe 1 + event.key.workflowId shouldBe rootWorkflowId + event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus + event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(Failed) } testProbe.expectTerminated(testDeleteWorkflowFilesActor, 10.seconds) } - it should "send success when the failure is FileNotFound" in { testDeleteWorkflowFilesActor ! StartWorkflowFilesDeletion - serviceRegistryActor.expectMsgPF(10.seconds) { - case m: PutMetadataAction => - val event = m.events.head - m.events.size shouldBe 1 - event.key.workflowId shouldBe rootWorkflowId - event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus - event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(InProgress) + serviceRegistryActor.expectMsgPF(10.seconds) { case m: PutMetadataAction => + val event = m.events.head + m.events.size shouldBe 1 + event.key.workflowId shouldBe rootWorkflowId + event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus + event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(InProgress) } - ioActor.expectMsgPF(10.seconds) { - case cmd: IoDeleteCommand => cmd.file shouldBe gcsFilePath + ioActor.expectMsgPF(10.seconds) { case cmd: IoDeleteCommand => + cmd.file shouldBe gcsFilePath } eventually { testDeleteWorkflowFilesActor.stateName shouldBe WaitingForIoResponses } - val fileNotFoundException = EnhancedCromwellIoException(s"File not found", new FileNotFoundException(gcsFilePath.pathAsString)) + val fileNotFoundException = + EnhancedCromwellIoException(s"File not found", new FileNotFoundException(gcsFilePath.pathAsString)) testDeleteWorkflowFilesActor ! IoFailure( command = GcsBatchDeleteCommand.forPath(gcsFilePath, swallowIOExceptions = false).get, - failure = fileNotFoundException, + failure = fileNotFoundException ) - serviceRegistryActor.expectMsgPF(10.seconds) { - case m: PutMetadataAction => - val event = m.events.head - m.events.size shouldBe 1 - event.key.workflowId shouldBe rootWorkflowId - event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus - event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(Succeeded) + serviceRegistryActor.expectMsgPF(10.seconds) { case m: PutMetadataAction => + val event = m.events.head + m.events.size shouldBe 1 + event.key.workflowId shouldBe rootWorkflowId + event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus + event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(Succeeded) } testProbe.expectTerminated(testDeleteWorkflowFilesActor, 10.seconds) } - it should "remove any non-file intermediate outputs" in { val expectedIntermediateFiles = Set(gcsFilePath) @@ -258,16 +311,33 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite actualIntermediateFiles shouldBe expectedIntermediateFiles } - it should "delete all file outputs if there are no final outputs" in { finalOutputs = CallOutputs.empty - val gcsFilePath1: Path = mockPathBuilder.build(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt").get - val gcsFilePath2: Path = mockPathBuilder.build(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file2.txt").get + val gcsFilePath1: Path = mockPathBuilder + .build( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt" + ) + .get + val gcsFilePath2: Path = mockPathBuilder + .build( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file2.txt" + ) + .get val expectedIntermediateFiles = Set(gcsFilePath1, gcsFilePath2) - testDeleteWorkflowFilesActor = TestFSMRef(new MockDeleteWorkflowFilesActor(rootWorkflowId, emptyWorkflowIdSet, rootWorkflowRoots, finalOutputs, allOutputs, mockPathBuilders, serviceRegistryActor.ref, ioActor.ref)) + testDeleteWorkflowFilesActor = TestFSMRef( + new MockDeleteWorkflowFilesActor(rootWorkflowId, + emptyWorkflowIdSet, + rootWorkflowRoots, + finalOutputs, + allOutputs, + mockPathBuilders, + serviceRegistryActor.ref, + ioActor.ref + ) + ) val actualIntermediateFiles = testDeleteWorkflowFilesActor.underlyingActor.gatherIntermediateOutputFiles( allOutputs.outputs.values.toSet, @@ -280,14 +350,15 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite it should "send failure when delete command creation is unsuccessful for a file" in { val partialIoCommandBuilder = new PartialIoCommandBuilder { - override def deleteCommand: PartialFunction[(Path, Boolean), Try[IoDeleteCommand]] = { - case _ => Failure(new Exception("everything's fine, I am an expected delete fail") with NoStackTrace) + override def deleteCommand: PartialFunction[(Path, Boolean), Try[IoDeleteCommand]] = { case _ => + Failure(new Exception("everything's fine, I am an expected delete fail") with NoStackTrace) } } val ioCommandBuilder = new IoCommandBuilder(List(partialIoCommandBuilder)) - testDeleteWorkflowFilesActor = - TestFSMRef(new MockDeleteWorkflowFilesActor(rootWorkflowId, + testDeleteWorkflowFilesActor = TestFSMRef( + new MockDeleteWorkflowFilesActor( + rootWorkflowId, rootAndSubworkflowIds = emptyWorkflowIdSet, rootWorkflowRoots = rootWorkflowRoots, workflowFinalOutputs = finalOutputs, @@ -295,19 +366,19 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite pathBuilders = mockPathBuilders, serviceRegistryActor = serviceRegistryActor.ref, ioActor = ioActor.ref, - gcsCommandBuilder = ioCommandBuilder, - )) + gcsCommandBuilder = ioCommandBuilder + ) + ) testProbe.watch(testDeleteWorkflowFilesActor) testDeleteWorkflowFilesActor ! StartWorkflowFilesDeletion - serviceRegistryActor.expectMsgPF(10.seconds) { - case m: PutMetadataAction => - val event = m.events.head - m.events.size shouldBe 1 - event.key.workflowId shouldBe rootWorkflowId - event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus - event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(InProgress) + serviceRegistryActor.expectMsgPF(10.seconds) { case m: PutMetadataAction => + val event = m.events.head + m.events.size shouldBe 1 + event.key.workflowId shouldBe rootWorkflowId + event.key.key shouldBe WorkflowMetadataKeys.FileDeletionStatus + event.value.get.value shouldBe FileDeletionStatus.toDatabaseValue(InProgress) } testProbe.expectTerminated(testDeleteWorkflowFilesActor, 10.seconds) @@ -316,14 +387,40 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite it should "terminate if root workflow has no intermediate outputs to delete" in { - finalOutputs = CallOutputs(Map( - GraphNodeOutputPort(WomIdentifier(LocalName("main_output_1"),FullyQualifiedName("main_workflow.main_output_1")), WomSingleFileType, null) - -> WomSingleFile(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt") , - GraphNodeOutputPort(WomIdentifier(LocalName("main_output_2"),FullyQualifiedName("main_workflow.main_output_2")), WomSingleFileType, null) - -> WomSingleFile(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file2.txt") - )) + finalOutputs = CallOutputs( + Map( + GraphNodeOutputPort(WomIdentifier(LocalName("main_output_1"), + FullyQualifiedName("main_workflow.main_output_1") + ), + WomSingleFileType, + null + ) + -> WomSingleFile( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt" + ), + GraphNodeOutputPort(WomIdentifier(LocalName("main_output_2"), + FullyQualifiedName("main_workflow.main_output_2") + ), + WomSingleFileType, + null + ) + -> WomSingleFile( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file2.txt" + ) + ) + ) - val testDeleteWorkflowFilesActor = TestFSMRef(new MockDeleteWorkflowFilesActor(rootWorkflowId, emptyWorkflowIdSet, rootWorkflowRoots, finalOutputs, allOutputs, mockPathBuilders, serviceRegistryActor.ref, ioActor.ref)) + val testDeleteWorkflowFilesActor = TestFSMRef( + new MockDeleteWorkflowFilesActor(rootWorkflowId, + emptyWorkflowIdSet, + rootWorkflowRoots, + finalOutputs, + allOutputs, + mockPathBuilders, + serviceRegistryActor.ref, + ioActor.ref + ) + ) testProbe watch testDeleteWorkflowFilesActor @@ -332,21 +429,56 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite testProbe.expectTerminated(testDeleteWorkflowFilesActor, 10.seconds) } - it should "remove values that are file names in form of string" in { - allOutputs = allOutputs.copy(outputs = allOutputs.outputs ++ Map( - GraphNodeOutputPort(WomIdentifier(LocalName("file_with_file_path_string_output"),FullyQualifiedName("first_sub_workflow.file_with_file_path_string_output")), WomStringType, null) - -> WomString(s"gs://my_bucket/non_existent_file.txt"), - ExpressionBasedOutputPort(WomIdentifier(LocalName("first_task.first_task_file_with_file_path_output"),FullyQualifiedName("first_sub_workflow.first_task.first_task_file_with_file_path_output")), WomSingleFileType, null, null) - -> WomSingleFile(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/file_with_file_path.txt") - )) + allOutputs = allOutputs.copy(outputs = + allOutputs.outputs ++ Map( + GraphNodeOutputPort( + WomIdentifier(LocalName("file_with_file_path_string_output"), + FullyQualifiedName("first_sub_workflow.file_with_file_path_string_output") + ), + WomStringType, + null + ) + -> WomString(s"gs://my_bucket/non_existent_file.txt"), + ExpressionBasedOutputPort( + WomIdentifier( + LocalName("first_task.first_task_file_with_file_path_output"), + FullyQualifiedName("first_sub_workflow.first_task.first_task_file_with_file_path_output") + ), + WomSingleFileType, + null, + null + ) + -> WomSingleFile( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/file_with_file_path.txt" + ) + ) + ) - val gcsFilePath1: Path = mockPathBuilder.build(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt").get - val gcsFilePath2: Path = mockPathBuilder.build(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/file_with_file_path.txt").get + val gcsFilePath1: Path = mockPathBuilder + .build( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt" + ) + .get + val gcsFilePath2: Path = mockPathBuilder + .build( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/file_with_file_path.txt" + ) + .get val expectedIntermediateFiles = Set(gcsFilePath1, gcsFilePath2) - val testDeleteWorkflowFilesActor = TestFSMRef(new MockDeleteWorkflowFilesActor(rootWorkflowId, emptyWorkflowIdSet, rootWorkflowRoots, finalOutputs, allOutputs, mockPathBuilders, serviceRegistryActor.ref, ioActor.ref)) + val testDeleteWorkflowFilesActor = TestFSMRef( + new MockDeleteWorkflowFilesActor(rootWorkflowId, + emptyWorkflowIdSet, + rootWorkflowRoots, + finalOutputs, + allOutputs, + mockPathBuilders, + serviceRegistryActor.ref, + ioActor.ref + ) + ) val actualIntermediateFiles = testDeleteWorkflowFilesActor.underlyingActor.gatherIntermediateOutputFiles( allOutputs.outputs.values.toSet, @@ -356,25 +488,76 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite actualIntermediateFiles shouldBe expectedIntermediateFiles } - it should "identify and gather glob files" in { - allOutputs = allOutputs.copy(outputs = allOutputs.outputs ++ Map( - GraphNodeOutputPort(WomIdentifier(LocalName("glob_output"),FullyQualifiedName("first_sub_workflow.glob_output")), WomMaybeEmptyArrayType(WomSingleFileType), null) - -> WomArray(Seq(WomSingleFile(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/glob-random_id/intermediate_file1.txt"), - WomSingleFile(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/glob-random_id/intermediate_file2.txt"))), - ExpressionBasedOutputPort(WomIdentifier(LocalName("first_task.first_task_glob"),FullyQualifiedName("first_sub_workflow.first_task.first_task_glob")), WomMaybeEmptyArrayType(WomSingleFileType), null, null) - -> WomArray(Seq(WomSingleFile(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/glob-random_id/intermediate_file1.txt"), - WomSingleFile(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/glob-random_id/intermediate_file2.txt"))) - )) + allOutputs = allOutputs.copy(outputs = + allOutputs.outputs ++ Map( + GraphNodeOutputPort(WomIdentifier(LocalName("glob_output"), + FullyQualifiedName("first_sub_workflow.glob_output") + ), + WomMaybeEmptyArrayType(WomSingleFileType), + null + ) + -> WomArray( + Seq( + WomSingleFile( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/glob-random_id/intermediate_file1.txt" + ), + WomSingleFile( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/glob-random_id/intermediate_file2.txt" + ) + ) + ), + ExpressionBasedOutputPort( + WomIdentifier(LocalName("first_task.first_task_glob"), + FullyQualifiedName("first_sub_workflow.first_task.first_task_glob") + ), + WomMaybeEmptyArrayType(WomSingleFileType), + null, + null + ) + -> WomArray( + Seq( + WomSingleFile( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/glob-random_id/intermediate_file1.txt" + ), + WomSingleFile( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/glob-random_id/intermediate_file2.txt" + ) + ) + ) + ) + ) - val gcsFilePath1: Path = mockPathBuilder.build(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt").get - val gcsGlobFilePath1: Path = mockPathBuilder.build(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/glob-random_id/intermediate_file1.txt").get - val gcsGlobFilePath2: Path = mockPathBuilder.build(s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/glob-random_id/intermediate_file2.txt").get + val gcsFilePath1: Path = mockPathBuilder + .build( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/intermediate_file1.txt" + ) + .get + val gcsGlobFilePath1: Path = mockPathBuilder + .build( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/glob-random_id/intermediate_file1.txt" + ) + .get + val gcsGlobFilePath2: Path = mockPathBuilder + .build( + s"$rootWorkflowExecutionDir/call-first_sub_workflow/firstSubWf.first_sub_workflow/$subworkflowId/call-first_task/glob-random_id/intermediate_file2.txt" + ) + .get val expectedIntermediateFiles = Set(gcsFilePath1, gcsGlobFilePath1, gcsGlobFilePath2) - val testDeleteWorkflowFilesActor = TestFSMRef(new MockDeleteWorkflowFilesActor(rootWorkflowId, emptyWorkflowIdSet, rootWorkflowRoots, finalOutputs, allOutputs, mockPathBuilders, serviceRegistryActor.ref, ioActor.ref)) + val testDeleteWorkflowFilesActor = TestFSMRef( + new MockDeleteWorkflowFilesActor(rootWorkflowId, + emptyWorkflowIdSet, + rootWorkflowRoots, + finalOutputs, + allOutputs, + mockPathBuilders, + serviceRegistryActor.ref, + ioActor.ref + ) + ) val actualIntermediateFiles = testDeleteWorkflowFilesActor.underlyingActor.gatherIntermediateOutputFiles( allOutputs.outputs.values.toSet, @@ -384,7 +567,6 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite actualIntermediateFiles shouldBe expectedIntermediateFiles } - it should "sanity check in multiple rootWorkflowRoots" in { val expectedIntermediateFiles = Set(gcsFilePath) @@ -393,7 +575,17 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite mockPathBuilder.build(s"gs://my_bucket/main_workflow/yyyy").get, mockPathBuilder.build(rootWorkflowExecutionDir).get ) - val testDeleteWorkflowFilesActor = TestFSMRef(new MockDeleteWorkflowFilesActor(rootWorkflowId, emptyWorkflowIdSet, rootWorkflowRoots, finalOutputs, allOutputs, mockPathBuilders, serviceRegistryActor.ref, ioActor.ref)) + val testDeleteWorkflowFilesActor = TestFSMRef( + new MockDeleteWorkflowFilesActor(rootWorkflowId, + emptyWorkflowIdSet, + rootWorkflowRoots, + finalOutputs, + allOutputs, + mockPathBuilders, + serviceRegistryActor.ref, + ioActor.ref + ) + ) val actualIntermediateFiles = testDeleteWorkflowFilesActor.underlyingActor.gatherIntermediateOutputFiles( allOutputs.outputs.values.toSet, @@ -403,10 +595,19 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite actualIntermediateFiles shouldBe expectedIntermediateFiles } - it should "not delete outputs if they are outside the root workflow execution directory" in { val rootWorkflowRoots = Set[Path](mockPathBuilder.build(s"gs://my_bucket/main_workflow/xxxx").get) - val testDeleteWorkflowFilesActor = TestFSMRef(new MockDeleteWorkflowFilesActor(rootWorkflowId, emptyWorkflowIdSet, rootWorkflowRoots, finalOutputs, allOutputs, mockPathBuilders, serviceRegistryActor.ref, ioActor.ref)) + val testDeleteWorkflowFilesActor = TestFSMRef( + new MockDeleteWorkflowFilesActor(rootWorkflowId, + emptyWorkflowIdSet, + rootWorkflowRoots, + finalOutputs, + allOutputs, + mockPathBuilders, + serviceRegistryActor.ref, + ioActor.ref + ) + ) val actualIntermediateFiles = testDeleteWorkflowFilesActor.underlyingActor.gatherIntermediateOutputFiles( allOutputs.outputs.values.toSet, @@ -417,9 +618,6 @@ class DeleteWorkflowFilesActorSpec extends TestKitSuite } } - - - class MockDeleteWorkflowFilesActor(rootWorkflowId: RootWorkflowId, rootAndSubworkflowIds: Set[WorkflowId], rootWorkflowRoots: Set[Path], @@ -428,19 +626,18 @@ class MockDeleteWorkflowFilesActor(rootWorkflowId: RootWorkflowId, pathBuilders: PathBuilders, serviceRegistryActor: ActorRef, ioActor: ActorRef, - gcsCommandBuilder: IoCommandBuilder = GcsBatchCommandBuilder, - ) extends - DeleteWorkflowFilesActor( - rootWorkflowId, - rootAndSubworkflowIds, - rootWorkflowRoots, - workflowFinalOutputs.outputs.values.toSet, - workflowAllOutputs.outputs.values.toSet, - pathBuilders, - serviceRegistryActor, - ioActor, - gcsCommandBuilder, - ) { + gcsCommandBuilder: IoCommandBuilder = GcsBatchCommandBuilder +) extends DeleteWorkflowFilesActor( + rootWorkflowId, + rootAndSubworkflowIds, + rootWorkflowRoots, + workflowFinalOutputs.outputs.values.toSet, + workflowAllOutputs.outputs.values.toSet, + pathBuilders, + serviceRegistryActor, + ioActor, + gcsCommandBuilder + ) { // Override the IO actor backoff for the benefit of the backpressure tests: - override def initialBackoff(): Backoff = SimpleExponentialBackoff(100.millis, 1.second, 1.2D) + override def initialBackoff(): Backoff = SimpleExponentialBackoff(100.millis, 1.second, 1.2d) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala index 6feb8338bab..797321d7ba9 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala @@ -34,28 +34,33 @@ object ExecutionStoreBenchmark extends Bench[Double] with DefaultJsonProtocol { val inputJson = Option(SampleWdl.PrepareScatterGatherWdl().rawInputs.toJson.compactPrint) val namespace = WdlNamespaceWithWorkflow.load(SampleWdl.PrepareScatterGatherWdl().workflowSource(), Seq.empty).get - val graph = namespace.toWomExecutable(inputJson, NoIoFunctionSet, strictValidation = true).getOrElse(throw new Exception("Failed to build womExecutable")).graph + val graph = namespace + .toWomExecutable(inputJson, NoIoFunctionSet, strictValidation = true) + .getOrElse(throw new Exception("Failed to build womExecutable")) + .graph val prepareCall: CommandCallNode = graph.calls.find(_.localName == "do_prepare").get.asInstanceOf[CommandCallNode] val scatterCall: CommandCallNode = graph.allNodes.find(_.localName == "do_scatter").get.asInstanceOf[CommandCallNode] val scatter: ScatterNode = graph.scatters.head - private def makeKey(call: CommandCallNode, executionStatus: ExecutionStatus)(index: Int) = { + private def makeKey(call: CommandCallNode, executionStatus: ExecutionStatus)(index: Int) = BackendJobDescriptorKey(call, Option(index), 1) -> executionStatus - } // Generates executionStores using the given above sizes // Each execution store contains X simulated shards of "prepareCall" in status Done and X simulated shards of "scatterCall" in status NotStarted // This provides a good starting point to evaluate the speed of "runnableCalls", as it needs to iterate over all "NotStarted" keys, and for each one // look for their upstreams keys in status "Done" - private def stores(sizes: Gen[Int]): Gen[ActiveExecutionStore] = { + private def stores(sizes: Gen[Int]): Gen[ActiveExecutionStore] = for { size <- sizes doneMap = (0 until size map makeKey(prepareCall, ExecutionStatus.Done)).toMap - collectorKeys = scatter.outputMapping.map(om => ScatterCollectorKey(om, size, ScatterNode.DefaultScatterCollectionFunction) -> ExecutionStatus.NotStarted).toMap + collectorKeys = scatter.outputMapping + .map(om => + ScatterCollectorKey(om, size, ScatterNode.DefaultScatterCollectionFunction) -> ExecutionStatus.NotStarted + ) + .toMap notStartedMap = (0 until size map makeKey(scatterCall, ExecutionStatus.NotStarted)).toMap ++ collectorKeys finalMap = doneMap ++ notStartedMap } yield ActiveExecutionStore(finalMap.toMap, needsUpdate = true) - } performance of "ExecutionStore" in { // Measures how fast the execution store can find runnable calls with lots of "Done" calls and "NotStarted" calls. diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala index fa90e796d95..ebd76a8e5fb 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala @@ -23,7 +23,11 @@ import cromwell.engine.workflow.lifecycle.execution.keys.SubWorkflowKey import cromwell.engine.workflow.lifecycle.execution.stores.ValueStore import cromwell.engine.workflow.workflowstore.{RestartableRunning, StartableState, Submitted} import cromwell.engine.{ContinueWhilePossible, EngineIoFunctions, EngineWorkflowDescriptor} -import cromwell.services.metadata.MetadataService.{MetadataWriteFailure, MetadataWriteSuccess, PutMetadataActionAndRespond} +import cromwell.services.metadata.MetadataService.{ + MetadataWriteFailure, + MetadataWriteSuccess, + PutMetadataActionAndRespond +} import cromwell.subworkflowstore.SubWorkflowStoreActor.{QuerySubWorkflow, SubWorkflowFound, SubWorkflowNotFound} import cromwell.util.WomMocks import org.scalatest.BeforeAndAfterAll @@ -37,8 +41,13 @@ import scala.concurrent.duration._ import scala.language.postfixOps import scala.util.control.NoStackTrace -class SubWorkflowExecutionActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with MockSugar - with Eventually with BeforeAndAfterAll { +class SubWorkflowExecutionActorSpec + extends TestKitSuite + with AnyFlatSpecLike + with Matchers + with MockSugar + with Eventually + with BeforeAndAfterAll { behavior of "SubWorkflowExecutionActor" @@ -90,35 +99,40 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with AnyFlatSpecLike wi parentProbe = TestProbe() } - private def buildSWEA(startState: StartableState = Submitted) = { - new TestFSMRef[SubWorkflowExecutionActorState, SubWorkflowExecutionActorData, SubWorkflowExecutionActor](system, Props( - new SubWorkflowExecutionActor( - subKey, - parentWorkflowDescriptor, - new EngineIoFunctions(List.empty, new AsyncIo(simpleIoActor, DefaultIoCommandBuilder), system.dispatcher), - Map.empty, - ioActorProbe.ref, - serviceRegistryProbe.ref, - jobStoreProbe.ref, - subWorkflowStoreProbe.ref, - callCacheReadActorProbe.ref, - callCacheWriteActorProbe.ref, - dockerHashActorProbe.ref, - jobRestartCheckTokenDispenserProbe.ref, - jobExecutionTokenDispenserProbe.ref, - BackendSingletonCollection(Map.empty), - AllBackendInitializationData(Map.empty), - startState, - rootConfig, - new AtomicInteger(), - fileHashCacheActor = None, - blacklistCache = None - ) { - override def createSubWorkflowPreparationActor(subWorkflowId: WorkflowId): ActorRef = preparationActor.ref - override def createSubWorkflowActor(createSubWorkflowActor: EngineWorkflowDescriptor): ActorRef = - subWorkflowActor.ref - }), parentProbe.ref, s"SubWorkflowExecutionActorSpec-${UUID.randomUUID()}") - } + private def buildSWEA(startState: StartableState = Submitted) = + new TestFSMRef[SubWorkflowExecutionActorState, SubWorkflowExecutionActorData, SubWorkflowExecutionActor]( + system, + Props( + new SubWorkflowExecutionActor( + subKey, + parentWorkflowDescriptor, + new EngineIoFunctions(List.empty, new AsyncIo(simpleIoActor, DefaultIoCommandBuilder), system.dispatcher), + Map.empty, + ioActorProbe.ref, + serviceRegistryProbe.ref, + jobStoreProbe.ref, + subWorkflowStoreProbe.ref, + callCacheReadActorProbe.ref, + callCacheWriteActorProbe.ref, + dockerHashActorProbe.ref, + jobRestartCheckTokenDispenserProbe.ref, + jobExecutionTokenDispenserProbe.ref, + BackendSingletonCollection(Map.empty), + AllBackendInitializationData(Map.empty), + startState, + rootConfig, + new AtomicInteger(), + fileHashCacheActor = None, + blacklistCache = None + ) { + override def createSubWorkflowPreparationActor(subWorkflowId: WorkflowId): ActorRef = preparationActor.ref + override def createSubWorkflowActor(createSubWorkflowActor: EngineWorkflowDescriptor): ActorRef = + subWorkflowActor.ref + } + ), + parentProbe.ref, + s"SubWorkflowExecutionActorSpec-${UUID.randomUUID()}" + ) it should "Check the sub workflow store when restarting" in { val swea = buildSWEA(startState = RestartableRunning) @@ -138,7 +152,16 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with AnyFlatSpecLike wi swea.setState(SubWorkflowCheckingStoreState) val subWorkflowUuid = WorkflowId.randomId() - swea ! SubWorkflowFound(SubWorkflowStoreEntry(Option(0), parentWorkflowId.toString, subKey.node.fullyQualifiedName, subKey.index.fromIndex, subKey.attempt, subWorkflowUuid.toString, None)) + swea ! SubWorkflowFound( + SubWorkflowStoreEntry(Option(0), + parentWorkflowId.toString, + subKey.node.fullyQualifiedName, + subKey.index.fromIndex, + subKey.attempt, + subWorkflowUuid.toString, + None + ) + ) parentProbe.expectMsg(RequestValueStore) eventually { @@ -210,8 +233,8 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with AnyFlatSpecLike wi val preparationFailedMessage: CallPreparationFailed = CallPreparationFailed(subWorkflowKey, throwable) swea ! preparationFailedMessage - serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { - case PutMetadataActionAndRespond(events, _, _) => swea ! MetadataWriteSuccess(events) + serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { case PutMetadataActionAndRespond(events, _, _) => + swea ! MetadataWriteSuccess(events) } parentProbe.expectMsg(SubWorkflowFailedResponse(subKey, Map.empty, throwable)) deathWatch.expectTerminated(swea, awaitTimeout) @@ -228,8 +251,8 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with AnyFlatSpecLike wi val outputs: CallOutputs = CallOutputs.empty val workflowSuccessfulMessage = WorkflowExecutionSucceededResponse(jobExecutionMap, Set.empty[WorkflowId], outputs) swea ! workflowSuccessfulMessage - serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { - case PutMetadataActionAndRespond(events, _, _) => swea ! MetadataWriteSuccess(events) + serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { case PutMetadataActionAndRespond(events, _, _) => + swea ! MetadataWriteSuccess(events) } parentProbe.expectMsg(SubWorkflowSucceededResponse(subKey, jobExecutionMap, Set.empty[WorkflowId], outputs)) deathWatch.expectTerminated(swea, awaitTimeout) @@ -246,8 +269,8 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with AnyFlatSpecLike wi val workflowFailedMessage = WorkflowExecutionFailedResponse(jobExecutionMap, expectedException) swea ! workflowFailedMessage - serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { - case PutMetadataActionAndRespond(events, _, _) => swea ! MetadataWriteSuccess(events) + serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { case PutMetadataActionAndRespond(events, _, _) => + swea ! MetadataWriteSuccess(events) } parentProbe.expectMsg(SubWorkflowFailedResponse(subKey, jobExecutionMap, expectedException)) deathWatch.expectTerminated(swea, awaitTimeout) @@ -265,33 +288,31 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with AnyFlatSpecLike wi val outputs: CallOutputs = CallOutputs.empty val workflowSuccessfulMessage = WorkflowExecutionSucceededResponse(jobExecutionMap, Set.empty[WorkflowId], outputs) swea ! workflowSuccessfulMessage - serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { - case PutMetadataActionAndRespond(events, _, _) => swea ! MetadataWriteFailure(expectedException, events) + serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { case PutMetadataActionAndRespond(events, _, _) => + swea ! MetadataWriteFailure(expectedException, events) } import ManyTimes.intWithTimes 10.times { - serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { - case PutMetadataActionAndRespond(events, _, _) => - events.size should be(1) - events.head.key.key should be("status") - events.head.value.get.value should be("Failed") - swea ! MetadataWriteFailure(expectedException, events) + serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { case PutMetadataActionAndRespond(events, _, _) => + events.size should be(1) + events.head.key.key should be("status") + events.head.value.get.value should be("Failed") + swea ! MetadataWriteFailure(expectedException, events) } // Check there are no messages going to the parent yet: parentProbe.expectNoMessage(10.millis) } // Now let's say eventually the write does somehow get through. - serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { - case PutMetadataActionAndRespond(events, _, _) => swea ! MetadataWriteSuccess(events) + serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { case PutMetadataActionAndRespond(events, _, _) => + swea ! MetadataWriteSuccess(events) } // The workflow is now considered failed since lots of metadata is probably lost: - parentProbe.expectMsgPF(awaitTimeout) { - case SubWorkflowFailedResponse(`subKey`, `jobExecutionMap`, reason) => - reason.getMessage should be("Sub workflow execution actor unable to write final state to metadata") - reason.getCause should be(expectedException) + parentProbe.expectMsgPF(awaitTimeout) { case SubWorkflowFailedResponse(`subKey`, `jobExecutionMap`, reason) => + reason.getMessage should be("Sub workflow execution actor unable to write final state to metadata") + reason.getCause should be(expectedException) } deathWatch.expectTerminated(swea, awaitTimeout) } @@ -305,8 +326,8 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with AnyFlatSpecLike wi val jobExecutionMap: JobExecutionMap = Map.empty val workflowAbortedMessage = WorkflowExecutionAbortedResponse(jobExecutionMap) swea ! workflowAbortedMessage - serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { - case PutMetadataActionAndRespond(events, _, _) => swea ! MetadataWriteSuccess(events) + serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { case PutMetadataActionAndRespond(events, _, _) => + swea ! MetadataWriteSuccess(events) } parentProbe.expectMsg(SubWorkflowAbortedResponse(subKey, jobExecutionMap)) deathWatch.expectTerminated(swea, awaitTimeout) @@ -316,7 +337,7 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with AnyFlatSpecLike wi val swea = buildSWEA() swea.setState( SubWorkflowRunningState, - SubWorkflowExecutionActorLiveData(Option(WorkflowId.randomId()), Option(subWorkflowActor.ref)), + SubWorkflowExecutionActorLiveData(Option(WorkflowId.randomId()), Option(subWorkflowActor.ref)) ) deathWatch watch swea @@ -325,8 +346,8 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with AnyFlatSpecLike wi swea ! EngineLifecycleActorAbortCommand subWorkflowActor.expectMsg(EngineLifecycleActorAbortCommand) subWorkflowActor.reply(WorkflowExecutionAbortedResponse(jobExecutionMap)) - serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { - case PutMetadataActionAndRespond(events, _, _) => swea ! MetadataWriteSuccess(events) + serviceRegistryProbe.fishForSpecificMessage(awaitTimeout) { case PutMetadataActionAndRespond(events, _, _) => + swea ! MetadataWriteSuccess(events) } parentProbe.expectMsg(SubWorkflowAbortedResponse(subKey, jobExecutionMap)) deathWatch.expectTerminated(swea, awaitTimeout) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActorSpec.scala index 61877de8a1a..094a41c78a5 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActorSpec.scala @@ -15,7 +15,12 @@ import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers import spray.json.{JsArray, JsField, JsObject, JsString, JsValue} -class CallCacheDiffActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with ImplicitSender with Eventually { +class CallCacheDiffActorSpec + extends TestKitSuite + with AnyFlatSpecLike + with Matchers + with ImplicitSender + with Eventually { behavior of "CallCacheDiffActor" @@ -50,23 +55,37 @@ class CallCacheDiffActorSpec extends TestKitSuite with AnyFlatSpecLike with Matc ) val eventsA = List( - MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "executionStatus"), MetadataValue("Done")), - MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:allowResultReuse"), MetadataValue(true)), - MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes:hash in only in A"), MetadataValue("hello from A")), - MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes:hash in A and B with same value"), MetadataValue("we are thinking the same thought")), - MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes:hash in A and B with different value"), MetadataValue("I'm the hash for A !")) + MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "executionStatus"), MetadataValue("Done")), + MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:allowResultReuse"), MetadataValue(true)), + MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes:hash in only in A"), + MetadataValue("hello from A") + ), + MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes:hash in A and B with same value"), + MetadataValue("we are thinking the same thought") + ), + MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes:hash in A and B with different value"), + MetadataValue("I'm the hash for A !") + ) ) - val workflowMetadataA: JsObject = MetadataBuilderActor.workflowMetadataResponse(workflowIdA, eventsA, includeCallsIfEmpty = false, Map.empty) + val workflowMetadataA: JsObject = + MetadataBuilderActor.workflowMetadataResponse(workflowIdA, eventsA, includeCallsIfEmpty = false, Map.empty) val responseForA = SuccessfulMetadataJsonResponse(MetadataService.GetMetadataAction(queryA), workflowMetadataA) val eventsB = List( MetadataEvent(MetadataKey(workflowIdB, metadataJobKeyB, "executionStatus"), MetadataValue("Failed")), MetadataEvent(MetadataKey(workflowIdB, metadataJobKeyB, "callCaching:allowResultReuse"), MetadataValue(false)), - MetadataEvent(MetadataKey(workflowIdB, metadataJobKeyB, "callCaching:hashes:hash in only in B"), MetadataValue("hello from B")), - MetadataEvent(MetadataKey(workflowIdB, metadataJobKeyB, "callCaching:hashes:hash in A and B with same value"), MetadataValue("we are thinking the same thought")), - MetadataEvent(MetadataKey(workflowIdB, metadataJobKeyB, "callCaching:hashes:hash in A and B with different value"), MetadataValue("I'm the hash for B !")) + MetadataEvent(MetadataKey(workflowIdB, metadataJobKeyB, "callCaching:hashes:hash in only in B"), + MetadataValue("hello from B") + ), + MetadataEvent(MetadataKey(workflowIdB, metadataJobKeyB, "callCaching:hashes:hash in A and B with same value"), + MetadataValue("we are thinking the same thought") + ), + MetadataEvent(MetadataKey(workflowIdB, metadataJobKeyB, "callCaching:hashes:hash in A and B with different value"), + MetadataValue("I'm the hash for B !") + ) ) - val workflowMetadataB: JsObject = MetadataBuilderActor.workflowMetadataResponse(workflowIdB, eventsB, includeCallsIfEmpty = false, Map.empty) + val workflowMetadataB: JsObject = + MetadataBuilderActor.workflowMetadataResponse(workflowIdB, eventsB, includeCallsIfEmpty = false, Map.empty) val responseForB = SuccessfulMetadataJsonResponse(MetadataService.GetMetadataAction(queryB), workflowMetadataB) it should "send correct queries to MetadataService when receiving a CallCacheDiffRequest" in { @@ -90,7 +109,12 @@ class CallCacheDiffActorSpec extends TestKitSuite with AnyFlatSpecLike with Matc actor ! responseForA eventually { - actor.stateData shouldBe CallCacheDiffWithRequest(queryA, queryB, Some(WorkflowMetadataJson(workflowMetadataA)), None, self) + actor.stateData shouldBe CallCacheDiffWithRequest(queryA, + queryB, + Some(WorkflowMetadataJson(workflowMetadataA)), + None, + self + ) actor.stateName shouldBe WaitingForMetadata } @@ -106,7 +130,12 @@ class CallCacheDiffActorSpec extends TestKitSuite with AnyFlatSpecLike with Matc actor ! responseForB eventually { - actor.stateData shouldBe CallCacheDiffWithRequest(queryA, queryB, None, Some(WorkflowMetadataJson(workflowMetadataB)), self) + actor.stateData shouldBe CallCacheDiffWithRequest(queryA, + queryB, + None, + Some(WorkflowMetadataJson(workflowMetadataB)), + self + ) actor.stateName shouldBe WaitingForMetadata } @@ -118,7 +147,9 @@ class CallCacheDiffActorSpec extends TestKitSuite with AnyFlatSpecLike with Matc val actor = TestFSMRef(new CallCacheDiffActor(mockServiceRegistryActor.ref)) watch(actor) - actor.setState(WaitingForMetadata, CallCacheDiffWithRequest(queryA, queryB, None, Some(WorkflowMetadataJson(workflowMetadataB)), self)) + actor.setState(WaitingForMetadata, + CallCacheDiffWithRequest(queryA, queryB, None, Some(WorkflowMetadataJson(workflowMetadataB)), self) + ) actor ! responseForA @@ -131,7 +162,9 @@ class CallCacheDiffActorSpec extends TestKitSuite with AnyFlatSpecLike with Matc val actor = TestFSMRef(new CallCacheDiffActor(mockServiceRegistryActor.ref)) watch(actor) - actor.setState(WaitingForMetadata, CallCacheDiffWithRequest(queryA, queryB, Some(WorkflowMetadataJson(workflowMetadataA)), None, self)) + actor.setState(WaitingForMetadata, + CallCacheDiffWithRequest(queryA, queryB, Some(WorkflowMetadataJson(workflowMetadataA)), None, self) + ) actor ! responseForB @@ -194,11 +227,11 @@ class CallCacheDiffActorSpec extends TestKitSuite with AnyFlatSpecLike with Matc expectMsgPF() { case r: SuccessfulCallCacheDiffResponse => withClue(s""" - |Expected: - |${correctCallCacheDiff.prettyPrint} - | - |Actual: - |${r.toJson.prettyPrint}""".stripMargin) { + |Expected: + |${correctCallCacheDiff.prettyPrint} + | + |Actual: + |${r.toJson.prettyPrint}""".stripMargin) { r.toJson should be(correctCallCacheDiff) } case other => fail(s"Expected SuccessfulCallCacheDiffResponse but got $other") @@ -219,19 +252,30 @@ class CallCacheDiffActorSpec extends TestKitSuite with AnyFlatSpecLike with Matc val eventsAAttempt1 = List( MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "executionStatus"), MetadataValue("Failed")), MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:allowResultReuse"), MetadataValue(false)), - MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes:hash in only in A"), MetadataValue("ouch!")), - MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes:hash in A and B with same value"), MetadataValue("ouch!")), - MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes:hash in A and B with different value"), MetadataValue("ouch!")) + MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes:hash in only in A"), + MetadataValue("ouch!") + ), + MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes:hash in A and B with same value"), + MetadataValue("ouch!") + ), + MetadataEvent( + MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes:hash in A and B with different value"), + MetadataValue("ouch!") + ) ) // And update the old "eventsA" to represent attempt 2: - val eventsAAttempt2 = eventsA.map(event => event.copy(key = event.key.copy(jobKey = event.key.jobKey.map(_.copy(attempt = 2))))) + val eventsAAttempt2 = + eventsA.map(event => event.copy(key = event.key.copy(jobKey = event.key.jobKey.map(_.copy(attempt = 2))))) val modifiedEventsA = eventsAAttempt1 ++ eventsAAttempt2 - val workflowMetadataA: JsObject = MetadataBuilderActor.workflowMetadataResponse(workflowIdA, modifiedEventsA, includeCallsIfEmpty = false, Map.empty) + val workflowMetadataA: JsObject = MetadataBuilderActor.workflowMetadataResponse(workflowIdA, + modifiedEventsA, + includeCallsIfEmpty = false, + Map.empty + ) val responseForA = SuccessfulMetadataJsonResponse(MetadataService.GetMetadataAction(queryA), workflowMetadataA) - actor ! responseForB actor ! responseForA @@ -267,9 +311,8 @@ class CallCacheDiffActorSpec extends TestKitSuite with AnyFlatSpecLike with Matc actor ! responseA - expectMsgPF(1 second) { - case FailedCallCacheDiffResponse(e: Throwable) => - e.getMessage shouldBe "Query lookup failed - but it's ok ! this is a test !" + expectMsgPF(1 second) { case FailedCallCacheDiffResponse(e: Throwable) => + e.getMessage shouldBe "Query lookup failed - but it's ok ! this is a test !" } expectTerminated(actor) @@ -278,18 +321,20 @@ class CallCacheDiffActorSpec extends TestKitSuite with AnyFlatSpecLike with Matc it should "respond with an appropriate error if calls' hashes are missing" in { testExpectedErrorForModifiedMetadata( metadataFilter = _.key.key.contains("hashes"), - error = s"""Failed to calculate diff for call A and call B: - |Failed to extract relevant metadata for call A (971652a6-139c-4ef3-96b5-aeb611a40dbf / callFqnA:1) (reason 1 of 1): No 'hashes' field found - |Failed to extract relevant metadata for call B (bb85b3ec-e179-4f12-b90f-5191216da598 / callFqnB:-1) (reason 1 of 1): No 'hashes' field found""".stripMargin + error = + s"""Failed to calculate diff for call A and call B: + |Failed to extract relevant metadata for call A (971652a6-139c-4ef3-96b5-aeb611a40dbf / callFqnA:1) (reason 1 of 1): No 'hashes' field found + |Failed to extract relevant metadata for call B (bb85b3ec-e179-4f12-b90f-5191216da598 / callFqnB:-1) (reason 1 of 1): No 'hashes' field found""".stripMargin ) } it should "respond with an appropriate error if both calls are missing" in { testExpectedErrorForModifiedMetadata( metadataFilter = _.key.jobKey.nonEmpty, - error = s"""Failed to calculate diff for call A and call B: - |Failed to extract relevant metadata for call A (971652a6-139c-4ef3-96b5-aeb611a40dbf / callFqnA:1) (reason 1 of 1): No 'calls' field found - |Failed to extract relevant metadata for call B (bb85b3ec-e179-4f12-b90f-5191216da598 / callFqnB:-1) (reason 1 of 1): No 'calls' field found""".stripMargin + error = + s"""Failed to calculate diff for call A and call B: + |Failed to extract relevant metadata for call A (971652a6-139c-4ef3-96b5-aeb611a40dbf / callFqnA:1) (reason 1 of 1): No 'calls' field found + |Failed to extract relevant metadata for call B (bb85b3ec-e179-4f12-b90f-5191216da598 / callFqnB:-1) (reason 1 of 1): No 'calls' field found""".stripMargin ) } @@ -301,7 +346,9 @@ class CallCacheDiffActorSpec extends TestKitSuite with AnyFlatSpecLike with Matc def str(s: String): JsString = JsString(s) it should "handle nested JsObjects if field names collide" in { - val objectToParse = obj("hashes" -> obj("subObj" -> obj("field" -> str("fieldValue1"), "subObj" -> obj("field" -> str("fieldValue2"))))) + val objectToParse = obj( + "hashes" -> obj("subObj" -> obj("field" -> str("fieldValue1"), "subObj" -> obj("field" -> str("fieldValue2")))) + ) val res = CallCacheDiffActor.extractHashes(objectToParse).toOption res should be(defined) @@ -324,9 +371,16 @@ class CallCacheDiffActorSpec extends TestKitSuite with AnyFlatSpecLike with Matc val actor = TestFSMRef(new CallCacheDiffActor(mockServiceRegistryActor.ref)) watch(actor) - def getModifiedResponse(workflowId: WorkflowId, query: MetadataQuery, events: Seq[MetadataEvent]): SuccessfulMetadataJsonResponse = { + def getModifiedResponse(workflowId: WorkflowId, + query: MetadataQuery, + events: Seq[MetadataEvent] + ): SuccessfulMetadataJsonResponse = { val modifiedEvents = events.filterNot(metadataFilter) // filters out any "call" level metadata - val modifiedWorkflowMetadata = MetadataBuilderActor.workflowMetadataResponse(workflowId, modifiedEvents, includeCallsIfEmpty = false, Map.empty) + val modifiedWorkflowMetadata = MetadataBuilderActor.workflowMetadataResponse(workflowId, + modifiedEvents, + includeCallsIfEmpty = false, + Map.empty + ) SuccessfulMetadataJsonResponse(MetadataService.GetMetadataAction(query), modifiedWorkflowMetadata) } @@ -335,8 +389,8 @@ class CallCacheDiffActorSpec extends TestKitSuite with AnyFlatSpecLike with Matc actor ! getModifiedResponse(workflowIdA, queryA, eventsA) actor ! getModifiedResponse(workflowIdB, queryB, eventsB) - expectMsgPF(1 second) { - case FailedCallCacheDiffResponse(e) => e.getMessage shouldBe error + expectMsgPF(1 second) { case FailedCallCacheDiffResponse(e) => + e.getMessage shouldBe error } expectTerminated(actor) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorDataSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorDataSpec.scala index 4cb279602c2..13b1a2205de 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorDataSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorDataSpec.scala @@ -5,13 +5,24 @@ import cromwell.backend._ import cromwell.backend.standard.callcaching.StandardFileHashingActor.SingleFileHashRequest import cromwell.core.TestKitSuite import cromwell.core.callcaching._ -import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{CallCacheHashingJobActorData, CompleteFileHashingResult, NoFileHashesResult, PartialFileHashingResult} +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{ + CallCacheHashingJobActorData, + CompleteFileHashingResult, + NoFileHashesResult, + PartialFileHashingResult +} import org.scalatest.concurrent.Eventually import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks -class CallCacheHashingJobActorDataSpec extends TestKitSuite with AnyFlatSpecLike with BackendSpec with Matchers with Eventually with TableDrivenPropertyChecks { +class CallCacheHashingJobActorDataSpec + extends TestKitSuite + with AnyFlatSpecLike + with BackendSpec + with Matchers + with Eventually + with TableDrivenPropertyChecks { behavior of "CallCacheReadingJobActorData" private val fileHash1 = HashResult(HashKey("key"), HashValue("value")) @@ -20,71 +31,107 @@ class CallCacheHashingJobActorDataSpec extends TestKitSuite with AnyFlatSpecLike private val fileHashRequest1 = SingleFileHashRequest(null, fileHash1.hashKey, null, null) private val fileHashRequest2 = SingleFileHashRequest(null, fileHash2.hashKey, null, null) private val fileHashRequest3 = SingleFileHashRequest(null, fileHash3.hashKey, null, null) - + private val testCases = Table( ("dataBefore", "dataAfter", "result"), // No fileHashRequestsRemaining ( CallCacheHashingJobActorData( - List.empty, List.empty, None, 50 + List.empty, + List.empty, + None, + 50 ), CallCacheHashingJobActorData( - List.empty, List(fileHash1), None, 50 + List.empty, + List(fileHash1), + None, + 50 ), Option(NoFileHashesResult) ), // Last fileHashRequestsRemaining ( CallCacheHashingJobActorData( - List(List(fileHashRequest1)), List.empty, None, 50 + List(List(fileHashRequest1)), + List.empty, + None, + 50 ), CallCacheHashingJobActorData( - List.empty, List(fileHash1), None, 50 + List.empty, + List(fileHash1), + None, + 50 ), Option(CompleteFileHashingResult(Set(fileHash1), "6A02F950958AEDA3DBBF83FBB306A030")) ), // Last batch and not last value ( CallCacheHashingJobActorData( - List(List(fileHashRequest1, fileHashRequest2)), List.empty, None, 50 + List(List(fileHashRequest1, fileHashRequest2)), + List.empty, + None, + 50 ), CallCacheHashingJobActorData( - List(List(fileHashRequest2)), List(fileHash1), None, 50 + List(List(fileHashRequest2)), + List(fileHash1), + None, + 50 ), None ), // Not last batch but last value of this batch ( CallCacheHashingJobActorData( - List(List(fileHashRequest1), List(fileHashRequest2)), List.empty, None, 50 + List(List(fileHashRequest1), List(fileHashRequest2)), + List.empty, + None, + 50 ), CallCacheHashingJobActorData( - List(List(fileHashRequest2)), List(fileHash1), None, 50 + List(List(fileHashRequest2)), + List(fileHash1), + None, + 50 ), Option(PartialFileHashingResult(NonEmptyList.of(fileHash1))) ), // Not last batch and not last value of this batch ( CallCacheHashingJobActorData( - List(List(fileHashRequest1, fileHashRequest2), List(fileHashRequest3)), List.empty, None, 50 + List(List(fileHashRequest1, fileHashRequest2), List(fileHashRequest3)), + List.empty, + None, + 50 ), CallCacheHashingJobActorData( - List(List(fileHashRequest2), List(fileHashRequest3)), List(fileHash1), None, 50 + List(List(fileHashRequest2), List(fileHashRequest3)), + List(fileHash1), + None, + 50 ), None ), // Makes sure new hash is added at the front of the list ( CallCacheHashingJobActorData( - List(List(fileHashRequest1, fileHashRequest2), List(fileHashRequest3)), List(fileHash2), None, 50 + List(List(fileHashRequest1, fileHashRequest2), List(fileHashRequest3)), + List(fileHash2), + None, + 50 ), CallCacheHashingJobActorData( - List(List(fileHashRequest2), List(fileHashRequest3)), List(fileHash1, fileHash2), None, 50 + List(List(fileHashRequest2), List(fileHashRequest3)), + List(fileHash1, fileHash2), + None, + 50 ), None ) ) - + it should "process new file hashes" in { forAll(testCases) { case (oldData, newData, result) => oldData.withFileHash(fileHash1) shouldBe (newData -> result) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorSpec.scala index 602d04c145d..b70722c4200 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorSpec.scala @@ -10,7 +10,17 @@ import cromwell.backend._ import cromwell.backend.standard.callcaching.StandardFileHashingActor.{FileHashResponse, SingleFileHashRequest} import cromwell.core.TestKitSuite import cromwell.core.callcaching.{HashingFailedMessage, _} -import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{CCHJAFileHashResponse, CallCacheHashingJobActorData, CompleteFileHashingResult, HashingFiles, InitialHashingResult, NextBatchOfFileHashesRequest, NoFileHashesResult, PartialFileHashingResult, WaitingForHashFileRequest} +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{ + CallCacheHashingJobActorData, + CCHJAFileHashResponse, + CompleteFileHashingResult, + HashingFiles, + InitialHashingResult, + NextBatchOfFileHashesRequest, + NoFileHashesResult, + PartialFileHashingResult, + WaitingForHashFileRequest +} import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CacheMiss import cromwell.util.WomMocks import org.scalatest.Assertion @@ -24,12 +34,20 @@ import wom.values.{WomInteger, WomSingleFile, WomString, WomValue} import scala.util.control.NoStackTrace -class CallCacheHashingJobActorSpec extends TestKitSuite with AnyFlatSpecLike with BackendSpec with Matchers - with Eventually with TableDrivenPropertyChecks with MockSugar { +class CallCacheHashingJobActorSpec + extends TestKitSuite + with AnyFlatSpecLike + with BackendSpec + with Matchers + with Eventually + with TableDrivenPropertyChecks + with MockSugar { behavior of "CallCacheReadingJobActor" def templateJobDescriptor(inputs: Map[LocallyQualifiedName, WomValue] = Map.empty): BackendJobDescriptor = { - val task = WomMocks.mockTaskDefinition("task").copy(commandTemplateBuilder = Function.const(List(StringCommandPart("Do the stuff... now!!")).validNel)) + val task = WomMocks + .mockTaskDefinition("task") + .copy(commandTemplateBuilder = Function.const(List(StringCommandPart("Do the stuff... now!!")).validNel)) val call = WomMocks.mockTaskCall(WomIdentifier("call"), definition = task) val workflowDescriptor = mock[BackendWorkflowDescriptor] val runtimeAttributes = Map( @@ -38,24 +56,34 @@ class CallCacheHashingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit "continueOnReturnCode" -> WomInteger(0), "docker" -> WomString("ubuntu:latest") ) - val jobDescriptor = BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, None, 1), runtimeAttributes, fqnWdlMapToDeclarationMap(inputs), NoDocker, None, Map.empty) + val jobDescriptor = BackendJobDescriptor(workflowDescriptor, + BackendJobDescriptorKey(call, None, 1), + runtimeAttributes, + fqnWdlMapToDeclarationMap(inputs), + NoDocker, + None, + Map.empty + ) jobDescriptor } it should "die immediately if created without cache read actor and write to cache turned off" in { val parent = TestProbe() - val testActor = TestFSMRef(new CallCacheHashingJobActor( - templateJobDescriptor(), - None, - None, - Set.empty, - "backedName", - Props.empty, - DockerWithHash("ubuntu@sha256:blablablba"), - CallCachingActivity(readWriteMode = ReadCache), - callCachePathPrefixes = None, - fileHashBatchSize = 100 - ), parent.ref) + val testActor = TestFSMRef( + new CallCacheHashingJobActor( + templateJobDescriptor(), + None, + None, + Set.empty, + "backedName", + Props.empty, + DockerWithHash("ubuntu@sha256:blablablba"), + CallCachingActivity(readWriteMode = ReadCache), + callCachePathPrefixes = None, + fileHashBatchSize = 100 + ), + parent.ref + ) watch(testActor) expectTerminated(testActor) parent.expectMsgClass(classOf[InitialHashingResult]) @@ -80,18 +108,21 @@ class CallCacheHashingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit ) val callCacheRead = TestProbe() val jobDescriptor: BackendJobDescriptor = templateJobDescriptor(inputs) - val actorUnderTest = TestFSMRef(new CallCacheHashingJobActor( - jobDescriptor, - Option(callCacheRead.ref), - None, - runtimeAttributeDefinitions, - "backedName", - Props.empty, - DockerWithHash("ubuntu@sha256:blablablba"), - CallCachingActivity(readWriteMode = ReadAndWriteCache), - callCachePathPrefixes = None, - fileHashBatchSize = 100 - ), parent.ref) + val actorUnderTest = TestFSMRef( + new CallCacheHashingJobActor( + jobDescriptor, + Option(callCacheRead.ref), + None, + runtimeAttributeDefinitions, + "backedName", + Props.empty, + DockerWithHash("ubuntu@sha256:blablablba"), + CallCachingActivity(readWriteMode = ReadAndWriteCache), + callCachePathPrefixes = None, + fileHashBatchSize = 100 + ), + parent.ref + ) val expectedInitialHashes = Set( // md5 of Do the stuff... now @@ -104,7 +135,9 @@ class CallCacheHashingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit HashResult(HashKey("output count"), HashValue("CFCD208495D565EF66E7DFF9F98764DA")), HashResult(HashKey("runtime attribute", "failOnStderr"), HashValue("N/A")), // md5 of 1 - HashResult(HashKey(checkForHitOrMiss = false, "runtime attribute", "cpu"), HashValue("C4CA4238A0B923820DCC509A6F75849B")), + HashResult(HashKey(checkForHitOrMiss = false, "runtime attribute", "cpu"), + HashValue("C4CA4238A0B923820DCC509A6F75849B") + ), // md5 of 0 HashResult(HashKey("runtime attribute", "continueOnReturnCode"), HashValue("CFCD208495D565EF66E7DFF9F98764DA")), // md5 of "hello" (with quotes) @@ -128,25 +161,32 @@ class CallCacheHashingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit testFileHashingActor: ActorRef, parent: ActorRef, writeToCache: Boolean = true, - addFileHashMockResult: Option[(CallCacheHashingJobActorData, Option[CCHJAFileHashResponse])] = None): TestFSMRef[CallCacheHashingJobActor.CallCacheHashingJobActorState, CallCacheHashingJobActorData, CallCacheHashingJobActor] = { - TestFSMRef(new CallCacheHashingJobActor( - templateJobDescriptor(), - callCacheReader, - None, - Set.empty, - "backend", - Props.empty, - DockerWithHash("ubuntu@256:blablabla"), - CallCachingActivity(readWriteMode = if (writeToCache) ReadAndWriteCache else ReadCache), - callCachePathPrefixes = None, - fileHashBatchSize = 100 - ) { - override def makeFileHashingActor(): ActorRef = testFileHashingActor - override def addFileHash(hashResult: HashResult, data: CallCacheHashingJobActorData): (CallCacheHashingJobActorData, Option[CCHJAFileHashResponse]) = { - addFileHashMockResult.getOrElse(super.addFileHash(hashResult, data)) - } - }, parent) - } + addFileHashMockResult: Option[(CallCacheHashingJobActorData, Option[CCHJAFileHashResponse])] = None + ): TestFSMRef[CallCacheHashingJobActor.CallCacheHashingJobActorState, + CallCacheHashingJobActorData, + CallCacheHashingJobActor + ] = + TestFSMRef( + new CallCacheHashingJobActor( + templateJobDescriptor(), + callCacheReader, + None, + Set.empty, + "backend", + Props.empty, + DockerWithHash("ubuntu@256:blablabla"), + CallCachingActivity(readWriteMode = if (writeToCache) ReadAndWriteCache else ReadCache), + callCachePathPrefixes = None, + fileHashBatchSize = 100 + ) { + override def makeFileHashingActor(): ActorRef = testFileHashingActor + override def addFileHash(hashResult: HashResult, + data: CallCacheHashingJobActorData + ): (CallCacheHashingJobActorData, Option[CCHJAFileHashResponse]) = + addFileHashMockResult.getOrElse(super.addFileHash(hashResult, data)) + }, + parent + ) it should "send hash file requests when receiving a NextBatchOfFileHashesRequest" in { val callCacheReadProbe = TestProbe() @@ -198,7 +238,12 @@ class CallCacheHashingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit // still gives a CCReader when instantiating the actor, but not in the data (above) // This ensures the check is done with the data and not the actor attribute, as the data will change if the ccreader dies but the actor attribute // will stay Some(...) - val cchja = makeCCHJA(Option(TestProbe().ref), fileHashingActor.ref, TestProbe().ref, writeToCache = true, Option(newData -> Option(result))) + val cchja = makeCCHJA(Option(TestProbe().ref), + fileHashingActor.ref, + TestProbe().ref, + writeToCache = true, + Option(newData -> Option(result)) + ) watch(cchja) cchja.setState(HashingFiles) @@ -211,19 +256,24 @@ class CallCacheHashingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit List( ("send itself a NextBatchOfFileHashesRequest when a batch is complete and there is no CC Reader", None), - ("send itself a NextBatchOfFileHashesRequest when a batch is complete and there is a CC Reader", Option(TestProbe().ref)) - ) foreach { - case (description, ccReader) => - it should description in selfSendNextBatchRequest(ccReader) + ("send itself a NextBatchOfFileHashesRequest when a batch is complete and there is a CC Reader", + Option(TestProbe().ref) + ) + ) foreach { case (description, ccReader) => + it should description in selfSendNextBatchRequest(ccReader) } it should "send FinalFileHashingResult to parent and CCReader and die" in { val parent = TestProbe() val callCacheReadProbe = TestProbe() - List(CompleteFileHashingResult(Set(mock[HashResult]), "AggregatedFileHash"), NoFileHashesResult) foreach - { result => + List(CompleteFileHashingResult(Set(mock[HashResult]), "AggregatedFileHash"), NoFileHashesResult) foreach { result => val newData = CallCacheHashingJobActorData(List.empty, List.empty, Option(callCacheReadProbe.ref), 50) - val cchja = makeCCHJA(Option(callCacheReadProbe.ref), TestProbe().ref, parent.ref, writeToCache = true, Option(newData -> Option(result))) + val cchja = makeCCHJA(Option(callCacheReadProbe.ref), + TestProbe().ref, + parent.ref, + writeToCache = true, + Option(newData -> Option(result)) + ) parent.expectMsgClass(classOf[InitialHashingResult]) callCacheReadProbe.expectMsgClass(classOf[InitialHashingResult]) @@ -242,8 +292,14 @@ class CallCacheHashingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit it should "wait for next file hash if the batch is not complete yet" in { val callCacheReadProbe = TestProbe() val parent = TestProbe() - val newData: CallCacheHashingJobActorData = CallCacheHashingJobActorData(List.empty, List.empty, Option(callCacheReadProbe.ref), 50) - val cchja = makeCCHJA(Option(callCacheReadProbe.ref), TestProbe().ref, parent.ref, writeToCache = true, Option(newData -> None)) + val newData: CallCacheHashingJobActorData = + CallCacheHashingJobActorData(List.empty, List.empty, Option(callCacheReadProbe.ref), 50) + val cchja = makeCCHJA(Option(callCacheReadProbe.ref), + TestProbe().ref, + parent.ref, + writeToCache = true, + Option(newData -> None) + ) parent.expectMsgClass(classOf[InitialHashingResult]) callCacheReadProbe.expectMsgClass(classOf[InitialHashingResult]) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActorSpec.scala index a39597fa347..78106628ca3 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActorSpec.scala @@ -2,10 +2,19 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.testkit.{TestFSMRef, TestProbe} import cromwell.core.TestKitSuite -import cromwell.core.callcaching.{HashKey, HashResult, HashValue, HashingFailedMessage} -import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{CompleteFileHashingResult, InitialHashingResult, NextBatchOfFileHashesRequest, NoFileHashesResult} +import cromwell.core.callcaching.{HashingFailedMessage, HashKey, HashResult, HashValue} +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{ + CompleteFileHashingResult, + InitialHashingResult, + NextBatchOfFileHashesRequest, + NoFileHashesResult +} import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor._ -import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadingJobActor.{CCRJAWithData, WaitingForCacheHitOrMiss, _} +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadingJobActor.{ + CCRJAWithData, + WaitingForCacheHitOrMiss, + _ +} import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, HashError} import cromwell.services.CallCaching.CallCachingEntryId import org.scalatest.concurrent.Eventually @@ -35,7 +44,9 @@ class CallCacheReadingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit val callCacheReadProbe = TestProbe() val callCacheHashingActor = TestProbe() val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None)) - actorUnderTest.setState(WaitingForHashCheck, CCRJAWithData(callCacheHashingActor.ref, "AggregatedInitialHash", None, Set.empty)) + actorUnderTest.setState(WaitingForHashCheck, + CCRJAWithData(callCacheHashingActor.ref, "AggregatedInitialHash", None, Set.empty) + ) callCacheReadProbe.send(actorUnderTest, HasMatchingEntries) callCacheHashingActor.expectMsg(NextBatchOfFileHashesRequest) @@ -49,10 +60,13 @@ class CallCacheReadingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit val callCacheHashingActor = TestProbe() val parent = TestProbe() - val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), parent.ref) + val actorUnderTest = + TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), parent.ref) parent.watch(actorUnderTest) - actorUnderTest.setState(WaitingForHashCheck, CCRJAWithData(callCacheHashingActor.ref, "AggregatedInitialHash", None, Set.empty)) + actorUnderTest.setState(WaitingForHashCheck, + CCRJAWithData(callCacheHashingActor.ref, "AggregatedInitialHash", None, Set.empty) + ) callCacheReadProbe.send(actorUnderTest, NoMatchingEntries) parent.expectMsg(CacheMiss) @@ -63,19 +77,31 @@ class CallCacheReadingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit val callCacheReadProbe = TestProbe() val callCacheHashingActor = TestProbe() - val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), TestProbe().ref) + val actorUnderTest = + TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), TestProbe().ref) val aggregatedInitialHash: String = "AggregatedInitialHash" val aggregatedFileHash: String = "AggregatedFileHash" - actorUnderTest.setState(WaitingForFileHashes, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, Set.empty)) + actorUnderTest.setState(WaitingForFileHashes, + CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, Set.empty) + ) val fileHashes = Set(HashResult(HashKey("f1"), HashValue("h1")), HashResult(HashKey("f2"), HashValue("h2"))) callCacheHashingActor.send(actorUnderTest, CompleteFileHashingResult(fileHashes, aggregatedFileHash)) - callCacheReadProbe.expectMsg(CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, aggregatedFileHash), Set.empty, prefixesHint = None)) + callCacheReadProbe.expectMsg( + CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, aggregatedFileHash), + Set.empty, + prefixesHint = None + ) + ) eventually { actorUnderTest.stateName shouldBe WaitingForCacheHitOrMiss - actorUnderTest.stateData shouldBe CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, Some(aggregatedFileHash), Set.empty) + actorUnderTest.stateData shouldBe CCRJAWithData(callCacheHashingActor.ref, + aggregatedInitialHash, + Some(aggregatedFileHash), + Set.empty + ) } } @@ -83,13 +109,18 @@ class CallCacheReadingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit val callCacheReadProbe = TestProbe() val callCacheHashingActor = TestProbe() - val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), TestProbe().ref) + val actorUnderTest = + TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), TestProbe().ref) val aggregatedInitialHash: String = "AggregatedInitialHash" - actorUnderTest.setState(WaitingForFileHashes, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, Set.empty)) + actorUnderTest.setState(WaitingForFileHashes, + CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, Set.empty) + ) callCacheHashingActor.send(actorUnderTest, NoFileHashesResult) - callCacheReadProbe.expectMsg(CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, None), Set.empty, prefixesHint = None)) + callCacheReadProbe.expectMsg( + CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, None), Set.empty, prefixesHint = None) + ) eventually { actorUnderTest.stateName shouldBe WaitingForCacheHitOrMiss @@ -102,10 +133,13 @@ class CallCacheReadingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit val callCacheHashingActor = TestProbe() val parent = TestProbe() - val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), parent.ref) + val actorUnderTest = + TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), parent.ref) val aggregatedInitialHash: String = "AggregatedInitialHash" - actorUnderTest.setState(WaitingForCacheHitOrMiss, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, Set.empty)) + actorUnderTest.setState(WaitingForCacheHitOrMiss, + CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, Set.empty) + ) val id: CallCachingEntryId = CallCachingEntryId(8) callCacheReadProbe.send(actorUnderTest, CacheLookupNextHit(id)) @@ -122,11 +156,14 @@ class CallCacheReadingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit val callCacheHashingActor = TestProbe() val parent = TestProbe() - val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), parent.ref) + val actorUnderTest = + TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), parent.ref) parent.watch(actorUnderTest) val aggregatedInitialHash: String = "AggregatedInitialHash" - actorUnderTest.setState(WaitingForCacheHitOrMiss, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, Set.empty)) + actorUnderTest.setState(WaitingForCacheHitOrMiss, + CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, Set.empty) + ) callCacheReadProbe.send(actorUnderTest, CacheLookupNoHit) parent.expectMsg(CacheMiss) @@ -138,14 +175,19 @@ class CallCacheReadingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit val callCacheReadProbe = TestProbe() val callCacheHashingActor = TestProbe() - val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), TestProbe().ref) + val actorUnderTest = + TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), TestProbe().ref) val aggregatedInitialHash: String = "AggregatedInitialHash" val seenCaches: Set[CallCachingEntryId] = Set(CallCachingEntryId(0)) - actorUnderTest.setState(WaitingForCacheHitOrMiss, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, seenCaches)) + actorUnderTest.setState(WaitingForCacheHitOrMiss, + CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, seenCaches) + ) actorUnderTest ! NextHit - callCacheReadProbe.expectMsg(CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, None), seenCaches, prefixesHint = None)) + callCacheReadProbe.expectMsg( + CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, None), seenCaches, prefixesHint = None) + ) actorUnderTest.stateName shouldBe WaitingForCacheHitOrMiss } @@ -154,15 +196,24 @@ class CallCacheReadingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit val callCacheReadProbe = TestProbe() val callCacheHashingActor = TestProbe() - val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), TestProbe().ref) + val actorUnderTest = + TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), TestProbe().ref) val aggregatedInitialHash: String = "AggregatedInitialHash" val aggregatedFileHash: String = "AggregatedFileHash" val seenCaches: Set[CallCachingEntryId] = Set(CallCachingEntryId(0)) - actorUnderTest.setState(WaitingForCacheHitOrMiss, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, Option(aggregatedFileHash), seenCaches)) + actorUnderTest.setState( + WaitingForCacheHitOrMiss, + CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, Option(aggregatedFileHash), seenCaches) + ) actorUnderTest ! NextHit - callCacheReadProbe.expectMsg(CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, Option(aggregatedFileHash)), seenCaches, prefixesHint = None)) + callCacheReadProbe.expectMsg( + CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, Option(aggregatedFileHash)), + seenCaches, + prefixesHint = None + ) + ) actorUnderTest.stateName shouldBe WaitingForCacheHitOrMiss } @@ -172,11 +223,14 @@ class CallCacheReadingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit val callCacheHashingActor = TestProbe() val parent = TestProbe() - val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), parent.ref) + val actorUnderTest = + TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), parent.ref) parent.watch(actorUnderTest) val aggregatedInitialHash: String = "AggregatedInitialHash" - actorUnderTest.setState(WaitingForCacheHitOrMiss, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, Set.empty)) + actorUnderTest.setState(WaitingForCacheHitOrMiss, + CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, Set.empty) + ) callCacheHashingActor.send(actorUnderTest, HashingFailedMessage("file", new Exception("Hashing failed"))) parent.expectMsg(CacheMiss) @@ -189,11 +243,14 @@ class CallCacheReadingJobActorSpec extends TestKitSuite with AnyFlatSpecLike wit val callCacheHashingActor = TestProbe() val parent = TestProbe() - val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), parent.ref) + val actorUnderTest = + TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref, prefixesHint = None), parent.ref) parent.watch(actorUnderTest) val aggregatedInitialHash: String = "AggregatedInitialHash" - actorUnderTest.setState(WaitingForCacheHitOrMiss, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, Set.empty)) + actorUnderTest.setState(WaitingForCacheHitOrMiss, + CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, Set.empty) + ) val reason: Exception = new Exception("Lookup failed") callCacheHashingActor.send(actorUnderTest, CacheResultLookupFailure(reason)) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala index 279ee80681c..d3dcf58e3d3 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala @@ -18,7 +18,11 @@ import org.scalatest.time.{Millis, Seconds, Span} import scala.concurrent.ExecutionContext class CallCachingSlickDatabaseSpec - extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalaFutures with BeforeAndAfterAll + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalaFutures + with BeforeAndAfterAll with TableDrivenPropertyChecks { implicit val ec: ExecutionContext = ExecutionContext.global @@ -30,7 +34,7 @@ class CallCachingSlickDatabaseSpec ("description", "prefixOption"), ("without prefixes", None), ("with some prefixes", Option(List("prefix1", "prefix2", "prefix3", "prefix4"))), - ("with thousands of prefixes", Option((1 to 10000).map("prefix" + _).toList)), + ("with thousands of prefixes", Option((1 to 10000).map("prefix" + _).toList)) ) DatabaseSystem.All foreach { databaseSystem => @@ -38,14 +42,14 @@ class CallCachingSlickDatabaseSpec val containerOpt: Option[Container] = DatabaseTestKit.getDatabaseTestContainer(databaseSystem) - lazy val dataAccess = DatabaseTestKit.initializeDatabaseByContainerOptTypeAndSystem(containerOpt, EngineDatabaseType, databaseSystem) + lazy val dataAccess = + DatabaseTestKit.initializeDatabaseByContainerOptTypeAndSystem(containerOpt, EngineDatabaseType, databaseSystem) it should "start container if required" taggedAs DbmsTest in { - containerOpt.foreach { _.start } + containerOpt.foreach(_.start) } forAll(allowResultReuseTests) { (description, prefixOption) => - val idA = WorkflowId.randomId().toString val callA = "AwesomeWorkflow.GoodJob" val callCachingEntryA = CallCachingEntry( @@ -85,14 +89,15 @@ class CallCachingSlickDatabaseSpec it should s"honor allowResultReuse $description" taggedAs DbmsTest in { (for { _ <- dataAccess.addCallCaching(Seq( - CallCachingJoin( - callCachingEntryA, - callCachingHashEntriesA, - aggregation, - callCachingSimpletonsA, callCachingDetritusesA - ) - ), - 100 + CallCachingJoin( + callCachingEntryA, + callCachingHashEntriesA, + aggregation, + callCachingSimpletonsA, + callCachingDetritusesA + ) + ), + 100 ) hasBaseAggregation <- dataAccess.hasMatchingCallCachingEntriesForBaseAggregation( "BASE_AGGREGATION", @@ -115,20 +120,16 @@ class CallCachingSlickDatabaseSpec _ = join shouldBe defined getJoin = join.get // We can't compare directly because the ones out from the DB have IDs filled in, so just compare the relevant values - _ = getJoin - .callCachingHashEntries + _ = getJoin.callCachingHashEntries .map(e => (e.hashKey, e.hashValue)) should contain theSameElementsAs callCachingHashEntriesA.map(e => (e.hashKey, e.hashValue)) - _ = getJoin - .callCachingSimpletonEntries + _ = getJoin.callCachingSimpletonEntries .map(e => (e.simpletonKey, e.simpletonValue.map(_.toRawString))) should contain theSameElementsAs callCachingSimpletonsA.map(e => (e.simpletonKey, e.simpletonValue.map(_.toRawString))) - _ = getJoin - .callCachingAggregationEntry + _ = getJoin.callCachingAggregationEntry .map(e => (e.baseAggregation, e.inputFilesAggregation)) shouldBe aggregation.map(e => (e.baseAggregation, e.inputFilesAggregation)) - _ = getJoin - .callCachingDetritusEntries + _ = getJoin.callCachingDetritusEntries .map(e => (e.detritusKey, e.detritusValue.map(_.toRawString))) should contain theSameElementsAs callCachingDetritusesA.map(e => (e.detritusKey, e.detritusValue.map(_.toRawString))) } yield ()).futureValue @@ -141,7 +142,7 @@ class CallCachingSlickDatabaseSpec } it should "stop container if required" taggedAs DbmsTest in { - containerOpt.foreach { _.stop() } + containerOpt.foreach(_.stop()) } } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala index 1d78da0e37b..5e3bff33671 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala @@ -6,7 +6,11 @@ import cats.syntax.validated._ import cromwell.backend._ import cromwell.core._ import cromwell.core.callcaching._ -import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{CompleteFileHashingResult, InitialHashingResult, NoFileHashesResult} +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{ + CompleteFileHashingResult, + InitialHashingResult, + NoFileHashesResult +} import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadingJobActor.NextHit import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor._ import cromwell.services.metadata.MetadataService.PutMetadataAction @@ -21,24 +25,42 @@ import wom.graph.WomIdentifier import wom.values.WomValue import common.mock.MockSugar -class EngineJobHashingActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with BackendSpec - with MockSugar with TableDrivenPropertyChecks with Eventually { +class EngineJobHashingActorSpec + extends TestKitSuite + with AnyFlatSpecLike + with Matchers + with BackendSpec + with MockSugar + with TableDrivenPropertyChecks + with Eventually { behavior of "EngineJobHashingActor" def templateJobDescriptor(inputs: Map[LocallyQualifiedName, WomValue] = Map.empty): BackendJobDescriptor = { - val task = WomMocks.mockTaskDefinition("hello").copy( - commandTemplateBuilder = Function.const(List(StringCommandPart("Do the stuff... now!!")).validNel) - ) + val task = WomMocks + .mockTaskDefinition("hello") + .copy( + commandTemplateBuilder = Function.const(List(StringCommandPart("Do the stuff... now!!")).validNel) + ) val call = WomMocks.mockTaskCall(WomIdentifier("hello", "workflow.hello")).copy(callable = task) val workflowDescriptor = mock[BackendWorkflowDescriptor] workflowDescriptor.id returns WorkflowId.randomId() - val jobDescriptor = BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, None, 1), Map.empty, fqnWdlMapToDeclarationMap(inputs), NoDocker, None, Map.empty) + val jobDescriptor = BackendJobDescriptor(workflowDescriptor, + BackendJobDescriptorKey(call, None, 1), + Map.empty, + fqnWdlMapToDeclarationMap(inputs), + NoDocker, + None, + Map.empty + ) jobDescriptor } val serviceRegistryActorProbe: TestProbe = TestProbe() - def makeEJHA(receiver: ActorRef, activity: CallCachingActivity, ccReaderProps: Props = Props.empty): TestActorRef[EngineJobHashingActor] = { + def makeEJHA(receiver: ActorRef, + activity: CallCachingActivity, + ccReaderProps: Props = Props.empty + ): TestActorRef[EngineJobHashingActor] = TestActorRef[EngineJobHashingActor]( EngineJobHashingActorTest.props( receiver, @@ -54,7 +76,6 @@ class EngineJobHashingActorSpec extends TestKitSuite with AnyFlatSpecLike with M fileHashBatchSize = 100 ) ) - } it should "record initial hashes" in { val receiver = TestProbe() @@ -106,7 +127,9 @@ class EngineJobHashingActorSpec extends TestKitSuite with AnyFlatSpecLike with M actorUnderTest ! initialResult actorUnderTest ! fileResult - receiver.expectMsg(CallCacheHashes(initialHashes, initialAggregatedHash, Option(FileHashes(fileHashes, fileAggregatedHash)))) + receiver.expectMsg( + CallCacheHashes(initialHashes, initialAggregatedHash, Option(FileHashes(fileHashes, fileAggregatedHash))) + ) } it should "forward CacheMiss to receiver" in { @@ -133,8 +156,8 @@ class EngineJobHashingActorSpec extends TestKitSuite with AnyFlatSpecLike with M val activity = CallCachingActivity(ReadAndWriteCache) val monitorProbe = TestProbe() val ccReadActorProps = Props(new Actor { - override def receive: Receive = { - case NextHit => monitorProbe.ref forward NextHit + override def receive: Receive = { case NextHit => + monitorProbe.ref forward NextHit } }) @@ -188,18 +211,22 @@ class EngineJobHashingActorSpec extends TestKitSuite with AnyFlatSpecLike with M backendName: String, activity: CallCachingActivity, callCachingEligible: CallCachingEligible, - fileHashBatchSize: Int): Props = Props(new EngineJobHashingActorTest( - receiver = receiver, - serviceRegistryActor = serviceRegistryActor, - jobDescriptor = jobDescriptor, - initializationData = initializationData, - fileHashingActorProps = fileHashingActorProps, - callCacheReadingJobActorProps = callCacheReadingJobActorProps, - runtimeAttributeDefinitions = runtimeAttributeDefinitions, - backendName = backendName, - activity = activity, - callCachingEligible = callCachingEligible, - fileHashBatchSize = fileHashBatchSize)) + fileHashBatchSize: Int + ): Props = Props( + new EngineJobHashingActorTest( + receiver = receiver, + serviceRegistryActor = serviceRegistryActor, + jobDescriptor = jobDescriptor, + initializationData = initializationData, + fileHashingActorProps = fileHashingActorProps, + callCacheReadingJobActorProps = callCacheReadingJobActorProps, + runtimeAttributeDefinitions = runtimeAttributeDefinitions, + backendName = backendName, + activity = activity, + callCachingEligible = callCachingEligible, + fileHashBatchSize = fileHashBatchSize + ) + ) } class EngineJobHashingActorTest(receiver: ActorRef, @@ -212,19 +239,21 @@ class EngineJobHashingActorSpec extends TestKitSuite with AnyFlatSpecLike with M backendName: String, activity: CallCachingActivity, callCachingEligible: CallCachingEligible, - fileHashBatchSize: Int) extends EngineJobHashingActor( - receiver = receiver, - serviceRegistryActor = serviceRegistryActor, - jobDescriptor = jobDescriptor, - initializationData = initializationData, - fileHashingActorProps = fileHashingActorProps, - callCacheReadingJobActorProps = callCacheReadingJobActorProps, - runtimeAttributeDefinitions = runtimeAttributeDefinitions, - backendNameForCallCachingPurposes = backendName, - activity = activity, - callCachingEligible = callCachingEligible, - callCachePathPrefixes = None, - fileHashBatchSize = fileHashBatchSize) { + fileHashBatchSize: Int + ) extends EngineJobHashingActor( + receiver = receiver, + serviceRegistryActor = serviceRegistryActor, + jobDescriptor = jobDescriptor, + initializationData = initializationData, + fileHashingActorProps = fileHashingActorProps, + callCacheReadingJobActorProps = callCacheReadingJobActorProps, + runtimeAttributeDefinitions = runtimeAttributeDefinitions, + backendNameForCallCachingPurposes = backendName, + activity = activity, + callCachingEligible = callCachingEligible, + callCachePathPrefixes = None, + fileHashBatchSize = fileHashBatchSize + ) { // override preStart to nothing to prevent the creation of the CCHJA. // This way it doesn't interfere with the tests and we can manually inject the messages we want override def preStart(): Unit = () diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/CallPreparationSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/CallPreparationSpec.scala index 7d0d40fd6f7..5b876319a8e 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/CallPreparationSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/CallPreparationSpec.scala @@ -19,7 +19,8 @@ class CallPreparationSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc it should "disallow empty Strings being input as Files" in { val callKey = mock[CallKey] - val inputExpressionPointer: InputDefinitionPointer = Coproduct[InputDefinitionPointer](WomString("").asWomExpression: WomExpression) + val inputExpressionPointer: InputDefinitionPointer = + Coproduct[InputDefinitionPointer](WomString("").asWomExpression: WomExpression) val inputs: InputDefinitionMappings = List( (RequiredInputDefinition("inputVal", WomSingleFileType), inputExpressionPointer) ) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/JobPreparationActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/JobPreparationActorSpec.scala index 3744ea9e3cd..23e44e32900 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/JobPreparationActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/JobPreparationActorSpec.scala @@ -4,10 +4,14 @@ import akka.testkit.{ImplicitSender, TestActorRef} import cats.syntax.validated._ import cromwell.core.TestKitSuite import cromwell.core.callcaching.{DockerWithHash, FloatingDockerTagWithoutHash} -import cromwell.docker.DockerInfoActor.{DockerInfoSuccessResponse, DockerInformation, DockerSize} +import cromwell.docker.DockerInfoActor.{DockerInformation, DockerInfoSuccessResponse, DockerSize} import cromwell.docker.{DockerHashResult, DockerImageIdentifier, DockerImageIdentifierWithoutHash, DockerInfoRequest} import cromwell.engine.workflow.WorkflowDockerLookupActor.WorkflowDockerLookupFailure -import cromwell.engine.workflow.lifecycle.execution.job.preparation.CallPreparation.{BackendJobPreparationSucceeded, CallPreparationFailed, Start} +import cromwell.engine.workflow.lifecycle.execution.job.preparation.CallPreparation.{ + BackendJobPreparationSucceeded, + CallPreparationFailed, + Start +} import cromwell.engine.workflow.lifecycle.execution.stores.ValueStore import cromwell.services.keyvalue.KeyValueServiceActor.{KvGet, KvKeyLookupFailed, KvPair} import org.scalatest.BeforeAndAfter @@ -24,7 +28,12 @@ import scala.language.postfixOps import scala.util.control.NoStackTrace class JobPreparationActorSpec - extends TestKitSuite with AnyFlatSpecLike with Matchers with ImplicitSender with BeforeAndAfter with MockSugar { + extends TestKitSuite + with AnyFlatSpecLike + with Matchers + with ImplicitSender + with BeforeAndAfter + with MockSugar { behavior of "JobPreparationActor" @@ -41,8 +50,8 @@ class JobPreparationActorSpec val error = "Failed to prepare inputs/attributes - part of test flow" val actor = TestActorRef(helper.buildTestJobPreparationActor(null, null, null, error.invalidNel, List.empty), self) actor ! Start(ValueStore.empty) - expectMsgPF(1.second) { - case CallPreparationFailed(_, ex) => ex.getMessage shouldBe "Call input and runtime attributes evaluation failed for JobPreparationSpec_call:\nFailed to prepare inputs/attributes - part of test flow" + expectMsgPF(1.second) { case CallPreparationFailed(_, ex) => + ex.getMessage shouldBe "Call input and runtime attributes evaluation failed for JobPreparationSpec_call:\nFailed to prepare inputs/attributes - part of test flow" } helper.workflowDockerLookupActor.expectNoMessage(100 millis) } @@ -50,11 +59,11 @@ class JobPreparationActorSpec it should "prepare successfully a job without docker attribute" in { val attributes = Map.empty[LocallyQualifiedName, WomValue] val inputsAndAttributes = (inputs, attributes).validNel - val actor = TestActorRef(helper.buildTestJobPreparationActor(null, null, null, inputsAndAttributes, List.empty), self) + val actor = + TestActorRef(helper.buildTestJobPreparationActor(null, null, null, inputsAndAttributes, List.empty), self) actor ! Start(ValueStore.empty) - expectMsgPF(5 seconds) { - case success: BackendJobPreparationSucceeded => - success.jobDescriptor.maybeCallCachingEligible.dockerHash shouldBe None + expectMsgPF(5 seconds) { case success: BackendJobPreparationSucceeded => + success.jobDescriptor.maybeCallCachingEligible.dockerHash shouldBe None } helper.workflowDockerLookupActor.expectNoMessage(1 second) } @@ -65,7 +74,8 @@ class JobPreparationActorSpec "docker" -> WomString(dockerValue) ) val inputsAndAttributes = (inputs, attributes).validNel - val actor = TestActorRef(helper.buildTestJobPreparationActor(null, null, null, inputsAndAttributes, List.empty), self) + val actor = + TestActorRef(helper.buildTestJobPreparationActor(null, null, null, inputsAndAttributes, List.empty), self) actor ! Start(ValueStore.empty) helper.workflowDockerLookupActor.expectMsgClass(classOf[DockerInfoRequest]) actor ! DockerInfoSuccessResponse( @@ -75,17 +85,18 @@ class JobPreparationActorSpec ), null ) - expectMsgPF(5 seconds) { - case success: BackendJobPreparationSucceeded => - success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe dockerValue - success.jobDescriptor.maybeCallCachingEligible shouldBe DockerWithHash("ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950") - success.jobDescriptor.dockerSize shouldBe Option(DockerSize(100)) + expectMsgPF(5 seconds) { case success: BackendJobPreparationSucceeded => + success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe dockerValue + success.jobDescriptor.maybeCallCachingEligible shouldBe DockerWithHash( + "ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950" + ) + success.jobDescriptor.dockerSize shouldBe Option(DockerSize(100)) } } it should "lookup any requested key/value prefetches after (not) performing a docker hash lookup" in { val dockerValue = "ubuntu:latest" - val attributes = Map ( + val attributes = Map( "docker" -> WomString(dockerValue) ) val hashResult = DockerHashResult("sha256", "71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950") @@ -96,73 +107,88 @@ class JobPreparationActorSpec val prefetchedVal2 = KvKeyLookupFailed(KvGet(helper.scopedKeyMaker(prefetchedKey2))) val prefetchedValues = Map(prefetchedKey1 -> prefetchedVal1, prefetchedKey2 -> prefetchedVal2) var keysToPrefetch = List(prefetchedKey1, prefetchedKey2) - val actor = TestActorRef(helper.buildTestJobPreparationActor(1 minute, 1 minutes, List.empty, inputsAndAttributes, List(prefetchedKey1, prefetchedKey2)), self) + val actor = TestActorRef(helper.buildTestJobPreparationActor(1 minute, + 1 minutes, + List.empty, + inputsAndAttributes, + List(prefetchedKey1, prefetchedKey2) + ), + self + ) actor ! Start(ValueStore.empty) val req = helper.workflowDockerLookupActor.expectMsgClass(classOf[DockerInfoRequest]) helper.workflowDockerLookupActor.reply(DockerInfoSuccessResponse(DockerInformation(hashResult, None), req)) - def respondFromKv(): Unit = { + def respondFromKv(): Unit = helper.serviceRegistryProbe.expectMsgPF(max = 100 milliseconds) { case KvGet(k) if keysToPrefetch.contains(k.key) => actor.tell(msg = prefetchedValues(k.key), sender = helper.serviceRegistryProbe.ref) keysToPrefetch = keysToPrefetch diff List(k.key) } - } respondFromKv() helper.workflowDockerLookupActor.expectNoMessage(max = 100 milliseconds) respondFromKv() - expectMsgPF(5 seconds) { - case success: BackendJobPreparationSucceeded => - success.jobDescriptor.prefetchedKvStoreEntries should be(Map(prefetchedKey1 -> prefetchedVal1, prefetchedKey2 -> prefetchedVal2)) + expectMsgPF(5 seconds) { case success: BackendJobPreparationSucceeded => + success.jobDescriptor.prefetchedKvStoreEntries should be( + Map(prefetchedKey1 -> prefetchedVal1, prefetchedKey2 -> prefetchedVal2) + ) } } it should "leave the docker attribute as is and provide a DockerWithHash value" in { val dockerValue = "ubuntu:latest" - val attributes = Map ( + val attributes = Map( "docker" -> WomString(dockerValue) ) val hashResult = DockerHashResult("sha256", "71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950") val inputsAndAttributes = (inputs, attributes).validNel val finalValue = "ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950" - val actor = TestActorRef(helper.buildTestJobPreparationActor(1 minute, 1 minutes, List.empty, inputsAndAttributes, List.empty), self) + val actor = TestActorRef( + helper.buildTestJobPreparationActor(1 minute, 1 minutes, List.empty, inputsAndAttributes, List.empty), + self + ) actor ! Start(ValueStore.empty) helper.workflowDockerLookupActor.expectMsgClass(classOf[DockerInfoRequest]) helper.workflowDockerLookupActor.reply( DockerInfoSuccessResponse(DockerInformation(hashResult, None), mock[DockerInfoRequest]) ) - expectMsgPF(5 seconds) { - case success: BackendJobPreparationSucceeded => - success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe dockerValue - success.jobDescriptor.maybeCallCachingEligible shouldBe DockerWithHash(finalValue) + expectMsgPF(5 seconds) { case success: BackendJobPreparationSucceeded => + success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe dockerValue + success.jobDescriptor.maybeCallCachingEligible shouldBe DockerWithHash(finalValue) } } it should "not provide a DockerWithHash value if it can't get the docker hash" in { val dockerValue = "ubuntu:latest" - val request = DockerInfoRequest(DockerImageIdentifier.fromString(dockerValue).get.asInstanceOf[DockerImageIdentifierWithoutHash]) - val attributes = Map ( + val request = DockerInfoRequest( + DockerImageIdentifier.fromString(dockerValue).get.asInstanceOf[DockerImageIdentifierWithoutHash] + ) + val attributes = Map( "docker" -> WomString(dockerValue) ) val inputsAndAttributes = (inputs, attributes).validNel - val actor = TestActorRef(helper.buildTestJobPreparationActor(1 minute, 1 minutes, List.empty, inputsAndAttributes, List.empty), self) + val actor = TestActorRef( + helper.buildTestJobPreparationActor(1 minute, 1 minutes, List.empty, inputsAndAttributes, List.empty), + self + ) actor ! Start(ValueStore.empty) helper.workflowDockerLookupActor.expectMsgClass(classOf[DockerInfoRequest]) - helper.workflowDockerLookupActor.reply(WorkflowDockerLookupFailure( - new Exception("Failed to get docker hash - part of test flow") with NoStackTrace, - request - )) - expectMsgPF(5 seconds) { - case success: BackendJobPreparationSucceeded => - success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe dockerValue - success.jobDescriptor.maybeCallCachingEligible shouldBe FloatingDockerTagWithoutHash("ubuntu:latest") + helper.workflowDockerLookupActor.reply( + WorkflowDockerLookupFailure( + new Exception("Failed to get docker hash - part of test flow") with NoStackTrace, + request + ) + ) + expectMsgPF(5 seconds) { case success: BackendJobPreparationSucceeded => + success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe dockerValue + success.jobDescriptor.maybeCallCachingEligible shouldBe FloatingDockerTagWithoutHash("ubuntu:latest") } } it should "lookup MemoryMultiplier key/value if available and accordingly update runtime attributes" in { - val attributes = Map ( + val attributes = Map( "memory" -> WomString("1.1 GB") ) val inputsAndAttributes = (inputs, attributes).validNel @@ -170,7 +196,10 @@ class JobPreparationActorSpec val prefetchedVal = KvPair(helper.scopedKeyMaker(prefetchedKey), "1.1") val prefetchedValues = Map(prefetchedKey -> prefetchedVal) var keysToPrefetch = List(prefetchedKey) - val actor = TestActorRef(helper.buildTestJobPreparationActor(1 minute, 1 minutes, List.empty, inputsAndAttributes, List(prefetchedKey)), self) + val actor = TestActorRef( + helper.buildTestJobPreparationActor(1 minute, 1 minutes, List.empty, inputsAndAttributes, List(prefetchedKey)), + self + ) actor ! Start(ValueStore.empty) helper.serviceRegistryProbe.expectMsgPF(max = 100 milliseconds) { @@ -179,10 +208,11 @@ class JobPreparationActorSpec keysToPrefetch = keysToPrefetch diff List(k.key) } - expectMsgPF(5 seconds) { - case success: BackendJobPreparationSucceeded => - success.jobDescriptor.prefetchedKvStoreEntries should be(Map(prefetchedKey -> prefetchedVal)) - success.jobDescriptor.runtimeAttributes(RuntimeAttributesKeys.MemoryKey) shouldBe WomString("1.2100000000000002 GB") + expectMsgPF(5 seconds) { case success: BackendJobPreparationSucceeded => + success.jobDescriptor.prefetchedKvStoreEntries should be(Map(prefetchedKey -> prefetchedVal)) + success.jobDescriptor.runtimeAttributes(RuntimeAttributesKeys.MemoryKey) shouldBe WomString( + "1.2100000000000002 GB" + ) } } @@ -190,7 +220,7 @@ class JobPreparationActorSpec val prefetchedKey = "MemoryMultiplier" val retryFactor = 1.1 val taskMemory = 1.0 - val attributes = Map ("memory" -> WomString(s"$taskMemory GB")) + val attributes = Map("memory" -> WomString(s"$taskMemory GB")) val inputsAndAttributes = (inputs, attributes).validNel var previousMultiplier = 1.0 @@ -207,7 +237,10 @@ class JobPreparationActorSpec val prefetchedValues = Map(prefetchedKey -> prefetchedVal) var keysToPrefetch = List(prefetchedKey) - val actor = TestActorRef(helper.buildTestJobPreparationActor(1 minute, 1 minutes, List.empty, inputsAndAttributes, List(prefetchedKey)), self) + val actor = TestActorRef( + helper.buildTestJobPreparationActor(1 minute, 1 minutes, List.empty, inputsAndAttributes, List(prefetchedKey)), + self + ) actor ! Start(ValueStore.empty) helper.serviceRegistryProbe.expectMsgPF(max = 100 milliseconds) { @@ -216,10 +249,11 @@ class JobPreparationActorSpec keysToPrefetch = keysToPrefetch diff List(k.key) } - expectMsgPF(5 seconds) { - case success: BackendJobPreparationSucceeded => - success.jobDescriptor.prefetchedKvStoreEntries should be(Map(prefetchedKey -> prefetchedVal)) - success.jobDescriptor.runtimeAttributes(RuntimeAttributesKeys.MemoryKey) shouldBe WomString(s"${taskMemory * nextMultiplier} GB") + expectMsgPF(5 seconds) { case success: BackendJobPreparationSucceeded => + success.jobDescriptor.prefetchedKvStoreEntries should be(Map(prefetchedKey -> prefetchedVal)) + success.jobDescriptor.runtimeAttributes(RuntimeAttributesKeys.MemoryKey) shouldBe WomString( + s"${taskMemory * nextMultiplier} GB" + ) } } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/JobPreparationTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/JobPreparationTestHelper.scala index d4e62c262f6..58498d5ca4c 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/JobPreparationTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/job/preparation/JobPreparationTestHelper.scala @@ -36,48 +36,56 @@ class JobPreparationTestHelper(implicit val system: ActorSystem) extends MockSug val ioActor: TestProbe = TestProbe() val workflowDockerLookupActor: TestProbe = TestProbe() - val scopedKeyMaker: ScopedKeyMaker = key => ScopedKey(workflowId, KvJobKey("correct.horse.battery.staple", None, 1), key) + val scopedKeyMaker: ScopedKeyMaker = key => + ScopedKey(workflowId, KvJobKey("correct.horse.battery.staple", None, 1), key) - def buildTestJobPreparationActor(backpressureTimeout: FiniteDuration, - noResponseTimeout: FiniteDuration, - dockerHashCredentials: List[Any], - inputsAndAttributes: ErrorOr[(WomEvaluatedCallInputs, Map[LocallyQualifiedName, WomValue])], - kvStoreKeysForPrefetch: List[String], - jobKey: BackendJobDescriptorKey = mockJobKey): Props = { + def buildTestJobPreparationActor( + backpressureTimeout: FiniteDuration, + noResponseTimeout: FiniteDuration, + dockerHashCredentials: List[Any], + inputsAndAttributes: ErrorOr[(WomEvaluatedCallInputs, Map[LocallyQualifiedName, WomValue])], + kvStoreKeysForPrefetch: List[String], + jobKey: BackendJobDescriptorKey = mockJobKey + ): Props = + Props( + new TestJobPreparationActor( + kvStoreKeysForPrefetch = kvStoreKeysForPrefetch, + dockerHashCredentialsInput = dockerHashCredentials, + backpressureWaitTimeInput = backpressureTimeout, + dockerNoResponseTimeoutInput = noResponseTimeout, + inputsAndAttributes = inputsAndAttributes, + workflowDescriptor = workflowDescriptor, + jobKey = jobKey, + workflowDockerLookupActor = workflowDockerLookupActor.ref, + serviceRegistryActor = serviceRegistryProbe.ref, + ioActor = ioActor.ref, + scopedKeyMaker + ) + ) +} - Props(new TestJobPreparationActor( - kvStoreKeysForPrefetch = kvStoreKeysForPrefetch, - dockerHashCredentialsInput = dockerHashCredentials, - backpressureWaitTimeInput = backpressureTimeout, - dockerNoResponseTimeoutInput = noResponseTimeout, - inputsAndAttributes = inputsAndAttributes, +private[preparation] class TestJobPreparationActor( + kvStoreKeysForPrefetch: List[String], + dockerHashCredentialsInput: List[Any], + backpressureWaitTimeInput: FiniteDuration, + dockerNoResponseTimeoutInput: FiniteDuration, + inputsAndAttributes: ErrorOr[(WomEvaluatedCallInputs, Map[LocallyQualifiedName, WomValue])], + workflowDescriptor: EngineWorkflowDescriptor, + jobKey: BackendJobDescriptorKey, + workflowDockerLookupActor: ActorRef, + serviceRegistryActor: ActorRef, + ioActor: ActorRef, + scopedKeyMaker: ScopedKeyMaker +) extends JobPreparationActor( workflowDescriptor = workflowDescriptor, jobKey = jobKey, - workflowDockerLookupActor = workflowDockerLookupActor.ref, - serviceRegistryActor = serviceRegistryProbe.ref, - ioActor = ioActor.ref, - scopedKeyMaker)) - } -} - -private[preparation] class TestJobPreparationActor(kvStoreKeysForPrefetch: List[String], - dockerHashCredentialsInput: List[Any], - backpressureWaitTimeInput: FiniteDuration, - dockerNoResponseTimeoutInput: FiniteDuration, - inputsAndAttributes: ErrorOr[(WomEvaluatedCallInputs, Map[LocallyQualifiedName, WomValue])], - workflowDescriptor: EngineWorkflowDescriptor, - jobKey: BackendJobDescriptorKey, - workflowDockerLookupActor: ActorRef, - serviceRegistryActor: ActorRef, - ioActor: ActorRef, - scopedKeyMaker: ScopedKeyMaker) extends JobPreparationActor(workflowDescriptor = workflowDescriptor, - jobKey = jobKey, - factory = null, - workflowDockerLookupActor = workflowDockerLookupActor, - initializationData = None, - serviceRegistryActor = serviceRegistryActor, - ioActor = ioActor, - backendSingletonActor = None) { + factory = null, + workflowDockerLookupActor = workflowDockerLookupActor, + initializationData = None, + serviceRegistryActor = serviceRegistryActor, + ioActor = ioActor, + backendSingletonActor = None + ) { override private[preparation] lazy val kvStoreKeysToPrefetch = kvStoreKeysForPrefetch @@ -93,7 +101,8 @@ private[preparation] class TestJobPreparationActor(kvStoreKeysForPrefetch: List[ initializationData: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef, - backendSingletonActor: Option[ActorRef]) = Props.empty + backendSingletonActor: Option[ActorRef] + ) = Props.empty } object JobPreparationTestHelper { diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/stores/ExecutionStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/stores/ExecutionStoreSpec.scala index 20b1ae40c0a..a1a741c27ff 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/stores/ExecutionStoreSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/stores/ExecutionStoreSpec.scala @@ -13,20 +13,23 @@ import scala.util.Random class ExecutionStoreSpec extends AnyFlatSpec with Matchers with BeforeAndAfter { - it should "allow 10000 unconnected call keys to be enqueued and started in small batches" in { - def jobKeys: Map[JobKey, ExecutionStatus] = (0.until(10000).toList map { - i => BackendJobDescriptorKey(noConnectionsGraphNode, Option(i), 1) -> NotStarted }) - .toMap + def jobKeys: Map[JobKey, ExecutionStatus] = (0.until(10000).toList map { i => + BackendJobDescriptorKey(noConnectionsGraphNode, Option(i), 1) -> NotStarted + }).toMap var store: ExecutionStore = ActiveExecutionStore(jobKeys, needsUpdate = true) var iterationNumber = 0 while (store.needsUpdate) { // Assert that we're increasing the queue size by 1000 each time - store.store.getOrElse(NotStarted, List.empty).size should be(10000 - iterationNumber * ExecutionStore.MaxJobsToStartPerTick) - store.store.getOrElse(QueuedInCromwell, List.empty).size should be(iterationNumber * ExecutionStore.MaxJobsToStartPerTick) + store.store.getOrElse(NotStarted, List.empty).size should be( + 10000 - iterationNumber * ExecutionStore.MaxJobsToStartPerTick + ) + store.store.getOrElse(QueuedInCromwell, List.empty).size should be( + iterationNumber * ExecutionStore.MaxJobsToStartPerTick + ) val update = store.update store = update.updatedStore.updateKeys(update.runnableKeys.map(_ -> QueuedInCromwell).toMap) iterationNumber = iterationNumber + 1 @@ -36,7 +39,7 @@ class ExecutionStoreSpec extends AnyFlatSpec with Matchers with BeforeAndAfter { var previouslyRunning = store.store.getOrElse(Running, List.empty).size previouslyRunning should be(0) - while(store.store.getOrElse(Running, List.empty).size < 10000) { + while (store.store.getOrElse(Running, List.empty).size < 10000) { val toStartRunning = store.store(QueuedInCromwell).take(Random.nextInt(1000)) store = store.updateKeys(toStartRunning.map(j => j -> Running).toMap) val nowRunning = store.store.getOrElse(Running, List.empty).size @@ -60,7 +63,7 @@ object ExecutionStoreSpec { val noConnectionsGraphNode: CommandCallNode = CommandCallNode( identifier = WomIdentifier("mock_task", "mock_wf.mock_task"), callable = null, - inputPorts = Set.empty[GraphNodePort.InputPort], + inputPorts = Set.empty[GraphNodePort.InputPort], inputDefinitionMappings = List.empty, nonInputBasedPrerequisites = Set.empty[GraphNode], outputIdentifierCompoundingFunction = (wi, _) => wi, diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/finalization/CopyWorkflowLogsActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/finalization/CopyWorkflowLogsActorSpec.scala index 554ad52abd0..f2b2a615e58 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/finalization/CopyWorkflowLogsActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/finalization/CopyWorkflowLogsActorSpec.scala @@ -14,8 +14,7 @@ import scala.concurrent.duration._ import scala.util.control.NoStackTrace import scala.util.{Failure, Try} -class CopyWorkflowLogsActorSpec - extends TestKitSuite with AnyFlatSpecLike with Matchers { +class CopyWorkflowLogsActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers { behavior of "CopyWorkflowLogsActor" @@ -25,9 +24,8 @@ class CopyWorkflowLogsActorSpec private val deathWatch = TestProbe("deathWatch") private val tempDir = DefaultPathBuilder.createTempDirectory("tempDir.") - override protected def beforeAll(): Unit = { + override protected def beforeAll(): Unit = super.beforeAll() - } override protected def afterAll(): Unit = { super.afterAll() @@ -43,7 +41,7 @@ class CopyWorkflowLogsActorSpec ioActor = ioActor.ref, workflowLogConfigurationOption = WorkflowLogger.workflowLogConfiguration, copyCommandBuilder = DefaultIoCommandBuilder, - deleteCommandBuilder = DefaultIoCommandBuilder, + deleteCommandBuilder = DefaultIoCommandBuilder ) val copyWorkflowLogsActor = system.actorOf(props, "testCopyWorkflowLogsActor") @@ -60,7 +58,7 @@ class CopyWorkflowLogsActorSpec copyWorkflowLogsActor ! (( workflowId, - IoFailure(copyCommand, new Exception("everything's fine, I am an expected copy fail") with NoStackTrace), + IoFailure(copyCommand, new Exception("everything's fine, I am an expected copy fail") with NoStackTrace) )) // There should now be a delete command sent to the ioActor @@ -83,8 +81,8 @@ class CopyWorkflowLogsActorSpec val workflowId = WorkflowId.randomId() val destinationPath = DefaultPathBuilder.createTempFile(s"test_file_$workflowId.", ".file", Option(tempDir)) val partialIoCommandBuilder = new PartialIoCommandBuilder { - override def copyCommand: PartialFunction[(Path, Path), Try[IoCopyCommand]] = { - case _ => Failure(new Exception("everything's fine, I am an expected copy fail") with NoStackTrace) + override def copyCommand: PartialFunction[(Path, Path), Try[IoCopyCommand]] = { case _ => + Failure(new Exception("everything's fine, I am an expected copy fail") with NoStackTrace) } } val ioCommandBuilder = new IoCommandBuilder(List(partialIoCommandBuilder)) @@ -93,7 +91,7 @@ class CopyWorkflowLogsActorSpec serviceRegistryActor = serviceRegistryActor.ref, ioActor = ioActor.ref, workflowLogConfigurationOption = WorkflowLogger.workflowLogConfiguration, - copyCommandBuilder = ioCommandBuilder, + copyCommandBuilder = ioCommandBuilder ) val copyWorkflowLogsActor = system.actorOf(props, "testCopyWorkflowLogsActorFailCopy") @@ -105,7 +103,9 @@ class CopyWorkflowLogsActorSpec val deleteCommand = DefaultIoDeleteCommand(workflowLogPath, swallowIOExceptions = true) ioActor.expectMsg(msgWait, deleteCommand) - copyWorkflowLogsActor ! IoFailure(deleteCommand, new Exception("everything's fine, I am an expected delete fail") with NoStackTrace) + copyWorkflowLogsActor ! IoFailure(deleteCommand, + new Exception("everything's fine, I am an expected delete fail") with NoStackTrace + ) // Send a shutdown after the delete deathWatch.watch(copyWorkflowLogsActor) @@ -119,8 +119,8 @@ class CopyWorkflowLogsActorSpec val workflowId = WorkflowId.randomId() val destinationPath = DefaultPathBuilder.createTempFile(s"test_file_$workflowId.", ".file", Option(tempDir)) val partialIoCommandBuilder = new PartialIoCommandBuilder { - override def copyCommand: PartialFunction[(Path, Path), Try[IoCopyCommand]] = { - case _ => Failure(new Exception("everything's fine, I am an expected copy fail") with NoStackTrace) + override def copyCommand: PartialFunction[(Path, Path), Try[IoCopyCommand]] = { case _ => + Failure(new Exception("everything's fine, I am an expected copy fail") with NoStackTrace) } } val ioCommandBuilder = new IoCommandBuilder(List(partialIoCommandBuilder)) @@ -128,7 +128,7 @@ class CopyWorkflowLogsActorSpec serviceRegistryActor = serviceRegistryActor.ref, ioActor = ioActor.ref, workflowLogConfigurationOption = WorkflowLogger.workflowLogConfiguration, - copyCommandBuilder = ioCommandBuilder, + copyCommandBuilder = ioCommandBuilder ) val copyWorkflowLogsActor = system.actorOf(props, "testCopyWorkflowLogsActorFailCopyShutdown") @@ -146,7 +146,10 @@ class CopyWorkflowLogsActorSpec // Test that the actor is still alive and receiving messages even after a shutdown was requested EventFilter.error(pattern = "Failed to delete workflow logs", occurrences = 1).intercept { - copyWorkflowLogsActor ! IoFailure(deleteCommand, new Exception("everything's fine, I am an expected delete fail") with NoStackTrace) + copyWorkflowLogsActor ! IoFailure(deleteCommand, + new Exception("everything's fine, I am an expected delete fail") + with NoStackTrace + ) } // Then the actor should shutdown @@ -157,12 +160,12 @@ class CopyWorkflowLogsActorSpec val workflowId = WorkflowId.randomId() val destinationPath = DefaultPathBuilder.createTempFile(s"test_file_$workflowId.", ".file", Option(tempDir)) val partialIoCommandBuilder = new PartialIoCommandBuilder { - override def copyCommand: PartialFunction[(Path, Path), Try[IoCopyCommand]] = { - case _ => Failure(new Exception("everything's fine, I am an expected copy fail") with NoStackTrace) + override def copyCommand: PartialFunction[(Path, Path), Try[IoCopyCommand]] = { case _ => + Failure(new Exception("everything's fine, I am an expected copy fail") with NoStackTrace) } - override def deleteCommand: PartialFunction[(Path, Boolean), Try[IoDeleteCommand]] = { - case _ => Failure(new Exception("everything's fine, I am an expected delete fail") with NoStackTrace) + override def deleteCommand: PartialFunction[(Path, Boolean), Try[IoDeleteCommand]] = { case _ => + Failure(new Exception("everything's fine, I am an expected delete fail") with NoStackTrace) } } val ioCommandBuilder = new IoCommandBuilder(List(partialIoCommandBuilder)) @@ -172,7 +175,7 @@ class CopyWorkflowLogsActorSpec ioActor = ioActor.ref, workflowLogConfigurationOption = WorkflowLogger.workflowLogConfiguration, copyCommandBuilder = ioCommandBuilder, - deleteCommandBuilder = ioCommandBuilder, + deleteCommandBuilder = ioCommandBuilder ) val copyWorkflowLogsActor = system.actorOf(props, "testCopyWorkflowLogsActorFailDelete") diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackActorSpec.scala index 97479d348ae..976a8994c9e 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/finalization/WorkflowCallbackActorSpec.scala @@ -1,42 +1,49 @@ package cromwell.engine.workflow.lifecycle.finalization -import akka.testkit._ import akka.http.scaladsl.client.RequestBuilding.Post import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import akka.http.scaladsl.model.{HttpResponse, StatusCodes} -import akka.testkit.TestProbe +import akka.testkit.{TestProbe, _} import common.mock.MockSugar import cromwell.core.retry.SimpleExponentialBackoff -import org.mockito.Mockito._ -import cromwell.core.{CallOutputs, TestKitSuite, WorkflowFailed, WorkflowId, WorkflowSucceeded} +import cromwell.core._ import cromwell.engine.workflow.lifecycle.finalization.WorkflowCallbackActor.PerformCallbackCommand import cromwell.engine.workflow.lifecycle.finalization.WorkflowCallbackJsonSupport._ import cromwell.services.metadata.MetadataService.PutMetadataAction import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} -import cromwell.util.{GracefulShutdownHelper, WomMocks} +import cromwell.util.GracefulShutdownHelper +import org.mockito.Mockito._ import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers +import wom.graph.GraphNodePort.GraphNodeOutputPort +import wom.graph.WomIdentifier +import wom.types.WomStringType import wom.values.WomString import java.net.URI import java.time.Instant -import scala.concurrent.duration._ import scala.concurrent.Future +import scala.concurrent.duration._ -class WorkflowCallbackActorSpec - extends TestKitSuite with AnyFlatSpecLike with Matchers with MockSugar { +class WorkflowCallbackActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with MockSugar { behavior of "WorkflowCallbackActor" - private implicit val ec = system.dispatcher + implicit private val ec = system.dispatcher private val msgWait = 10.second.dilated private val awaitAlmostNothing = 1.second private val serviceRegistryActor = TestProbe("testServiceRegistryActor") private val deathWatch = TestProbe("deathWatch") private val mockUri = new URI("http://example.com") - private val basicConfig = WorkflowCallbackConfig.empty.copy(enabled = true).copy(retryBackoff = SimpleExponentialBackoff(100.millis, 200.millis, 1.1)) - private val basicOutputs = WomMocks.mockOutputExpectations(List("foo" -> WomString("bar")).toMap) + private val basicConfig = WorkflowCallbackConfig.empty + .copy(enabled = true) + .copy(retryBackoff = SimpleExponentialBackoff(100.millis, 200.millis, 1.1)) + private val basicOutputs = CallOutputs( + Map( + GraphNodeOutputPort(WomIdentifier("foo", "wf.foo"), WomStringType, null) -> WomString("bar") + ) + ) private val httpSuccess = Future.successful(HttpResponse.apply(StatusCodes.OK)) private val httpFailure = Future.successful(HttpResponse.apply(StatusCodes.GatewayTimeout)) @@ -64,7 +71,7 @@ class WorkflowCallbackActorSpec val expectedPostBody = CallbackMessage( workflowId.toString, WorkflowSucceeded.toString, - basicOutputs.outputs.map(entry => (entry._1.name, entry._2)), + Map(("wf.foo", WomString("bar"))), List.empty ) val expectedRequest = Post(mockUri.toString, expectedPostBody) @@ -81,7 +88,11 @@ class WorkflowCallbackActorSpec // Do the thing val cmd = PerformCallbackCommand( - workflowId = workflowId, uri = None, terminalState = WorkflowSucceeded, workflowOutputs = basicOutputs, List.empty + workflowId = workflowId, + uri = None, + terminalState = WorkflowSucceeded, + workflowOutputs = basicOutputs, + List.empty ) workflowCallbackActor ! cmd @@ -93,7 +104,7 @@ class WorkflowCallbackActorSpec uriEvent.key shouldBe expectedUriMetadata.key uriEvent.value shouldBe expectedUriMetadata.value timestampEvent.key shouldBe expectedTimestampMetadata.key - // Not checking timestamp value because it won't match + // Not checking timestamp value because it won't match case _ => } @@ -114,7 +125,7 @@ class WorkflowCallbackActorSpec val expectedPostBody = CallbackMessage( workflowId.toString, WorkflowSucceeded.toString, - basicOutputs.outputs.map(entry => (entry._1.name, entry._2)), + Map(("wf.foo", WomString("bar"))), List.empty ) val expectedRequest = Post(mockUri.toString, expectedPostBody) @@ -132,7 +143,11 @@ class WorkflowCallbackActorSpec // Do the thing val cmd = PerformCallbackCommand( - workflowId = workflowId, uri = None, terminalState = WorkflowSucceeded, workflowOutputs = basicOutputs, List.empty + workflowId = workflowId, + uri = None, + terminalState = WorkflowSucceeded, + workflowOutputs = basicOutputs, + List.empty ) workflowCallbackActor ! cmd @@ -179,7 +194,7 @@ class WorkflowCallbackActorSpec serviceRegistryActor.ref, basicConfig.copy( retryBackoff = SimpleExponentialBackoff(500.millis, 1.minute, 1.1), - maxRetries = 5, + maxRetries = 5 ), httpClient = mockHttpClient ) @@ -230,7 +245,11 @@ class WorkflowCallbackActorSpec // Do the thing val cmd = PerformCallbackCommand( - workflowId = workflowId, uri = None, terminalState = WorkflowSucceeded, workflowOutputs = basicOutputs, List.empty + workflowId = workflowId, + uri = None, + terminalState = WorkflowSucceeded, + workflowOutputs = basicOutputs, + List.empty ) workflowCallbackActor ! cmd diff --git a/engine/src/test/scala/cromwell/engine/workflow/mocks/DeclarationMock.scala b/engine/src/test/scala/cromwell/engine/workflow/mocks/DeclarationMock.scala index be7b0d741a3..5f8f846d7d4 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/mocks/DeclarationMock.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/mocks/DeclarationMock.scala @@ -9,9 +9,7 @@ object DeclarationMock { } trait DeclarationMock extends MockSugar { - def mockDeclaration(name: String, - womType: WomType, - expression: WdlExpression): Declaration = { + def mockDeclaration(name: String, womType: WomType, expression: WdlExpression): Declaration = { val declaration = mock[Declaration] declaration.unqualifiedName returns name declaration.expression returns Option(expression) diff --git a/engine/src/test/scala/cromwell/engine/workflow/mocks/TaskMock.scala b/engine/src/test/scala/cromwell/engine/workflow/mocks/TaskMock.scala index 7ce565a4b6c..77aaa449c3b 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/mocks/TaskMock.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/mocks/TaskMock.scala @@ -12,15 +12,15 @@ trait TaskMock extends MockSugar { runtimeAttributes: WdlRuntimeAttributes = new WdlRuntimeAttributes(Map.empty), commandTemplateString: String = "!!shazam!!", outputs: Seq[DeclarationMockType] = Seq.empty - ): WdlTask = { + ): WdlTask = { val task = mock[WdlTask] task.declarations returns declarations task.runtimeAttributes returns runtimeAttributes task.commandTemplateString returns commandTemplateString task.name returns name task.unqualifiedName returns name - task.outputs returns (outputs map { - case (outputName, womType, expression) => TaskOutput(outputName, womType, expression, mock[Ast], Option(task)) + task.outputs returns (outputs map { case (outputName, womType, expression) => + TaskOutput(outputName, womType, expression, mock[Ast], Option(task)) }) task } diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobTokenDispenserActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobTokenDispenserActorSpec.scala index adaadb0fef7..ca2409b4931 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobTokenDispenserActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobTokenDispenserActorSpec.scala @@ -18,8 +18,14 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.duration._ import scala.util.Random -class JobTokenDispenserActorSpec extends TestKitSuite - with ImplicitSender with AnyFlatSpecLike with Matchers with BeforeAndAfter with BeforeAndAfterAll with Eventually { +class JobTokenDispenserActorSpec + extends TestKitSuite + with ImplicitSender + with AnyFlatSpecLike + with Matchers + with BeforeAndAfter + with BeforeAndAfterAll + with Eventually { val MaxWaitTime: FiniteDuration = 10.seconds implicit val pc: PatienceConfig = PatienceConfig(MaxWaitTime) @@ -30,43 +36,49 @@ class JobTokenDispenserActorSpec extends TestKitSuite val hogGroupB: HogGroup = HogGroup("hogGroupB") private def getActorRefUnderTest(serviceRegistryActorName: String, - jobExecutionTokenDispenserActorName: String, - ): TestActorRef[JobTokenDispenserActor] = { + jobExecutionTokenDispenserActorName: String + ): TestActorRef[JobTokenDispenserActor] = TestActorRef( - factory = - new JobTokenDispenserActor( - serviceRegistryActor = TestProbe(serviceRegistryActorName).ref, - dispensingRate = Rate(10, 100.millis), - logInterval = None, - dispenserType = "execution", - tokenAllocatedDescription = "Running" - ), - name = jobExecutionTokenDispenserActorName, + factory = new JobTokenDispenserActor( + serviceRegistryActor = TestProbe(serviceRegistryActorName).ref, + dispensingRate = Rate(10, 100.millis), + logInterval = None, + dispenserType = "execution", + tokenAllocatedDescription = "Running" + ), + name = jobExecutionTokenDispenserActorName ) - } it should "dispense an infinite token correctly" in { val actorRefUnderTest = getActorRefUnderTest( serviceRegistryActorName = "serviceRegistryActor-dispense-infinite", - jobExecutionTokenDispenserActorName = "dispense-infinite", + jobExecutionTokenDispenserActorName = "dispense-infinite" ) actorRefUnderTest ! JobTokenRequest(hogGroupA, TestInfiniteTokenType) expectMsg(max = MaxWaitTime, JobTokenDispensed) actorRefUnderTest.underlyingActor.tokenAssignments.contains(self) shouldBe true - actorRefUnderTest.underlyingActor.tokenAssignments(self).tokenLease.get().jobTokenType shouldBe TestInfiniteTokenType + actorRefUnderTest.underlyingActor + .tokenAssignments(self) + .tokenLease + .get() + .jobTokenType shouldBe TestInfiniteTokenType } it should "accept return of an infinite token correctly" in { val actorRefUnderTest = getActorRefUnderTest( serviceRegistryActorName = "serviceRegistryActor-accept-return", - jobExecutionTokenDispenserActorName = "accept-return", + jobExecutionTokenDispenserActorName = "accept-return" ) actorRefUnderTest ! JobTokenRequest(hogGroupA, TestInfiniteTokenType) expectMsg(max = MaxWaitTime, JobTokenDispensed) actorRefUnderTest.underlyingActor.tokenAssignments.contains(self) shouldBe true - actorRefUnderTest.underlyingActor.tokenAssignments(self).tokenLease.get().jobTokenType shouldBe TestInfiniteTokenType + actorRefUnderTest.underlyingActor + .tokenAssignments(self) + .tokenLease + .get() + .jobTokenType shouldBe TestInfiniteTokenType actorRefUnderTest ! JobTokenReturn actorRefUnderTest.underlyingActor.tokenAssignments.contains(self) shouldBe false } @@ -75,10 +87,12 @@ class JobTokenDispenserActorSpec extends TestKitSuite val actorRefUnderTest = getActorRefUnderTest( serviceRegistryActorName = "serviceRegistryActor-dispense-indefinitely", - jobExecutionTokenDispenserActorName = "dispense-indefinitely", + jobExecutionTokenDispenserActorName = "dispense-indefinitely" ) val senders = (1 to 20).map(index => TestProbe(s"sender-dispense-indefinitely-$index")) - senders.foreach(sender => actorRefUnderTest.tell(msg = JobTokenRequest(hogGroupA, TestInfiniteTokenType), sender = sender.ref)) + senders.foreach(sender => + actorRefUnderTest.tell(msg = JobTokenRequest(hogGroupA, TestInfiniteTokenType), sender = sender.ref) + ) senders.foreach(_.expectMsg(max = MaxWaitTime, JobTokenDispensed)) actorRefUnderTest.underlyingActor.tokenAssignments.size shouldBe 20 } @@ -87,19 +101,20 @@ class JobTokenDispenserActorSpec extends TestKitSuite // Override with a slower distribution rate for this one test: val actorRefUnderTest = TestActorRef( - factory = - new JobTokenDispenserActor( - serviceRegistryActor = TestProbe("serviceRegistryActor-dispense-correct-amount").ref, - dispensingRate = Rate(10, 4.seconds), - logInterval = None, - dispenserType = "execution", - tokenAllocatedDescription = "Running" - ), - name = "dispense-correct-amount", + factory = new JobTokenDispenserActor( + serviceRegistryActor = TestProbe("serviceRegistryActor-dispense-correct-amount").ref, + dispensingRate = Rate(10, 4.seconds), + logInterval = None, + dispenserType = "execution", + tokenAllocatedDescription = "Running" + ), + name = "dispense-correct-amount" ) val senders = (1 to 20).map(index => TestProbe(s"sender-dispense-correct-amount-$index")) - senders.foreach(sender => actorRefUnderTest.tell(msg = JobTokenRequest(hogGroupA, TestInfiniteTokenType), sender = sender.ref)) + senders.foreach(sender => + actorRefUnderTest.tell(msg = JobTokenRequest(hogGroupA, TestInfiniteTokenType), sender = sender.ref) + ) // The first 10 should get their token senders.take(10).foreach(_.expectMsg(max = MaxWaitTime, JobTokenDispensed)) // Couldn't figure out a cleaner way to "verify that none of this probes gets a message in the next X seconds" @@ -113,7 +128,7 @@ class JobTokenDispenserActorSpec extends TestKitSuite val actorRefUnderTest = getActorRefUnderTest( serviceRegistryActorName = "serviceRegistryActor-dispense-limited", - jobExecutionTokenDispenserActorName = "dispense-limited", + jobExecutionTokenDispenserActorName = "dispense-limited" ) actorRefUnderTest ! JobTokenRequest(hogGroupA, LimitedTo5Tokens) expectMsg(max = MaxWaitTime, JobTokenDispensed) @@ -125,7 +140,7 @@ class JobTokenDispenserActorSpec extends TestKitSuite val actorRefUnderTest = getActorRefUnderTest( serviceRegistryActorName = "serviceRegistryActor-accept-return-limited", - jobExecutionTokenDispenserActorName = "accept-return-limited", + jobExecutionTokenDispenserActorName = "accept-return-limited" ) actorRefUnderTest ! JobTokenRequest(hogGroupA, LimitedTo5Tokens) expectMsg(max = MaxWaitTime, JobTokenDispensed) @@ -142,23 +157,32 @@ class JobTokenDispenserActorSpec extends TestKitSuite val actorRefUnderTest = getActorRefUnderTest( serviceRegistryActorName = "serviceRegistryActor-limit-dispensing-limited", - jobExecutionTokenDispenserActorName = "limit-dispensing-limited", + jobExecutionTokenDispenserActorName = "limit-dispensing-limited" ) val senders = (1 to 15).map(index => TestProbe(s"sender-limit-dispensing-limited-$index")) // Ask for 20 tokens - senders.foreach(sender => actorRefUnderTest.tell(msg = JobTokenRequest(hogGroupA, LimitedTo5Tokens), sender = sender.ref)) + senders.foreach(sender => + actorRefUnderTest.tell(msg = JobTokenRequest(hogGroupA, LimitedTo5Tokens), sender = sender.ref) + ) // Force token distribution actorRefUnderTest ! TokensAvailable(100) senders.take(5).foreach(_.expectMsg(JobTokenDispensed)) actorRefUnderTest.underlyingActor.tokenAssignments.size shouldBe 5 - actorRefUnderTest.underlyingActor.tokenAssignments.keySet should contain theSameElementsAs senders.map(_.ref).take(5).toSet + actorRefUnderTest.underlyingActor.tokenAssignments.keySet should contain theSameElementsAs senders + .map(_.ref) + .take(5) + .toSet // The last 10 should be queued // At this point [0, 1, 2, 3, 4] are the ones with tokens, and [5, 14] are still queued actorRefUnderTest.underlyingActor.tokenQueues(LimitedTo5Tokens).size shouldBe 10 - actorRefUnderTest.underlyingActor.tokenQueues(LimitedTo5Tokens).queues.flatMap(_._2).toList should contain theSameElementsInOrderAs senders.drop(5).map(asHogGroupAPlaceholder) + actorRefUnderTest.underlyingActor + .tokenQueues(LimitedTo5Tokens) + .queues + .flatMap(_._2) + .toList should contain theSameElementsInOrderAs senders.drop(5).map(asHogGroupAPlaceholder) // Force token distribution actorRefUnderTest ! TokensAvailable(100) @@ -174,7 +198,11 @@ class JobTokenDispenserActorSpec extends TestKitSuite senders.slice(5, 8).foreach(_.expectMsg(JobTokenDispensed)) // At this point [3, 4, 5, 6, 7] are the ones with tokens, and [8, 19] are still queued - actorRefUnderTest.underlyingActor.tokenQueues(LimitedTo5Tokens).queues.flatMap(_._2).toList should contain theSameElementsInOrderAs senders.slice(8, 20).map(asHogGroupAPlaceholder) + actorRefUnderTest.underlyingActor + .tokenQueues(LimitedTo5Tokens) + .queues + .flatMap(_._2) + .toList should contain theSameElementsInOrderAs senders.slice(8, 20).map(asHogGroupAPlaceholder) // Double-check the queue state: when we request a token now, we should still be denied: actorRefUnderTest ! JobTokenRequest(hogGroupA, LimitedTo5Tokens) @@ -182,13 +210,19 @@ class JobTokenDispenserActorSpec extends TestKitSuite actorRefUnderTest ! TokensAvailable(100) expectNoMessage() // We should be enqueued and the last in the queue though - actorRefUnderTest.underlyingActor.tokenQueues(LimitedTo5Tokens).queues.flatMap(_._2).last shouldBe TokenQueuePlaceholder(self, "hogGroupA") + actorRefUnderTest.underlyingActor + .tokenQueues(LimitedTo5Tokens) + .queues + .flatMap(_._2) + .last shouldBe TokenQueuePlaceholder(self, "hogGroupA") // Release all currently owned tokens senders.slice(3, 8).foreach(_.send(actorRefUnderTest, JobTokenReturn)) // Force token distribution actorRefUnderTest ! TokensAvailable(100) - actorRefUnderTest.underlyingActor.tokenAssignments.keySet should contain theSameElementsAs senders.map(_.ref).slice(8, 13) + actorRefUnderTest.underlyingActor.tokenAssignments.keySet should contain theSameElementsAs senders + .map(_.ref) + .slice(8, 13) // Keep accepting and returning tokens immediately senders.slice(8, 13).foreach(_.expectMsg(JobTokenDispensed)) senders.slice(8, 13).foreach(_.reply(JobTokenReturn)) @@ -203,14 +237,16 @@ class JobTokenDispenserActorSpec extends TestKitSuite actorRefUnderTest.underlyingActor.tokenQueues(LimitedTo5Tokens).size shouldBe 0 // There should be 3 assigned tokens: index 18, 19, and this test actor - actorRefUnderTest.underlyingActor.tokenAssignments.keySet should contain theSameElementsAs senders.map(_.ref).slice(13, 15) :+ self + actorRefUnderTest.underlyingActor.tokenAssignments.keySet should contain theSameElementsAs senders + .map(_.ref) + .slice(13, 15) :+ self } it should "resend the same token to an actor which already has one" in { val actorRefUnderTest = getActorRefUnderTest( serviceRegistryActorName = "serviceRegistryActor-resend-same", - jobExecutionTokenDispenserActorName = "resend-same", + jobExecutionTokenDispenserActorName = "resend-same" ) 5 indexedTimes { _ => actorRefUnderTest ! JobTokenRequest(hogGroupA, LimitedTo5Tokens) @@ -225,17 +261,18 @@ class JobTokenDispenserActorSpec extends TestKitSuite actorRefUnderTest.underlyingActor.tokenQueues(LimitedTo5Tokens).pool.leased() shouldBe 1 } - - //Incidentally, also covers: it should "not be fooled if the wrong actor returns a token" + // Incidentally, also covers: it should "not be fooled if the wrong actor returns a token" it should "not be fooled by a doubly-returned token" in { val actorRefUnderTest = getActorRefUnderTest( serviceRegistryActorName = "serviceRegistryActor-not-fooled", - jobExecutionTokenDispenserActorName = "not-fooled", + jobExecutionTokenDispenserActorName = "not-fooled" ) val senders = (1 to 7).map(index => TestProbe(s"sender-not-fooled-$index")) // Ask for 7 tokens - senders.foreach(sender => actorRefUnderTest.tell(msg = JobTokenRequest(hogGroupA, LimitedTo5Tokens), sender = sender.ref)) + senders.foreach(sender => + actorRefUnderTest.tell(msg = JobTokenRequest(hogGroupA, LimitedTo5Tokens), sender = sender.ref) + ) // Force token distribution actorRefUnderTest ! TokensAvailable(5) @@ -261,16 +298,21 @@ class JobTokenDispenserActorSpec extends TestKitSuite it should s"recover tokens lost to actors which are $name before they hand back their token" in { val actorRefUnderTest = TestActorRef( - new JobTokenDispenserActor(TestProbe(s"serviceRegistryActor-$name").ref, Rate(10, 100.millis), None, - dispenserType = "execution", - tokenAllocatedDescription = "Running" + new JobTokenDispenserActor(TestProbe(s"serviceRegistryActor-$name").ref, + Rate(10, 100.millis), + None, + dispenserType = "execution", + tokenAllocatedDescription = "Running" ), - s"lost-to-$name", + s"lost-to-$name" ) val grabberSupervisor = TestActorRef(new StoppingSupervisor(), s"lost-to-$name-supervisor") // The first 5 get a token and the 6th one is queued val tokenGrabbingActors = (1 to 6).map { i => - TestActorRef[TestTokenGrabbingActor](TestTokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"grabber_${name}_" + i) + TestActorRef[TestTokenGrabbingActor](TestTokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), + grabberSupervisor, + s"grabber_${name}_" + i + ) } // Force token distribution @@ -289,7 +331,7 @@ class JobTokenDispenserActorSpec extends TestKitSuite deathwatch watch actorToStop stopMethod(actorToStop) deathwatch.expectTerminated(actorToStop) - eventually { nextInLine.underlyingActor.hasToken shouldBe true } + eventually(nextInLine.underlyingActor.hasToken shouldBe true) } } @@ -297,12 +339,15 @@ class JobTokenDispenserActorSpec extends TestKitSuite val actorRefUnderTest = getActorRefUnderTest( serviceRegistryActorName = "serviceRegistryActor-skip-dead", - jobExecutionTokenDispenserActorName = "skip-dead", + jobExecutionTokenDispenserActorName = "skip-dead" ) val grabberSupervisor = TestActorRef(new StoppingSupervisor(), "skip-dead-supervisor") // The first 5 get a token and the 6th and 7h one are queued val tokenGrabbingActors = (1 to 7).map { i => - TestActorRef[TestTokenGrabbingActor](TestTokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"grabber_" + i) + TestActorRef[TestTokenGrabbingActor](TestTokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), + grabberSupervisor, + s"grabber_" + i + ) } // Force token distribution @@ -315,7 +360,11 @@ class JobTokenDispenserActorSpec extends TestKitSuite // Check that the next in lines have no tokens and are indeed in the queue nextInLine1.underlyingActor.hasToken shouldBe false nextInLine2.underlyingActor.hasToken shouldBe false - actorRefUnderTest.underlyingActor.tokenQueues(LimitedTo5Tokens).queues.flatMap(_._2).toList should contain theSameElementsInOrderAs List(nextInLine1, nextInLine2).map(asHogGroupAPlaceholder) + actorRefUnderTest.underlyingActor + .tokenQueues(LimitedTo5Tokens) + .queues + .flatMap(_._2) + .toList should contain theSameElementsInOrderAs List(nextInLine1, nextInLine2).map(asHogGroupAPlaceholder) // First, kill off the actor which would otherwise be first in line: val deathwatch = TestProbe("death-watch-skip-dead") @@ -327,19 +376,22 @@ class JobTokenDispenserActorSpec extends TestKitSuite actorRefUnderTest.tell(msg = JobTokenReturn, sender = tokenGrabbingActors.head) // Force token distribution actorRefUnderTest ! TokensAvailable(1) - eventually { nextInLine2.underlyingActor.hasToken shouldBe true } + eventually(nextInLine2.underlyingActor.hasToken shouldBe true) } it should "skip over dead actors repeatedly when assigning tokens to the actor queue" in { val actorRefUnderTest = getActorRefUnderTest( serviceRegistryActorName = "serviceRegistryActor-skip-dead-repeatedly", - jobExecutionTokenDispenserActorName = "skip-dead-repeatedly", + jobExecutionTokenDispenserActorName = "skip-dead-repeatedly" ) val grabberSupervisor = TestActorRef(new StoppingSupervisor(), "skip-dead-repeatedly-supervisor") // The first 5 get a token and the 6th and 7th one are queued val tokenGrabbingActors = (0 until 1000).toVector.map { i => - TestActorRef[TestTokenGrabbingActor](TestTokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"grabber_" + i) + TestActorRef[TestTokenGrabbingActor](TestTokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), + grabberSupervisor, + s"grabber_" + i + ) } // Create a sliding window of 10 actors, skipping by 10 so the windows do not overlap. @@ -374,7 +426,7 @@ class JobTokenDispenserActorSpec extends TestKitSuite actorRefUnderTest.tell(msg = JobTokenReturn, sender = withTokens(3)) actorRefUnderTest ! TokensAvailable(100) - eventually { nextInLine(3).underlyingActor.hasToken shouldBe true } + eventually(nextInLine(3).underlyingActor.hasToken shouldBe true) // And kill off the rest of the actors: (withTokens :+ nextInLine(3)) foreach { actor => actor ! PoisonPill } @@ -387,7 +439,7 @@ class JobTokenDispenserActorSpec extends TestKitSuite val actorRefUnderTest = getActorRefUnderTest( serviceRegistryActorName = "serviceRegistryActor-resilient-last-request", - jobExecutionTokenDispenserActorName = "resilient-last-request", + jobExecutionTokenDispenserActorName = "resilient-last-request" ) val tokenType = JobTokenType(s"mini", maxPoolSize = Option(6), hogFactor = 2) @@ -404,7 +456,9 @@ class JobTokenDispenserActorSpec extends TestKitSuite actorRefUnderTest.underlyingActor.tokenAssignments.keys should contain(probe.ref) } // And both groups should have one item in the queue: - actorRefUnderTest.underlyingActor.tokenQueues(tokenType).queues.values.foreach { queue => queue.size should be(1) } + actorRefUnderTest.underlyingActor.tokenQueues(tokenType).queues.values.foreach { queue => + queue.size should be(1) + } } // Group B gets bored and aborts all jobs (in reverse order to make sure ): @@ -430,6 +484,7 @@ object JobTokenDispenserActorSpec { } val TestInfiniteTokenType: JobTokenType = JobTokenType("infinite", maxPoolSize = None, hogFactor = 1) - def limitedTokenType(limit: Int): JobTokenType = JobTokenType(s"$limit-limit", maxPoolSize = Option(limit), hogFactor = 1) + def limitedTokenType(limit: Int): JobTokenType = + JobTokenType(s"$limit-limit", maxPoolSize = Option(limit), hogFactor = 1) val LimitedTo5Tokens: JobTokenType = limitedTokenType(5) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/RoundRobinQueueIteratorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/RoundRobinQueueIteratorSpec.scala index b2c22f3bebc..d26cde04faf 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/RoundRobinQueueIteratorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/RoundRobinQueueIteratorSpec.scala @@ -40,7 +40,9 @@ class RoundRobinQueueIteratorSpec extends TestKitSuite with AnyFlatSpecLike with val probe2 = TestProbe("probe-2").ref val probe3 = TestProbe("probe-3").ref val queues = List( - TokenQueue(InfiniteTokenType, tokenEventLogger).enqueue(TokenQueuePlaceholder(probe1, "hogGroupA")).enqueue(TokenQueuePlaceholder(probe3, "hogGroupA")), + TokenQueue(InfiniteTokenType, tokenEventLogger) + .enqueue(TokenQueuePlaceholder(probe1, "hogGroupA")) + .enqueue(TokenQueuePlaceholder(probe3, "hogGroupA")), TokenQueue(Pool2, tokenEventLogger).enqueue(TokenQueuePlaceholder(probe2, "hogGroupA")) ) val iterator = new RoundRobinQueueIterator(queues, 0) diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala index caaef991994..2fe028910ef 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala @@ -13,8 +13,8 @@ class TestTokenGrabbingActor(tokenDispenser: ActorRef, tokenType: JobTokenType) var hasToken: Boolean = false - override def receive = stoppingReceive orElse { - case JobTokenDispensed => hasToken = true + override def receive = stoppingReceive orElse { case JobTokenDispensed => + hasToken = true } tokenDispenser ! JobTokenRequest(HogGroup("hogGroupA"), tokenType) @@ -22,7 +22,9 @@ class TestTokenGrabbingActor(tokenDispenser: ActorRef, tokenType: JobTokenType) object TestTokenGrabbingActor { - def props(tokenDispenserActor: ActorRef, tokenType: JobTokenType) = Props(new TestTokenGrabbingActor(tokenDispenserActor, tokenType)) + def props(tokenDispenserActor: ActorRef, tokenType: JobTokenType) = Props( + new TestTokenGrabbingActor(tokenDispenserActor, tokenType) + ) class StoppingSupervisor extends Actor { override val supervisorStrategy = SupervisorStrategy.stoppingStrategy diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenQueueSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenQueueSpec.scala index 6786c02ae83..306e589a241 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenQueueSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenQueueSpec.scala @@ -156,7 +156,7 @@ class TokenQueueSpec extends TestKitSuite with AnyFlatSpecLike with Matchers { val expectedOrder = (0 until 23).toVector.map(i => s"hogGroup${i + 2}") ++ Vector("hogGroup0", "hogGroup1") usedQueue.queueOrder should be(expectedOrder) - usedQueue.size should be(jobCount - poolSize) + usedQueue.size should be(jobCount - poolSize) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/UnhoggableTokenPoolSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/UnhoggableTokenPoolSpec.scala index f9bbd73fd34..d120b87b605 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/UnhoggableTokenPoolSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/UnhoggableTokenPoolSpec.scala @@ -2,7 +2,12 @@ package cromwell.engine.workflow.tokens import common.assertion.CromwellTimeoutSpec import cromwell.core.JobToken.JobTokenType -import cromwell.engine.workflow.tokens.UnhoggableTokenPool.{HogLimitExceeded, TokenHoggingLease, TokenTypeExhausted, TokensAvailable} +import cromwell.engine.workflow.tokens.UnhoggableTokenPool.{ + HogLimitExceeded, + TokenHoggingLease, + TokensAvailable, + TokenTypeExhausted +} import org.scalatest.concurrent.Eventually import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -25,7 +30,7 @@ class UnhoggableTokenPoolSpec extends AnyFlatSpec with CromwellTimeoutSpec with JobTokenType("backend", Some(150), 200) -> Some(1), JobTokenType("backend", None, 1) -> None, JobTokenType("backend", None, 150) -> None - ) ++ (hogLimitingTokenTypeToHogLimit map { case (k,v) => (k, Some(v)) }) + ) ++ (hogLimitingTokenTypeToHogLimit map { case (k, v) => (k, Some(v)) }) tokenTypeToHogLimit foreach { case (tokenType, expectedHogLimit) => it should s"correctly calculate hogLimit for $tokenType as $expectedHogLimit" in { @@ -44,8 +49,12 @@ class UnhoggableTokenPoolSpec extends AnyFlatSpec with CromwellTimeoutSpec with (0 until hogLimit) foreach { index => pool.tryAcquire(hogGroup) match { case _: TokenHoggingLease => // great! - case TokenTypeExhausted => fail(s"Unhoggable token pool ran out after $index tokens distributed to $hogGroupNumber") - case HogLimitExceeded => fail(s"Unhoggable token pool making unfounded accusations of hogging after $index tokens distributed to $hogGroupNumber") + case TokenTypeExhausted => + fail(s"Unhoggable token pool ran out after $index tokens distributed to $hogGroupNumber") + case HogLimitExceeded => + fail( + s"Unhoggable token pool making unfounded accusations of hogging after $index tokens distributed to $hogGroupNumber" + ) } val acquiredTokensForGroup = index + 1 @@ -89,7 +98,7 @@ class UnhoggableTokenPoolSpec extends AnyFlatSpec with CromwellTimeoutSpec with hogLimitPool.tryAcquire("group1") should be(HogLimitExceeded) lease1.release() - eventually { hogLimitPool.available("group1") shouldBe TokensAvailable } + eventually(hogLimitPool.available("group1") shouldBe TokensAvailable) hogLimitPool.tryAcquire("group1") match { case _: TokenHoggingLease => // Great! case other => fail(s"expected lease but got $other") @@ -97,7 +106,7 @@ class UnhoggableTokenPoolSpec extends AnyFlatSpec with CromwellTimeoutSpec with hogLimitPool.tryAcquire("group1") should be(HogLimitExceeded) lease2.release() - eventually { hogLimitPool.available("group1") shouldBe TokensAvailable } + eventually(hogLimitPool.available("group1") shouldBe TokensAvailable) hogLimitPool.tryAcquire("group1") match { case _: TokenHoggingLease => // Great! case other => fail(s"expected lease but got $other") diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/large/LargeScaleJobTokenDispenserActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/large/LargeScaleJobTokenDispenserActorSpec.scala index af93aaaa73e..779dd98065a 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/large/LargeScaleJobTokenDispenserActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/large/LargeScaleJobTokenDispenserActorSpec.scala @@ -17,7 +17,14 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.duration._ -class LargeScaleJobTokenDispenserActorSpec extends TestKit(ActorSystem("LSJETDASpec")) with ImplicitSender with AnyFlatSpecLike with Matchers with BeforeAndAfter with BeforeAndAfterAll with Eventually { +class LargeScaleJobTokenDispenserActorSpec + extends TestKit(ActorSystem("LSJETDASpec")) + with ImplicitSender + with AnyFlatSpecLike + with Matchers + with BeforeAndAfter + with BeforeAndAfterAll + with Eventually { val multipleTokenUsingActorIndex: AtomicInteger = new AtomicInteger(0) def multipleTokenUsingActorName() = s"multipleTokenUsingActor${multipleTokenUsingActorIndex.getAndIncrement()}" @@ -32,14 +39,35 @@ class LargeScaleJobTokenDispenserActorSpec extends TestKit(ActorSystem("LSJETDAS val totalJobsPerWorkflow = maxConcurrencyToTest + 1 val tokenType = JobTokenType(backendName, Some(maxConcurrencyToTest), hogFactor) - val tokenDispenserUnderTest = TestActorRef(new JobTokenDispenserActor(TestProbe().ref, Rate(maxConcurrencyToTest + 1, 100.millis), None, - dispenserType = "execution", - tokenAllocatedDescription = "Running" - ), "tokenDispenserUnderTest1") + val tokenDispenserUnderTest = TestActorRef( + new JobTokenDispenserActor(TestProbe().ref, + Rate(maxConcurrencyToTest + 1, 100.millis), + None, + dispenserType = "execution", + tokenAllocatedDescription = "Running" + ), + "tokenDispenserUnderTest1" + ) val globalRunningJobsCounter = new RunningJobCounter() - val bigWorkflow1 = TestActorRef(new MultipleTokenUsingActor(tokenDispenserUnderTest, tokenType, totalJobsPerWorkflow, hogGroup = "hogGroupA", globalRunningJobsCounter), multipleTokenUsingActorName()) - val bigWorkflow2 = TestActorRef(new MultipleTokenUsingActor(tokenDispenserUnderTest, tokenType, totalJobsPerWorkflow, hogGroup = "hogGroupA", globalRunningJobsCounter), multipleTokenUsingActorName()) + val bigWorkflow1 = TestActorRef( + new MultipleTokenUsingActor(tokenDispenserUnderTest, + tokenType, + totalJobsPerWorkflow, + hogGroup = "hogGroupA", + globalRunningJobsCounter + ), + multipleTokenUsingActorName() + ) + val bigWorkflow2 = TestActorRef( + new MultipleTokenUsingActor(tokenDispenserUnderTest, + tokenType, + totalJobsPerWorkflow, + hogGroup = "hogGroupA", + globalRunningJobsCounter + ), + multipleTokenUsingActorName() + ) val parentProbe = new TestProbe(system, "parent") @@ -47,11 +75,12 @@ class LargeScaleJobTokenDispenserActorSpec extends TestKit(ActorSystem("LSJETDAS parentProbe.send(bigWorkflow2, Begin) (0 until 2) foreach { _ => - parentProbe.expectMsgPF(100.seconds) { - case TokenUsingActorCompletion(queueWaits, maximumConcurrency, errors) => - Assertions.assert(maximumConcurrency <= maxConcurrencyToTest, "(asserting maxActualConcurrency <= maxRequestedConcurrency)") - queueWaits.size should be(totalJobsPerWorkflow) - errors shouldBe List.empty + parentProbe.expectMsgPF(100.seconds) { case TokenUsingActorCompletion(queueWaits, maximumConcurrency, errors) => + Assertions.assert(maximumConcurrency <= maxConcurrencyToTest, + "(asserting maxActualConcurrency <= maxRequestedConcurrency)" + ) + queueWaits.size should be(totalJobsPerWorkflow) + errors shouldBe List.empty } } @@ -69,14 +98,35 @@ class LargeScaleJobTokenDispenserActorSpec extends TestKit(ActorSystem("LSJETDAS val totalJobsPerWorkflow = maxConcurrencyExpected + 1 val tokenType = JobTokenType(backendName, Some(totalTokensAvailable), hogFactor) - val tokenDispenserUnderTest = TestActorRef(new JobTokenDispenserActor(TestProbe().ref, Rate(maxConcurrencyExpected + 1, 100.millis), None, - dispenserType = "execution", - tokenAllocatedDescription = "Running" - ), "tokenDispenserUnderTest2") + val tokenDispenserUnderTest = TestActorRef( + new JobTokenDispenserActor(TestProbe().ref, + Rate(maxConcurrencyExpected + 1, 100.millis), + None, + dispenserType = "execution", + tokenAllocatedDescription = "Running" + ), + "tokenDispenserUnderTest2" + ) val globalRunningJobsCounter = new RunningJobCounter() - val bigWorkflow1 = TestActorRef(new MultipleTokenUsingActor(tokenDispenserUnderTest, tokenType, totalJobsPerWorkflow, hogGroup = "hogGroupA", globalRunningJobsCounter), multipleTokenUsingActorName()) - val bigWorkflow2 = TestActorRef(new MultipleTokenUsingActor(tokenDispenserUnderTest, tokenType, totalJobsPerWorkflow, hogGroup = "hogGroupA", globalRunningJobsCounter), multipleTokenUsingActorName()) + val bigWorkflow1 = TestActorRef( + new MultipleTokenUsingActor(tokenDispenserUnderTest, + tokenType, + totalJobsPerWorkflow, + hogGroup = "hogGroupA", + globalRunningJobsCounter + ), + multipleTokenUsingActorName() + ) + val bigWorkflow2 = TestActorRef( + new MultipleTokenUsingActor(tokenDispenserUnderTest, + tokenType, + totalJobsPerWorkflow, + hogGroup = "hogGroupA", + globalRunningJobsCounter + ), + multipleTokenUsingActorName() + ) val parentProbe = new TestProbe(system, "parent") @@ -84,11 +134,12 @@ class LargeScaleJobTokenDispenserActorSpec extends TestKit(ActorSystem("LSJETDAS parentProbe.send(bigWorkflow2, Begin) (0 until 2) foreach { _ => - parentProbe.expectMsgPF(100.seconds) { - case TokenUsingActorCompletion(queueWaits, maximumConcurrency, errors) => - Assertions.assert(maximumConcurrency <= maxConcurrencyExpected, "(asserting maxActualConcurrency <= maxRequestedConcurrency)") - queueWaits.size should be(totalJobsPerWorkflow) - errors shouldBe List.empty + parentProbe.expectMsgPF(100.seconds) { case TokenUsingActorCompletion(queueWaits, maximumConcurrency, errors) => + Assertions.assert(maximumConcurrency <= maxConcurrencyExpected, + "(asserting maxActualConcurrency <= maxRequestedConcurrency)" + ) + queueWaits.size should be(totalJobsPerWorkflow) + errors shouldBe List.empty } } @@ -106,14 +157,35 @@ class LargeScaleJobTokenDispenserActorSpec extends TestKit(ActorSystem("LSJETDAS val totalJobsPerWorkflow = maxConcurrencyPerWorkflow + 1 val tokenType = JobTokenType(backendName, Some(totalTokensAvailable), hogFactor) - val tokenDispenserUnderTest = TestActorRef(new JobTokenDispenserActor(TestProbe().ref, Rate(maxConcurrencyOverall + 1, 100.millis), None, - dispenserType = "execution", - tokenAllocatedDescription = "Running" - ), "tokenDispenserUnderTest3") + val tokenDispenserUnderTest = TestActorRef( + new JobTokenDispenserActor(TestProbe().ref, + Rate(maxConcurrencyOverall + 1, 100.millis), + None, + dispenserType = "execution", + tokenAllocatedDescription = "Running" + ), + "tokenDispenserUnderTest3" + ) val globalRunningJobsCounter = new RunningJobCounter() - val bigWorkflow1 = TestActorRef(new MultipleTokenUsingActor(tokenDispenserUnderTest, tokenType, totalJobsPerWorkflow, hogGroup = "hogGroupA", globalRunningJobsCounter), multipleTokenUsingActorName()) - val bigWorkflow2 = TestActorRef(new MultipleTokenUsingActor(tokenDispenserUnderTest, tokenType, totalJobsPerWorkflow, hogGroup = "hogGroupB", globalRunningJobsCounter), multipleTokenUsingActorName()) + val bigWorkflow1 = TestActorRef( + new MultipleTokenUsingActor(tokenDispenserUnderTest, + tokenType, + totalJobsPerWorkflow, + hogGroup = "hogGroupA", + globalRunningJobsCounter + ), + multipleTokenUsingActorName() + ) + val bigWorkflow2 = TestActorRef( + new MultipleTokenUsingActor(tokenDispenserUnderTest, + tokenType, + totalJobsPerWorkflow, + hogGroup = "hogGroupB", + globalRunningJobsCounter + ), + multipleTokenUsingActorName() + ) val parentProbe = new TestProbe(system, "parent") @@ -121,11 +193,12 @@ class LargeScaleJobTokenDispenserActorSpec extends TestKit(ActorSystem("LSJETDAS parentProbe.send(bigWorkflow2, Begin) (0 until 2) foreach { _ => - parentProbe.expectMsgPF(100.seconds) { - case TokenUsingActorCompletion(queueWaits, maximumConcurrency, errors) => - Assertions.assert(maximumConcurrency == maxConcurrencyPerWorkflow, "(asserting maxActualConcurrency <= maxRequestedConcurrency)") - queueWaits.size should be(totalJobsPerWorkflow) - errors shouldBe List.empty + parentProbe.expectMsgPF(100.seconds) { case TokenUsingActorCompletion(queueWaits, maximumConcurrency, errors) => + Assertions.assert(maximumConcurrency == maxConcurrencyPerWorkflow, + "(asserting maxActualConcurrency <= maxRequestedConcurrency)" + ) + queueWaits.size should be(totalJobsPerWorkflow) + errors shouldBe List.empty } } @@ -143,27 +216,41 @@ class LargeScaleJobTokenDispenserActorSpec extends TestKit(ActorSystem("LSJETDAS val totalJobsPerWorkflow = maxConcurrencyPerWorkflow * 2 val tokenType = JobTokenType(backendName, Some(totalTokensAvailable), hogFactor) - val tokenDispenserUnderTest = TestActorRef(new JobTokenDispenserActor(TestProbe().ref, Rate(maxConcurrencyOverall + 1, 100.millis), None, - dispenserType = "execution", - tokenAllocatedDescription = "Running" - ), "tokenDispenserUnderTest4") + val tokenDispenserUnderTest = TestActorRef( + new JobTokenDispenserActor(TestProbe().ref, + Rate(maxConcurrencyOverall + 1, 100.millis), + None, + dispenserType = "execution", + tokenAllocatedDescription = "Running" + ), + "tokenDispenserUnderTest4" + ) val globalRunningJobsCounter = new RunningJobCounter() val workflows = (0 until 100) map { i => - TestActorRef(new MultipleTokenUsingActor(tokenDispenserUnderTest, tokenType, totalJobsPerWorkflow, hogGroup = s"hogGroup$i", globalRunningJobsCounter), multipleTokenUsingActorName()) + TestActorRef( + new MultipleTokenUsingActor(tokenDispenserUnderTest, + tokenType, + totalJobsPerWorkflow, + hogGroup = s"hogGroup$i", + globalRunningJobsCounter + ), + multipleTokenUsingActorName() + ) } val parentProbe = new TestProbe(system, "parent") - workflows foreach { parentProbe.send(_, Begin)} + workflows foreach { parentProbe.send(_, Begin) } workflows.indices foreach { _ => - parentProbe.expectMsgPF(100.seconds) { - case TokenUsingActorCompletion(queueWaits, maximumConcurrency, errors) => - Assertions.assert(maximumConcurrency == maxConcurrencyPerWorkflow, "(asserting maxActualConcurrency <= maxRequestedConcurrency)") - queueWaits.size should be(totalJobsPerWorkflow) - errors shouldBe List.empty + parentProbe.expectMsgPF(100.seconds) { case TokenUsingActorCompletion(queueWaits, maximumConcurrency, errors) => + Assertions.assert(maximumConcurrency == maxConcurrencyPerWorkflow, + "(asserting maxActualConcurrency <= maxRequestedConcurrency)" + ) + queueWaits.size should be(totalJobsPerWorkflow) + errors shouldBe List.empty } } @@ -182,28 +269,42 @@ class LargeScaleJobTokenDispenserActorSpec extends TestKit(ActorSystem("LSJETDAS val totalJobsPerWorkflow = maxConcurrencyPerHogGroup * 2 val tokenType = JobTokenType(backendName, Some(totalTokensAvailable), hogFactor) - val tokenDispenserUnderTest = TestActorRef(new JobTokenDispenserActor(TestProbe().ref, Rate(maxConcurrencyOverall + 1, 100.millis), None, - dispenserType = "execution", - tokenAllocatedDescription = "Running" - ), "tokenDispenserUnderTest5") + val tokenDispenserUnderTest = TestActorRef( + new JobTokenDispenserActor(TestProbe().ref, + Rate(maxConcurrencyOverall + 1, 100.millis), + None, + dispenserType = "execution", + tokenAllocatedDescription = "Running" + ), + "tokenDispenserUnderTest5" + ) val hogGroupConcurrencyCounters = (0 until totalHogGroups).toVector map { _ => new RunningJobCounter() } val workflows = (0 until totalWorkflows) map { i => val hogGroupNumber = i % totalHogGroups - TestActorRef(new MultipleTokenUsingActor(tokenDispenserUnderTest, tokenType, totalJobsPerWorkflow, hogGroup = s"hogGroup$hogGroupNumber", hogGroupConcurrencyCounters(hogGroupNumber)), multipleTokenUsingActorName()) + TestActorRef( + new MultipleTokenUsingActor(tokenDispenserUnderTest, + tokenType, + totalJobsPerWorkflow, + hogGroup = s"hogGroup$hogGroupNumber", + hogGroupConcurrencyCounters(hogGroupNumber) + ), + multipleTokenUsingActorName() + ) } val parentProbe = new TestProbe(system, "parent") - workflows foreach { parentProbe.send(_, Begin)} + workflows foreach { parentProbe.send(_, Begin) } workflows.indices foreach { _ => - parentProbe.expectMsgPF(100.seconds) { - case TokenUsingActorCompletion(queueWaits, maximumConcurrency, errors) => - Assertions.assert(maximumConcurrency <= maxConcurrencyPerHogGroup, "(asserting maxActualConcurrency per workflow <= maxRequestedConcurrency per hog group)") - queueWaits.size should be(totalJobsPerWorkflow) - errors shouldBe List.empty + parentProbe.expectMsgPF(100.seconds) { case TokenUsingActorCompletion(queueWaits, maximumConcurrency, errors) => + Assertions.assert(maximumConcurrency <= maxConcurrencyPerHogGroup, + "(asserting maxActualConcurrency per workflow <= maxRequestedConcurrency per hog group)" + ) + queueWaits.size should be(totalJobsPerWorkflow) + errors shouldBe List.empty } } @@ -219,12 +320,21 @@ class LargeScaleJobTokenDispenserActorSpec extends TestKit(ActorSystem("LSJETDAS (0 until totalHogGroups).toVector foreach { hogGroupNumber => val c: RunningJobCounter = hogGroupConcurrencyCounters(hogGroupNumber) - if (c.getMax == maxConcurrencyPerHogGroup) { exactlyAtLimit += 1 } else { - Assertions.assert(c.getMax <= maxConcurrencyPerHogGroup, s"(asserting maxActualConcurrency for each hog group <= maxRequestedConcurrency per hog group)") - Assertions.assert(c.getMax >= maxConcurrencyPerHogGroup * 0.95, s"(asserting maxActualConcurrency for each hog group >= (95% of maxRequestedConcurrency per hog group))") + if (c.getMax == maxConcurrencyPerHogGroup) { exactlyAtLimit += 1 } + else { + Assertions.assert( + c.getMax <= maxConcurrencyPerHogGroup, + s"(asserting maxActualConcurrency for each hog group <= maxRequestedConcurrency per hog group)" + ) + Assertions.assert( + c.getMax >= maxConcurrencyPerHogGroup * 0.95, + s"(asserting maxActualConcurrency for each hog group >= (95% of maxRequestedConcurrency per hog group))" + ) } } - Assertions.assert(exactlyAtLimit >= (totalHogGroups * 0.95), "(at least 95% of the hog groups reached the full concurrency limit)") + Assertions.assert(exactlyAtLimit >= (totalHogGroups * 0.95), + "(at least 95% of the hog groups reached the full concurrency limit)" + ) workflows foreach { system.stop(_) } system.stop(tokenDispenserUnderTest) diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/large/MultipleTokenUsingActor.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/large/MultipleTokenUsingActor.scala index 79bf6951948..c35315c19ca 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/large/MultipleTokenUsingActor.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/large/MultipleTokenUsingActor.scala @@ -1,6 +1,5 @@ package cromwell.engine.workflow.tokens.large - import akka.actor.{Actor, ActorRef} import cromwell.core.JobToken.JobTokenType import cromwell.engine.workflow.tokens.large.LargeScaleJobTokenDispenserActorSpec.RunningJobCounter @@ -15,7 +14,12 @@ import cromwell.engine.workflow.tokens.large.PatientTokenNeedingActor.{AllDone, * * Because I'm a good citizen, I'm going to record and return a value representing my "peak concurrent tokens distributed" */ -class MultipleTokenUsingActor(tokenDispenser: ActorRef, tokenType: JobTokenType, totalJobs: Int, hogGroup: String, globalRunningJobCounter: RunningJobCounter) extends Actor { +class MultipleTokenUsingActor(tokenDispenser: ActorRef, + tokenType: JobTokenType, + totalJobs: Int, + hogGroup: String, + globalRunningJobCounter: RunningJobCounter +) extends Actor { var hasToken: Boolean = false @@ -32,7 +36,9 @@ class MultipleTokenUsingActor(tokenDispenser: ActorRef, tokenType: JobTokenType, case Begin => starter = sender() (0 until totalJobs) foreach { i => - val jobActor = context.actorOf(PatientTokenNeedingActor.props(tokenDispenser, tokenType, hogGroup), name = self.path.name + s"job$i") + val jobActor = context.actorOf(PatientTokenNeedingActor.props(tokenDispenser, tokenType, hogGroup), + name = self.path.name + s"job$i" + ) jobActor ! Begin } startedJobs = totalJobs @@ -55,7 +61,6 @@ class MultipleTokenUsingActor(tokenDispenser: ActorRef, tokenType: JobTokenType, } } - object MultipleTokenUsingActor { final case class TokenUsingActorCompletion(queueWaits: Seq[Long], maximumConcurrency: Int, errors: List[String]) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/large/PatientTokenNeedingActor.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/large/PatientTokenNeedingActor.scala index debe5fe2537..817fa46934e 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/large/PatientTokenNeedingActor.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/large/PatientTokenNeedingActor.scala @@ -60,5 +60,7 @@ object PatientTokenNeedingActor { // Indicate to myself that I'm done (and gets forwarded to my parent) case object AllDone - def props(tokenDispenser: ActorRef, tokenType: JobTokenType, hogGroup: String): Props = Props(new PatientTokenNeedingActor(tokenDispenser, tokenType, hogGroup)) + def props(tokenDispenser: ActorRef, tokenType: JobTokenType, hogGroup: String): Props = Props( + new PatientTokenNeedingActor(tokenDispenser, tokenType, hogGroup) + ) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/large/TokenDispenserBenchmark.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/large/TokenDispenserBenchmark.scala index 8a5cbe6729f..590b5624b20 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/large/TokenDispenserBenchmark.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/large/TokenDispenserBenchmark.scala @@ -38,7 +38,7 @@ object TokenDispenserBenchmark extends Bench[Double] with DefaultJsonProtocol { def fillQueue(tokenQueueIn: TokenQueue, jobsPerGroup: Int, hogGroups: List[String]): TokenQueue = { var tokenQueue = tokenQueueIn hogGroups foreach { hogGroup => - (0 until jobsPerGroup) foreach { _ => + (0 until jobsPerGroup) foreach { _ => tokenQueue = tokenQueue.enqueue(TokenQueuePlaceholder(actorToQueue, hogGroup)) } } @@ -71,7 +71,8 @@ object TokenDispenserBenchmark extends Bench[Double] with DefaultJsonProtocol { measure method "enqueuing and dequeuing with multiple hog groups" in { val poolSize = 5 * ScaleFactor - val jobCounts: Gen[Int] = Gen.range("initialJobsInQueue")(from = 1 * ScaleFactor, upto = 15 * ScaleFactor, hop = 3 * ScaleFactor) + val jobCounts: Gen[Int] = + Gen.range("initialJobsInQueue")(from = 1 * ScaleFactor, upto = 15 * ScaleFactor, hop = 3 * ScaleFactor) val jobsAtATime = 50 val queues = for { diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala index fe51c9b065b..497482ddcc2 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala @@ -20,9 +20,12 @@ import spray.json.{JsObject, JsString} import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext, Future} - -class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalaFutures - with BeforeAndAfterAll { +class SqlWorkflowStoreSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalaFutures + with BeforeAndAfterAll { implicit val ec: ExecutionContextExecutor = ExecutionContext.global implicit val defaultPatience: PatienceConfig = PatienceConfig(scaled(Span(20, Seconds)), scaled(Span(100, Millis))) @@ -112,25 +115,27 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat ) DatabaseSystem.All foreach { databaseSystem => - behavior of s"SqlWorkflowStore on ${databaseSystem.name}" val containerOpt: Option[Container] = DatabaseTestKit.getDatabaseTestContainer(databaseSystem) - lazy val dataAccess = DatabaseTestKit.initializeDatabaseByContainerOptTypeAndSystem(containerOpt, EngineDatabaseType, databaseSystem) - lazy val metadataDataAccess = DatabaseTestKit.initializeDatabaseByContainerOptTypeAndSystem(containerOpt, MetadataDatabaseType, databaseSystem) + lazy val dataAccess = + DatabaseTestKit.initializeDatabaseByContainerOptTypeAndSystem(containerOpt, EngineDatabaseType, databaseSystem) + lazy val metadataDataAccess = + DatabaseTestKit.initializeDatabaseByContainerOptTypeAndSystem(containerOpt, MetadataDatabaseType, databaseSystem) lazy val workflowStore = SqlWorkflowStore(dataAccess, metadataDataAccess) - def updateWfToRunning(startableWorkflows: List[WorkflowToStart]): Unit = { + def updateWfToRunning(startableWorkflows: List[WorkflowToStart]): Unit = startableWorkflows.foreach { wf => Await.result(workflowStore.sqlDatabase.updateWorkflowState( - wf.id.toString, - WorkflowStoreState.Submitted.toString, - WorkflowStoreState.Running.toString - ), 5.seconds) + wf.id.toString, + WorkflowStoreState.Submitted.toString, + WorkflowStoreState.Running.toString + ), + 5.seconds + ) } - } it should "start container if required" taggedAs DbmsTest in { containerOpt.foreach { @@ -257,52 +262,72 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat it should "start workflows from hog group with lowest count of running workflows" taggedAs DbmsTest in { // first submission of 50 workflows for hogGroup "Goldfinger" - val goldFingerWorkflowIds = (for (_ <- 1 to 50) yield Await.result(workflowStore.add(includedGroupSourceFilesCollection1), 5.seconds)).flatMap(_.map(_.id).toList) + val goldFingerWorkflowIds = + (for (_ <- 1 to 50) yield Await.result(workflowStore.add(includedGroupSourceFilesCollection1), 5.seconds)) + .flatMap(_.map(_.id).toList) // second submission of 50 workflows for hogGroup "Highlander" - val highlanderWorkflowIds = (for (_ <- 1 to 50) yield Await.result(workflowStore.add(includedGroupSourceFilesCollection2), 5.seconds)).flatMap(_.map(_.id).toList) - - for (_ <- 1 to 10) yield { - (for { - // since both hog groups have 0 workflows running, the hog group with oldest submission time is picked first - startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String]) - _ = startableWorkflows1.map(_.hogGroup.value).toSet.head should be("Goldfinger") - _ = startableWorkflows1.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x)) - _ = updateWfToRunning(startableWorkflows1) - - startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String]) - _ = startableWorkflows2.map(_.hogGroup.value).toSet.head should be("Highlander") - _ = startableWorkflows2.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x)) - _ = updateWfToRunning(startableWorkflows2) - } yield ()).futureValue - } + val highlanderWorkflowIds = + (for (_ <- 1 to 50) yield Await.result(workflowStore.add(includedGroupSourceFilesCollection2), 5.seconds)) + .flatMap(_.map(_.id).toList) + + for (_ <- 1 to 10) yield (for { + // since both hog groups have 0 workflows running, the hog group with oldest submission time is picked first + startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String]) + _ = startableWorkflows1.map(_.hogGroup.value).toSet.head should be("Goldfinger") + _ = startableWorkflows1.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x)) + _ = updateWfToRunning(startableWorkflows1) + + startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, Set.empty[String]) + _ = startableWorkflows2.map(_.hogGroup.value).toSet.head should be("Highlander") + _ = startableWorkflows2.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x)) + _ = updateWfToRunning(startableWorkflows2) + } yield ()).futureValue // remove entries from WorkflowStore - (goldFingerWorkflowIds ++ highlanderWorkflowIds).foreach(id => Await.result(workflowStore.deleteFromStore(id), 5.seconds)) + (goldFingerWorkflowIds ++ highlanderWorkflowIds).foreach(id => + Await.result(workflowStore.deleteFromStore(id), 5.seconds) + ) } it should "respect excludedHogGroups and start workflows from hog group with lowest count of running workflows" taggedAs DbmsTest in { (for { // first submission of 10 workflows for hogGroup "Goldfinger" - goldFingerSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection1)) + goldFingerSubmissions <- Future.sequence( + for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection1) + ) goldFingerWorkflowIds = goldFingerSubmissions.flatMap(_.map(_.id).toList) // second submission of 10 workflows for hogGroup "Zardoz" - zardozSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(excludedGroupSourceFilesCollection)) + zardozSubmissions <- Future.sequence( + for (_ <- 1 to 10) yield workflowStore.add(excludedGroupSourceFilesCollection) + ) zardozWorkflowIds = zardozSubmissions.flatMap(_.map(_.id).toList) - startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set("Zardoz")) + startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, + "A08", + 5.minutes, + excludedGroups = Set("Zardoz") + ) _ = startableWorkflows1.map(_.hogGroup.value).toSet.head should be("Goldfinger") _ = startableWorkflows1.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x)) _ = updateWfToRunning(startableWorkflows1) - startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set("Zardoz")) + startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, + "A08", + 5.minutes, + excludedGroups = Set("Zardoz") + ) _ = startableWorkflows2.map(_.hogGroup.value).toSet.head should be("Goldfinger") _ = startableWorkflows2.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x)) _ = updateWfToRunning(startableWorkflows2) // there are 10 workflows from hog group "Zardoz" in the store, but since the group is excluded, 0 workflows are returned here - startableWorkflows3 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set("Zardoz")) + startableWorkflows3 <- workflowStore.fetchStartableWorkflows(5, + "A08", + 5.minutes, + excludedGroups = Set("Zardoz") + ) _ = startableWorkflows3.size should be(0) // hog group "Zardoz" has tokens to run workflows, hence don't exclude it @@ -319,59 +344,93 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat // remove entries from WorkflowStore workflowsList = goldFingerWorkflowIds ++ zardozWorkflowIds _ = workflowsList.foreach(id => Await.result(workflowStore.deleteFromStore(id), 5.seconds)) - } yield()).futureValue + } yield ()).futureValue } it should "start workflows from hog group with lowest count of running workflows for multiple hog groups" taggedAs DbmsTest in { (for { // first submission of 10 workflows for hogGroup "Goldfinger" - goldFingerSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection1)) + goldFingerSubmissions <- Future.sequence( + for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection1) + ) goldFingerWorkflowIds = goldFingerSubmissions.flatMap(_.map(_.id).toList) // second submission of 10 workflows for hogGroup "Highlander" - highlanderSubmissions <- Future.sequence(for (_ <- 1 to 15) yield workflowStore.add(includedGroupSourceFilesCollection2)) + highlanderSubmissions <- Future.sequence( + for (_ <- 1 to 15) yield workflowStore.add(includedGroupSourceFilesCollection2) + ) highlanderWorkflowIds = highlanderSubmissions.flatMap(_.map(_.id).toList) // since both hog groups have 0 workflows running, the hog group with oldest submission time is picked first - startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String]) + startableWorkflows1 <- workflowStore.fetchStartableWorkflows(5, + "A08", + 5.minutes, + excludedGroups = Set.empty[String] + ) _ = startableWorkflows1.map(_.hogGroup.value).toSet.head should be("Goldfinger") _ = startableWorkflows1.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x)) _ = updateWfToRunning(startableWorkflows1) - startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String]) + startableWorkflows2 <- workflowStore.fetchStartableWorkflows(5, + "A08", + 5.minutes, + excludedGroups = Set.empty[String] + ) _ = startableWorkflows2.map(_.hogGroup.value).toSet.head should be("Highlander") _ = startableWorkflows2.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x)) _ = updateWfToRunning(startableWorkflows2) // new submission for hog group "Finding Forrester" - foresterSubmissions <- Future.sequence(for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection3)) + foresterSubmissions <- Future.sequence( + for (_ <- 1 to 10) yield workflowStore.add(includedGroupSourceFilesCollection3) + ) foresterWorkflowIds = foresterSubmissions.flatMap(_.map(_.id).toList) // now hog group "Finding Forrester" has 0 workflows running, hence it is picked to run - startableWorkflows3 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String]) + startableWorkflows3 <- workflowStore.fetchStartableWorkflows(5, + "A08", + 5.minutes, + excludedGroups = Set.empty[String] + ) _ = startableWorkflows3.map(_.hogGroup.value).toSet.head should be("Finding Forrester") _ = startableWorkflows3.map(_.id).foreach(x => foresterWorkflowIds.toList should contain(x)) _ = updateWfToRunning(startableWorkflows3) // since all 3 hog groups have 5 workflows running each, the hog group with oldest submission time is picked first - startableWorkflows5 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String]) + startableWorkflows5 <- workflowStore.fetchStartableWorkflows(5, + "A08", + 5.minutes, + excludedGroups = Set.empty[String] + ) _ = startableWorkflows5.map(_.hogGroup.value).toSet.head should be("Goldfinger") _ = startableWorkflows5.map(_.id).foreach(x => goldFingerWorkflowIds.toList should contain(x)) _ = updateWfToRunning(startableWorkflows5) // since both "Highlander" and "Finding Forrester" have 5 workflows in Running state, the hog group with oldest submission time is picked first - startableWorkflows6 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String]) + startableWorkflows6 <- workflowStore.fetchStartableWorkflows(5, + "A08", + 5.minutes, + excludedGroups = Set.empty[String] + ) _ = startableWorkflows6.map(_.hogGroup.value).toSet.head should be("Highlander") _ = startableWorkflows6.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x)) _ = updateWfToRunning(startableWorkflows6) // "Finding Forrester" is now the hog group with least running workflows and has 5 more workflows to run, hence it is picked to run - startableWorkflows4 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String]) + startableWorkflows4 <- workflowStore.fetchStartableWorkflows(5, + "A08", + 5.minutes, + excludedGroups = Set.empty[String] + ) _ = startableWorkflows4.map(_.hogGroup.value).toSet.head should be("Finding Forrester") _ = startableWorkflows4.map(_.id).foreach(x => foresterWorkflowIds.toList should contain(x)) _ = updateWfToRunning(startableWorkflows4) - startableWorkflows7 <- workflowStore.fetchStartableWorkflows(5, "A08", 5.minutes, excludedGroups = Set.empty[String]) + startableWorkflows7 <- workflowStore.fetchStartableWorkflows(5, + "A08", + 5.minutes, + excludedGroups = Set.empty[String] + ) _ = startableWorkflows7.map(_.hogGroup.value).toSet.head should be("Highlander") _ = startableWorkflows7.map(_.id).foreach(x => highlanderWorkflowIds.toList should contain(x)) _ = updateWfToRunning(startableWorkflows7) @@ -385,10 +444,13 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat it should "accept and honor a requested workflow ID" taggedAs DbmsTest in { val requestedId = WorkflowId.randomId() - val sourcesToSubmit = onHoldSourceFilesCollection.map(c => c.asInstanceOf[WorkflowSourceFilesWithoutImports].copy( - requestedWorkflowId = Option(requestedId), - workflowOnHold = false - )) + val sourcesToSubmit = onHoldSourceFilesCollection.map(c => + c.asInstanceOf[WorkflowSourceFilesWithoutImports] + .copy( + requestedWorkflowId = Option(requestedId), + workflowOnHold = false + ) + ) (for { submissionResponses <- workflowStore.add(sourcesToSubmit) @@ -402,9 +464,11 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat it should "not accept a duplicate workflow ID" taggedAs DbmsTest in { val requestedId = WorkflowId.randomId() - val workflowSourceFilesTemplate = onHoldSourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports].copy( - requestedWorkflowId = Option(requestedId) - ) + val workflowSourceFilesTemplate = onHoldSourceFilesCollection.head + .asInstanceOf[WorkflowSourceFilesWithoutImports] + .copy( + requestedWorkflowId = Option(requestedId) + ) val sourcesToSubmit1 = NonEmptyList.of(workflowSourceFilesTemplate) val sourcesToSubmit2 = NonEmptyList.of(workflowSourceFilesTemplate.copy(workflowOnHold = false)) @@ -412,14 +476,16 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat ((for { _ <- workflowStore.add(sourcesToSubmit1) _ <- workflowStore.add(sourcesToSubmit2) - } yield "incorrectly accepted") recoverWith { - case error => for { + } yield "incorrectly accepted") recoverWith { case error => + for { message <- Future { error.getMessage should be(s"Requested workflow IDs are already in use: $requestedId") "duplicate ID correctly detected" } stats <- workflowStore.stats - _ = stats should be(Map(WorkflowStoreState.OnHold -> 1)) // Only the original (on-hold) version of requested ID 1 should be in the store + _ = stats should be( + Map(WorkflowStoreState.OnHold -> 1) + ) // Only the original (on-hold) version of requested ID 1 should be in the store _ <- workflowStore.deleteFromStore(requestedId) // tidy up } yield message }).futureValue should be("duplicate ID correctly detected") @@ -430,29 +496,35 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat val requestedId2 = WorkflowId.randomId() val requestedId3 = WorkflowId.randomId() - val workflowSourceFilesTemplate = onHoldSourceFilesCollection.head.asInstanceOf[WorkflowSourceFilesWithoutImports].copy( - requestedWorkflowId = Option(requestedId1) - ) + val workflowSourceFilesTemplate = onHoldSourceFilesCollection.head + .asInstanceOf[WorkflowSourceFilesWithoutImports] + .copy( + requestedWorkflowId = Option(requestedId1) + ) val sourcesToSubmit1 = NonEmptyList.of(workflowSourceFilesTemplate) val sourcesToSubmit2 = NonEmptyList.of( workflowSourceFilesTemplate.copy(requestedWorkflowId = Option(requestedId2), workflowOnHold = false), workflowSourceFilesTemplate.copy(requestedWorkflowId = Option(requestedId3), workflowOnHold = false), - workflowSourceFilesTemplate.copy(requestedWorkflowId = Option(requestedId1), workflowOnHold = false) // duplicates the existing ID. + workflowSourceFilesTemplate.copy(requestedWorkflowId = Option(requestedId1), + workflowOnHold = false + ) // duplicates the existing ID. ) ((for { _ <- workflowStore.add(sourcesToSubmit1) _ <- workflowStore.add(sourcesToSubmit2) - } yield "incorrectly accepted") recoverWith { - case error => for { + } yield "incorrectly accepted") recoverWith { case error => + for { message <- Future { error.getMessage should be(s"Requested workflow IDs are already in use: $requestedId1") "duplicate ID correctly detected" } stats <- workflowStore.stats - _ = stats should be(Map(WorkflowStoreState.OnHold -> 1)) // Only the original (on-hold) version of requested ID 1 should be in the store + _ = stats should be( + Map(WorkflowStoreState.OnHold -> 1) + ) // Only the original (on-hold) version of requested ID 1 should be in the store _ <- workflowStore.deleteFromStore(requestedId1) } yield message @@ -470,13 +542,15 @@ class SqlWorkflowStoreSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat workflowSourceFilesTemplate.copy(requestedWorkflowId = Option(requestedId1), workflowOnHold = false), workflowSourceFilesTemplate.copy(requestedWorkflowId = Option(requestedId2), workflowOnHold = false), workflowSourceFilesTemplate.copy(requestedWorkflowId = Option(requestedId3), workflowOnHold = false), - workflowSourceFilesTemplate.copy(requestedWorkflowId = Option(requestedId1), workflowOnHold = false) // duplicates an ID already in the set + workflowSourceFilesTemplate.copy(requestedWorkflowId = Option(requestedId1), + workflowOnHold = false + ) // duplicates an ID already in the set ) ((for { _ <- workflowStore.add(sourcesToSubmit) - } yield "incorrectly accepted") recoverWith { - case error => for { + } yield "incorrectly accepted") recoverWith { case error => + for { message <- Future { error.getMessage should be(s"Requested workflow IDs are duplicated: $requestedId1") "duplicate ID correctly detected" diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowHeartbeatConfigSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowHeartbeatConfigSpec.scala index fe4b951ec06..768731b815c 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowHeartbeatConfigSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowHeartbeatConfigSpec.scala @@ -9,7 +9,11 @@ import org.scalatest.prop.TableDrivenPropertyChecks import scala.concurrent.duration._ -class WorkflowHeartbeatConfigSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with TableDrivenPropertyChecks { +class WorkflowHeartbeatConfigSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with TableDrivenPropertyChecks { behavior of "WorkflowHeartbeatConfig" @@ -17,7 +21,7 @@ class WorkflowHeartbeatConfigSpec extends AnyFlatSpec with CromwellTimeoutSpec w val workflowHeartbeatConfig = WorkflowHeartbeatConfig(WorkflowHeartbeatConfigSpec.DefaultConfig) workflowHeartbeatConfig.cromwellId should startWith("cromid-") workflowHeartbeatConfig.cromwellId should have length 14 - workflowHeartbeatConfig.heartbeatInterval should be (2.minutes) + workflowHeartbeatConfig.heartbeatInterval should be(2.minutes) workflowHeartbeatConfig.ttl should be(10.minutes) workflowHeartbeatConfig.failureShutdownDuration should be(5.minutes) workflowHeartbeatConfig.writeBatchSize should be(10000) @@ -29,22 +33,22 @@ class WorkflowHeartbeatConfigSpec extends AnyFlatSpec with CromwellTimeoutSpec w ( "from an empty config", "system.cromwell_id_random_suffix = false", - WorkflowHeartbeatConfig("cromid", 2.minutes, 10.minutes, 5.minutes, 10000, 10000), + WorkflowHeartbeatConfig("cromid", 2.minutes, 10.minutes, 5.minutes, 10000, 10000) ), ( "with a specified cromid", """system.cromwell_id = "new_crom_name"""", - WorkflowHeartbeatConfig("new_crom_name", 2.minutes, 10.minutes, 5.minutes, 10000, 10000), + WorkflowHeartbeatConfig("new_crom_name", 2.minutes, 10.minutes, 5.minutes, 10000, 10000) ), ( "with a specified heartbeat interval", "system.workflow-heartbeats.heartbeat-interval = 3 minutes", - WorkflowHeartbeatConfig("cromid", 3.minutes, 10.minutes, 5.minutes, 10000, 10000), + WorkflowHeartbeatConfig("cromid", 3.minutes, 10.minutes, 5.minutes, 10000, 10000) ), ( "with a specified ttl", "system.workflow-heartbeats.ttl = 5 minutes", - WorkflowHeartbeatConfig("cromid", 2.minutes, 5.minutes, 5.minutes, 10000, 10000), + WorkflowHeartbeatConfig("cromid", 2.minutes, 5.minutes, 5.minutes, 10000, 10000) ), ( "with a ttl less than the default heartbeat interval", @@ -52,22 +56,22 @@ class WorkflowHeartbeatConfigSpec extends AnyFlatSpec with CromwellTimeoutSpec w |system.workflow-heartbeats.heartbeat-interval = 59 seconds |system.workflow-heartbeats.write-failure-shutdown-duration = 0 minutes |""".stripMargin, - WorkflowHeartbeatConfig("cromid", 59.seconds, 1.minutes, 0.minutes, 10000, 10000), + WorkflowHeartbeatConfig("cromid", 59.seconds, 1.minutes, 0.minutes, 10000, 10000) ), ( "with a specified shutdown duration", "system.workflow-heartbeats.write-failure-shutdown-duration = 1 minute", - WorkflowHeartbeatConfig("cromid", 2.minutes, 10.minutes, 1.minute, 10000, 10000), + WorkflowHeartbeatConfig("cromid", 2.minutes, 10.minutes, 1.minute, 10000, 10000) ), ( "with a specified batch size", "system.workflow-heartbeats.write-batch-size = 2000", - WorkflowHeartbeatConfig("cromid", 2.minutes, 10.minutes, 5.minutes, 2000, 10000), + WorkflowHeartbeatConfig("cromid", 2.minutes, 10.minutes, 5.minutes, 2000, 10000) ), ( "with a specified threshold", "system.workflow-heartbeats.write-threshold = 5000", - WorkflowHeartbeatConfig("cromid", 2.minutes, 10.minutes, 5.minutes, 10000, 5000), + WorkflowHeartbeatConfig("cromid", 2.minutes, 10.minutes, 5.minutes, 10000, 5000) ), ( "when trying to set the ttl below the minimum", @@ -75,28 +79,30 @@ class WorkflowHeartbeatConfigSpec extends AnyFlatSpec with CromwellTimeoutSpec w |system.workflow-heartbeats.heartbeat-interval = 8 seconds |system.workflow-heartbeats.write-failure-shutdown-duration = 0 minutes |""".stripMargin, - WorkflowHeartbeatConfig("cromid", 8.seconds, 10.seconds, 0.minutes, 10000, 10000), + WorkflowHeartbeatConfig("cromid", 8.seconds, 10.seconds, 0.minutes, 10000, 10000) ), ( "when trying to set the interval below the minimum", "system.workflow-heartbeats.heartbeat-interval = 3 seconds", - WorkflowHeartbeatConfig("cromid", 10.seconds / 3, 10.minutes, 5.minutes, 10000, 10000), + WorkflowHeartbeatConfig("cromid", 10.seconds / 3, 10.minutes, 5.minutes, 10000, 10000) ), ( "when trying to set a negative shutdown duration", "system.workflow-heartbeats.write-failure-shutdown-duration = -1 seconds", - WorkflowHeartbeatConfig("cromid", 2.minutes, 10.minutes, 0.minutes, 10000, 10000), - ), + WorkflowHeartbeatConfig("cromid", 2.minutes, 10.minutes, 0.minutes, 10000, 10000) + ) ) forAll(validConfigTests) { (description, configString, expected) => it should s"create an instance $description" in { - val config = ConfigFactory.parseString( - // Remove the randomness from the cromid - s"""|system.cromwell_id_random_suffix = false - |$configString - |""".stripMargin - ).withFallback(WorkflowHeartbeatConfigSpec.DefaultConfig) + val config = ConfigFactory + .parseString( + // Remove the randomness from the cromid + s"""|system.cromwell_id_random_suffix = false + |$configString + |""".stripMargin + ) + .withFallback(WorkflowHeartbeatConfigSpec.DefaultConfig) WorkflowHeartbeatConfig(config) should be(expected) } } @@ -114,7 +120,7 @@ class WorkflowHeartbeatConfigSpec extends AnyFlatSpec with CromwellTimeoutSpec w "Errors parsing WorkflowHeartbeatConfig", List( "The system.workflow-heartbeats.heartbeat-interval (2 minutes)" + - " is not less than the system.workflow-heartbeats.ttl (2 minutes).", + " is not less than the system.workflow-heartbeats.ttl (2 minutes)." ) ) ), @@ -129,7 +135,7 @@ class WorkflowHeartbeatConfigSpec extends AnyFlatSpec with CromwellTimeoutSpec w "Errors parsing WorkflowHeartbeatConfig", List( "The system.workflow-heartbeats.heartbeat-interval (2 minutes)" + - " is not less than the system.workflow-heartbeats.ttl (1 minute).", + " is not less than the system.workflow-heartbeats.ttl (1 minute)." ) ) ), @@ -144,10 +150,10 @@ class WorkflowHeartbeatConfigSpec extends AnyFlatSpec with CromwellTimeoutSpec w "Errors parsing WorkflowHeartbeatConfig", List( "The system.workflow-heartbeats.write-failure-shutdown-duration (301 seconds)" + - " is greater than the system.workflow-heartbeats.ttl (5 minutes).", + " is greater than the system.workflow-heartbeats.ttl (5 minutes)." ) ) - ), + ) ) forAll(invalidConfigTests) { (description, configString, expected: AggregatedMessageException) => diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActorSpec.scala index b68a7f9e64b..dcc4f87f897 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreCoordinatedAccessActorSpec.scala @@ -10,7 +10,12 @@ import cats.data.NonEmptyVector import cromwell.core._ import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.WorkflowStoreAbortResponse import cromwell.engine.workflow.workflowstore.SqlWorkflowStore.WorkflowStoreAbortResponse.WorkflowStoreAbortResponse -import cromwell.engine.workflow.workflowstore.WorkflowStoreCoordinatedAccessActor.{Abort, DeleteFromStore, FetchStartableWorkflows, WriteHeartbeats} +import cromwell.engine.workflow.workflowstore.WorkflowStoreCoordinatedAccessActor.{ + Abort, + DeleteFromStore, + FetchStartableWorkflows, + WriteHeartbeats +} import org.scalatest.flatspec.AsyncFlatSpecLike import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks @@ -18,13 +23,16 @@ import org.scalatest.prop.TableDrivenPropertyChecks import scala.concurrent.duration._ import scala.concurrent.{ExecutionContext, Future} -class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite - with AsyncFlatSpecLike with Matchers with TableDrivenPropertyChecks { +class WorkflowStoreCoordinatedAccessActorSpec + extends TestKitSuite + with AsyncFlatSpecLike + with Matchers + with TableDrivenPropertyChecks { behavior of "WorkflowStoreCoordinatedWriteActor" // So that we can timeout the asks below, change from the serial execution context to a parallel one - override implicit def executionContext: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global + implicit override def executionContext: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global def sleepAndThrow: Nothing = { Thread.sleep(30.seconds.dilated.toMillis) @@ -35,10 +43,9 @@ class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite val expected = 12345 val workflowStore = new InMemoryWorkflowStore { override def writeWorkflowHeartbeats(workflowIds: Set[(WorkflowId, OffsetDateTime)], - heartbeatDateTime: OffsetDateTime) - (implicit ec: ExecutionContext): Future[Int] = { + heartbeatDateTime: OffsetDateTime + )(implicit ec: ExecutionContext): Future[Int] = Future.successful(expected) - } } val actor = TestActorRef(new WorkflowStoreCoordinatedAccessActor(workflowStore)) val request = WriteHeartbeats(NonEmptyVector.of((WorkflowId.randomId(), OffsetDateTime.now)), OffsetDateTime.now) @@ -64,12 +71,15 @@ class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite requestedWorkflowId = None ) val now = OffsetDateTime.now() - val expected: List[WorkflowToStart] = List(WorkflowToStart(WorkflowId.randomId(), now, collection, Submitted, HogGroup("foo"))) + val expected: List[WorkflowToStart] = + List(WorkflowToStart(WorkflowId.randomId(), now, collection, Submitted, HogGroup("foo"))) val workflowStore = new InMemoryWorkflowStore { - override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String]) - (implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = { + override def fetchStartableWorkflows(n: Int, + cromwellId: String, + heartbeatTtl: FiniteDuration, + excludedGroups: Set[String] + )(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = Future.successful(expected) - } } val actor = TestActorRef(new WorkflowStoreCoordinatedAccessActor(workflowStore)) val request = FetchStartableWorkflows(1, "test fetchStartableWorkflows success", 1.second, Set.empty) @@ -95,15 +105,19 @@ class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite requestedWorkflowId = None ) val now = OffsetDateTime.now() - val expected: List[WorkflowToStart] = List(WorkflowToStart(WorkflowId.randomId(), now, collection, Submitted, HogGroup("foo"))) + val expected: List[WorkflowToStart] = + List(WorkflowToStart(WorkflowId.randomId(), now, collection, Submitted, HogGroup("foo"))) val workflowStore = new InMemoryWorkflowStore { - override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String]) - (implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = { + override def fetchStartableWorkflows(n: Int, + cromwellId: String, + heartbeatTtl: FiniteDuration, + excludedGroups: Set[String] + )(implicit ec: ExecutionContext): Future[List[WorkflowToStart]] = Future.successful(expected) - } } val actor = TestActorRef(new WorkflowStoreCoordinatedAccessActor(workflowStore)) - val request = FetchStartableWorkflows(1, "test fetchStartableWorkflows with workflow url success", 1.second, Set.empty) + val request = + FetchStartableWorkflows(1, "test fetchStartableWorkflows with workflow url success", 1.second, Set.empty) implicit val timeout: Timeout = Timeout(2.seconds.dilated) actor.ask(request).mapTo[List[WorkflowToStart]] map { actual => actual should be(expected) @@ -113,9 +127,8 @@ class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite it should "abort workflows" in { val expected = WorkflowStoreAbortResponse.AbortRequested val workflowStore = new InMemoryWorkflowStore { - override def abort(id: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowStoreAbortResponse] = { + override def abort(id: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowStoreAbortResponse] = Future.successful(WorkflowStoreAbortResponse.AbortRequested) - } } val actor = TestActorRef(new WorkflowStoreCoordinatedAccessActor(workflowStore)) val request = Abort(WorkflowId.fromString("00001111-2222-3333-aaaa-bbbbccccdddd")) @@ -128,9 +141,8 @@ class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite it should "delete workflow store entries" in { val expected = 1 // 1 row deleted val workflowStore = new InMemoryWorkflowStore { - override def deleteFromStore(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[Int] = { + override def deleteFromStore(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[Int] = Future.successful(1) - } } val actor = TestActorRef(new WorkflowStoreCoordinatedAccessActor(workflowStore)) val request = DeleteFromStore(WorkflowId.fromString("00001111-2222-3333-aaaa-bbbbccccdddd")) @@ -143,17 +155,16 @@ class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite val failureResponses = Table( ("description", "result", "expectedException", "expectedMessagePrefix"), ("a failure", () => Future.failed(new IOException("expected")), classOf[IOException], "expected"), - ("a timeout", () => Future(sleepAndThrow), classOf[AskTimeoutException], "Ask timed out"), + ("a timeout", () => Future(sleepAndThrow), classOf[AskTimeoutException], "Ask timed out") ) forAll(failureResponses) { (description, result, expectedException, expectedMessagePrefix) => it should s"fail to writeHeartBeats due to $description" in { val workflowStore = new InMemoryWorkflowStore { override def writeWorkflowHeartbeats(workflowIds: Set[(WorkflowId, OffsetDateTime)], - heartbeatDateTime: OffsetDateTime) - (implicit ec: ExecutionContext): Future[Nothing] = { + heartbeatDateTime: OffsetDateTime + )(implicit ec: ExecutionContext): Future[Nothing] = result() - } } val actor = TestActorRef(new WorkflowStoreCoordinatedAccessActor(workflowStore)) val request = WriteHeartbeats(NonEmptyVector.of((WorkflowId.randomId(), OffsetDateTime.now)), OffsetDateTime.now) @@ -166,14 +177,17 @@ class WorkflowStoreCoordinatedAccessActorSpec extends TestKitSuite it should s"fail to fetchStartableWorkflows due to $description" in { val workflowStore = new InMemoryWorkflowStore { - override def fetchStartableWorkflows(n: Int, cromwellId: String, heartbeatTtl: FiniteDuration, excludedGroups: Set[String]) - (implicit ec: ExecutionContext): Future[Nothing] = { + override def fetchStartableWorkflows(n: Int, + cromwellId: String, + heartbeatTtl: FiniteDuration, + excludedGroups: Set[String] + )(implicit ec: ExecutionContext): Future[Nothing] = result() - } } val actor = TestActorRef(new WorkflowStoreCoordinatedAccessActor(workflowStore)) val heartbeatTtlNotReallyUsed = 1.second - val request = FetchStartableWorkflows(1, s"test $description fetchStartableWorkflows", heartbeatTtlNotReallyUsed, Set.empty) + val request = + FetchStartableWorkflows(1, s"test $description fetchStartableWorkflows", heartbeatTtlNotReallyUsed, Set.empty) implicit val timeout: Timeout = Timeout(2.seconds.dilated) actor.ask(request).failed map { actual => actual.getMessage should startWith(expectedMessagePrefix) diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreHeartbeatWriteActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreHeartbeatWriteActorSpec.scala index 6749f53931c..71b6e440470 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreHeartbeatWriteActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreHeartbeatWriteActorSpec.scala @@ -16,8 +16,7 @@ import scala.concurrent.duration._ import scala.concurrent.{ExecutionContext, Future} import scala.util.control.NoStackTrace -class WorkflowStoreHeartbeatWriteActorSpec extends TestKitSuite - with AnyFlatSpecLike with Matchers with Eventually { +class WorkflowStoreHeartbeatWriteActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with Eventually { behavior of "WorkflowStoreHeartbeatWriteActor" @@ -27,22 +26,23 @@ class WorkflowStoreHeartbeatWriteActorSpec extends TestKitSuite val workflowStore = new InMemoryWorkflowStore { override def writeWorkflowHeartbeats(workflowIds: Set[(WorkflowId, OffsetDateTime)], - heartbeatDateTime: OffsetDateTime) - (implicit ec: ExecutionContext): Future[Int] = { + heartbeatDateTime: OffsetDateTime + )(implicit ec: ExecutionContext): Future[Int] = Future.failed(new RuntimeException("this is expected") with NoStackTrace) - } } val workflowStoreAccess = UncoordinatedWorkflowStoreAccess(workflowStore) - val workflowHeartbeatTypesafeConfig = ConfigFactory.parseString( - """|danger.debug.only.minimum-heartbeat-ttl = 10 ms - |system.workflow-heartbeats { - | heartbeat-interval = 500 ms - | write-batch-size = 1 - | write-failure-shutdown-duration = 1 s - |} - |""".stripMargin - ).withFallback(ConfigFactory.load()) + val workflowHeartbeatTypesafeConfig = ConfigFactory + .parseString( + """|danger.debug.only.minimum-heartbeat-ttl = 10 ms + |system.workflow-heartbeats { + | heartbeat-interval = 500 ms + | write-batch-size = 1 + | write-failure-shutdown-duration = 1 s + |} + |""".stripMargin + ) + .withFallback(ConfigFactory.load()) val workflowHeartbeatConfig = WorkflowHeartbeatConfig(workflowHeartbeatTypesafeConfig) val terminator = new CromwellTerminator { override def beginCromwellShutdown(reason: CoordinatedShutdown.Reason): Future[Done] = { diff --git a/engine/src/test/scala/cromwell/jobstore/JobResultSpec.scala b/engine/src/test/scala/cromwell/jobstore/JobResultSpec.scala index d623e9eee23..a951c479449 100644 --- a/engine/src/test/scala/cromwell/jobstore/JobResultSpec.scala +++ b/engine/src/test/scala/cromwell/jobstore/JobResultSpec.scala @@ -23,7 +23,16 @@ class JobResultSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "write more complicated WdlValues" in { - val success = JobResultSuccess(Some(0), WomMocks.mockOutputExpectations(Map("abc" -> WomMap(WomMapType(WomStringType, WomIntegerType), Map(WomString("hello") -> WomInteger(4), WomString("goodbye") -> WomInteger(6)))))) + val success = JobResultSuccess( + Some(0), + WomMocks.mockOutputExpectations( + Map( + "abc" -> WomMap(WomMapType(WomStringType, WomIntegerType), + Map(WomString("hello") -> WomInteger(4), WomString("goodbye") -> WomInteger(6)) + ) + ) + ) + ) val asJson = success.toJson val jsonString = asJson.toString() diff --git a/engine/src/test/scala/cromwell/webservice/EngineStatsActorSpec.scala b/engine/src/test/scala/cromwell/webservice/EngineStatsActorSpec.scala index 9f7567e8eee..08e5125e55c 100644 --- a/engine/src/test/scala/cromwell/webservice/EngineStatsActorSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/EngineStatsActorSpec.scala @@ -49,8 +49,8 @@ class EngineStatsActorSpec extends TestKitSuite with AnyFlatSpecLike with Matche object EngineStatsActorSpec { final case class FakeWorkflowActor(jobs: Int) extends Actor { - override def receive = { - case JobCountQuery => sender() ! JobCount(jobs) + override def receive = { case JobCountQuery => + sender() ! JobCount(jobs) } } } diff --git a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala index 700c0274026..b0669721518 100644 --- a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala @@ -22,8 +22,12 @@ import scala.concurrent.Future import scala.concurrent.duration._ import scala.util.Random -class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers - with TableDrivenPropertyChecks with ImplicitSender { +class MetadataBuilderActorSpec + extends TestKitSuite + with AsyncFlatSpecLike + with Matchers + with TableDrivenPropertyChecks + with ImplicitSender { behavior of "MetadataBuilderActor" @@ -37,42 +41,40 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with expectedRes: String, metadataBuilderActorName: String, failedTasks: Boolean = false - ): Future[Assertion] = { + ): Future[Assertion] = { val mockReadMetadataWorkerActor = TestProbe("mockReadMetadataWorkerActor") def readMetadataWorkerMaker = () => mockReadMetadataWorkerActor.props - val mba = system.actorOf( props = MetadataBuilderActor.props(readMetadataWorkerMaker, 1000000), - name = metadataBuilderActorName, + name = metadataBuilderActorName ) val response = mba.ask(action).mapTo[MetadataJsonResponse] mockReadMetadataWorkerActor.expectMsg(defaultTimeout, action) mockReadMetadataWorkerActor.reply( - if(failedTasks) FetchFailedJobsMetadataLookupResponse(events) else MetadataLookupResponse(queryReply, events) + if (failedTasks) FetchFailedJobsMetadataLookupResponse(events) else MetadataLookupResponse(queryReply, events) ) - response map { r => r shouldBe a [SuccessfulMetadataJsonResponse] } - response.mapTo[SuccessfulMetadataJsonResponse] map { b => b.responseJson shouldBe expectedRes.parseJson} + response map { r => r shouldBe a[SuccessfulMetadataJsonResponse] } + response.mapTo[SuccessfulMetadataJsonResponse] map { b => b.responseJson shouldBe expectedRes.parseJson } } - def assertMetadataFailureResponse(action: MetadataServiceAction, metadataServiceResponse: MetadataServiceResponse, expectedException: Exception, - metadataBuilderActorName: String, - ): Future[Assertion] = { + metadataBuilderActorName: String + ): Future[Assertion] = { val mockReadMetadataWorkerActor = TestProbe("mockReadMetadataWorkerActor") val mba = system.actorOf( props = MetadataBuilderActor.props(() => mockReadMetadataWorkerActor.props, defaultSafetyRowNumberThreshold), - name = metadataBuilderActorName, + name = metadataBuilderActorName ) val response = mba.ask(action).mapTo[MetadataServiceResponse] mockReadMetadataWorkerActor.expectMsg(defaultTimeout, action) mockReadMetadataWorkerActor.reply(metadataServiceResponse) - response map { r => r shouldBe a [FailedMetadataJsonResponse] } + response map { r => r shouldBe a[FailedMetadataJsonResponse] } response.mapTo[FailedMetadataJsonResponse] map { b => b.reason.getClass shouldBe expectedException.getClass b.reason.getMessage shouldBe expectedException.getMessage @@ -80,9 +82,8 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with } it should "build workflow scope tree from metadata events" in { - def makeEvent(workflow: WorkflowId, key: Option[MetadataJobKey]) = { + def makeEvent(workflow: WorkflowId, key: Option[MetadataJobKey]) = MetadataEvent(MetadataKey(workflow, key, "NOT_CHECKED"), MetadataValue("NOT_CHECKED")) - } val workflowA = WorkflowId.randomId() @@ -145,30 +146,32 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with type EventBuilder = (String, String, OffsetDateTime) - def makeEvent(workflow: WorkflowId)(key: String, value: MetadataValue, offsetDateTime: OffsetDateTime): MetadataEvent = { + def makeEvent( + workflow: WorkflowId + )(key: String, value: MetadataValue, offsetDateTime: OffsetDateTime): MetadataEvent = MetadataEvent(MetadataKey(workflow, None, key), Option(value), offsetDateTime) - } - def makeCallEvent(workflow: WorkflowId) - (key: String, value: MetadataValue, offsetDateTime: OffsetDateTime): MetadataEvent = { + def makeCallEvent( + workflow: WorkflowId + )(key: String, value: MetadataValue, offsetDateTime: OffsetDateTime): MetadataEvent = { val jobKey = MetadataJobKey("fqn", None, 1) MetadataEvent(MetadataKey(workflow, Option(jobKey), key), Option(value), offsetDateTime) } - //noinspection ScalaUnusedSymbol - def makeEmptyValue(workflow: WorkflowId) - (key: String, value: MetadataValue, offsetDateTime: OffsetDateTime): MetadataEvent = { + // noinspection ScalaUnusedSymbol + def makeEmptyValue( + workflow: WorkflowId + )(key: String, value: MetadataValue, offsetDateTime: OffsetDateTime): MetadataEvent = MetadataEvent(MetadataKey(workflow, None, key), None, offsetDateTime) - } def assertMetadataKeyStructure(eventList: List[EventBuilder], expectedJson: String, workflow: WorkflowId = WorkflowId.randomId(), eventMaker: WorkflowId => (String, MetadataValue, OffsetDateTime) => MetadataEvent = - makeEvent, + makeEvent, metadataBuilderActorName: String, isFailedTaskFetch: Boolean = false - ): Future[Assertion] = { + ): Future[Assertion] = { val events = eventList map { e => (e._1, MetadataValue(e._2), e._3) } map Function.tupled(eventMaker(workflow)) val expectedRes = s"""{ "calls": {}, $expectedJson, "id":"$workflow" }""" @@ -180,9 +183,8 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with it should "build the call list for failed tasks when prompted" in { - def makeEvent(workflow: WorkflowId, key: Option[MetadataJobKey]) = { + def makeEvent(workflow: WorkflowId, key: Option[MetadataJobKey]) = MetadataEvent(MetadataKey(workflow, key, "NOT_CHECKED"), MetadataValue("NOT_CHECKED")) - } val workflowA = WorkflowId.randomId() @@ -242,7 +244,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with events = workflowAEvents, expectedRes = expectedRes, failedTasks = true, - metadataBuilderActorName = "mba-failed-tasks", + metadataBuilderActorName = "mba-failed-tasks" ) } @@ -257,7 +259,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with assertMetadataKeyStructure( eventList = eventBuilderList, expectedJson = expectedRes, - metadataBuilderActorName = "mba-same-key", + metadataBuilderActorName = "mba-same-key" ) } @@ -272,7 +274,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with assertMetadataKeyStructure( eventList = eventBuilderList, expectedJson = expectedRes, - metadataBuilderActorName = "mba-not-workflow-state", + metadataBuilderActorName = "mba-not-workflow-state" ) } @@ -284,30 +286,29 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with val workflowId = WorkflowId.randomId() val expectedRes = s""""calls": { - | "fqn": [{ - | "attempt": 1, - | "executionStatus": "Done", - | "shardIndex": -1 - | }] - | }, - | "id": "$workflowId"""".stripMargin + | "fqn": [{ + | "attempt": 1, + | "executionStatus": "Done", + | "shardIndex": -1 + | }] + | }, + | "id": "$workflowId"""".stripMargin assertMetadataKeyStructure( eventList = eventBuilderList, expectedJson = expectedRes, workflow = workflowId, eventMaker = makeCallEvent, - metadataBuilderActorName = "mba-not-execution-status", + metadataBuilderActorName = "mba-not-execution-status" ) } - it should "use reverse date ordering (oldest first) for event start and stop values" in { val eventBuilderList = List( ("start", "1990-12-20T12:30:00.000Z", OffsetDateTime.now), ("start", "1990-12-20T12:30:01.000Z", OffsetDateTime.now.plusSeconds(1)), ("end", "2018-06-02T12:30:00.000Z", OffsetDateTime.now.plusSeconds(2)), - ("end", "2018-06-02T12:30:01.000Z", OffsetDateTime.now.plusSeconds(3)), + ("end", "2018-06-02T12:30:01.000Z", OffsetDateTime.now.plusSeconds(3)) ) val workflowId = WorkflowId.randomId() val expectedRes = @@ -326,11 +327,10 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with expectedJson = expectedRes, workflow = workflowId, eventMaker = makeCallEvent, - metadataBuilderActorName = "mba-start-end-values", + metadataBuilderActorName = "mba-start-end-values" ) } - it should "build JSON object structure from dotted key syntax" in { val eventBuilderList = List( ("a:b:c", "abc", OffsetDateTime.now), @@ -352,11 +352,10 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with assertMetadataKeyStructure( eventList = eventBuilderList, expectedJson = expectedRes, - metadataBuilderActorName = "mba-object-key", + metadataBuilderActorName = "mba-object-key" ) } - it should "build numerically sorted JSON list structure from dotted key syntax" in { val eventBuilderList = List( ("l[1]", "l1", OffsetDateTime.now), @@ -375,7 +374,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with assertMetadataKeyStructure( eventList = eventBuilderList, expectedJson = expectedRes, - metadataBuilderActorName = "mba-list-key", + metadataBuilderActorName = "mba-list-key" ) } @@ -397,7 +396,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with assertMetadataKeyStructure( eventList = eventBuilderList, expectedJson = expectedRes, - metadataBuilderActorName = "mba-same-index", + metadataBuilderActorName = "mba-same-index" ) } @@ -442,7 +441,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with assertMetadataKeyStructure( eventList = eventBuilderList, expectedJson = expectedRes, - metadataBuilderActorName = "mba-nest-objects", + metadataBuilderActorName = "mba-nest-objects" ) } @@ -467,7 +466,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with assertMetadataKeyStructure( eventList = eventBuilderList, expectedJson = expectedRes, - metadataBuilderActorName = "mba-nest-lists", + metadataBuilderActorName = "mba-nest-lists" ) } @@ -486,7 +485,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with eventList = eventBuilderList, expectedJson = expectedRes, eventMaker = makeEmptyValue, - metadataBuilderActorName = "mba-nest-empty", + metadataBuilderActorName = "mba-nest-empty" ) } @@ -497,19 +496,18 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with val kiv4 = ("key[0]", "value4", OffsetDateTime.now.plusSeconds(3)) val tuples = List( - ("mba-json-1", List(kv), """"key": "value""""), - ("mba-json-2", List(kv, ksv2), """"key": { "subkey": "value2" }"""), - ("mba-json-3", List(kv, ksv2, kisv3), """"key": [ { "subkey": "value3" } ]"""), - ("mba-json-4", List(kv, ksv2, kisv3, kiv4), """"key": [ "value4" ]""") - ) - - Future.sequence(tuples map { - case (metadataBuilderActorName, eventList, expectedJson) => - assertMetadataKeyStructure( - eventList = eventList, - expectedJson = expectedJson, - metadataBuilderActorName = metadataBuilderActorName, - ) + ("mba-json-1", List(kv), """"key": "value""""), + ("mba-json-2", List(kv, ksv2), """"key": { "subkey": "value2" }"""), + ("mba-json-3", List(kv, ksv2, kisv3), """"key": [ { "subkey": "value3" } ]"""), + ("mba-json-4", List(kv, ksv2, kisv3, kiv4), """"key": [ "value4" ]""") + ) + + Future.sequence(tuples map { case (metadataBuilderActorName, eventList, expectedJson) => + assertMetadataKeyStructure( + eventList = eventList, + expectedJson = expectedJson, + metadataBuilderActorName = metadataBuilderActorName + ) }) map { assertions => assertions should contain only Succeeded } @@ -530,17 +528,17 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with val expectedResponse = s"""{ - | "calls": {}, - | "a": 2, - | "b": 2, - | "c": 2, - | "d": 2.9, - | "e": 2.9, - | "f": true, - | "g": false, - | "h": "false", - | "id":"$workflowId" - | } + | "calls": {}, + | "a": 2, + | "b": 2, + | "c": 2, + | "d": 2.9, + | "e": 2.9, + | "f": true, + | "g": false, + | "h": "false", + | "id":"$workflowId" + | } """.stripMargin val mdQuery = MetadataQuery(workflowId, None, None, None, None, expandSubWorkflows = false) @@ -550,7 +548,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with queryReply = mdQuery, events = events, expectedRes = expectedResponse, - metadataBuilderActorName = "mba-coerce-type", + metadataBuilderActorName = "mba-coerce-type" ) } @@ -564,10 +562,10 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with val expectedResponse = s"""{ - | "calls": {}, - | "i": "UnknownClass(50)", - | "id":"$workflowId" - |} + | "calls": {}, + | "i": "UnknownClass(50)", + | "id":"$workflowId" + |} """.stripMargin val mdQuery = MetadataQuery(workflowId, None, None, None, None, expandSubWorkflows = false) @@ -577,7 +575,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with queryReply = mdQuery, events = events, expectedRes = expectedResponse, - metadataBuilderActorName = "mba-unknown-type", + metadataBuilderActorName = "mba-unknown-type" ) } @@ -590,10 +588,10 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with val expectedResponse = s"""{ - | "calls": {}, - | "i": "notAnInt", - | "id":"$workflowId" - |} + | "calls": {}, + | "i": "notAnInt", + | "id":"$workflowId" + |} """.stripMargin val mdQuery = MetadataQuery(workflowId, None, None, None, None, expandSubWorkflows = false) @@ -603,7 +601,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with queryReply = mdQuery, events = events, expectedRes = expectedResponse, - metadataBuilderActorName = "mba-coerce-fails", + metadataBuilderActorName = "mba-coerce-fails" ) } @@ -624,11 +622,11 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with val expectedEmptyResponse = s"""{ - | "calls": {}, - | "hey": {}, - | "emptyList": [], - | "id":"$workflowId" - |} + | "calls": {}, + | "hey": {}, + | "emptyList": [], + | "id":"$workflowId" + |} """.stripMargin val mdQuery = MetadataQuery(workflowId, None, None, None, None, expandSubWorkflows = false) @@ -638,16 +636,16 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with queryReply = mdQuery, events = emptyEvents, expectedRes = expectedEmptyResponse, - metadataBuilderActorName = "mba-empty-values", + metadataBuilderActorName = "mba-empty-values" ) val expectedNonEmptyResponse = s"""{ - | "calls": {}, - | "hey": "something", - | "emptyList": ["something", "something"], - | "id":"$workflowId" - |} + | "calls": {}, + | "hey": "something", + | "emptyList": ["something", "something"], + | "id":"$workflowId" + |} """.stripMargin assertMetadataResponse( @@ -655,7 +653,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with queryReply = mdQuery, events = valueEvents, expectedRes = expectedNonEmptyResponse, - metadataBuilderActorName = "mba-non-empty-values", + metadataBuilderActorName = "mba-non-empty-values" ) } @@ -664,7 +662,9 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with val subWorkflowId = WorkflowId.randomId() val mainEvents = List( - MetadataEvent(MetadataKey(mainWorkflowId, Option(MetadataJobKey("callA", None, 1)), "subWorkflowId"), MetadataValue(subWorkflowId)) + MetadataEvent(MetadataKey(mainWorkflowId, Option(MetadataJobKey("callA", None, 1)), "subWorkflowId"), + MetadataValue(subWorkflowId) + ) ) val subEvents = List( @@ -686,7 +686,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with TestActorRef( props = MetadataBuilderActor.props(readMetadataWorkerMaker, 1000000), supervisor = parentProbe.ref, - name = s"MetadataActor-$mainWorkflowId", + name = s"MetadataActor-$mainWorkflowId" ) val response = metadataBuilder.ask(mainQueryAction).mapTo[MetadataJsonResponse] mockReadMetadataWorkerActor.expectMsg(defaultTimeout, mainQueryAction) @@ -714,9 +714,9 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with |} """.stripMargin - response map { r => r shouldBe a [SuccessfulMetadataJsonResponse] } + response map { r => r shouldBe a[SuccessfulMetadataJsonResponse] } val bmr = response.mapTo[SuccessfulMetadataJsonResponse] - bmr map { b => b.responseJson shouldBe expandedRes.parseJson} + bmr map { b => b.responseJson shouldBe expandedRes.parseJson } } it should "NOT expand sub workflow metadata when NOT asked for" in { @@ -724,7 +724,9 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with val subWorkflowId = WorkflowId.randomId() val mainEvents = List( - MetadataEvent(MetadataKey(mainWorkflowId, Option(MetadataJobKey("callA", None, 1)), "subWorkflowId"), MetadataValue(subWorkflowId)) + MetadataEvent(MetadataKey(mainWorkflowId, Option(MetadataJobKey("callA", None, 1)), "subWorkflowId"), + MetadataValue(subWorkflowId) + ) ) val queryNoExpand = MetadataQuery(mainWorkflowId, None, None, None, None, expandSubWorkflows = false) @@ -733,18 +735,17 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with val parentProbe = TestProbe("parentProbe") val mockReadMetadataWorkerActor = TestProbe("mockReadMetadataWorkerActor") - def readMetadataWorkerMaker= () => mockReadMetadataWorkerActor.props + def readMetadataWorkerMaker = () => mockReadMetadataWorkerActor.props val metadataBuilder = TestActorRef( props = MetadataBuilderActor.props(readMetadataWorkerMaker, 1000000), supervisor = parentProbe.ref, - name = s"MetadataActor-$mainWorkflowId", + name = s"MetadataActor-$mainWorkflowId" ) val response = metadataBuilder.ask(queryNoExpandAction).mapTo[MetadataJsonResponse] mockReadMetadataWorkerActor.expectMsg(defaultTimeout, queryNoExpandAction) mockReadMetadataWorkerActor.reply(MetadataLookupResponse(queryNoExpand, mainEvents)) - val nonExpandedRes = s""" |{ @@ -761,9 +762,9 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with |} """.stripMargin - response map { r => r shouldBe a [SuccessfulMetadataJsonResponse] } + response map { r => r shouldBe a[SuccessfulMetadataJsonResponse] } val bmr = response.mapTo[SuccessfulMetadataJsonResponse] - bmr map { b => b.responseJson shouldBe nonExpandedRes.parseJson} + bmr map { b => b.responseJson shouldBe nonExpandedRes.parseJson } } @@ -788,7 +789,6 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with ee(workflowId, Bar, eventIndex = 5, StartTime, Interval4.start), ee(workflowId, Bar, eventIndex = 5, EndTime, Interval4.end), ee(workflowId, Bar, eventIndex = 5, Grouping, Delocalizing), - ee(workflowId, Baz, eventIndex = 6, StartTime, Interval2.start), ee(workflowId, Baz, eventIndex = 6, EndTime, Interval2.end), ee(workflowId, Baz, eventIndex = 6, Grouping, Localizing), @@ -804,7 +804,6 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with // cause problems. pe(workflowId, Quux, StartTime, Interval8.start), pe(workflowId, Quux, EndTime, Interval8.end), - pe(workflowId, Corge, StartTime, Interval9.start), pe(workflowId, Corge, EndTime, Interval9.end) ) @@ -817,36 +816,32 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with ee(workflowId, Foo, eventIndex = 1, StartTime, Interval1.start), ee(workflowId, Foo, eventIndex = 1, EndTime, Interval2.end), ee(workflowId, Foo, eventIndex = 1, Description, Localizing.name), - ee(workflowId, Bar, eventIndex = 3, StartTime, Interval3.start), ee(workflowId, Bar, eventIndex = 3, Description, Delocalizing.name), - ee(workflowId, Baz, eventIndex = 6, StartTime, Interval1.start), ee(workflowId, Baz, eventIndex = 6, EndTime, Interval2.end), ee(workflowId, Baz, eventIndex = 6, Description, Localizing.name), - ee(workflowId, Qux, eventIndex = 8, StartTime, Interval7.start), ee(workflowId, Qux, eventIndex = 8, EndTime, Interval7.end), - pe(workflowId, Quux, StartTime, Interval8.start), pe(workflowId, Quux, EndTime, Interval8.end), - pe(workflowId, Corge, StartTime, Interval9.start), pe(workflowId, Corge, EndTime, Interval9.end) ) val actual = MetadataBuilderActor.groupEvents(events).toSet - def filterEventsByCall(events: Iterable[MetadataEvent])(call: Call): Iterable[MetadataEvent] = { - events collect { case e@MetadataEvent(MetadataKey(_, Some(MetadataJobKey(n, _, _)), _), _, _) if call.name == n => e} - } + def filterEventsByCall(events: Iterable[MetadataEvent])(call: Call): Iterable[MetadataEvent] = + events collect { + case e @ MetadataEvent(MetadataKey(_, Some(MetadataJobKey(n, _, _)), _), _, _) if call.name == n => e + } val calls = List(Foo, Bar, Baz, Qux, Quux, Corge) val actuals = calls map filterEventsByCall(actual) val expecteds = calls map filterEventsByCall(expectations) - val matchesExpectations = (actuals zip expecteds) map { - case (as, es) => as.toList.map(_.toString).sorted == es.toList.map(_.toString).sorted + val matchesExpectations = (actuals zip expecteds) map { case (as, es) => + as.toList.map(_.toString).sorted == es.toList.map(_.toString).sorted } matchesExpectations.reduceLeft(_ && _) shouldBe true } @@ -854,9 +849,10 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with it should "correctly order statuses (even unused ones)" in { val workflowId = WorkflowId.randomId() - def statusEvent(callName: String, status: String) = { - MetadataEvent(MetadataKey(workflowId, Option(MetadataJobKey(callName, None, 1)), "executionStatus"), MetadataValue(status)) - } + def statusEvent(callName: String, status: String) = + MetadataEvent(MetadataKey(workflowId, Option(MetadataJobKey(callName, None, 1)), "executionStatus"), + MetadataValue(status) + ) // Combines standard "setup" statuses plus the conclusion status(es). def setupStatusesPlusConclusion(callName: String, conclusionStatuses: String*): Vector[MetadataEvent] = Vector( @@ -874,11 +870,11 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with val events = setupStatusesPlusConclusion("Foo", "Done") ++ - setupStatusesPlusConclusion("Bar", "Aborting", "Aborted") ++ - setupStatusesPlusConclusion("Baz", "Failed") ++ - setupStatusesPlusConclusion("Qux", "RetryableFailure") ++ - setupStatusesPlusConclusion("Quux", "Bypassed") ++ - setupStatusesPlusConclusion("Quuux", "Unstartable") + setupStatusesPlusConclusion("Bar", "Aborting", "Aborted") ++ + setupStatusesPlusConclusion("Baz", "Failed") ++ + setupStatusesPlusConclusion("Qux", "RetryableFailure") ++ + setupStatusesPlusConclusion("Quux", "Bypassed") ++ + setupStatusesPlusConclusion("Quuux", "Unstartable") val expectedRes = s"""{ @@ -909,12 +905,13 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with val action = GetMetadataAction(mdQuery) val metadataRowNumber = 100500 - val expectedException = new MetadataTooLargeNumberOfRowsException(workflowId, metadataRowNumber, defaultSafetyRowNumberThreshold) + val expectedException = + new MetadataTooLargeNumberOfRowsException(workflowId, metadataRowNumber, defaultSafetyRowNumberThreshold) assertMetadataFailureResponse( action = action, metadataServiceResponse = MetadataLookupFailedTooLargeResponse(mdQuery, metadataRowNumber), expectedException = expectedException, - metadataBuilderActorName = "mba-too-large", + metadataBuilderActorName = "mba-too-large" ) } @@ -929,7 +926,7 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with action = action, metadataServiceResponse = MetadataLookupFailedTimeoutResponse(mdQuery), expectedException = expectedException, - metadataBuilderActorName = "mba-read-timeout", + metadataBuilderActorName = "mba-read-timeout" ) } } @@ -990,8 +987,8 @@ object MetadataBuilderActorSpec { def executionEventName(i: Int, a: Attr): String = s"executionEvents[$i]:${a.name}" - def executionEventKey(workflowId: WorkflowId, call: Call, eventIndex: Int, attr: Attr): MetadataKey = MetadataKey(workflowId, Option(MetadataJobKey(call.name, None, 1)), executionEventName(eventIndex, attr)) - + def executionEventKey(workflowId: WorkflowId, call: Call, eventIndex: Int, attr: Attr): MetadataKey = + MetadataKey(workflowId, Option(MetadataJobKey(call.name, None, 1)), executionEventName(eventIndex, attr)) def ee(workflowId: WorkflowId, call: Call, eventIndex: Int, attr: Attr, value: Any): MetadataEvent = { val metadataValue = value match { @@ -1001,7 +998,9 @@ object MetadataBuilderActorSpec { new MetadataEvent(executionEventKey(workflowId, call, eventIndex, attr), Option(MetadataValue(metadataValue)), y2k) } - def eventKey(workflowId: WorkflowId, call: Call, attr: Attr): MetadataKey = MetadataKey(workflowId, Option(MetadataJobKey(call.name, None, 1)), attr.name) + def eventKey(workflowId: WorkflowId, call: Call, attr: Attr): MetadataKey = + MetadataKey(workflowId, Option(MetadataJobKey(call.name, None, 1)), attr.name) - def pe(workflowId: WorkflowId, call: Call, attr: Attr, value: Any): MetadataEvent = new MetadataEvent(eventKey(workflowId, call, attr), Option(MetadataValue(value)), y2k) + def pe(workflowId: WorkflowId, call: Call, attr: Attr, value: Any): MetadataEvent = + new MetadataEvent(eventKey(workflowId, call, attr), Option(MetadataValue(value)), y2k) } diff --git a/engine/src/test/scala/cromwell/webservice/PartialWorkflowSourcesSpec.scala b/engine/src/test/scala/cromwell/webservice/PartialWorkflowSourcesSpec.scala index f19e3c667ab..d1b86bfa216 100644 --- a/engine/src/test/scala/cromwell/webservice/PartialWorkflowSourcesSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/PartialWorkflowSourcesSpec.scala @@ -16,13 +16,13 @@ class PartialWorkflowSourcesSpec extends AnyFlatSpec with CromwellTimeoutSpec wi val input1 = Map("wf.a1" -> "hello", "wf.a2" -> "world").toJson.toString val input2 = Map.empty[String, String].toJson.toString val overrideInput1 = Map("wf.a2" -> "universe").toJson.toString - val mergedMapsErrorOr = PartialWorkflowSources.mergeMaps(Seq(Option(input1), Option(input2), Option(overrideInput1))) + val mergedMapsErrorOr = + PartialWorkflowSources.mergeMaps(Seq(Option(input1), Option(input2), Option(overrideInput1))) mergedMapsErrorOr match { - case Valid(inputs) => { - inputs.fields.keys should contain allOf("wf.a1", "wf.a2") + case Valid(inputs) => + inputs.fields.keys should contain allOf ("wf.a1", "wf.a2") inputs.fields("wf.a2") should be(JsString("universe")) - } case Invalid(error) => fail(s"This is unexpected! This test should pass! Error: $error") } } @@ -33,7 +33,8 @@ class PartialWorkflowSourcesSpec extends AnyFlatSpec with CromwellTimeoutSpec wi mergedMapsErrorOr match { case Valid(_) => fail("This is unexpected! This test is designed to fail!") - case Invalid(error) => error.head shouldBe "Submitted input '\"invalidInput\"' of type JsString is not a valid JSON object." + case Invalid(error) => + error.head shouldBe "Submitted input '\"invalidInput\"' of type JsString is not a valid JSON object." } } @@ -44,7 +45,8 @@ class PartialWorkflowSourcesSpec extends AnyFlatSpec with CromwellTimeoutSpec wi mergedMapsErrorOr match { case Valid(_) => fail("This is unexpected! This test is designed to fail!") - case Invalid(error) => error.head shouldBe "Failed to parse input: 'invalidInput', which is not a valid json. Please check for syntactical errors. (reason 1 of 1): Unexpected character 'i' at input index 0 (line 1, position 1), expected JSON Value:\ninvalidInput\n^\n" + case Invalid(error) => + error.head shouldBe "Failed to parse input: 'invalidInput', which is not a valid json. Please check for syntactical errors. (reason 1 of 1): Unexpected character 'i' at input index 0 (line 1, position 1), expected JSON Value:\ninvalidInput\n^\n" } } @@ -76,9 +78,8 @@ class PartialWorkflowSourcesSpec extends AnyFlatSpec with CromwellTimeoutSpec wi |}] |""".stripMargin - val expected = Vector( - """{"mywf.inInt":1,"mywf.inString":"one"}""", - """{"mywf.inInt":2,"mywf.inString":"two"}""").validNel + val expected = + Vector("""{"mywf.inInt":1,"mywf.inString":"one"}""", """{"mywf.inInt":2,"mywf.inString":"two"}""").validNel val actual = PartialWorkflowSources.workflowInputsValidation(input) diff --git a/engine/src/test/scala/cromwell/webservice/SwaggerServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/SwaggerServiceSpec.scala index 24cce8ae932..897e4beeec0 100644 --- a/engine/src/test/scala/cromwell/webservice/SwaggerServiceSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/SwaggerServiceSpec.scala @@ -15,8 +15,13 @@ import org.yaml.snakeyaml.{LoaderOptions, Yaml => SnakeYaml} import scala.jdk.CollectionConverters._ -class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with SwaggerService with ScalatestRouteTest with Matchers - with TableDrivenPropertyChecks { +class SwaggerServiceSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with SwaggerService + with ScalatestRouteTest + with Matchers + with TableDrivenPropertyChecks { def actorRefFactory = system behavior of "SwaggerService" @@ -28,7 +33,8 @@ class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Swagg status should be(StatusCodes.OK) val body = responseAs[String] - val yaml = new SnakeYaml(new UniqueKeyConstructor(new LoaderOptions)).loadAs(body, classOf[java.util.Map[String, AnyRef]]) + val yaml = new SnakeYaml(new UniqueKeyConstructor(new LoaderOptions)) + .loadAs(body, classOf[java.util.Map[String, AnyRef]]) yaml.get("swagger") should be("2.0") } @@ -53,27 +59,42 @@ class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Swagg resultWithInfo.getSwagger.getDefinitions.asScala foreach { // If no properties, `getProperties` returns `null` instead of an empty map - case (defKey, defVal) => Option(defVal.getProperties).map(_.asScala).getOrElse(Map.empty) foreach { - /* + case (defKey, defVal) => + Option(defVal.getProperties).map(_.asScala).getOrElse(Map.empty) foreach { + /* Two against one. Swagger parser implementation lets a RefProperty have descriptions. http://swagger.io/specification/#referenceObject & http://editor.swagger.io both say it's ref ONLY! - */ - case (propKey, propVal: RefProperty) => - withClue(s"RefProperty $defKey.$propKey has a description: ") { - propVal.getDescription should be(null) - } - case _ => /* ignore */ - } + */ + case (propKey, propVal: RefProperty) => + withClue(s"RefProperty $defKey.$propKey has a description: ") { + propVal.getDescription should be(null) + } + case _ => /* ignore */ + } } } } it should "return status OK when getting OPTIONS on paths" in { - val pathExamples = Table("path", "/", "/swagger", "/swagger/cromwell.yaml", "/swagger/index.html", "/api", - "/api/workflows/", "/api/workflows/v1", "/workflows/v1/outputs", "/workflows/v1/status", - "/api/workflows/v1/validate", "/workflows", "/workflows/v1", "/workflows/v1/outputs", "/workflows/v1/status", - "/workflows/v1/validate") + val pathExamples = Table( + "path", + "/", + "/swagger", + "/swagger/cromwell.yaml", + "/swagger/index.html", + "/api", + "/api/workflows/", + "/api/workflows/v1", + "/workflows/v1/outputs", + "/workflows/v1/status", + "/api/workflows/v1/validate", + "/workflows", + "/workflows/v1", + "/workflows/v1/outputs", + "/workflows/v1/status", + "/workflows/v1/validate" + ) forAll(pathExamples) { path => Options(path) ~> diff --git a/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala index 6afcf12bac8..e574f93dd0a 100644 --- a/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala @@ -11,18 +11,37 @@ import org.scalatest.prop.TableDrivenPropertyChecks import scala.concurrent.duration._ -trait SwaggerUiHttpServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with SwaggerUiHttpService - -trait SwaggerResourceHttpServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with -TableDrivenPropertyChecks with SwaggerResourceHttpService { - val testPathsForOptions = Table("endpoint", "/", "/swagger", "/swagger/index.html", "/api", "/api/example", - "/api/example?with=param", "/api/example/path") +trait SwaggerUiHttpServiceSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalatestRouteTest + with SwaggerUiHttpService + +trait SwaggerResourceHttpServiceSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalatestRouteTest + with TableDrivenPropertyChecks + with SwaggerResourceHttpService { + val testPathsForOptions = Table("endpoint", + "/", + "/swagger", + "/swagger/index.html", + "/api", + "/api/example", + "/api/example?with=param", + "/api/example/path" + ) implicit val timeout: RouteTestTimeout = RouteTestTimeout(5.seconds) } -trait SwaggerUiResourceHttpServiceSpec extends SwaggerUiHttpServiceSpec with SwaggerResourceHttpServiceSpec with -SwaggerUiResourceHttpService +trait SwaggerUiResourceHttpServiceSpec + extends SwaggerUiHttpServiceSpec + with SwaggerResourceHttpServiceSpec + with SwaggerUiResourceHttpService object SwaggerUiHttpServiceSpec { val SwaggerIndexPreamble = @@ -95,7 +114,6 @@ class OverrideBasePathSwaggerUiHttpServiceSpec extends SwaggerResourceHttpServic } } - class YamlSwaggerResourceHttpServiceSpec extends SwaggerResourceHttpServiceSpec { override def swaggerServiceName = "testservice" @@ -194,7 +212,6 @@ class YamlSwaggerUiResourceHttpServiceSpec extends SwaggerUiResourceHttpServiceS } } - class JsonSwaggerUiResourceHttpServiceSpec extends SwaggerUiResourceHttpServiceSpec { override def swaggerServiceName = "testservice" diff --git a/engine/src/test/scala/cromwell/webservice/WorkflowJsonSupportSpec.scala b/engine/src/test/scala/cromwell/webservice/WorkflowJsonSupportSpec.scala index fc25bd3cd2f..c6d944c3285 100644 --- a/engine/src/test/scala/cromwell/webservice/WorkflowJsonSupportSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/WorkflowJsonSupportSpec.scala @@ -8,7 +8,8 @@ import spray.json._ class WorkflowJsonSupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { - val sampleSuccessResponse1 = SuccessResponse("good", "msg", Option(JsArray(Vector(JsString("data1"), JsString("data2"))))) + val sampleSuccessResponse1 = + SuccessResponse("good", "msg", Option(JsArray(Vector(JsString("data1"), JsString("data2"))))) val sampleSuccessResponse2 = SuccessResponse("good", "msg", None) val sampleSuccessResponseJson1 = """{ diff --git a/engine/src/test/scala/cromwell/webservice/routes/CromwellApiServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/routes/CromwellApiServiceSpec.scala index 4e7caf932ef..9088879bc63 100644 --- a/engine/src/test/scala/cromwell/webservice/routes/CromwellApiServiceSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/routes/CromwellApiServiceSpec.scala @@ -9,14 +9,24 @@ import akka.stream.ActorMaterializer import com.typesafe.scalalogging.StrictLogging import common.util.VersionUtil import cromwell.core._ -import cromwell.core.abort.{WorkflowAbortFailureResponse, WorkflowAbortRequestedResponse, WorkflowAbortedResponse} +import cromwell.core.abort.{WorkflowAbortedResponse, WorkflowAbortFailureResponse, WorkflowAbortRequestedResponse} import cromwell.engine.workflow.WorkflowManagerActor import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._ -import cromwell.engine.workflow.workflowstore.WorkflowStoreEngineActor.{WorkflowOnHoldToSubmittedFailure, WorkflowOnHoldToSubmittedSuccess} -import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.{WorkflowSubmittedToStore, WorkflowsBatchSubmittedToStore} +import cromwell.engine.workflow.workflowstore.WorkflowStoreEngineActor.{ + WorkflowOnHoldToSubmittedFailure, + WorkflowOnHoldToSubmittedSuccess +} +import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.{ + WorkflowsBatchSubmittedToStore, + WorkflowSubmittedToStore +} import cromwell.services._ -import cromwell.services.healthmonitor.ProtoHealthMonitorServiceActor.{GetCurrentStatus, StatusCheckResponse, SubsystemStatus} +import cromwell.services.healthmonitor.ProtoHealthMonitorServiceActor.{ + GetCurrentStatus, + StatusCheckResponse, + SubsystemStatus +} import cromwell.services.instrumentation.InstrumentationService.InstrumentationServiceMessage import cromwell.services.metadata.MetadataArchiveStatus._ import cromwell.services.metadata.MetadataService._ @@ -71,64 +81,62 @@ class CromwellApiServiceSpec extends AsyncFlatSpec with ScalatestRouteTest with "REST ENGINE /status endpoint" should "return 200 for status when all is well" in { Get(s"/engine/$version/status") ~> akkaHttpService.engineRoutes ~> - check { - status should be(StatusCodes.OK) - contentType should be(ContentTypes.`application/json`) - val resp = responseAs[JsObject] - val db = resp.fields("Engine Database").asJsObject - db.fields("ok").asInstanceOf[JsBoolean].value should be(true) - } + check { + status should be(StatusCodes.OK) + contentType should be(ContentTypes.`application/json`) + val resp = responseAs[JsObject] + val db = resp.fields("Engine Database").asJsObject + db.fields("ok").asInstanceOf[JsBoolean].value should be(true) + } } + behavior of "REST API /abort endpoint" + it should "return 404 for abort of unknown workflow" in { + val workflowId = CromwellApiServiceSpec.UnrecognizedWorkflowId - behavior of "REST API /abort endpoint" - it should "return 404 for abort of unknown workflow" in { - val workflowId = CromwellApiServiceSpec.UnrecognizedWorkflowId - - Post(s"/workflows/$version/$workflowId/abort") ~> - akkaHttpService.workflowRoutes ~> - check { - assertResult(StatusCodes.NotFound) { - status - } - assertResult { - s"""|{ - | "status": "error", - | "message": "Couldn't abort $workflowId because no workflow with that ID is in progress" - |} - |""".stripMargin.trim - } { - responseAs[String] - } - assertResult(ContentTypes.`application/json`)(contentType) + Post(s"/workflows/$version/$workflowId/abort") ~> + akkaHttpService.workflowRoutes ~> + check { + assertResult(StatusCodes.NotFound) { + status } - } - - it should "return 400 for abort of a malformed workflow id" in { - Post(s"/workflows/$version/foobar/abort") ~> - akkaHttpService.workflowRoutes ~> - check { - assertResult(StatusCodes.BadRequest) { - status - } - assertResult( - """{ - | "status": "fail", - | "message": "Invalid workflow ID: 'foobar'." - |}""".stripMargin - ) { - responseAs[String] - } - assertResult(ContentTypes.`application/json`)(contentType) + assertResult { + s"""|{ + | "status": "error", + | "message": "Couldn't abort $workflowId because no workflow with that ID is in progress" + |} + |""".stripMargin.trim + } { + responseAs[String] } - } + assertResult(ContentTypes.`application/json`)(contentType) + } + } - it should "return 200 Aborted for abort of a workflow which a workflow is in OnHold state" in { - Post(s"/workflows/$version/${CromwellApiServiceSpec.OnHoldWorkflowId}/abort") ~> - akkaHttpService.workflowRoutes ~> + it should "return 400 for abort of a malformed workflow id" in { + Post(s"/workflows/$version/foobar/abort") ~> + akkaHttpService.workflowRoutes ~> check { + assertResult(StatusCodes.BadRequest) { + status + } assertResult( - s"""{"id":"${CromwellApiServiceSpec.OnHoldWorkflowId.toString}","status":"Aborted"}""") { + """{ + | "status": "fail", + | "message": "Invalid workflow ID: 'foobar'." + |}""".stripMargin + ) { + responseAs[String] + } + assertResult(ContentTypes.`application/json`)(contentType) + } + } + + it should "return 200 Aborted for abort of a workflow which a workflow is in OnHold state" in { + Post(s"/workflows/$version/${CromwellApiServiceSpec.OnHoldWorkflowId}/abort") ~> + akkaHttpService.workflowRoutes ~> + check { + assertResult(s"""{"id":"${CromwellApiServiceSpec.OnHoldWorkflowId.toString}","status":"Aborted"}""") { responseAs[String] } assertResult(StatusCodes.OK) { @@ -136,14 +144,13 @@ class CromwellApiServiceSpec extends AsyncFlatSpec with ScalatestRouteTest with } assertResult(ContentTypes.`application/json`)(contentType) } - } + } it should "return 200 Aborted for abort of a workflow which a workflow is in Submitted state" in { Post(s"/workflows/$version/${CromwellApiServiceSpec.SubmittedWorkflowId}/abort") ~> akkaHttpService.workflowRoutes ~> check { - assertResult( - s"""{"id":"${CromwellApiServiceSpec.SubmittedWorkflowId.toString}","status":"Aborted"}""") { + assertResult(s"""{"id":"${CromwellApiServiceSpec.SubmittedWorkflowId.toString}","status":"Aborted"}""") { responseAs[String] } assertResult(StatusCodes.OK) { @@ -153,55 +160,63 @@ class CromwellApiServiceSpec extends AsyncFlatSpec with ScalatestRouteTest with } } - it should "return 200 Aborting for abort of a known workflow id which is currently running" in { - Post(s"/workflows/$version/${CromwellApiServiceSpec.AbortingWorkflowId}/abort") ~> - akkaHttpService.workflowRoutes ~> - check { - assertResult( - s"""{"id":"${CromwellApiServiceSpec.AbortingWorkflowId.toString}","status":"Aborting"}""") { - responseAs[String] - } - assertResult(StatusCodes.OK) { - status - } - assertResult(ContentTypes.`application/json`)(contentType) + it should "return 200 Aborting for abort of a known workflow id which is currently running" in { + Post(s"/workflows/$version/${CromwellApiServiceSpec.AbortingWorkflowId}/abort") ~> + akkaHttpService.workflowRoutes ~> + check { + assertResult(s"""{"id":"${CromwellApiServiceSpec.AbortingWorkflowId.toString}","status":"Aborting"}""") { + responseAs[String] } - } + assertResult(StatusCodes.OK) { + status + } + assertResult(ContentTypes.`application/json`)(contentType) + } + } - behavior of "REST API submission endpoint" - it should "return 201 for a successful workflow submission " in { - val workflowSource = Multipart.FormData.BodyPart("workflowSource", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource())) - val workflowInputs = Multipart.FormData.BodyPart("workflowInputs", HttpEntity(MediaTypes.`application/json`, HelloWorld.rawInputs.toJson.toString())) - val formData = Multipart.FormData(workflowSource, workflowInputs).toEntity() - Post(s"/workflows/$version", formData) ~> - akkaHttpService.workflowRoutes ~> - check { - assertResult( - s"""{ - | "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}", - | "status": "Submitted" - |}""".stripMargin) { - responseAs[String].parseJson.prettyPrint - } - assertResult(StatusCodes.Created) { - status - } - headers should be(Seq.empty) + behavior of "REST API submission endpoint" + it should "return 201 for a successful workflow submission " in { + val workflowSource = + Multipart.FormData.BodyPart("workflowSource", + HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()) + ) + val workflowInputs = + Multipart.FormData.BodyPart("workflowInputs", + HttpEntity(MediaTypes.`application/json`, HelloWorld.rawInputs.toJson.toString()) + ) + val formData = Multipart.FormData(workflowSource, workflowInputs).toEntity() + Post(s"/workflows/$version", formData) ~> + akkaHttpService.workflowRoutes ~> + check { + assertResult(s"""{ + | "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}", + | "status": "Submitted" + |}""".stripMargin) { + responseAs[String].parseJson.prettyPrint } - } + assertResult(StatusCodes.Created) { + status + } + headers should be(Seq.empty) + } + } it should "return 201 for a successful workflow submission using workflowUrl" in { - val workflowUrl = Multipart.FormData.BodyPart("workflowUrl", HttpEntity("https://raw.githubusercontent.com/broadinstitute/cromwell/develop/womtool/src/test/resources/validate/wdl_draft3/valid/callable_imports/my_workflow.wdl")) + val workflowUrl = Multipart.FormData.BodyPart( + "workflowUrl", + HttpEntity( + "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/womtool/src/test/resources/validate/wdl_draft3/valid/callable_imports/my_workflow.wdl" + ) + ) val formData = Multipart.FormData(workflowUrl).toEntity() Post(s"/workflows/$version", formData) ~> akkaHttpService.workflowRoutes ~> check { - assertResult( - s"""{ - | "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}", - | "status": "Submitted" - |}""".stripMargin) { + assertResult(s"""{ + | "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}", + | "status": "Submitted" + |}""".stripMargin) { responseAs[String].parseJson.prettyPrint } assertResult(StatusCodes.Created) { @@ -212,17 +227,21 @@ class CromwellApiServiceSpec extends AsyncFlatSpec with ScalatestRouteTest with } it should "return 400 for a workflow submission using workflowUrl with invalid protocol" in { - val workflowUrl = Multipart.FormData.BodyPart("workflowUrl", HttpEntity("htpps://raw.githubusercontent.com/broadinstitute/cromwell/develop/womtool/src/test/resources/validate/wdl_draft3/valid/callable_imports/my_workflow.wdl")) + val workflowUrl = Multipart.FormData.BodyPart( + "workflowUrl", + HttpEntity( + "htpps://raw.githubusercontent.com/broadinstitute/cromwell/develop/womtool/src/test/resources/validate/wdl_draft3/valid/callable_imports/my_workflow.wdl" + ) + ) val formData = Multipart.FormData(workflowUrl).toEntity() Post(s"/workflows/$version", formData) ~> akkaHttpService.workflowRoutes ~> check { - assertResult( - s"""{ - | "message": "Error(s): Error while validating workflow url: unknown protocol: htpps", - | "status": "fail" - |}""".stripMargin) { + assertResult(s"""{ + | "message": "Error(s): Error while validating workflow url: unknown protocol: htpps", + | "status": "fail" + |}""".stripMargin) { responseAs[String].parseJson.prettyPrint } assertResult(StatusCodes.BadRequest) { @@ -233,18 +252,23 @@ class CromwellApiServiceSpec extends AsyncFlatSpec with ScalatestRouteTest with } it should "return 201 for a successful workflow submission with onHold = true" in { - val workflowSource = Multipart.FormData.BodyPart("workflowSource", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource())) - val workflowInputs = Multipart.FormData.BodyPart("workflowInputs", HttpEntity(MediaTypes.`application/json`, HelloWorld.rawInputs.toJson.toString())) - val onHold = Multipart.FormData.BodyPart("workflowOnHold", HttpEntity("true")) + val workflowSource = + Multipart.FormData.BodyPart("workflowSource", + HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()) + ) + val workflowInputs = + Multipart.FormData.BodyPart("workflowInputs", + HttpEntity(MediaTypes.`application/json`, HelloWorld.rawInputs.toJson.toString()) + ) + val onHold = Multipart.FormData.BodyPart("workflowOnHold", HttpEntity("true")) val formData = Multipart.FormData(workflowSource, workflowInputs, onHold).toEntity() Post(s"/workflows/$version", formData) ~> akkaHttpService.workflowRoutes ~> check { - assertResult( - s"""{ - | "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}", - | "status": "On Hold" - |}""".stripMargin) { + assertResult(s"""{ + | "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}", + | "status": "On Hold" + |}""".stripMargin) { responseAs[String].parseJson.prettyPrint } assertResult(StatusCodes.Created) { @@ -259,11 +283,10 @@ class CromwellApiServiceSpec extends AsyncFlatSpec with ScalatestRouteTest with Post(s"/workflows/$version/$id/releaseHold") ~> akkaHttpService.workflowRoutes ~> check { - assertResult( - s"""{ - | "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}", - | "status": "Submitted" - |}""".stripMargin) { + assertResult(s"""{ + | "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}", + | "status": "Submitted" + |}""".stripMargin) { responseAs[String].parseJson.prettyPrint } assertResult(StatusCodes.OK) { @@ -282,7 +305,8 @@ class CromwellApiServiceSpec extends AsyncFlatSpec with ScalatestRouteTest with s"""{ | "message": "Unrecognized workflow ID: ${CromwellApiServiceSpec.UnrecognizedWorkflowId.toString}", | "status": "fail" - |}""".stripMargin) { + |}""".stripMargin + ) { responseAs[String].parseJson.prettyPrint } assertResult(StatusCodes.NotFound) { @@ -293,8 +317,8 @@ class CromwellApiServiceSpec extends AsyncFlatSpec with ScalatestRouteTest with } it should "return 201 with warnings for a successful v1 workflow submission still using wdlSource" in { - val workflowSource = Multipart.FormData.BodyPart("wdlSource", - HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource())) + val workflowSource = + Multipart.FormData.BodyPart("wdlSource", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource())) val formData = Multipart.FormData(workflowSource).toEntity() Post(s"/workflows/v1", formData) ~> akkaHttpService.workflowRoutes ~> @@ -313,205 +337,230 @@ class CromwellApiServiceSpec extends AsyncFlatSpec with ScalatestRouteTest with warningHeader shouldNot be(empty) warningHeader.get.value should fullyMatch regex s"""299 cromwell/(\\d+-([0-9a-f]){7}(-SNAP)?|${VersionUtil.defaultMessage("cromwell-engine")}) """ + - "\"The 'wdlSource' parameter name has been deprecated in favor of 'workflowSource'. " + - "Support for 'wdlSource' will be removed from future versions of Cromwell. " + - "Please switch to using 'workflowSource' in future submissions.\"" + "\"The 'wdlSource' parameter name has been deprecated in favor of 'workflowSource'. " + + "Support for 'wdlSource' will be removed from future versions of Cromwell. " + + "Please switch to using 'workflowSource' in future submissions.\"" } } - it should "return 400 for an unrecognized form data request parameter " in { - val formData = Multipart.FormData(Map( - "incorrectParameter" -> HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()), - "incorrectParameter2" -> HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()) - )).toEntity - - Post(s"/workflows/$version", formData) ~> - akkaHttpService.workflowRoutes ~> - check { - assertResult( - s"""{ - | "status": "fail", - | "message": "Error(s): Unexpected body part name: incorrectParameter\\nUnexpected body part name: incorrectParameter2\\nworkflowSource or workflowUrl needs to be supplied" - |}""".stripMargin) { - responseAs[String] - } - assertResult(StatusCodes.BadRequest) { - status - } - assertResult(ContentTypes.`application/json`)(contentType) + it should "return 400 for an unrecognized form data request parameter " in { + val formData = Multipart + .FormData( + Map( + "incorrectParameter" -> HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()), + "incorrectParameter2" -> HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()) + ) + ) + .toEntity + + Post(s"/workflows/$version", formData) ~> + akkaHttpService.workflowRoutes ~> + check { + assertResult( + s"""{ + | "status": "fail", + | "message": "Error(s): Unexpected body part name: incorrectParameter\\nUnexpected body part name: incorrectParameter2\\nworkflowSource or workflowUrl needs to be supplied" + |}""".stripMargin + ) { + responseAs[String] } - } + assertResult(StatusCodes.BadRequest) { + status + } + assertResult(ContentTypes.`application/json`)(contentType) + } + } - it should "return 400 for a workflow submission with unsupported workflow option keys" in { - val options = """ - |{ - | "defaultRuntimeOptions": { - | "cpu":1 - | } - |} - |""".stripMargin - - val workflowSource = Multipart.FormData.BodyPart("workflowSource", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource())) - val workflowInputs = Multipart.FormData.BodyPart("workflowOptions", HttpEntity(MediaTypes.`application/json`, options)) - val formData = Multipart.FormData(workflowSource, workflowInputs).toEntity() - - Post(s"/workflows/$version", formData) ~> - akkaHttpService.workflowRoutes ~> - check { - assertResult(StatusCodes.BadRequest) { - status - } - assertResult { - """|{ - | "status": "fail", - | "message": "Error(s): Invalid workflow options provided: Unsupported key/value pair in WorkflowOptions: defaultRuntimeOptions -> {\"cpu\":1}" - |} - |""".stripMargin.trim - } { - responseAs[String] - } - assertResult(ContentTypes.`application/json`)(contentType) + it should "return 400 for a workflow submission with unsupported workflow option keys" in { + val options = """ + |{ + | "defaultRuntimeOptions": { + | "cpu":1 + | } + |} + |""".stripMargin + + val workflowSource = + Multipart.FormData.BodyPart("workflowSource", + HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()) + ) + val workflowInputs = + Multipart.FormData.BodyPart("workflowOptions", HttpEntity(MediaTypes.`application/json`, options)) + val formData = Multipart.FormData(workflowSource, workflowInputs).toEntity() + + Post(s"/workflows/$version", formData) ~> + akkaHttpService.workflowRoutes ~> + check { + assertResult(StatusCodes.BadRequest) { + status } - } + assertResult { + """|{ + | "status": "fail", + | "message": "Error(s): Invalid workflow options provided: Unsupported key/value pair in WorkflowOptions: defaultRuntimeOptions -> {\"cpu\":1}" + |} + |""".stripMargin.trim + } { + responseAs[String] + } + assertResult(ContentTypes.`application/json`)(contentType) + } + } + + it should "return 400 for a workflow submission with malformed workflow options json" in { + val options = s""" + |{"read_from_cache": "true" + |""".stripMargin + + val workflowSource = + Multipart.FormData.BodyPart("workflowSource", + HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()) + ) + val workflowInputs = + Multipart.FormData.BodyPart("workflowOptions", HttpEntity(MediaTypes.`application/json`, options)) + val formData = Multipart.FormData(workflowSource, workflowInputs).toEntity() - it should "return 400 for a workflow submission with malformed workflow options json" in { - val options = s""" - |{"read_from_cache": "true" - |""".stripMargin - - val workflowSource = Multipart.FormData.BodyPart("workflowSource", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource())) - val workflowInputs = Multipart.FormData.BodyPart("workflowOptions", HttpEntity(MediaTypes.`application/json`, options)) - val formData = Multipart.FormData(workflowSource, workflowInputs).toEntity() - - Post(s"/workflows/$version", formData) ~> - akkaHttpService.workflowRoutes ~> - check { - assertResult(StatusCodes.BadRequest) { - status - } - assertResult { - """|{ - | "status": "fail", - | "message": "Error(s): Invalid workflow options provided: Unexpected end-of-input at input index 28 (line 3, position 1), expected '}':\n\n^\n" - |} - |""".stripMargin.trim - } { - responseAs[String] - } - assertResult(ContentTypes.`application/json`)(contentType) + Post(s"/workflows/$version", formData) ~> + akkaHttpService.workflowRoutes ~> + check { + assertResult(StatusCodes.BadRequest) { + status } - } + assertResult { + """|{ + | "status": "fail", + | "message": "Error(s): Invalid workflow options provided: Unexpected end-of-input at input index 28 (line 3, position 1), expected '}':\n\n^\n" + |} + |""".stripMargin.trim + } { + responseAs[String] + } + assertResult(ContentTypes.`application/json`)(contentType) + } + } - it should "return 400 for a workflow submission with invalid workflow custom labels" in { - val labels = s""" - |{"key with more than 255 characters-at vero eos et accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias excepturi sint occaecati cupiditate non provident, similique sunt in culpas":"value with more than 255 characters-at vero eos et accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias excepturi sint occaecati cupiditate non provident, similique sunt in culpa"} - |""".stripMargin + it should "return 400 for a workflow submission with invalid workflow custom labels" in { + val labels = + s""" + |{"key with more than 255 characters-at vero eos et accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias excepturi sint occaecati cupiditate non provident, similique sunt in culpas":"value with more than 255 characters-at vero eos et accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias excepturi sint occaecati cupiditate non provident, similique sunt in culpa"} + |""".stripMargin - val workflowSource = Multipart.FormData.BodyPart("workflowSource", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource())) - val customLabels = Multipart.FormData.BodyPart("labels", HttpEntity(MediaTypes.`application/json`, labels)) - val onHold = Multipart.FormData.BodyPart("workflowOnHold", HttpEntity("true")) - val formData = Multipart.FormData(workflowSource, customLabels, onHold).toEntity() + val workflowSource = + Multipart.FormData.BodyPart("workflowSource", + HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()) + ) + val customLabels = Multipart.FormData.BodyPart("labels", HttpEntity(MediaTypes.`application/json`, labels)) + val onHold = Multipart.FormData.BodyPart("workflowOnHold", HttpEntity("true")) + val formData = Multipart.FormData(workflowSource, customLabels, onHold).toEntity() - Post(s"/workflows/$version", formData) ~> - akkaHttpService.workflowRoutes ~> - check { - assertResult(StatusCodes.BadRequest) { - status - } - assertResult(ContentTypes.`application/json`)(contentType) + Post(s"/workflows/$version", formData) ~> + akkaHttpService.workflowRoutes ~> + check { + assertResult(StatusCodes.BadRequest) { + status } - } + assertResult(ContentTypes.`application/json`)(contentType) + } + } + + behavior of "REST API batch submission endpoint" + it should "return 200 for a successful workflow submission " in { + val inputs = HelloWorld.rawInputs.toJson + val workflowSource = + Multipart.FormData.BodyPart("workflowSource", + HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()) + ) + val workflowInputs = + Multipart.FormData.BodyPart("workflowInputs", HttpEntity(MediaTypes.`application/json`, s"[$inputs, $inputs]")) + val formData = Multipart.FormData(workflowSource, workflowInputs).toEntity() - behavior of "REST API batch submission endpoint" - it should "return 200 for a successful workflow submission " in { - val inputs = HelloWorld.rawInputs.toJson - val workflowSource = Multipart.FormData.BodyPart("workflowSource", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource())) - val workflowInputs = Multipart.FormData.BodyPart("workflowInputs", HttpEntity(MediaTypes.`application/json`, s"[$inputs, $inputs]")) - val formData = Multipart.FormData(workflowSource, workflowInputs).toEntity() - - Post(s"/workflows/$version/batch", formData) ~> - akkaHttpService.workflowRoutes ~> - check { - assertResult( - s"""[{ - | "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}", - | "status": "Submitted" - |}, { - | "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}", - | "status": "Submitted" - |}]""".stripMargin) { - responseAs[String].parseJson.prettyPrint - } - assertResult(StatusCodes.Created) { - status - } - assertResult(ContentTypes.`application/json`)(contentType) + Post(s"/workflows/$version/batch", formData) ~> + akkaHttpService.workflowRoutes ~> + check { + assertResult(s"""[{ + | "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}", + | "status": "Submitted" + |}, { + | "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}", + | "status": "Submitted" + |}]""".stripMargin) { + responseAs[String].parseJson.prettyPrint } - } + assertResult(StatusCodes.Created) { + status + } + assertResult(ContentTypes.`application/json`)(contentType) + } + } + + it should "return 400 for an submission with no inputs" in { + val formData = Multipart + .FormData( + Multipart.FormData.BodyPart("workflowSource", + HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()) + ) + ) + .toEntity() - it should "return 400 for an submission with no inputs" in { - val formData = Multipart.FormData(Multipart.FormData.BodyPart("workflowSource", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()))).toEntity() - - Post(s"/workflows/$version/batch", formData) ~> - akkaHttpService.workflowRoutes ~> - check { - assertResult( - s"""{ - | "status": "fail", - | "message": "Error(s): No inputs were provided" - |}""".stripMargin) { - responseAs[String] - } - assertResult(StatusCodes.BadRequest) { - status - } - assertResult(ContentTypes.`application/json`)(contentType) + Post(s"/workflows/$version/batch", formData) ~> + akkaHttpService.workflowRoutes ~> + check { + assertResult(s"""{ + | "status": "fail", + | "message": "Error(s): No inputs were provided" + |}""".stripMargin) { + responseAs[String] } - } + assertResult(StatusCodes.BadRequest) { + status + } + assertResult(ContentTypes.`application/json`)(contentType) + } + } - behavior of "REST API /timing endpoint" - it should "return 200 with an HTML document for the timings route" in { - Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/timing") ~> - akkaHttpService.workflowRoutes ~> - check { - assertResult(StatusCodes.OK) { status } - assertResult(`text/html(UTF-8)`) { contentType } - assertResult("") { - responseAs[String].substring(0, 6) - } + behavior of "REST API /timing endpoint" + it should "return 200 with an HTML document for the timings route" in { + Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/timing") ~> + akkaHttpService.workflowRoutes ~> + check { + assertResult(StatusCodes.OK)(status) + assertResult(`text/html(UTF-8)`)(contentType) + assertResult("") { + responseAs[String].substring(0, 6) } - } + } + } - it should "return 404 when unrecognized workflow id is submitted" in { - Get(s"/workflows/$version/${CromwellApiServiceSpec.UnrecognizedWorkflowId}/timing") ~> - akkaHttpService.workflowRoutes ~> - check { - assertResult(StatusCodes.NotFound) { status } - assertResult( - s"""{ - | "message": "Unrecognized workflow ID: ${CromwellApiServiceSpec.UnrecognizedWorkflowId.toString}", - | "status": "fail" - |}""".stripMargin) { - responseAs[String].parseJson.prettyPrint - } + it should "return 404 when unrecognized workflow id is submitted" in { + Get(s"/workflows/$version/${CromwellApiServiceSpec.UnrecognizedWorkflowId}/timing") ~> + akkaHttpService.workflowRoutes ~> + check { + assertResult(StatusCodes.NotFound)(status) + assertResult( + s"""{ + | "message": "Unrecognized workflow ID: ${CromwellApiServiceSpec.UnrecognizedWorkflowId.toString}", + | "status": "fail" + |}""".stripMargin + ) { + responseAs[String].parseJson.prettyPrint } - } + } + } - it should "return 400 when invalid workflow id is submitted" in { - Get(s"/workflows/$version/foo/timing") ~> - akkaHttpService.workflowRoutes ~> - check { - assertResult(StatusCodes.BadRequest) { status } - assertResult( - s"""{ - | "message": "Invalid workflow ID: 'foo'.", - | "status": "fail" - |}""".stripMargin) { - responseAs[String].parseJson.prettyPrint - } - assertResult(ContentTypes.`application/json`)(contentType) + it should "return 400 when invalid workflow id is submitted" in { + Get(s"/workflows/$version/foo/timing") ~> + akkaHttpService.workflowRoutes ~> + check { + assertResult(StatusCodes.BadRequest)(status) + assertResult(s"""{ + | "message": "Invalid workflow ID: 'foo'.", + | "status": "fail" + |}""".stripMargin) { + responseAs[String].parseJson.prettyPrint } - } + assertResult(ContentTypes.`application/json`)(contentType) + } + } } object CromwellApiServiceSpec { @@ -561,26 +610,25 @@ object CromwellApiServiceSpec { object MockServiceRegistryActor { - private def fullMetadataResponse(workflowId: WorkflowId) = { + private def fullMetadataResponse(workflowId: WorkflowId) = List( MetadataEvent(MetadataKey(workflowId, None, "testKey1a"), MetadataValue("myValue1a", MetadataString)), MetadataEvent(MetadataKey(workflowId, None, "testKey1b"), MetadataValue("myValue1b", MetadataString)), - MetadataEvent(MetadataKey(workflowId, None, "testKey2a"), MetadataValue("myValue2a", MetadataString)), + MetadataEvent(MetadataKey(workflowId, None, "testKey2a"), MetadataValue("myValue2a", MetadataString)) ) - } - private def wesFullMetadataResponse(workflowId: WorkflowId) = { + private def wesFullMetadataResponse(workflowId: WorkflowId) = List( MetadataEvent(MetadataKey(workflowId, None, "status"), MetadataValue("Running", MetadataString)), - MetadataEvent(MetadataKey(workflowId, None, "submittedFiles:workflow"), MetadataValue("myValue2a", MetadataString)), - + MetadataEvent(MetadataKey(workflowId, None, "submittedFiles:workflow"), + MetadataValue("myValue2a", MetadataString) + ) ) - } - def responseMetadataValues(workflowId: WorkflowId, withKeys: List[String], withoutKeys: List[String]): JsObject = { def keyFilter(keys: List[String])(m: MetadataEvent) = keys.exists(k => m.key.key.startsWith(k)) - val metadataEvents = if (workflowId == wesWorkflowId) wesFullMetadataResponse(workflowId) else fullMetadataResponse(workflowId) + val metadataEvents = + if (workflowId == wesWorkflowId) wesFullMetadataResponse(workflowId) else fullMetadataResponse(workflowId) val events = metadataEvents .filter(m => withKeys.isEmpty || keyFilter(withKeys)(m)) .filter(m => withoutKeys.isEmpty || !keyFilter(withoutKeys)(m)) @@ -592,9 +640,16 @@ object CromwellApiServiceSpec { MetadataQuery(workflowId, None, None, None, None, expandSubWorkflows = false) def logsEvents(id: WorkflowId) = { - val stdout = MetadataEvent(MetadataKey(id, Some(MetadataJobKey("mycall", None, 1)), CallMetadataKeys.Stdout), MetadataValue("stdout.txt", MetadataString)) - val stderr = MetadataEvent(MetadataKey(id, Some(MetadataJobKey("mycall", None, 1)), CallMetadataKeys.Stderr), MetadataValue("stderr.txt", MetadataString)) - val backend = MetadataEvent(MetadataKey(id, Some(MetadataJobKey("mycall", None, 1)), s"${CallMetadataKeys.BackendLogsPrefix}:log"), MetadataValue("backend.log", MetadataString)) + val stdout = MetadataEvent(MetadataKey(id, Some(MetadataJobKey("mycall", None, 1)), CallMetadataKeys.Stdout), + MetadataValue("stdout.txt", MetadataString) + ) + val stderr = MetadataEvent(MetadataKey(id, Some(MetadataJobKey("mycall", None, 1)), CallMetadataKeys.Stderr), + MetadataValue("stderr.txt", MetadataString) + ) + val backend = MetadataEvent( + MetadataKey(id, Some(MetadataJobKey("mycall", None, 1)), s"${CallMetadataKeys.BackendLogsPrefix}:log"), + MetadataValue("backend.log", MetadataString) + ) Vector(stdout, stderr, backend) } } @@ -604,13 +659,30 @@ object CromwellApiServiceSpec { override def receive = { case QueryForWorkflowsMatchingParameters(parameters) => - val labels: Option[Map[String, String]] = { - parameters.contains(("additionalQueryResultFields", "labels")).option( - Map("key1" -> "label1", "key2" -> "label2")) - } - - val response = WorkflowQuerySuccess(WorkflowQueryResponse(List(WorkflowQueryResult(ExistingWorkflowId.toString, - None, Some(WorkflowSucceeded.toString), None, None, None, labels, Option("pid"), Option("rid"), Unarchived)), 1), None) + val labels: Option[Map[String, String]] = + parameters + .contains(("additionalQueryResultFields", "labels")) + .option(Map("key1" -> "label1", "key2" -> "label2")) + + val response = WorkflowQuerySuccess( + WorkflowQueryResponse( + List( + WorkflowQueryResult(ExistingWorkflowId.toString, + None, + Some(WorkflowSucceeded.toString), + None, + None, + None, + labels, + Option("pid"), + Option("rid"), + Unarchived + ) + ), + 1 + ), + None + ) sender() ! response case ValidateWorkflowIdInMetadata(id) => if (RecognizedWorkflowIds.contains(id)) sender() ! MetadataService.RecognizedWorkflowId @@ -620,15 +692,16 @@ object CromwellApiServiceSpec { else sender() ! MetadataService.UnrecognizedWorkflowId case FetchWorkflowMetadataArchiveStatusAndEndTime(id) => id match { - case ArchivedAndDeletedWorkflowId => sender() ! WorkflowMetadataArchivedStatusAndEndTime(ArchivedAndDeleted, Option(OffsetDateTime.now)) - case ArchivedWorkflowId => sender() ! WorkflowMetadataArchivedStatusAndEndTime(Archived, Option(OffsetDateTime.now)) + case ArchivedAndDeletedWorkflowId => + sender() ! WorkflowMetadataArchivedStatusAndEndTime(ArchivedAndDeleted, Option(OffsetDateTime.now)) + case ArchivedWorkflowId => + sender() ! WorkflowMetadataArchivedStatusAndEndTime(Archived, Option(OffsetDateTime.now)) case _ => sender() ! WorkflowMetadataArchivedStatusAndEndTime(Unarchived, Option(OffsetDateTime.now)) } case GetCurrentStatus => - sender() ! StatusCheckResponse( - ok = true, - systems = Map( - "Engine Database" -> SubsystemStatus(ok = true, messages = None))) + sender() ! StatusCheckResponse(ok = true, + systems = Map("Engine Database" -> SubsystemStatus(ok = true, messages = None)) + ) case request @ GetStatus(id) => val status = id match { case OnHoldWorkflowId => WorkflowOnHold @@ -641,12 +714,22 @@ object CromwellApiServiceSpec { } sender() ! SuccessfulMetadataJsonResponse(request, MetadataBuilderActor.processStatusResponse(id, status)) case request @ GetLabels(id) => - sender() ! SuccessfulMetadataJsonResponse(request, MetadataBuilderActor.processLabelsResponse(id, Map("key1" -> "label1", "key2" -> "label2"))) + sender() ! SuccessfulMetadataJsonResponse( + request, + MetadataBuilderActor.processLabelsResponse(id, Map("key1" -> "label1", "key2" -> "label2")) + ) case request @ WorkflowOutputs(id) => - val event = Vector(MetadataEvent(MetadataKey(id, None, "outputs:test.hello.salutation"), MetadataValue("Hello foo!", MetadataString))) + val event = Vector( + MetadataEvent(MetadataKey(id, None, "outputs:test.hello.salutation"), + MetadataValue("Hello foo!", MetadataString) + ) + ) sender() ! SuccessfulMetadataJsonResponse(request, MetadataBuilderActor.processOutputsResponse(id, event)) case request @ GetLogs(id) => - sender() ! SuccessfulMetadataJsonResponse(request, MetadataBuilderActor.workflowMetadataResponse(id, logsEvents(id), includeCallsIfEmpty = false, Map.empty)) + sender() ! SuccessfulMetadataJsonResponse( + request, + MetadataBuilderActor.workflowMetadataResponse(id, logsEvents(id), includeCallsIfEmpty = false, Map.empty) + ) case request @ FetchFailedJobsMetadataWithWorkflowId(id) => sender() ! SuccessfulMetadataJsonResponse(request, responseMetadataValues(id, List.empty, List.empty)) case request @ GetMetadataAction(MetadataQuery(id, _, _, withKeys, withoutKeys, _), _) => @@ -657,11 +740,15 @@ object CromwellApiServiceSpec { events.head.key.workflowId match { case CromwellApiServiceSpec.ExistingWorkflowId => sender() ! MetadataWriteSuccess(events) case CromwellApiServiceSpec.SummarizedWorkflowId => sender() ! MetadataWriteSuccess(events) - case CromwellApiServiceSpec.AbortedWorkflowId => sender() ! MetadataWriteFailure(new Exception("mock exception of db failure"), events) - case WorkflowId(_) => throw new Exception("Something untoward happened, this situation is not believed to be possible at this time") + case CromwellApiServiceSpec.AbortedWorkflowId => + sender() ! MetadataWriteFailure(new Exception("mock exception of db failure"), events) + case WorkflowId(_) => + throw new Exception( + "Something untoward happened, this situation is not believed to be possible at this time" + ) case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") } - case DescribeRequest(sourceFiles) => + case DescribeRequest(sourceFiles, _) => sourceFiles.workflowSource match { case Some("fail to describe") => sender() ! DescribeFailure("as requested, failing to describe") @@ -677,7 +764,9 @@ object CromwellApiServiceSpec { s"[reading back DescribeRequest contents] version: ${sourceFiles.workflowTypeVersion}" ) - sender() ! DescribeSuccess(description = WorkflowDescription(valid = true, errors = readBack, validWorkflow = true)) + sender() ! DescribeSuccess(description = + WorkflowDescription(valid = true, errors = readBack, validWorkflow = true) + ) } case _: InstrumentationServiceMessage => // Do nothing. case m => logger.error("Unexpected message received by MockServiceRegistryActor: {}", m) @@ -698,7 +787,11 @@ object CromwellApiServiceSpec { val message = id match { case AbortingWorkflowId => WorkflowAbortRequestedResponse(id) case OnHoldWorkflowId | SubmittedWorkflowId => WorkflowAbortedResponse(id) - case UnrecognizedWorkflowId => WorkflowAbortFailureResponse(id, new WorkflowNotFoundException(s"Couldn't abort $id because no workflow with that ID is in progress")) + case UnrecognizedWorkflowId => + WorkflowAbortFailureResponse( + id, + new WorkflowNotFoundException(s"Couldn't abort $id because no workflow with that ID is in progress") + ) case WorkflowId(_) => throw new Exception("Something untoward happened") case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") } diff --git a/engine/src/test/scala/cromwell/webservice/routes/MetadataRouteSupportSpec.scala b/engine/src/test/scala/cromwell/webservice/routes/MetadataRouteSupportSpec.scala index 73cc257e828..64d5ae24376 100644 --- a/engine/src/test/scala/cromwell/webservice/routes/MetadataRouteSupportSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/routes/MetadataRouteSupportSpec.scala @@ -2,7 +2,7 @@ package cromwell.webservice.routes import akka.actor.{ActorSystem, Props} import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ -import akka.http.scaladsl.model.headers.{HttpEncodings, `Accept-Encoding`} +import akka.http.scaladsl.model.headers.{`Accept-Encoding`, HttpEncodings} import akka.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCodes} import akka.http.scaladsl.server.Route import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} @@ -83,7 +83,7 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit akkaHttpService.metadataRoutes ~> check { status should be(StatusCodes.OK) - responseAs[JsObject].fields.keys should contain allOf(WorkflowMetadataKeys.Id, WorkflowMetadataKeys.Outputs) + responseAs[JsObject].fields.keys should contain allOf (WorkflowMetadataKeys.Id, WorkflowMetadataKeys.Outputs) contentType should be(ContentTypes.`application/json`) } } @@ -116,19 +116,28 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit akkaHttpService.metadataRoutes ~> check { status should be(StatusCodes.OK) - responseAs[JsObject].fields.keys should contain allOf(WorkflowMetadataKeys.Id, WorkflowMetadataKeys.Outputs) + responseAs[JsObject].fields.keys should contain allOf (WorkflowMetadataKeys.Id, WorkflowMetadataKeys.Outputs) contentType should be(ContentTypes.`application/json`) } } - def validateArchivedMetadataResponseMessage(responseJson: JsObject, includeAvailabilityMessage: Boolean, includeLabelsMessage: Boolean) = { + def validateArchivedMetadataResponseMessage(responseJson: JsObject, + includeAvailabilityMessage: Boolean, + includeLabelsMessage: Boolean + ) = { val responseMessage = responseJson.fields("message").asInstanceOf[JsString].value val expectedSuffix = - (if (includeAvailabilityMessage) " It is available in the archive bucket, or via a support request in the case of a managed instance." else "") + - (if (includeLabelsMessage) " As a result, new labels can't be added or existing labels can't be updated for this workflow." else "") - - responseMessage should startWith("Cromwell has archived this workflow's metadata " + - "according to the lifecycle policy. The workflow completed at ") + (if (includeAvailabilityMessage) + " It is available in the archive bucket, or via a support request in the case of a managed instance." + else "") + + (if (includeLabelsMessage) + " As a result, new labels can't be added or existing labels can't be updated for this workflow." + else "") + + responseMessage should startWith( + "Cromwell has archived this workflow's metadata " + + "according to the lifecycle policy. The workflow completed at " + ) // The missing middle of the message looks like "The workflow completed at x timestamp, which was y milliseconds ago." responseMessage should endWith(s"ago.${expectedSuffix}") } @@ -139,10 +148,13 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit check { status should be(StatusCodes.OK) - val responseJson = responseAs[JsObject] - responseJson.fields.keys should contain allOf(WorkflowMetadataKeys.Id, WorkflowMetadataKeys.MetadataArchiveStatus, WorkflowMetadataKeys.Message) + val responseJson = responseAs[JsObject] + responseJson.fields.keys should contain allOf (WorkflowMetadataKeys.Id, WorkflowMetadataKeys.MetadataArchiveStatus, WorkflowMetadataKeys.Message) responseJson.fields("metadataArchiveStatus").asInstanceOf[JsString].value shouldBe "ArchivedAndDeleted" - validateArchivedMetadataResponseMessage(responseJson, includeAvailabilityMessage = true, includeLabelsMessage = false) + validateArchivedMetadataResponseMessage(responseJson, + includeAvailabilityMessage = true, + includeLabelsMessage = false + ) contentType should be(ContentTypes.`application/json`) } } @@ -154,11 +166,12 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit check { status should be(StatusCodes.OK) - val call = responseAs[JsObject].fields("calls").convertTo[JsObject].fields("mycall").convertTo[Seq[JsObject]].head + val call = + responseAs[JsObject].fields("calls").convertTo[JsObject].fields("mycall").convertTo[Seq[JsObject]].head call.fields("stdout") should be(JsString("stdout.txt")) call.fields("stderr") should be(JsString("stderr.txt")) call.fields("stdout") should be(JsString("stdout.txt")) - call.fields("backendLogs").convertTo[JsObject].fields("log") should be (JsString("backend.log")) + call.fields("backendLogs").convertTo[JsObject].fields("log") should be(JsString("backend.log")) } } @@ -178,11 +191,12 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit check { status should be(StatusCodes.OK) - val call = responseAs[JsObject].fields("calls").convertTo[JsObject].fields("mycall").convertTo[Seq[JsObject]].head + val call = + responseAs[JsObject].fields("calls").convertTo[JsObject].fields("mycall").convertTo[Seq[JsObject]].head call.fields("stdout") should be(JsString("stdout.txt")) call.fields("stderr") should be(JsString("stderr.txt")) call.fields("stdout") should be(JsString("stdout.txt")) - call.fields("backendLogs").convertTo[JsObject].fields("log") should be (JsString("backend.log")) + call.fields("backendLogs").convertTo[JsObject].fields("log") should be(JsString("backend.log")) } } @@ -192,11 +206,14 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit check { status should be(StatusCodes.OK) - val responseJson = responseAs[JsObject] - responseJson.fields.keys should contain allOf(WorkflowMetadataKeys.Id, WorkflowMetadataKeys.MetadataArchiveStatus, WorkflowMetadataKeys.Message) + val responseJson = responseAs[JsObject] + responseJson.fields.keys should contain allOf (WorkflowMetadataKeys.Id, WorkflowMetadataKeys.MetadataArchiveStatus, WorkflowMetadataKeys.Message) responseJson.fields("metadataArchiveStatus").asInstanceOf[JsString].value shouldBe "ArchivedAndDeleted" - validateArchivedMetadataResponseMessage(responseJson, includeAvailabilityMessage = true, includeLabelsMessage = false) - } + validateArchivedMetadataResponseMessage(responseJson, + includeAvailabilityMessage = true, + includeLabelsMessage = false + ) + } } behavior of "REST API /metadata endpoint" @@ -206,7 +223,7 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit check { status should be(StatusCodes.OK) val result = responseAs[JsObject] - result.fields.keys should contain allOf("testKey1a", "testKey1b", "testKey2a") + result.fields.keys should contain allOf ("testKey1a", "testKey1b", "testKey2a") result.fields.keys shouldNot contain("testKey3") result.fields("testKey1a") should be(JsString("myValue1a")) result.fields("testKey1b") should be(JsString("myValue1b")) @@ -220,7 +237,7 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit check { status should be(StatusCodes.OK) val result = responseAs[JsObject] - result.fields.keys should contain allOf("testKey1a", "testKey1b", "testKey2a") + result.fields.keys should contain allOf ("testKey1a", "testKey1b", "testKey2a") result.fields.keys shouldNot contain("testKey3") result.fields("testKey1a") should be(JsString("myValue1a")) result.fields("testKey1b") should be(JsString("myValue1b")) @@ -229,7 +246,8 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit } it should "return with gzip encoding when requested" in { - Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata").addHeader(`Accept-Encoding`(HttpEncodings.gzip)) ~> + Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata") + .addHeader(`Accept-Encoding`(HttpEncodings.gzip)) ~> akkaHttpService.metadataRoutes ~> check { response.headers.find(_.name == "Content-Encoding").get.value should be("gzip") @@ -245,13 +263,15 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit } it should "return with included metadata from the metadata route" in { - Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata?includeKey=testKey1&includeKey=testKey2a") ~> + Get( + s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata?includeKey=testKey1&includeKey=testKey2a" + ) ~> akkaHttpService.metadataRoutes ~> check { status should be(StatusCodes.OK) val result = responseAs[JsObject] - result.fields.keys should contain allOf("testKey1a", "testKey1b", "testKey2a") - result.fields.keys should contain noneOf("testKey2b", "testKey3") + result.fields.keys should contain allOf ("testKey1a", "testKey1b", "testKey2a") + result.fields.keys should contain noneOf ("testKey2b", "testKey3") result.fields("testKey1a") should be(JsString("myValue1a")) result.fields("testKey1b") should be(JsString("myValue1b")) result.fields("testKey2a") should be(JsString("myValue2a")) @@ -259,28 +279,32 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit } it should "return with excluded metadata from the metadata route" in { - Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata?excludeKey=testKey2&excludeKey=testKey3") ~> + Get( + s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata?excludeKey=testKey2&excludeKey=testKey3" + ) ~> akkaHttpService.metadataRoutes ~> check { status should be(StatusCodes.OK) val result = responseAs[JsObject] - result.fields.keys should contain allOf("testKey1a", "testKey1b") - result.fields.keys should contain noneOf("testKey2a", "testKey3") + result.fields.keys should contain allOf ("testKey1a", "testKey1b") + result.fields.keys should contain noneOf ("testKey2a", "testKey3") result.fields("testKey1a") should be(JsString("myValue1a")) result.fields("testKey1b") should be(JsString("myValue1b")) } } it should "correctly include and exclude metadata keys in workflow details requests" in { - Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata?includeKey=testKey1&excludeKey=testKey1a") ~> + Get( + s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata?includeKey=testKey1&excludeKey=testKey1a" + ) ~> akkaHttpService.metadataRoutes ~> check { val r = responseAs[String] withClue(s"From response $r") { status should be(StatusCodes.OK) val result = responseAs[JsObject] - result.fields.keys should contain allElementsOf(List("testKey1b")) - result.fields.keys should contain noneOf("testKey1a", "testKey2") + result.fields.keys should contain allElementsOf (List("testKey1b")) + result.fields.keys should contain noneOf ("testKey1a", "testKey2") result.fields("testKey1b") should be(JsString("myValue1b")) } } @@ -292,7 +316,7 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit check { status should be(StatusCodes.OK) val result = responseAs[JsObject] - result.fields.keys should contain allOf("testKey1a", "testKey1b", "testKey2a") + result.fields.keys should contain allOf ("testKey1a", "testKey1b", "testKey2a") result.fields.keys shouldNot contain("testKey3") result.fields("testKey1a") should be(JsString("myValue1a")) result.fields("testKey1b") should be(JsString("myValue1b")) @@ -306,10 +330,13 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit check { status should be(StatusCodes.OK) - val responseJson = responseAs[JsObject] - responseJson.fields.keys should contain allOf(WorkflowMetadataKeys.Id, WorkflowMetadataKeys.MetadataArchiveStatus, WorkflowMetadataKeys.Message) + val responseJson = responseAs[JsObject] + responseJson.fields.keys should contain allOf (WorkflowMetadataKeys.Id, WorkflowMetadataKeys.MetadataArchiveStatus, WorkflowMetadataKeys.Message) responseJson.fields("metadataArchiveStatus").asInstanceOf[JsString].value shouldBe "ArchivedAndDeleted" - validateArchivedMetadataResponseMessage(responseJson, includeAvailabilityMessage = true, includeLabelsMessage = false) + validateArchivedMetadataResponseMessage(responseJson, + includeAvailabilityMessage = true, + includeLabelsMessage = false + ) } } @@ -319,7 +346,7 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit check { status should be(StatusCodes.OK) val result = responseAs[JsObject] - result.fields.keys should contain allOf("testKey1a", "testKey1b", "testKey2a") + result.fields.keys should contain allOf ("testKey1a", "testKey1b", "testKey2a") result.fields.keys shouldNot contain("testKey3") result.fields("testKey1a") should be(JsString("myValue1a")) result.fields("testKey1b") should be(JsString("myValue1b")) @@ -341,7 +368,9 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit } it should "return labels if specified in additionalQueryResultFields param" in { - Get(s"/workflows/$version/query?additionalQueryResultFields=labels&id=${CromwellApiServiceSpec.ExistingWorkflowId}") ~> + Get( + s"/workflows/$version/query?additionalQueryResultFields=labels&id=${CromwellApiServiceSpec.ExistingWorkflowId}" + ) ~> akkaHttpService.metadataRoutes ~> check { status should be(StatusCodes.OK) @@ -355,7 +384,9 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit } it should "return parentWorkflowId if specified in additionalQueryResultFields param" in { - Get(s"/workflows/$version/query?additionalQueryResultFields=parentWorkflowId&id=${CromwellApiServiceSpec.ExistingWorkflowId}") ~> + Get( + s"/workflows/$version/query?additionalQueryResultFields=parentWorkflowId&id=${CromwellApiServiceSpec.ExistingWorkflowId}" + ) ~> akkaHttpService.metadataRoutes ~> check { status should be(StatusCodes.OK) @@ -382,7 +413,9 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit } it should "return labels if specified in additionalQueryResultFields param" in { - Post(s"/workflows/$version/query", HttpEntity(ContentTypes.`application/json`, """[{"additionalQueryResultFields":"labels"}]""")) ~> + Post(s"/workflows/$version/query", + HttpEntity(ContentTypes.`application/json`, """[{"additionalQueryResultFields":"labels"}]""") + ) ~> akkaHttpService.metadataRoutes ~> check { assertResult(StatusCodes.OK) { @@ -396,7 +429,9 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit } it should "return parentWorkflowId if specified in additionalQueryResultFields param" in { - Post(s"/workflows/$version/query", HttpEntity(ContentTypes.`application/json`, """[{"additionalQueryResultFields":"parentWorkflowId"}]""")) ~> + Post(s"/workflows/$version/query", + HttpEntity(ContentTypes.`application/json`, """[{"additionalQueryResultFields":"parentWorkflowId"}]""") + ) ~> akkaHttpService.metadataRoutes ~> check { assertResult(StatusCodes.OK) { @@ -480,9 +515,12 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit check { status shouldBe StatusCodes.BadRequest val actualResult = responseAs[JsObject] - actualResult.fields.keys should contain allOf(WorkflowMetadataKeys.Id, WorkflowMetadataKeys.MetadataArchiveStatus, WorkflowMetadataKeys.Message) + actualResult.fields.keys should contain allOf (WorkflowMetadataKeys.Id, WorkflowMetadataKeys.MetadataArchiveStatus, WorkflowMetadataKeys.Message) actualResult.fields("metadataArchiveStatus").asInstanceOf[JsString].value shouldBe "Archived" - validateArchivedMetadataResponseMessage(actualResult, includeAvailabilityMessage = false, includeLabelsMessage = true) + validateArchivedMetadataResponseMessage(actualResult, + includeAvailabilityMessage = false, + includeLabelsMessage = true + ) } } @@ -502,9 +540,12 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit check { status shouldBe StatusCodes.BadRequest val actualResult = responseAs[JsObject] - actualResult.fields.keys should contain allOf(WorkflowMetadataKeys.Id, WorkflowMetadataKeys.MetadataArchiveStatus, WorkflowMetadataKeys.Message) + actualResult.fields.keys should contain allOf (WorkflowMetadataKeys.Id, WorkflowMetadataKeys.MetadataArchiveStatus, WorkflowMetadataKeys.Message) actualResult.fields("metadataArchiveStatus").asInstanceOf[JsString].value shouldBe "ArchivedAndDeleted" - validateArchivedMetadataResponseMessage(actualResult, includeAvailabilityMessage = true, includeLabelsMessage = true) + validateArchivedMetadataResponseMessage(actualResult, + includeAvailabilityMessage = true, + includeLabelsMessage = true + ) } } @@ -518,7 +559,9 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit """.stripMargin val unsummarizedId = CromwellApiServiceSpec.ExistingWorkflowId - Patch(s"/workflows/$version/$unsummarizedId/labels", HttpEntity(ContentTypes.`application/json`, validLabelsJson)) ~> + Patch(s"/workflows/$version/$unsummarizedId/labels", + HttpEntity(ContentTypes.`application/json`, validLabelsJson) + ) ~> akkaHttpService.metadataRoutes ~> check { status shouldBe StatusCodes.NotFound @@ -528,7 +571,8 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit } object MetadataRouteSupportSpec { - class MockMetadataRouteSupport()(implicit val system: ActorSystem, routeTestTimeout: RouteTestTimeout) extends MetadataRouteSupport { + class MockMetadataRouteSupport()(implicit val system: ActorSystem, routeTestTimeout: RouteTestTimeout) + extends MetadataRouteSupport { override def actorRefFactory = system override val ec = system.dispatcher override val timeout = routeTestTimeout.duration diff --git a/engine/src/test/scala/cromwell/webservice/routes/WomtoolRouteSupportSpec.scala b/engine/src/test/scala/cromwell/webservice/routes/WomtoolRouteSupportSpec.scala index 24f8ce18ee7..06c48a42b77 100644 --- a/engine/src/test/scala/cromwell/webservice/routes/WomtoolRouteSupportSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/routes/WomtoolRouteSupportSpec.scala @@ -16,7 +16,6 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.duration._ - // N.B. this suite only tests the routing and creation of the WorkflowSourceFilesCollection, it uses the MockServiceRegistryActor // to return fake results instead of going to a real WomtoolServiceActor class WomtoolRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with Matchers { @@ -29,12 +28,24 @@ class WomtoolRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with behavior of "/describe endpoint" object BodyParts { - val workflowSource = Multipart.FormData.BodyPart("workflowSource", HttpEntity(ContentTypes.`text/plain(UTF-8)`, "This is not a WDL, but that's OK for this test of request routing.")) - val workflowUrl = Multipart.FormData.BodyPart("workflowUrl", HttpEntity(MediaTypes.`application/json`, - "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/womtool/src/test/resources/validate/wdl_draft3/valid/callable_imports/my_workflow.wdl")) - val workflowInputs = Multipart.FormData.BodyPart("workflowInputs", HttpEntity(MediaTypes.`application/json`, "{\"a\":\"is for apple\"}")) + val workflowSource = Multipart.FormData.BodyPart( + "workflowSource", + HttpEntity(ContentTypes.`text/plain(UTF-8)`, "This is not a WDL, but that's OK for this test of request routing.") + ) + val workflowUrl = Multipart.FormData.BodyPart( + "workflowUrl", + HttpEntity( + MediaTypes.`application/json`, + "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/womtool/src/test/resources/validate/wdl_draft3/valid/callable_imports/my_workflow.wdl" + ) + ) + val workflowInputs = Multipart.FormData.BodyPart( + "workflowInputs", + HttpEntity(MediaTypes.`application/json`, "{\"a\":\"is for apple\"}") + ) val workflowType = Multipart.FormData.BodyPart("workflowType", HttpEntity(ContentTypes.`text/plain(UTF-8)`, "WDL")) - val workflowVersion = Multipart.FormData.BodyPart("workflowTypeVersion", HttpEntity(ContentTypes.`text/plain(UTF-8)`, "1.0")) + val workflowVersion = + Multipart.FormData.BodyPart("workflowTypeVersion", HttpEntity(ContentTypes.`text/plain(UTF-8)`, "1.0")) val workflowSourceTriggerDescribeFailure = Multipart.FormData.BodyPart("workflowSource", HttpEntity(ContentTypes.`text/plain(UTF-8)`, "fail to describe")) @@ -43,16 +54,17 @@ class WomtoolRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with } it should "return Bad Request if the actor returns DescribeFailure" in { - Post(s"/womtool/$version/describe", Multipart.FormData(BodyParts.workflowSourceTriggerDescribeFailure).toEntity()) ~> + Post(s"/womtool/$version/describe", + Multipart.FormData(BodyParts.workflowSourceTriggerDescribeFailure).toEntity() + ) ~> akkaHttpService.womtoolRoutes ~> check { status should be(StatusCodes.BadRequest) - assertResult( - s"""{ - | "status": "fail", - | "message": "as requested, failing to describe" - |}""".stripMargin) { + assertResult(s"""{ + | "status": "fail", + | "message": "as requested, failing to describe" + |}""".stripMargin) { responseAs[String] } } @@ -75,7 +87,8 @@ class WomtoolRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with status should be(StatusCodes.OK) assertResult { - WorkflowDescription(valid = true, + WorkflowDescription( + valid = true, errors = List( "this is fake data from the mock SR actor", "[reading back DescribeRequest contents] workflow hashcode: Some(580529622)", @@ -86,7 +99,7 @@ class WomtoolRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with ), validWorkflow = true ) - } { responseAs[WorkflowDescription] } + }(responseAs[WorkflowDescription]) } } @@ -97,7 +110,8 @@ class WomtoolRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with status should be(StatusCodes.OK) assertResult { - WorkflowDescription(valid = true, + WorkflowDescription( + valid = true, errors = List( "this is fake data from the mock SR actor", "[reading back DescribeRequest contents] workflow hashcode: None", @@ -108,18 +122,24 @@ class WomtoolRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with ), validWorkflow = true ) - } { responseAs[WorkflowDescription] } + }(responseAs[WorkflowDescription]) } } it should "include inputs, workflow type, and workflow version in the WorkflowSourceFilesCollection" in { - Post(s"/womtool/$version/describe", Multipart.FormData(BodyParts.workflowSource, BodyParts.workflowInputs, BodyParts.workflowType, BodyParts.workflowVersion).toEntity()) ~> + Post( + s"/womtool/$version/describe", + Multipart + .FormData(BodyParts.workflowSource, BodyParts.workflowInputs, BodyParts.workflowType, BodyParts.workflowVersion) + .toEntity() + ) ~> akkaHttpService.womtoolRoutes ~> check { status should be(StatusCodes.OK) assertResult { - WorkflowDescription(valid = true, + WorkflowDescription( + valid = true, errors = List( "this is fake data from the mock SR actor", "[reading back DescribeRequest contents] workflow hashcode: Some(580529622)", @@ -130,17 +150,18 @@ class WomtoolRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with ), validWorkflow = true ) - } { responseAs[WorkflowDescription] } + }(responseAs[WorkflowDescription]) } } } object WomtoolRouteSupportSpec { - class MockWomtoolRouteSupport()(implicit val system: ActorSystem, routeTestTimeout: RouteTestTimeout) extends WomtoolRouteSupport { + class MockWomtoolRouteSupport()(implicit val system: ActorSystem, routeTestTimeout: RouteTestTimeout) + extends WomtoolRouteSupport { override def actorRefFactory = system override val ec = system.dispatcher override val timeout = routeTestTimeout.duration override val serviceRegistryActor = actorRefFactory.actorOf(Props(new MockServiceRegistryActor())) - override implicit val materializer = ActorMaterializer() + implicit override val materializer = ActorMaterializer() } } diff --git a/engine/src/test/scala/cromwell/webservice/routes/wes/ServiceInfoSpec.scala b/engine/src/test/scala/cromwell/webservice/routes/wes/ServiceInfoSpec.scala index ba60258476c..e75e3236254 100644 --- a/engine/src/test/scala/cromwell/webservice/routes/wes/ServiceInfoSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/routes/wes/ServiceInfoSpec.scala @@ -22,7 +22,8 @@ class ServiceInfoSpec extends AsyncFlatSpec with ScalatestRouteTest with Matcher behavior of "ServiceInfo" - val expectedResponse = WesStatusInfoResponse(Map("WDL" -> Set("draft-2", "1.0", "biscayne", "cascades")), + val expectedResponse = WesStatusInfoResponse( + Map("WDL" -> Set("draft-2", "1.0", "biscayne", "cascades")), List("1.0"), Set("ftp", "s3", "drs", "gcs", "http"), Map("Cromwell" -> CromwellApiService.cromwellVersion), @@ -30,8 +31,8 @@ class ServiceInfoSpec extends AsyncFlatSpec with ScalatestRouteTest with Matcher Map(WesState.Running -> 5, WesState.Queued -> 3, WesState.Canceling -> 2), "https://cromwell.readthedocs.io/en/stable/", "https://cromwell.readthedocs.io/en/stable/", - Map()) - + Map() + ) it should "should eventually build the right WesResponse" in { ServiceInfo.toWesResponse(workflowStoreActor) map { r => diff --git a/engine/src/test/scala/cromwell/webservice/routes/wes/WesRouteSupportSpec.scala b/engine/src/test/scala/cromwell/webservice/routes/wes/WesRouteSupportSpec.scala index 00a361a3174..80d4ec23f6a 100644 --- a/engine/src/test/scala/cromwell/webservice/routes/wes/WesRouteSupportSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/routes/wes/WesRouteSupportSpec.scala @@ -7,7 +7,11 @@ import akka.http.scaladsl.server.MethodRejection import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} import cromwell.util.SampleWdl.HelloWorld import cromwell.webservice.routes.CromwellApiServiceSpec -import cromwell.webservice.routes.CromwellApiServiceSpec.{MockServiceRegistryActor, MockWorkflowManagerActor, MockWorkflowStoreActor} +import cromwell.webservice.routes.CromwellApiServiceSpec.{ + MockServiceRegistryActor, + MockWorkflowManagerActor, + MockWorkflowStoreActor +} import cromwell.webservice.routes.wes.WesResponseJsonSupport._ import org.scalatest.flatspec.AsyncFlatSpec import org.scalatest.matchers.should.Matchers @@ -18,11 +22,10 @@ import scala.concurrent.duration._ class WesRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with Matchers with WesRouteSupport { val actorRefFactory = system - override implicit val ec = system.dispatcher + implicit override val ec = system.dispatcher override val timeout = routeTestTimeout.duration implicit def routeTestTimeout = RouteTestTimeout(5.seconds) - override val workflowStoreActor = actorRefFactory.actorOf(Props(new MockWorkflowStoreActor())) override val serviceRegistryActor = actorRefFactory.actorOf(Props(new MockServiceRegistryActor())) override val workflowManagerActor = actorRefFactory.actorOf(Props(new MockWorkflowManagerActor())) @@ -33,16 +36,20 @@ class WesRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with Mat it should "return PAUSED when on hold" in { Get(s"/ga4gh/wes/$version/runs/${CromwellApiServiceSpec.OnHoldWorkflowId}/status") ~> wesRoutes ~> - check { - responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.OnHoldWorkflowId.toString, WesState.Paused) - } + check { + responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.OnHoldWorkflowId.toString, + WesState.Paused + ) + } } it should "return QUEUED when submitted" in { Get(s"/ga4gh/wes/$version/runs/${CromwellApiServiceSpec.ExistingWorkflowId}/status") ~> wesRoutes ~> check { - responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.ExistingWorkflowId.toString, WesState.Queued) + responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.ExistingWorkflowId.toString, + WesState.Queued + ) } } @@ -50,7 +57,9 @@ class WesRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with Mat Get(s"/ga4gh/wes/$version/runs/${CromwellApiServiceSpec.RunningWorkflowId}/status") ~> wesRoutes ~> check { - responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.RunningWorkflowId.toString, WesState.Running) + responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.RunningWorkflowId.toString, + WesState.Running + ) } } @@ -58,7 +67,9 @@ class WesRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with Mat Get(s"/ga4gh/wes/$version/runs/${CromwellApiServiceSpec.AbortingWorkflowId}/status") ~> wesRoutes ~> check { - responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.AbortingWorkflowId.toString, WesState.Canceling) + responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.AbortingWorkflowId.toString, + WesState.Canceling + ) } } @@ -66,7 +77,9 @@ class WesRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with Mat Get(s"/ga4gh/wes/$version/runs/${CromwellApiServiceSpec.AbortedWorkflowId}/status") ~> wesRoutes ~> check { - responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.AbortedWorkflowId.toString, WesState.Canceled) + responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.AbortedWorkflowId.toString, + WesState.Canceled + ) } } @@ -74,7 +87,9 @@ class WesRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with Mat Get(s"/ga4gh/wes/$version/runs/${CromwellApiServiceSpec.SucceededWorkflowId}/status") ~> wesRoutes ~> check { - responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.SucceededWorkflowId.toString, WesState.Complete) + responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.SucceededWorkflowId.toString, + WesState.Complete + ) } } @@ -82,7 +97,9 @@ class WesRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with Mat Get(s"/ga4gh/wes/$version/runs/${CromwellApiServiceSpec.FailedWorkflowId}/status") ~> wesRoutes ~> check { - responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.FailedWorkflowId.toString, WesState.ExecutorError) + responseAs[WesRunStatus] shouldEqual WesRunStatus(CromwellApiServiceSpec.FailedWorkflowId.toString, + WesState.ExecutorError + ) } } @@ -107,7 +124,9 @@ class WesRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with Mat status } - responseAs[WesErrorResponse] shouldEqual WesErrorResponse("Invalid workflow ID: 'foobar'.", StatusCodes.InternalServerError.intValue) + responseAs[WesErrorResponse] shouldEqual WesErrorResponse("Invalid workflow ID: 'foobar'.", + StatusCodes.InternalServerError.intValue + ) } } @@ -157,16 +176,24 @@ class WesRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with Mat behavior of "WES API /runs POST endpoint" it should "return 201 for a successful workflow submission" in { - val workflowSource = Multipart.FormData.BodyPart("workflow_url", HttpEntity(MediaTypes.`application/json`, "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/womtool/src/test/resources/validate/wdl_draft3/valid/callable_imports/my_workflow.wdl")) - val workflowInputs = Multipart.FormData.BodyPart("workflow_params", HttpEntity(MediaTypes.`application/json`, HelloWorld.rawInputs.toJson.toString())) + val workflowSource = Multipart.FormData.BodyPart( + "workflow_url", + HttpEntity( + MediaTypes.`application/json`, + "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/womtool/src/test/resources/validate/wdl_draft3/valid/callable_imports/my_workflow.wdl" + ) + ) + val workflowInputs = + Multipart.FormData.BodyPart("workflow_params", + HttpEntity(MediaTypes.`application/json`, HelloWorld.rawInputs.toJson.toString()) + ) val formData = Multipart.FormData(workflowSource, workflowInputs).toEntity() Post(s"/ga4gh/wes/$version/runs", formData) ~> wesRoutes ~> check { - assertResult( - s"""{ - | "run_id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}" - |}""".stripMargin) { + assertResult(s"""{ + | "run_id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}" + |}""".stripMargin) { responseAs[String].parseJson.prettyPrint } assertResult(StatusCodes.Created) { @@ -176,7 +203,6 @@ class WesRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with Mat } } - behavior of "WES API /runs GET endpoint" it should "return results for a good query" in { Get(s"/ga4gh/wes/v1/runs") ~> @@ -197,9 +223,9 @@ class WesRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with Mat check { status should be(StatusCodes.OK) val result = responseAs[JsObject] - result.fields.keys should contain allOf("request", "run_id", "state") + result.fields.keys should contain allOf ("request", "run_id", "state") result.fields("state") should be(JsString("RUNNING")) result.fields("run_id") should be(JsString(CromwellApiServiceSpec.wesWorkflowId.toString)) } } -} \ No newline at end of file +} diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemConfig.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemConfig.scala index c5467c78ffe..172b3e8a9d0 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemConfig.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemConfig.scala @@ -10,12 +10,12 @@ import java.util.UUID // WSM config is needed for accessing WSM-managed blob containers created in Terra workspaces. // If the identity executing Cromwell has native access to the blob container, this can be ignored. -final case class WorkspaceManagerConfig(url: WorkspaceManagerURL, - overrideWsmAuthToken: Option[String]) // dev-only +final case class WorkspaceManagerConfig(url: WorkspaceManagerURL, overrideWsmAuthToken: Option[String]) // dev-only final case class BlobFileSystemConfig(subscriptionId: Option[SubscriptionId], expiryBufferMinutes: Long, - workspaceManagerConfig: Option[WorkspaceManagerConfig]) + workspaceManagerConfig: Option[WorkspaceManagerConfig] +) object BlobFileSystemConfig { @@ -36,8 +36,7 @@ object BlobFileSystemConfig { (wsmURL, overrideWsmAuthToken) .mapN(WorkspaceManagerConfig) .map(Option(_)) - } - else None.validNel + } else None.validNel (subscriptionId, expiryBufferMinutes, wsmConfig) .mapN(BlobFileSystemConfig.apply) @@ -45,16 +44,16 @@ object BlobFileSystemConfig { } private def parseString(config: Config, path: String) = - validate[String] { config.as[String](path) } + validate[String](config.as[String](path)) private def parseStringOpt(config: Config, path: String) = - validate[Option[String]] { config.as[Option[String]](path) } + validate[Option[String]](config.as[Option[String]](path)) private def parseUUIDOpt(config: Config, path: String) = - validate[Option[UUID]] { config.as[Option[String]](path).map(UUID.fromString) } + validate[Option[UUID]](config.as[Option[String]](path).map(UUID.fromString)) private def parseLongOpt(config: Config, path: String) = - validate[Option[Long]] { config.as[Option[Long]](path) } + validate[Option[Long]](config.as[Option[Long]](path)) } // Our filesystem setup magic can't use BlobFileSystemConfig.apply directly, so we need this diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index e3de6783d85..21316250146 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -15,6 +15,7 @@ import java.nio.file.spi.FileSystemProvider import java.time.temporal.ChronoUnit import java.time.{Duration, OffsetDateTime} import java.util.UUID +import scala.collection.mutable import scala.jdk.CollectionConverters._ import scala.util.{Failure, Success, Try} @@ -22,9 +23,12 @@ import scala.util.{Failure, Success, Try} // actually connecting to Blob storage. case class AzureFileSystemAPI(private val provider: FileSystemProvider = new AzureFileSystemProvider()) { def getFileSystem(uri: URI): Try[AzureFileSystem] = Try(provider.getFileSystem(uri).asInstanceOf[AzureFileSystem]) - def newFileSystem(uri: URI, config: Map[String, Object]): Try[AzureFileSystem] = Try(provider.newFileSystem(uri, config.asJava).asInstanceOf[AzureFileSystem]) + def newFileSystem(uri: URI, config: Map[String, Object]): Try[AzureFileSystem] = Try( + provider.newFileSystem(uri, config.asJava).asInstanceOf[AzureFileSystem] + ) def closeFileSystem(uri: URI): Option[Unit] = getFileSystem(uri).toOption.map(_.close) } + /** * The BlobFileSystemManager is an object that is responsible for managing the open filesystem, * and refreshing the SAS token that is used to access the blob container containing that filesystem. @@ -36,27 +40,33 @@ object BlobFileSystemManager { def buildConfigMap(credential: AzureSasCredential, container: BlobContainerName): Map[String, Object] = { // Special handling is done here to provide a special key value pair if the placeholder token is provided // This is due to the BlobClient requiring an auth token even for public blob paths. - val sasTuple = if (credential == PLACEHOLDER_TOKEN) (AzureFileSystem.AZURE_STORAGE_PUBLIC_ACCESS_CREDENTIAL, PLACEHOLDER_TOKEN) - else (AzureFileSystem.AZURE_STORAGE_SAS_TOKEN_CREDENTIAL, credential) + val sasTuple = + if (credential == PLACEHOLDER_TOKEN) (AzureFileSystem.AZURE_STORAGE_PUBLIC_ACCESS_CREDENTIAL, PLACEHOLDER_TOKEN) + else (AzureFileSystem.AZURE_STORAGE_SAS_TOKEN_CREDENTIAL, credential) - Map(sasTuple, (AzureFileSystem.AZURE_STORAGE_FILE_STORES, container.value), - (AzureFileSystem.AZURE_STORAGE_SKIP_INITIAL_CONTAINER_CHECK, java.lang.Boolean.TRUE)) + Map( + sasTuple, + (AzureFileSystem.AZURE_STORAGE_FILE_STORES, container.value), + (AzureFileSystem.AZURE_STORAGE_SKIP_INITIAL_CONTAINER_CHECK, java.lang.Boolean.TRUE) + ) } - def combinedEnpointContainerUri(endpoint: EndpointURL, container: BlobContainerName) = new URI("azb://?endpoint=" + endpoint + "/" + container.value) + def combinedEnpointContainerUri(endpoint: EndpointURL, container: BlobContainerName) = new URI( + "azb://?endpoint=" + endpoint + "/" + container.value + ) val PLACEHOLDER_TOKEN = new AzureSasCredential("this-is-a-public-sas") } class BlobFileSystemManager(val expiryBufferMinutes: Long, val blobTokenGenerator: BlobSasTokenGenerator, - val fileSystemAPI: AzureFileSystemAPI = AzureFileSystemAPI()) extends LazyLogging { + val fileSystemAPI: AzureFileSystemAPI = AzureFileSystemAPI() +) extends LazyLogging { - def this(config: BlobFileSystemConfig) = { + def this(config: BlobFileSystemConfig) = this( config.expiryBufferMinutes, BlobSasTokenGenerator.createBlobTokenGeneratorFromConfig(config) ) - } def this(rawConfig: Config) = this(BlobFileSystemConfig(rawConfig)) @@ -67,18 +77,16 @@ class BlobFileSystemManager(val expiryBufferMinutes: Long, synchronized { fileSystemAPI.getFileSystem(uri).filter(!_.isExpired(buffer)).recoverWith { // If no filesystem already exists, this will create a new connection, with the provided configs - case _: FileSystemNotFoundException => { + case _: FileSystemNotFoundException => logger.info(s"Creating new blob filesystem for URI $uri") generateFilesystem(uri, container, endpoint) - } - case _ : NoSuchElementException => { + case _: NoSuchElementException => // When the filesystem expires, the above filter results in a // NoSuchElementException. If expired, close the filesystem // and reopen the filesystem with the fresh token logger.info(s"Closing & regenerating token for existing blob filesystem at URI $uri") fileSystemAPI.closeFileSystem(uri) generateFilesystem(uri, container, endpoint) - } } } } @@ -92,15 +100,17 @@ class BlobFileSystemManager(val expiryBufferMinutes: Long, * @param endpoint the endpoint containing the storage account for the container to open * @return a try with either the successfully created filesystem, or a failure containing the exception */ - private def generateFilesystem(uri: URI, container: BlobContainerName, endpoint: EndpointURL): Try[AzureFileSystem] = { - blobTokenGenerator.generateBlobSasToken(endpoint, container) - .flatMap((token: AzureSasCredential) => { + private def generateFilesystem(uri: URI, container: BlobContainerName, endpoint: EndpointURL): Try[AzureFileSystem] = + blobTokenGenerator + .generateBlobSasToken(endpoint, container) + .flatMap { (token: AzureSasCredential) => fileSystemAPI.newFileSystem(uri, BlobFileSystemManager.buildConfigMap(token, container)) - }) - } + } } -sealed trait BlobSasTokenGenerator { def generateBlobSasToken(endpoint: EndpointURL, container: BlobContainerName): Try[AzureSasCredential] } +sealed trait BlobSasTokenGenerator { + def generateBlobSasToken(endpoint: EndpointURL, container: BlobContainerName): Try[AzureSasCredential] +} object BlobSasTokenGenerator { /** @@ -122,16 +132,18 @@ object BlobSasTokenGenerator { * @return An appropriate BlobSasTokenGenerator */ def createBlobTokenGeneratorFromConfig(config: BlobFileSystemConfig): BlobSasTokenGenerator = - config.workspaceManagerConfig.map { wsmConfig => - val wsmClient: WorkspaceManagerApiClientProvider = new HttpWorkspaceManagerClientProvider(wsmConfig.url) - - // WSM-mediated mediated SAS token generator - // parameterizing client instead of URL to make injecting mock client possible - BlobSasTokenGenerator.createBlobTokenGenerator(wsmClient, wsmConfig.overrideWsmAuthToken) - }.getOrElse( - // Native SAS token generator - BlobSasTokenGenerator.createBlobTokenGenerator(config.subscriptionId) - ) + config.workspaceManagerConfig + .map { wsmConfig => + val wsmClient: WorkspaceManagerApiClientProvider = new HttpWorkspaceManagerClientProvider(wsmConfig.url) + + // WSM-mediated mediated SAS token generator + // parameterizing client instead of URL to make injecting mock client possible + BlobSasTokenGenerator.createBlobTokenGenerator(wsmClient, wsmConfig.overrideWsmAuthToken) + } + .getOrElse( + // Native SAS token generator + BlobSasTokenGenerator.createBlobTokenGenerator(config.subscriptionId) + ) /** * Native SAS token generator, uses the DefaultAzureCredentialBuilder in the local environment @@ -142,9 +154,8 @@ object BlobSasTokenGenerator { * @return A NativeBlobTokenGenerator, able to produce a valid SAS token for accessing the provided blob * container and endpoint locally */ - def createBlobTokenGenerator(subscription: Option[SubscriptionId]): BlobSasTokenGenerator = { + def createBlobTokenGenerator(subscription: Option[SubscriptionId]): BlobSasTokenGenerator = NativeBlobSasTokenGenerator(subscription) - } /** * WSM-mediated SAS token generator, uses the DefaultAzureCredentialBuilder in the cloud environment @@ -159,14 +170,17 @@ object BlobSasTokenGenerator { * container and endpoint that is managed by WSM */ def createBlobTokenGenerator(workspaceManagerClient: WorkspaceManagerApiClientProvider, - overrideWsmAuthToken: Option[String]): BlobSasTokenGenerator = { - WSMBlobSasTokenGenerator(workspaceManagerClient, overrideWsmAuthToken) - } + overrideWsmAuthToken: Option[String] + ): BlobSasTokenGenerator = + new WSMBlobSasTokenGenerator(workspaceManagerClient, overrideWsmAuthToken) } -case class WSMBlobSasTokenGenerator(wsmClientProvider: WorkspaceManagerApiClientProvider, - overrideWsmAuthToken: Option[String]) extends BlobSasTokenGenerator { +case class WSMTerraCoordinates(wsmEndpoint: String, workspaceId: UUID, containerResourceId: UUID) + +class WSMBlobSasTokenGenerator(wsmClientProvider: WorkspaceManagerApiClientProvider, + overrideWsmAuthToken: Option[String] +) extends BlobSasTokenGenerator { /** * Generate a BlobSasToken by using the available authorization information @@ -178,32 +192,83 @@ case class WSMBlobSasTokenGenerator(wsmClientProvider: WorkspaceManagerApiClient * @return an AzureSasCredential for accessing a blob container */ def generateBlobSasToken(endpoint: EndpointURL, container: BlobContainerName): Try[AzureSasCredential] = { - val wsmAuthToken: Try[String] = overrideWsmAuthToken match { - case Some(t) => Success(t) - case None => AzureCredentials.getAccessToken(None).toTry - } + val wsmAuthToken: Try[String] = getWsmAuth container.workspaceId match { // If this is a Terra workspace, request a token from WSM - case Success(workspaceId) => { + case Success(workspaceId) => (for { wsmAuth <- wsmAuthToken wsmAzureResourceClient = wsmClientProvider.getControlledAzureResourceApi(wsmAuth) - resourceId <- getContainerResourceId(workspaceId, container, wsmAuth) + resourceId <- getContainerResourceId(workspaceId, container, Option(wsmAuth)) sasToken <- wsmAzureResourceClient.createAzureStorageContainerSasToken(workspaceId, resourceId) } yield sasToken).recoverWith { // If the storage account was still not found in WSM, this may be a public filesystem case exception: ApiException if exception.getCode == 404 => Try(BlobFileSystemManager.PLACEHOLDER_TOKEN) } - } // Otherwise assume that the container is public and use a placeholder // SAS token to bypass the BlobClient authentication requirement case Failure(_) => Try(BlobFileSystemManager.PLACEHOLDER_TOKEN) } } - def getContainerResourceId(workspaceId: UUID, container: BlobContainerName, wsmAuth : String): Try[UUID] = { - val wsmResourceClient = wsmClientProvider.getResourceApi(wsmAuth) - wsmResourceClient.findContainerResourceId(workspaceId, container) + private val cachedContainerResourceIds = new mutable.HashMap[BlobContainerName, UUID]() + + // Optionally provide wsmAuth to avoid acquiring it twice in generateBlobSasToken. + // In the case that the resourceId is not cached and no auth is provided, this function will acquire a new auth as necessary. + private def getContainerResourceId(workspaceId: UUID, + container: BlobContainerName, + precomputedWsmAuth: Option[String] + ): Try[UUID] = + cachedContainerResourceIds.get(container) match { + case Some(id) => Try(id) // cache hit + case _ => // cache miss + val auth: Try[String] = precomputedWsmAuth.map(auth => Try(auth)).getOrElse(getWsmAuth) + val resourceId = for { + wsmAuth <- auth + wsmResourceApi = wsmClientProvider.getResourceApi(wsmAuth) + resourceId <- wsmResourceApi.findContainerResourceId(workspaceId, container) + } yield resourceId + resourceId.map(id => cachedContainerResourceIds.put(container, id)) // NB: Modifying cache state here. + cachedContainerResourceIds.get(container) match { + case Some(uuid) => Try(uuid) + case _ => Failure(new NoSuchElementException("Could not retrieve container resource ID from WSM")) + } + } + + private def getWsmAuth: Try[String] = + overrideWsmAuthToken match { + case Some(t) => Success(t) + case None => AzureCredentials.getAccessToken(None).toTry + } + + private def parseTerraWorkspaceIdFromPath(blobPath: BlobPath): Try[UUID] = + if (blobPath.container.value.startsWith("sc-")) Try(UUID.fromString(blobPath.container.value.substring(3))) + else + Failure( + new Exception( + "Could not parse workspace ID from storage container. Are you sure this is a file in a Terra Workspace?" + ) + ) + + /** + * Return a REST endpoint that will reply with a sas token for the blob storage container associated with the provided blob path. + * @param blobPath A blob path of a file living in a blob container that WSM knows about (likely a workspace container). + * @param tokenDuration How long will the token last after being generated. Default is 8 hours. Sas tokens won't last longer than 24h. + * NOTE: If a blobPath is provided for a file in a container other than what this token generator was constructed for, + * this function will make two REST requests. Otherwise, the relevant data is already cached locally. + */ + def getWSMSasFetchEndpoint(blobPath: BlobPath, tokenDuration: Option[Duration] = None): Try[String] = { + val wsmEndpoint = wsmClientProvider.getBaseWorkspaceManagerUrl + val lifetimeQueryParameters: String = + tokenDuration.map(d => s"?sasExpirationDuration=${d.toSeconds.intValue}").getOrElse("") + val terraInfo: Try[WSMTerraCoordinates] = for { + workspaceId <- parseTerraWorkspaceIdFromPath(blobPath) + containerResourceId <- getContainerResourceId(workspaceId, blobPath.container, None) + coordinates = WSMTerraCoordinates(wsmEndpoint, workspaceId, containerResourceId) + } yield coordinates + terraInfo.map { terraCoordinates => + s"${terraCoordinates.wsmEndpoint}/api/workspaces/v1/${terraCoordinates.workspaceId.toString}/resources/controlled/azure/storageContainer/${terraCoordinates.containerResourceId.toString}/getSasToken${lifetimeQueryParameters}" + } } } @@ -223,11 +288,14 @@ case class NativeBlobSasTokenGenerator(subscription: Option[SubscriptionId] = No * @return an AzureSasCredential for accessing a blob container */ def generateBlobSasToken(endpoint: EndpointURL, container: BlobContainerName): Try[AzureSasCredential] = { - val c = AzureUtils.buildContainerClientFromLocalEnvironment(container.toString, endpoint.toString, subscription.map(_.toString)) + val c = AzureUtils.buildContainerClientFromLocalEnvironment(container.toString, + endpoint.toString, + subscription.map(_.toString) + ) c.map { bcc => - val bsssv = new BlobServiceSasSignatureValues(OffsetDateTime.now.plusDays(1), bcsp) - new AzureSasCredential(bcc.generateSas(bsssv)) - }.orElse(Try(BlobFileSystemManager.PLACEHOLDER_TOKEN)) + val bsssv = new BlobServiceSasSignatureValues(OffsetDateTime.now.plusDays(1), bcsp) + new AzureSasCredential(bcc.generateSas(bsssv)) + }.orElse(Try(BlobFileSystemManager.PLACEHOLDER_TOKEN)) } } diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilder.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilder.scala index 3aa26eb3c11..3c984d1f2c8 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilder.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilder.scala @@ -1,5 +1,6 @@ package cromwell.filesystems.blob +import akka.http.scaladsl.model.Uri import com.azure.storage.blob.nio.AzureBlobFileAttributes import com.google.common.net.UrlEscapers import cromwell.core.path.{NioPath, Path, PathBuilder} @@ -17,11 +18,19 @@ object BlobPathBuilder { case class ValidBlobPath(path: String, container: BlobContainerName, endpoint: EndpointURL) extends BlobPathValidation case class UnparsableBlobPath(errorMessage: Throwable) extends BlobPathValidation - def invalidBlobHostMessage(endpoint: EndpointURL) = s"Malformed Blob URL for this builder: The endpoint $endpoint doesn't contain the expected host string '{SA}.blob.core.windows.net/'" - def invalidBlobContainerMessage(endpoint: EndpointURL) = s"Malformed Blob URL for this builder: Could not parse container" + def invalidBlobHostMessage(endpoint: EndpointURL) = + s"Malformed Blob URL for this builder: The endpoint $endpoint doesn't contain the expected host string '{SA}.blob.core.windows.net/'" + def invalidBlobContainerMessage(endpoint: EndpointURL) = + s"Malformed Blob URL for this builder: Could not parse container" + val externalToken = + "Rejecting pre-signed SAS URL so that filesystem selection falls through to HTTP filesystem" def parseURI(string: String): Try[URI] = Try(URI.create(UrlEscapers.urlFragmentEscaper().escape(string))) - def parseStorageAccount(uri: URI): Try[StorageAccountName] = uri.getHost.split("\\.").find(_.nonEmpty).map(StorageAccountName(_)) - .map(Success(_)).getOrElse(Failure(new Exception("Could not parse storage account"))) + def parseStorageAccount(uri: URI): Try[StorageAccountName] = uri.getHost + .split("\\.") + .find(_.nonEmpty) + .map(StorageAccountName(_)) + .map(Success(_)) + .getOrElse(Failure(new Exception("Could not parse storage account"))) /** * Validates a that a path from a string is a valid BlobPath of the format: @@ -41,34 +50,46 @@ object BlobPathBuilder { * * If the configured container and storage account do not match, the string is considered unparsable */ - def validateBlobPath(string: String): BlobPathValidation = { + def validateBlobPath(string: String): BlobPathValidation = { val blobValidation = for { testUri <- parseURI(string) - testEndpoint = EndpointURL(testUri.getScheme + "://" + testUri.getHost()) - testStorageAccount <- parseStorageAccount(testUri) + testEndpoint = EndpointURL(testUri.getScheme + "://" + testUri.getHost) + _ <- parseStorageAccount(testUri) testContainer = testUri.getPath.split("/").find(_.nonEmpty) - isBlobHost = testUri.getHost().contains(blobHostnameSuffix) && testUri.getScheme().contains("https") - blobPathValidation = (isBlobHost, testContainer) match { - case (true, Some(container)) => ValidBlobPath( - testUri.getPath.replaceFirst("/" + container, ""), - BlobContainerName(container), - testEndpoint) - case (false, _) => UnparsableBlobPath(new MalformedURLException(invalidBlobHostMessage(testEndpoint))) - case (true, None) => UnparsableBlobPath(new MalformedURLException(invalidBlobContainerMessage(testEndpoint))) + isBlobHost = testUri.getHost.contains(blobHostnameSuffix) && testUri.getScheme.contains("https") + hasToken = hasSasToken(string) + blobPathValidation = (isBlobHost, testContainer, hasToken) match { + case (true, Some(container), false) => + ValidBlobPath(testUri.getPath.replaceFirst("/" + container, ""), BlobContainerName(container), testEndpoint) + case (false, _, false) => + UnparsableBlobPath(new MalformedURLException(invalidBlobHostMessage(testEndpoint))) + case (true, None, false) => + UnparsableBlobPath(new MalformedURLException(invalidBlobContainerMessage(testEndpoint))) + case (_, _, true) => + UnparsableBlobPath(new IllegalArgumentException(externalToken)) } } yield blobPathValidation blobValidation recover { case t => UnparsableBlobPath(t) } get } + + private def hasSasToken(uri: Uri) = { + // These keys are required for all SAS tokens. + // https://learn.microsoft.com/en-us/rest/api/storageservices/create-service-sas#construct-a-service-sas + val SignedVersionKey = "sv" + val SignatureKey = "sig" + + val query = uri.query().toMap + query.isDefinedAt(SignedVersionKey) && query.isDefinedAt(SignatureKey) + } } class BlobPathBuilder()(private val fsm: BlobFileSystemManager) extends PathBuilder { - def build(string: String): Try[BlobPath] = { + def build(string: String): Try[BlobPath] = validateBlobPath(string) match { case ValidBlobPath(path, container, endpoint) => Try(BlobPath(path, endpoint, container)(fsm)) case UnparsableBlobPath(errorMessage: Throwable) => Failure(errorMessage) } - } override def name: String = "Azure Blob Storage" } @@ -89,12 +110,12 @@ object BlobPath { // They do this for all files they touch, regardless of size, and the root/metadata property is authoritative over native. // // N.B. most if not virtually all large files in the wild will NOT have this key populated because they were not created - // by TES or its associated upload utility [4]. + // by TES or its associated upload utility [3]. // // [0] https://learn.microsoft.com/en-us/azure/storage/blobs/scalability-targets // [1] https://learn.microsoft.com/en-us/rest/api/storageservices/version-2019-12-12 // [2] https://github.com/microsoft/ga4gh-tes/blob/03feb746bb961b72fa91266a56db845e3b31be27/src/Tes.Runner/Transfer/BlobBlockApiHttpUtils.cs#L25 - // [4] https://github.com/microsoft/ga4gh-tes/blob/main/src/Tes.RunnerCLI/scripts/roothash.sh + // [3] https://github.com/microsoft/ga4gh-tes/blob/main/src/Tes.RunnerCLI/scripts/roothash.sh private val largeBlobFileMetadataKey = "md5_4mib_hashlist_root_hash" def cleanedNioPathString(nioString: String): String = { @@ -103,33 +124,36 @@ object BlobPath { s"${containerName}:/${pathInContainer}" case _ => nioString } - pathStr.substring(pathStr.indexOf(":")+1) + pathStr.substring(pathStr.indexOf(":") + 1) } def apply(nioPath: NioPath, endpoint: EndpointURL, container: BlobContainerName, - fsm: BlobFileSystemManager): BlobPath = { + fsm: BlobFileSystemManager + ): BlobPath = BlobPath(cleanedNioPathString(nioPath.toString), endpoint, container)(fsm) - } } -case class BlobPath private[blob](pathString: String, endpoint: EndpointURL, container: BlobContainerName)(private val fsm: BlobFileSystemManager) extends Path { +case class BlobPath private[blob] (pathString: String, endpoint: EndpointURL, container: BlobContainerName)( + private val fsm: BlobFileSystemManager +) extends Path { override def nioPath: NioPath = findNioPath(pathString) override protected def newPath(nioPath: NioPath): Path = BlobPath(nioPath, endpoint, container, fsm) override def pathAsString: String = List(endpoint, container, pathString.stripPrefix("/")).mkString("/") - //This is purposefully an unprotected get because if the endpoint cannot be parsed this should fail loudly rather than quietly - override def pathWithoutScheme: String = parseURI(endpoint.value).map(u => List(u.getHost, container, pathString.stripPrefix("/")).mkString("/")).get + // This is purposefully an unprotected get because if the endpoint cannot be parsed this should fail loudly rather than quietly + override def pathWithoutScheme: String = + parseURI(endpoint.value).map(u => List(u.getHost, container, pathString.stripPrefix("/")).mkString("/")).get private def findNioPath(path: String): NioPath = (for { fileSystem <- fsm.retrieveFilesystem(endpoint, container) // The Azure NIO library uses `{container}:` to represent the root of the path nioPath = fileSystem.getPath(s"${container.value}:", path) - // This is purposefully an unprotected get because the NIO API needing an unwrapped path object. - // If an error occurs the api expects a thrown exception + // This is purposefully an unprotected get because the NIO API needing an unwrapped path object. + // If an error occurs the api expects a thrown exception } yield nioPath).get def blobFileAttributes: Try[AzureBlobFileAttributes] = @@ -168,15 +192,13 @@ case class BlobPath private[blob](pathString: String, endpoint: EndpointURL, con * Return the pathString of this BlobPath, with the given prefix removed if this path shares that * prefix. */ - def pathStringWithoutPrefix(prefix: Path): String = { + def pathStringWithoutPrefix(prefix: Path): String = if (this.startsWith(prefix)) { prefix.relativize(this) match { case b: BlobPath => b.pathString // path inside the blob container case p: Path => p.pathAsString // full path } - } - else pathString - } + } else pathString /** * Returns the path relative to the container root. @@ -184,7 +206,10 @@ case class BlobPath private[blob](pathString: String, endpoint: EndpointURL, con * will be returned as path/to/my/file. * @return Path string relative to the container root. */ - def pathWithoutContainer : String = pathString - - override def getSymlinkSafePath(options: LinkOption*): Path = toAbsolutePath + def pathWithoutContainer: String = pathString + + def getFilesystemManager: BlobFileSystemManager = fsm + + override def getSymlinkSafePath(options: LinkOption*): Path = toAbsolutePath + } diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala index 47245552dc2..faa04481316 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala @@ -10,17 +10,16 @@ import java.util.UUID import scala.concurrent.{ExecutionContext, Future} import scala.util.Try -final case class SubscriptionId(value: UUID) {override def toString: String = value.toString} +final case class SubscriptionId(value: UUID) { override def toString: String = value.toString } final case class BlobContainerName(value: String) { override def toString: String = value - lazy val workspaceId: Try[UUID] = { - Try(UUID.fromString(value.replaceFirst("sc-",""))) - } + lazy val workspaceId: Try[UUID] = + Try(UUID.fromString(value.replaceFirst("sc-", ""))) } -final case class StorageAccountName(value: String) {override def toString: String = value} +final case class StorageAccountName(value: String) { override def toString: String = value } final case class EndpointURL(value: String) { override def toString: String = value - lazy val storageAccountName : Try[StorageAccountName] = { + lazy val storageAccountName: Try[StorageAccountName] = { val sa = for { host <- value.split("//").findLast(_.nonEmpty) storageAccountName <- host.split("\\.").find(_.nonEmpty) @@ -28,17 +27,18 @@ final case class EndpointURL(value: String) { sa.toRight(new Exception(s"Storage account name could not be parsed from $value")).toTry } } -final case class WorkspaceId(value: UUID) {override def toString: String = value.toString} -final case class ContainerResourceId(value: UUID) {override def toString: String = value.toString} -final case class WorkspaceManagerURL(value: String) {override def toString: String = value} -final case class BlobPathBuilderFactory(globalConfig: Config, instanceConfig: Config, fsm: BlobFileSystemManager) extends PathBuilderFactory { +final case class WorkspaceManagerURL(value: String) { override def toString: String = value } + +final case class BlobPathBuilderFactory(globalConfig: Config, instanceConfig: Config, fsm: BlobFileSystemManager) + extends PathBuilderFactory { - override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[BlobPathBuilder] = { + override def withOptions( + options: WorkflowOptions + )(implicit as: ActorSystem, ec: ExecutionContext): Future[BlobPathBuilder] = Future { new BlobPathBuilder()(fsm) } - } override def priority: Int = PriorityBlob } diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/WorkspaceManagerApiClientProvider.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/WorkspaceManagerApiClientProvider.scala index 276738c98b6..8698dc7f859 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/WorkspaceManagerApiClientProvider.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/WorkspaceManagerApiClientProvider.scala @@ -20,9 +20,11 @@ import scala.util.Try trait WorkspaceManagerApiClientProvider { def getControlledAzureResourceApi(token: String): WsmControlledAzureResourceApi def getResourceApi(token: String): WsmResourceApi + def getBaseWorkspaceManagerUrl: String } -class HttpWorkspaceManagerClientProvider(baseWorkspaceManagerUrl: WorkspaceManagerURL) extends WorkspaceManagerApiClientProvider { +class HttpWorkspaceManagerClientProvider(baseWorkspaceManagerUrl: WorkspaceManagerURL) + extends WorkspaceManagerApiClientProvider { private def getApiClient: ApiClient = { val client: ApiClient = new ApiClient() client.setBasePath(baseWorkspaceManagerUrl.value) @@ -40,29 +42,40 @@ class HttpWorkspaceManagerClientProvider(baseWorkspaceManagerUrl: WorkspaceManag apiClient.setAccessToken(token) WsmControlledAzureResourceApi(new ControlledAzureResourceApi(apiClient)) } + def getBaseWorkspaceManagerUrl: String = baseWorkspaceManagerUrl.value } -case class WsmResourceApi(resourcesApi : ResourceApi) { - def findContainerResourceId(workspaceId : UUID, container: BlobContainerName): Try[UUID] = { +case class WsmResourceApi(resourcesApi: ResourceApi) { + def findContainerResourceId(workspaceId: UUID, container: BlobContainerName): Try[UUID] = for { - workspaceResources <- Try(resourcesApi.enumerateResources(workspaceId, 0, 10, ResourceType.AZURE_STORAGE_CONTAINER, StewardshipType.CONTROLLED).getResources()) - workspaceStorageContainerOption = workspaceResources.asScala.find(r => r.getMetadata().getName() == container.value) - workspaceStorageContainer <- workspaceStorageContainerOption.toRight(new Exception("No storage container found for this workspace")).toTry + workspaceResources <- Try( + resourcesApi + .enumerateResources(workspaceId, 0, 10, ResourceType.AZURE_STORAGE_CONTAINER, StewardshipType.CONTROLLED) + .getResources() + ) + workspaceStorageContainerOption = workspaceResources.asScala.find(r => + r.getMetadata().getName() == container.value + ) + workspaceStorageContainer <- workspaceStorageContainerOption + .toRight(new Exception("No storage container found for this workspace")) + .toTry resourceId = workspaceStorageContainer.getMetadata().getResourceId() } yield resourceId - } } -case class WsmControlledAzureResourceApi(controlledAzureResourceApi : ControlledAzureResourceApi) { - def createAzureStorageContainerSasToken(workspaceId: UUID, resourceId: UUID): Try[AzureSasCredential] = { +case class WsmControlledAzureResourceApi(controlledAzureResourceApi: ControlledAzureResourceApi) { + def createAzureStorageContainerSasToken(workspaceId: UUID, resourceId: UUID): Try[AzureSasCredential] = for { - sas <- Try(controlledAzureResourceApi.createAzureStorageContainerSasToken( - workspaceId, - resourceId, - null, - null, - null, - null - ).getToken) + sas <- Try( + controlledAzureResourceApi + .createAzureStorageContainerSasToken( + workspaceId, + resourceId, + null, + null, + null, + null + ) + .getToken + ) } yield new AzureSasCredential(sas) - } } diff --git a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/AzureFileSystemSpec.scala b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/AzureFileSystemSpec.scala index 9b8362ced80..0626a008de0 100644 --- a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/AzureFileSystemSpec.scala +++ b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/AzureFileSystemSpec.scala @@ -23,34 +23,36 @@ class AzureFileSystemSpec extends AnyFlatSpec with Matchers { val combinedEndpoint = BlobFileSystemManager.combinedEnpointContainerUri(storageEndpoint, container) val provider = new AzureFileSystemProvider() - provider.newFileSystem( - combinedEndpoint, - BlobFileSystemManager.buildConfigMap(creds, container).asJava - ).asInstanceOf[AzureFileSystem] + provider + .newFileSystem( + combinedEndpoint, + BlobFileSystemManager.buildConfigMap(creds, container).asJava + ) + .asInstanceOf[AzureFileSystem] } it should "parse an expiration from a sas token" in { val now = Instant.now() - val filesystem : AzureFileSystem = makeFilesystemWithExpiration(now) + val filesystem: AzureFileSystem = makeFilesystemWithExpiration(now) filesystem.getExpiry.asScala shouldBe Some(now) filesystem.getFileStores.asScala.map(_.name()).exists(_ == "testContainer") shouldBe true } it should "not be expired when the token is fresh" in { val anHourFromNow = Instant.now().plusSeconds(3600) - val filesystem : AzureFileSystem = makeFilesystemWithExpiration(anHourFromNow) + val filesystem: AzureFileSystem = makeFilesystemWithExpiration(anHourFromNow) filesystem.isExpired(fiveMinutes) shouldBe false } it should "be expired when we're within the buffer" in { val threeMinutesFromNow = Instant.now().plusSeconds(180) - val filesystem : AzureFileSystem = makeFilesystemWithExpiration(threeMinutesFromNow) + val filesystem: AzureFileSystem = makeFilesystemWithExpiration(threeMinutesFromNow) filesystem.isExpired(fiveMinutes) shouldBe true } it should "be expired when the token is stale" in { val anHourAgo = Instant.now().minusSeconds(3600) - val filesystem : AzureFileSystem = makeFilesystemWithExpiration(anHourAgo) + val filesystem: AzureFileSystem = makeFilesystemWithExpiration(anHourAgo) filesystem.isExpired(fiveMinutes) shouldBe true } diff --git a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobFileSystemConfigSpec.scala b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobFileSystemConfigSpec.scala index 68804113763..30d580a6e49 100644 --- a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobFileSystemConfigSpec.scala +++ b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobFileSystemConfigSpec.scala @@ -12,8 +12,7 @@ class BlobFileSystemConfigSpec extends AnyFlatSpec with Matchers { it should "parse configs for a minimal functioning factory with native blob access" in { val config = BlobFileSystemConfig( - ConfigFactory.parseString( - s""" + ConfigFactory.parseString(s""" """.stripMargin) ) config.expiryBufferMinutes should equal(BlobFileSystemConfig.defaultExpiryBufferMinutes) @@ -21,14 +20,13 @@ class BlobFileSystemConfigSpec extends AnyFlatSpec with Matchers { it should "parse configs for a functioning factory with WSM-mediated blob access" in { val config = BlobFileSystemConfig( - ConfigFactory.parseString( - s""" - |expiry-buffer-minutes = "20" - |workspace-manager { - | url = "$workspaceManagerURL" - | b2cToken = "$b2cToken" - |} - | + ConfigFactory.parseString(s""" + |expiry-buffer-minutes = "20" + |workspace-manager { + | url = "$workspaceManagerURL" + | b2cToken = "$b2cToken" + |} + | """.stripMargin) ) config.expiryBufferMinutes should equal(20L) @@ -39,13 +37,12 @@ class BlobFileSystemConfigSpec extends AnyFlatSpec with Matchers { it should "fail when partial WSM config is supplied" in { val rawConfig = - ConfigFactory.parseString( - s""" - |expiry-buffer-minutes = "10" - |workspace-manager { - | b2cToken = "$b2cToken" - |} - | + ConfigFactory.parseString(s""" + |expiry-buffer-minutes = "10" + |workspace-manager { + | b2cToken = "$b2cToken" + |} + | """.stripMargin) val error = intercept[AggregatedMessageException](BlobFileSystemConfig(rawConfig)) diff --git a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala index 24783c15780..2b46a8b80b8 100644 --- a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala +++ b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala @@ -14,12 +14,11 @@ import java.time.{Duration, Instant, ZoneId} import java.util.UUID import scala.util.{Failure, Success, Try} - object BlobPathBuilderFactorySpec { def buildExampleSasToken(expiry: Instant): AzureSasCredential = { val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault()) val sv = formatter.format(expiry) - val se = expiry.toString().replace(":","%3A") + val se = expiry.toString().replace(":", "%3A") new AzureSasCredential(s"sv=$sv&se=$se&sr=c&sp=rcl") } } @@ -47,15 +46,15 @@ class BlobPathBuilderFactorySpec extends AnyFlatSpec with Matchers with MockSuga it should "test retrieveFileSystem with expired Terra filesystem" in { val endpoint = BlobPathBuilderSpec.buildEndpoint("storageAccount") - //val expiredToken = generateTokenExpiration(9L) + // val expiredToken = generateTokenExpiration(9L) val refreshedToken = generateTokenExpiration(69L) val sasToken = BlobPathBuilderFactorySpec.buildExampleSasToken(refreshedToken) val container = BlobContainerName("sc-" + UUID.randomUUID().toString()) val configMap = BlobFileSystemManager.buildConfigMap(sasToken, container) val azureUri = BlobFileSystemManager.combinedEnpointContainerUri(endpoint, container) - //Mocking this final class requires the plugin Mock Maker Inline plugin, configured here - //at filesystems/blob/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker + // Mocking this final class requires the plugin Mock Maker Inline plugin, configured here + // at filesystems/blob/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker val azureFileSystem = mock[AzureFileSystem] when(azureFileSystem.isExpired(Duration.ofMinutes(10L))).thenReturn(true) val fileSystems = mock[AzureFileSystemAPI] @@ -73,15 +72,15 @@ class BlobPathBuilderFactorySpec extends AnyFlatSpec with Matchers with MockSuga it should "test retrieveFileSystem with an unexpired Terra fileSystem" in { val endpoint = BlobPathBuilderSpec.buildEndpoint("storageAccount") - //val initialToken = generateTokenExpiration(11L) + // val initialToken = generateTokenExpiration(11L) val refreshedToken = generateTokenExpiration(71L) val sasToken = BlobPathBuilderFactorySpec.buildExampleSasToken(refreshedToken) val container = BlobContainerName("sc-" + UUID.randomUUID().toString()) val configMap = BlobFileSystemManager.buildConfigMap(sasToken, container) - val azureUri = BlobFileSystemManager.combinedEnpointContainerUri(endpoint,container) + val azureUri = BlobFileSystemManager.combinedEnpointContainerUri(endpoint, container) - //Mocking this final class requires the plugin Mock Maker Inline plugin, configured here - //at filesystems/blob/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker + // Mocking this final class requires the plugin Mock Maker Inline plugin, configured here + // at filesystems/blob/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker val azureFileSystem = mock[AzureFileSystem] when(azureFileSystem.isExpired(Duration.ofMinutes(10L))).thenReturn(false) val fileSystems = mock[AzureFileSystemAPI] @@ -106,8 +105,8 @@ class BlobPathBuilderFactorySpec extends AnyFlatSpec with Matchers with MockSuga val configMap = BlobFileSystemManager.buildConfigMap(sasToken, container) val azureUri = BlobFileSystemManager.combinedEnpointContainerUri(endpoint, container) - //Mocking this final class requires the plugin Mock Maker Inline plugin, configured here - //at filesystems/blob/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker + // Mocking this final class requires the plugin Mock Maker Inline plugin, configured here + // at filesystems/blob/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker val azureFileSystem = mock[AzureFileSystem] when(azureFileSystem.isExpired(Duration.ofMinutes(10L))).thenReturn(false) val fileSystems = mock[AzureFileSystemAPI] @@ -131,8 +130,8 @@ class BlobPathBuilderFactorySpec extends AnyFlatSpec with Matchers with MockSuga val configMap = BlobFileSystemManager.buildConfigMap(sasToken, container) val azureUri = BlobFileSystemManager.combinedEnpointContainerUri(endpoint, container) - //Mocking this final class requires the plugin Mock Maker Inline plugin, configured here - //at filesystems/blob/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker + // Mocking this final class requires the plugin Mock Maker Inline plugin, configured here + // at filesystems/blob/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker val azureFileSystem = mock[AzureFileSystem] when(azureFileSystem.isExpired(Duration.ofMinutes(10L))).thenReturn(true) val fileSystems = mock[AzureFileSystemAPI] @@ -153,10 +152,10 @@ class BlobPathBuilderFactorySpec extends AnyFlatSpec with Matchers with MockSuga val sasToken = BlobFileSystemManager.PLACEHOLDER_TOKEN val container = BlobContainerName("sc-" + UUID.randomUUID().toString()) val configMap = BlobFileSystemManager.buildConfigMap(sasToken, container) - val azureUri = BlobFileSystemManager.combinedEnpointContainerUri(endpoint,container) + val azureUri = BlobFileSystemManager.combinedEnpointContainerUri(endpoint, container) - //Mocking this final class requires the plugin Mock Maker Inline plugin, configured here - //at filesystems/blob/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker + // Mocking this final class requires the plugin Mock Maker Inline plugin, configured here + // at filesystems/blob/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker val azureFileSystem = mock[AzureFileSystem] when(azureFileSystem.isExpired(Duration.ofMinutes(10L))).thenReturn(false) val fileSystems = mock[AzureFileSystemAPI] @@ -180,8 +179,8 @@ class BlobPathBuilderFactorySpec extends AnyFlatSpec with Matchers with MockSuga val configMap = BlobFileSystemManager.buildConfigMap(sasToken, container) val azureUri = BlobFileSystemManager.combinedEnpointContainerUri(endpoint, container) - //Mocking this final class requires the plugin Mock Maker Inline plugin, configured here - //at filesystems/blob/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker + // Mocking this final class requires the plugin Mock Maker Inline plugin, configured here + // at filesystems/blob/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker val azureFileSystem = mock[AzureFileSystem] when(azureFileSystem.isExpired(Duration.ofMinutes(10L))).thenReturn(false) val fileSystems = mock[AzureFileSystemAPI] diff --git a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderSpec.scala b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderSpec.scala index a8ca7d58d6f..9093141054f 100644 --- a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderSpec.scala +++ b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderSpec.scala @@ -19,15 +19,31 @@ class BlobPathBuilderSpec extends AnyFlatSpec with Matchers with MockSugar { val evalPath = "/path/to/file" val testString = endpoint.value + "/" + container + evalPath BlobPathBuilder.validateBlobPath(testString) match { - case BlobPathBuilder.ValidBlobPath(path, parsedContainer, parsedEndpoint) => { + case BlobPathBuilder.ValidBlobPath(path, parsedContainer, parsedEndpoint) => path should equal(evalPath) parsedContainer should equal(container) parsedEndpoint should equal(endpoint) - } case BlobPathBuilder.UnparsableBlobPath(errorMessage) => fail(errorMessage) } } + it should "reject a path that is otherwise valid, but has a preexisting SAS token" in { + import cromwell.filesystems.blob.BlobPathBuilder.UnparsableBlobPath + + // The `.asInstanceOf[UnparsableBlobPath].errorMessage.getMessage` malarkey is necessary + // because Java exceptions compare by reference, while strings are by value + + val sasBlob = "https://lz304a1e79fd7359e5327eda.blob.core.windows.net/sc-705b830a-d699-478e-9da6-49661b326e77" + + "?sv=2021-12-02&spr=https&st=2023-12-13T20%3A27%3A55Z&se=2023-12-14T04%3A42%3A55Z&sr=c&sp=racwdlt&sig=blah&rscd=foo" + BlobPathBuilder.validateBlobPath(sasBlob).asInstanceOf[UnparsableBlobPath].errorMessage.getMessage should equal( + UnparsableBlobPath( + new IllegalArgumentException( + "Rejecting pre-signed SAS URL so that filesystem selection falls through to HTTP filesystem" + ) + ).errorMessage.getMessage + ) + } + it should "provide a readable error when getting an illegal nioPath" in { val endpoint = BlobPathBuilderSpec.buildEndpoint("storageAccount") val container = BlobContainerName("container") @@ -40,8 +56,9 @@ class BlobPathBuilderSpec extends AnyFlatSpec with Matchers with MockSugar { testException should contain(exception) } - private def testBlobNioStringCleaning(input: String, expected: String) = - BlobPath.cleanedNioPathString(input) shouldBe expected + // The following tests use the `centaurtesting` account injected into CI. They depend on access to the + // container specified below. You may need to log in to az cli locally to get them to pass. + private val subscriptionId: SubscriptionId = SubscriptionId(UUID.fromString("62b22893-6bc1-46d9-8a90-806bb3cce3c9")) it should "clean the NIO path string when it has a garbled http protocol" in { testBlobNioStringCleaning( @@ -70,10 +87,6 @@ class BlobPathBuilderSpec extends AnyFlatSpec with Matchers with MockSugar { "" ) } - - // The following tests use the `centaurtesting` account injected into CI. They depend on access to the - // container specified below. You may need to log in to az cli locally to get them to pass. - private val subscriptionId: SubscriptionId = SubscriptionId(UUID.fromString("62b22893-6bc1-46d9-8a90-806bb3cce3c9")) private val endpoint: EndpointURL = BlobPathBuilderSpec.buildEndpoint("centaurtesting") private val container: BlobContainerName = BlobContainerName("test-blob") @@ -83,6 +96,9 @@ class BlobPathBuilderSpec extends AnyFlatSpec with Matchers with MockSugar { new BlobPathBuilder()(fsm) } + private def testBlobNioStringCleaning(input: String, expected: String) = + BlobPath.cleanedNioPathString(input) shouldBe expected + it should "read md5 from small files <5g" in { val builder = makeBlobPathBuilder() val evalPath = "/testRead.txt" @@ -119,9 +135,14 @@ class BlobPathBuilderSpec extends AnyFlatSpec with Matchers with MockSugar { val builder = makeBlobPathBuilder() val rootString = s"${endpoint.value}/${container.value}/cromwell-execution" val blobRoot: BlobPath = builder build rootString getOrElse fail() - blobRoot.toAbsolutePath.pathAsString should equal ("https://centaurtesting.blob.core.windows.net/test-blob/cromwell-execution") - val otherFile = blobRoot.resolve("https://centaurtesting.blob.core.windows.net/test-blob/cromwell-execution/test/inputFile.txt") - otherFile.toAbsolutePath.pathAsString should equal ("https://centaurtesting.blob.core.windows.net/test-blob/cromwell-execution/test/inputFile.txt") + blobRoot.toAbsolutePath.pathAsString should equal( + "https://centaurtesting.blob.core.windows.net/test-blob/cromwell-execution" + ) + val otherFile = + blobRoot.resolve("https://centaurtesting.blob.core.windows.net/test-blob/cromwell-execution/test/inputFile.txt") + otherFile.toAbsolutePath.pathAsString should equal( + "https://centaurtesting.blob.core.windows.net/test-blob/cromwell-execution/test/inputFile.txt" + ) } it should "build a blob path from a test string and read a file" in { @@ -136,8 +157,8 @@ class BlobPathBuilderSpec extends AnyFlatSpec with Matchers with MockSugar { blobPath.pathAsString should equal(testString) blobPath.pathWithoutScheme should equal(endpointHost + "/" + container + evalPath) val is = blobPath.newInputStream() - val fileText = (is.readAllBytes.map(_.toChar)).mkString - fileText should include ("This is my test file!!!! Did it work?") + val fileText = is.readAllBytes.map(_.toChar).mkString + fileText should include("This is my test file!!!! Did it work?") } it should "build duplicate blob paths in the same filesystem" in { @@ -149,38 +170,47 @@ class BlobPathBuilderSpec extends AnyFlatSpec with Matchers with MockSugar { val blobPath2: BlobPath = builder build testString getOrElse fail() blobPath1 should equal(blobPath2) val is = blobPath1.newInputStream() - val fileText = (is.readAllBytes.map(_.toChar)).mkString - fileText should include ("This is my test file!!!! Did it work?") + val fileText = is.readAllBytes.map(_.toChar).mkString + fileText should include("This is my test file!!!! Did it work?") } it should "resolve a path without duplicating container name" in { val builder = makeBlobPathBuilder() val rootString = s"${endpoint.value}/${container.value}/cromwell-execution" val blobRoot: BlobPath = builder build rootString getOrElse fail() - blobRoot.toAbsolutePath.pathAsString should equal ("https://centaurtesting.blob.core.windows.net/test-blob/cromwell-execution") + blobRoot.toAbsolutePath.pathAsString should equal( + "https://centaurtesting.blob.core.windows.net/test-blob/cromwell-execution" + ) val otherFile = blobRoot.resolve("test/inputFile.txt") - otherFile.toAbsolutePath.pathAsString should equal ("https://centaurtesting.blob.core.windows.net/test-blob/cromwell-execution/test/inputFile.txt") + otherFile.toAbsolutePath.pathAsString should equal( + "https://centaurtesting.blob.core.windows.net/test-blob/cromwell-execution/test/inputFile.txt" + ) } it should "correctly remove a prefix from the blob path" in { val builder = makeBlobPathBuilder() val rootString = s"${endpoint.value}/${container.value}/cromwell-execution/" - val execDirString = s"${endpoint.value}/${container.value}/cromwell-execution/abc123/myworkflow/task1/def4356/execution/" - val fileString = s"${endpoint.value}/${container.value}/cromwell-execution/abc123/myworkflow/task1/def4356/execution/stdout" + val execDirString = + s"${endpoint.value}/${container.value}/cromwell-execution/abc123/myworkflow/task1/def4356/execution/" + val fileString = + s"${endpoint.value}/${container.value}/cromwell-execution/abc123/myworkflow/task1/def4356/execution/stdout" val blobRoot: BlobPath = builder build rootString getOrElse fail() val execDir: BlobPath = builder build execDirString getOrElse fail() val blobFile: BlobPath = builder build fileString getOrElse fail() - blobFile.pathStringWithoutPrefix(blobRoot) should equal ("abc123/myworkflow/task1/def4356/execution/stdout") - blobFile.pathStringWithoutPrefix(execDir) should equal ("stdout") - blobFile.pathStringWithoutPrefix(blobFile) should equal ("") + blobFile.pathStringWithoutPrefix(blobRoot) should equal("abc123/myworkflow/task1/def4356/execution/stdout") + blobFile.pathStringWithoutPrefix(execDir) should equal("stdout") + blobFile.pathStringWithoutPrefix(blobFile) should equal("") } it should "not change a path if it doesn't start with a prefix" in { val builder = makeBlobPathBuilder() val otherRootString = s"${endpoint.value}/${container.value}/foobar/" - val fileString = s"${endpoint.value}/${container.value}/cromwell-execution/abc123/myworkflow/task1/def4356/execution/stdout" + val fileString = + s"${endpoint.value}/${container.value}/cromwell-execution/abc123/myworkflow/task1/def4356/execution/stdout" val otherBlobRoot: BlobPath = builder build otherRootString getOrElse fail() val blobFile: BlobPath = builder build fileString getOrElse fail() - blobFile.pathStringWithoutPrefix(otherBlobRoot) should equal ("/cromwell-execution/abc123/myworkflow/task1/def4356/execution/stdout") + blobFile.pathStringWithoutPrefix(otherBlobRoot) should equal( + "/cromwell-execution/abc123/myworkflow/task1/def4356/execution/stdout" + ) } } diff --git a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPath.scala b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPath.scala index 5856e41f97b..59b056f7247 100644 --- a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPath.scala +++ b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPath.scala @@ -6,14 +6,12 @@ import cromwell.core.path.{NioPath, Path} import java.io.IOException - case class DrsPath(drsPath: CloudNioPath, requesterPaysProjectIdOption: Option[String]) extends Path { override def nioPath: NioPath = drsPath - override protected def newPath(nioPath: NioPath): Path = { + override protected def newPath(nioPath: NioPath): Path = DrsPath(nioPath.asInstanceOf[CloudNioPath], requesterPaysProjectIdOption) - } override def pathAsString: String = drsPath.cloudHost @@ -28,9 +26,15 @@ case class DrsPath(drsPath: CloudNioPath, requesterPaysProjectIdOption: Option[S case Some(fileAttributes) => fileAttributes.fileHash match { case Some(fileHash) => fileHash - case None => throw new IOException(s"Error while resolving DRS path $this. The response from DRS Resolver doesn't contain the 'md5' hash for the file.") + case None => + throw new IOException( + s"Error while resolving DRS path $this. The response from DRS Resolver doesn't contain the 'md5' hash for the file." + ) } - case None => throw new IOException(s"Error getting file hash of DRS path $this. Reason: File attributes class DrsCloudNioRegularFileAttributes wasn't defined in DrsCloudNioFileProvider.") + case None => + throw new IOException( + s"Error getting file hash of DRS path $this. Reason: File attributes class DrsCloudNioRegularFileAttributes wasn't defined in DrsCloudNioFileProvider." + ) } } } diff --git a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilder.scala b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilder.scala index b464772eec8..4b0b6c9758f 100644 --- a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilder.scala +++ b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilder.scala @@ -9,20 +9,20 @@ import scala.util.{Failure, Success, Try} case class DrsPathBuilder(fileSystemProvider: DrsCloudNioFileSystemProvider, requesterPaysProjectIdOption: Option[String], - preResolve: Boolean = false, - ) extends PreResolvePathBuilder with StrictLogging { + preResolve: Boolean = false +) extends PreResolvePathBuilder + with StrictLogging { private val drsScheme: String = fileSystemProvider.getScheme override def name: String = "DRS" - override def build(pathAsString: String, pathBuilders: PathBuilders): Try[Path] = { + override def build(pathAsString: String, pathBuilders: PathBuilders): Try[Path] = if (pathAsString.startsWith(s"$drsScheme://")) { Try(createDrsOrOtherPath(pathAsString, pathBuilders)) } else { Failure(new IllegalArgumentException(s"$pathAsString does not have a $drsScheme scheme.")) } - } private def createDrsOrOtherPath(pathAsString: String, pathBuilders: PathBuilders): Path = { def drsPath = DrsPath(fileSystemProvider.getCloudNioPath(pathAsString), requesterPaysProjectIdOption) @@ -35,17 +35,15 @@ case class DrsPathBuilder(fileSystemProvider: DrsCloudNioFileSystemProvider, private def maybeCreateOtherPath(pathAsString: String, pathBuilders: PathBuilders): Option[Path] = { - def logAttempt[A](description: String, attempt: => A): Option[A] = { + def logAttempt[A](description: String, attempt: => A): Option[A] = logTry(description, Try(attempt)) - } - def logTry[A](description: String, tried: Try[A]): Option[A] = { + def logTry[A](description: String, tried: Try[A]): Option[A] = tried match { case Success(result) => Option(result) case Failure(exception) => logFailure(description, exception) } - } def logFailure(description: String, reason: Any): None.type = { logger.debug(s"Unable to $description, will use a DrsPath to access '$pathAsString': $reason") @@ -66,7 +64,7 @@ case class DrsPathBuilder(fileSystemProvider: DrsCloudNioFileSystemProvider, gcsUrlWithNoCreds <- gsUriOption gcsPath <- logAttempt( s"create a GcsPath for '$gcsUrlWithNoCreds'", - PathFactory.buildPath(gcsUrlWithNoCreds, pathBuilders), + PathFactory.buildPath(gcsUrlWithNoCreds, pathBuilders) ) // Extra: Make sure the GcsPath _actually_ has permission to access the path _ <- logAttempt(s"access '$gcsPath' with GCS credentials", gcsPath.size) diff --git a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala index 873874912a0..b39dde25358 100644 --- a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala +++ b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala @@ -17,8 +17,8 @@ import scala.concurrent.{ExecutionContext, Future} */ class DrsFileSystemConfig(val config: Config) - -class DrsPathBuilderFactory(globalConfig: Config, instanceConfig: Config, singletonConfig: DrsFileSystemConfig) extends PathBuilderFactory { +class DrsPathBuilderFactory(globalConfig: Config, instanceConfig: Config, singletonConfig: DrsFileSystemConfig) + extends PathBuilderFactory { private lazy val googleConfiguration: GoogleConfiguration = GoogleConfiguration(globalConfig) private lazy val scheme = instanceConfig.getString("auth") @@ -26,7 +26,9 @@ class DrsPathBuilderFactory(globalConfig: Config, instanceConfig: Config, single // For Azure support - this should be the UAMI client id private val dataAccessIdentityKey = "data_access_identity" - override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[PathBuilder] = { + override def withOptions( + options: WorkflowOptions + )(implicit as: ActorSystem, ec: ExecutionContext): Future[PathBuilder] = Future { val drsResolverScopes = List( // Profile and Email scopes are requirements for interacting with DRS Resolvers @@ -36,13 +38,20 @@ class DrsPathBuilderFactory(globalConfig: Config, instanceConfig: Config, single val (googleAuthMode, drsCredentials) = scheme match { case "azure" => (None, AzureDrsCredentials(options.get(dataAccessIdentityKey).toOption)) - case googleAuthScheme => googleConfiguration.auth(googleAuthScheme) match { - case Valid(auth) => ( - Option(auth), - GoogleOauthDrsCredentials(auth.credentials(options.get(_).get, drsResolverScopes), singletonConfig.config) - ) - case Invalid(error) => throw new RuntimeException(s"Error while instantiating DRS path builder factory. Errors: ${error.toString}") - } + case googleAuthScheme => + googleConfiguration.auth(googleAuthScheme) match { + case Valid(auth) => + ( + Option(auth), + GoogleOauthDrsCredentials(auth.credentials(options.get(_).get, drsResolverScopes), + singletonConfig.config + ) + ) + case Invalid(error) => + throw new RuntimeException( + s"Error while instantiating DRS path builder factory. Errors: ${error.toString}" + ) + } } // Unlike PAPI we're not going to fall back to a "default" project from the backend config. @@ -58,8 +67,7 @@ class DrsPathBuilderFactory(globalConfig: Config, instanceConfig: Config, single .getBoolean("override_preresolve_for_test") .toOption .getOrElse( - singletonConfig - .config + singletonConfig.config .getBoolean("resolver.preresolve") ) @@ -67,15 +75,15 @@ class DrsPathBuilderFactory(globalConfig: Config, instanceConfig: Config, single new DrsCloudNioFileSystemProvider( singletonConfig.config, drsCredentials, - DrsReader.readInterpreter(googleAuthMode, options, requesterPaysProjectIdOption), + DrsReader.readInterpreter(googleAuthMode, options, requesterPaysProjectIdOption) ), requesterPaysProjectIdOption, - preResolve, + preResolve ) } - } } case class UrlNotFoundException(scheme: String) extends Exception(s"No $scheme url associated with given DRS path.") -case class DrsResolverResponseMissingKeyException(missingKey: String) extends Exception(s"The response from the DRS Resolver doesn't contain the key '$missingKey'.") +case class DrsResolverResponseMissingKeyException(missingKey: String) + extends Exception(s"The response from the DRS Resolver doesn't contain the key '$missingKey'.") diff --git a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsReader.scala b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsReader.scala index 3256d6da248..e82790b454c 100644 --- a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsReader.scala +++ b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsReader.scala @@ -21,49 +21,48 @@ object DrsReader { options: WorkflowOptions, requesterPaysProjectIdOption: Option[String], drsPathResolver: DrsPathResolver, - drsResolverResponse: DrsResolverResponse): IO[DrsReader] = { + drsResolverResponse: DrsResolverResponse + ): IO[DrsReader] = (drsResolverResponse.accessUrl, drsResolverResponse.gsUri, googleAuthMode) match { case (Some(accessUrl), _, _) => IO.pure(AccessUrlReader(drsPathResolver, accessUrl)) case (_, Some(gcsPath), Some(authMode)) => - IO.pure(GcsReader( - authMode, - options, - requesterPaysProjectIdOption, - gcsPath, - drsResolverResponse.googleServiceAccount, - )) + IO.pure( + GcsReader( + authMode, + options, + requesterPaysProjectIdOption, + gcsPath, + drsResolverResponse.googleServiceAccount + ) + ) case (_, Some(_), _) => IO.raiseError(new RuntimeException("GCS URI found in the DRS Resolver response, but no Google auth found!")) case _ => IO.raiseError(new RuntimeException(DrsPathResolver.ExtractUriErrorMsg)) } - } def readInterpreter(googleAuthMode: Option[GoogleAuthMode], options: WorkflowOptions, - requesterPaysProjectIdOption: Option[String]) - (drsPathResolver: DrsPathResolver, - drsResolverResponse: DrsResolverResponse): IO[ReadableByteChannel] = { + requesterPaysProjectIdOption: Option[String] + )(drsPathResolver: DrsPathResolver, drsResolverResponse: DrsResolverResponse): IO[ReadableByteChannel] = for { reader <- reader(googleAuthMode, options, requesterPaysProjectIdOption, drsPathResolver, drsResolverResponse) channel <- reader.read() } yield channel - } } case class AccessUrlReader(drsPathResolver: DrsPathResolver, accessUrl: AccessUrl) extends DrsReader { - override def read(): IO[ReadableByteChannel] = { + override def read(): IO[ReadableByteChannel] = drsPathResolver.openChannel(accessUrl) - } } case class GcsReader(googleAuthMode: GoogleAuthMode, options: WorkflowOptions, requesterPaysProjectIdOption: Option[String], gsUri: String, - googleServiceAccount: Option[SADataObject], - ) extends DrsReader { + googleServiceAccount: Option[SADataObject] +) extends DrsReader { override def read(): IO[ReadableByteChannel] = { val readScopes = List(StorageScopes.DEVSTORAGE_READ_ONLY) val credentialsIo = googleServiceAccount match { @@ -71,7 +70,7 @@ case class GcsReader(googleAuthMode: GoogleAuthMode, IO( UserServiceAccountMode("drs_resolver_service_account").credentials( Map(GoogleAuthMode.UserServiceAccountKey -> googleSA.data.noSpaces), - readScopes, + readScopes ) ) case None => @@ -86,25 +85,23 @@ case class GcsReader(googleAuthMode: GoogleAuthMode, private def gcsInputStream(gcsFile: String, credentials: OAuth2Credentials, - requesterPaysProjectIdOption: Option[String], - ): IO[ReadableByteChannel] = { + requesterPaysProjectIdOption: Option[String] + ): IO[ReadableByteChannel] = for { storage <- IO(StorageOptions.newBuilder().setCredentials(credentials).build().getService) gcsBucketAndName <- IO(getGcsBucketAndName(gcsFile)) (bucketName, objectName) = gcsBucketAndName - readChannel <- IO(storage.get(bucketName, objectName).reader()) handleErrorWith { - throwable => - (requesterPaysProjectIdOption, throwable) match { - case (Some(requesterPaysProjectId), storageException: StorageException) + readChannel <- IO(storage.get(bucketName, objectName).reader()) handleErrorWith { throwable => + (requesterPaysProjectIdOption, throwable) match { + case (Some(requesterPaysProjectId), storageException: StorageException) if storageException.getMessage.contains("requester pays bucket but no user project") => - IO( - storage - .get(bucketName, objectName, BlobGetOption.userProject(requesterPaysProjectId)) - .reader(Blob.BlobSourceOption.userProject(requesterPaysProjectId)) - ) - case _ => IO.raiseError(throwable) - } + IO( + storage + .get(bucketName, objectName, BlobGetOption.userProject(requesterPaysProjectId)) + .reader(Blob.BlobSourceOption.userProject(requesterPaysProjectId)) + ) + case _ => IO.raiseError(throwable) + } } } yield readChannel - } } diff --git a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsResolver.scala b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsResolver.scala index 4aa760f17e1..39115e1633e 100644 --- a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsResolver.scala +++ b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsResolver.scala @@ -8,7 +8,6 @@ import cromwell.core.path.{DefaultPathBuilder, Path} import org.apache.commons.lang3.exception.ExceptionUtils import shapeless.syntax.typeable._ - object DrsResolver { private val GcsScheme: String = "gs" @@ -16,18 +15,18 @@ object DrsResolver { private val DrsLocalizationPathsContainer = "drs_localization_paths" - private def resolveError[A](pathAsString: String)(throwable: Throwable): IO[A] = { + private def resolveError[A](pathAsString: String)(throwable: Throwable): IO[A] = IO.raiseError( new RuntimeException( s"Error while resolving DRS path: $pathAsString. Error: ${ExceptionUtils.getMessage(throwable)}" ) ) - } private def getDrsPathResolver(drsPath: DrsPath): IO[DrsPathResolver] = { val drsFileSystemProviderOption = drsPath.drsPath.getFileSystem.provider.cast[DrsCloudNioFileSystemProvider] - val noFileSystemForDrsError = s"Unable to cast file system provider to DrsCloudNioFileSystemProvider for DRS path $drsPath." + val noFileSystemForDrsError = + s"Unable to cast file system provider to DrsCloudNioFileSystemProvider for DRS path $drsPath." for { drsFileSystemProvider <- toIO(drsFileSystemProviderOption, noFileSystemForDrsError) @@ -37,11 +36,17 @@ object DrsResolver { case class DrsResolverLocalizationData(gsUri: Option[String], fileName: Option[String], bondProvider: Option[String], - localizationPath: Option[String]) + localizationPath: Option[String] + ) private def getDrsResolverLocalizationData(pathAsString: String, - drsPathResolver: DrsPathResolver): IO[DrsResolverLocalizationData] = { - val fields = NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.FileName, DrsResolverField.BondProvider, DrsResolverField.LocalizationPath) + drsPathResolver: DrsPathResolver + ): IO[DrsResolverLocalizationData] = { + val fields = NonEmptyList.of(DrsResolverField.GsUri, + DrsResolverField.FileName, + DrsResolverField.BondProvider, + DrsResolverField.LocalizationPath + ) drsPathResolver.resolveDrs(pathAsString, fields) map { r => DrsResolverLocalizationData(r.gsUri, r.fileName, r.bondProvider, r.localizationPath) @@ -49,7 +54,7 @@ object DrsResolver { } /** Returns the `gsUri` if it ends in the `fileName` and the `bondProvider` is empty. */ - private def getSimpleGsUri(localizationData: DrsResolverLocalizationData): Option[String] = { + private def getSimpleGsUri(localizationData: DrsResolverLocalizationData): Option[String] = localizationData match { // `gsUri` not defined so no gsUri can be returned. case DrsResolverLocalizationData(None, _, _, _) => None @@ -60,11 +65,9 @@ object DrsResolver { // Barring any of the situations above return the `gsUri`. case DrsResolverLocalizationData(Some(gsUri), _, _, _) => Option(gsUri) } - } /** Returns the `gsUri` if it ends in the `fileName` and the `bondProvider` is empty. */ - def getSimpleGsUri(pathAsString: String, - drsPathResolver: DrsPathResolver): IO[Option[String]] = { + def getSimpleGsUri(pathAsString: String, drsPathResolver: DrsPathResolver): IO[Option[String]] = { val gsUriIO = getDrsResolverLocalizationData(pathAsString, drsPathResolver) map getSimpleGsUri @@ -72,12 +75,11 @@ object DrsResolver { } /** Returns the `gsUri` if it ends in the `fileName` and the `bondProvider` is empty. */ - def getSimpleGsUri(drsPath: DrsPath): IO[Option[String]] = { + def getSimpleGsUri(drsPath: DrsPath): IO[Option[String]] = for { drsPathResolver <- getDrsPathResolver(drsPath) gsUri <- getSimpleGsUri(drsPath.pathAsString, drsPathResolver) } yield gsUri - } def getContainerRelativePath(drsPath: DrsPath): IO[String] = { val pathIO = for { diff --git a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderFactorySpec.scala b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderFactorySpec.scala index d207bba8d86..5fc3257ed9d 100644 --- a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderFactorySpec.scala +++ b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderFactorySpec.scala @@ -6,7 +6,7 @@ import cromwell.core.filesystem.CromwellFileSystems import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -class DrsPathBuilderFactorySpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers{ +class DrsPathBuilderFactorySpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "DrsPathBuilderFactory" diff --git a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderSpec.scala b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderSpec.scala index 315e51b5d04..65f71e029c7 100644 --- a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderSpec.scala +++ b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderSpec.scala @@ -51,9 +51,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "a path with non-ascii", path = s"drs://$bucket/hello/world/with non ascii £€", @@ -66,9 +65,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "a gs uri path with encoded characters", path = s"drs://$bucket/hello/world/encoded%20spaces", @@ -81,9 +79,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "a file at the top of the bucket", path = s"drs://$bucket/hello", @@ -96,9 +93,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "a path ending in /", path = s"drs://$bucket/hello/world/", @@ -111,7 +107,7 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), // Special paths @@ -128,9 +124,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "a bucket with a path ..", path = s"drs://$bucket/..", @@ -143,9 +138,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "a bucket including . in the path", path = s"drs://$bucket/hello/./world", @@ -158,9 +152,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "a bucket including .. in the path", path = s"drs://$bucket/hello/../world", @@ -173,7 +166,7 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), // Normalized @@ -190,9 +183,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "a bucket with a normalized path ..", path = s"drs://$bucket/..", @@ -205,9 +197,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "a bucket including . in the normalized path", path = s"drs://$bucket/hello/./world", @@ -220,9 +211,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "a bucket including .. in the normalized path", path = s"drs://$bucket/hello/../world", @@ -235,9 +225,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "a bucket with an underscore", path = s"drs://hello_underscore/world", @@ -250,9 +239,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "a bucket named .", path = s"drs://./hello/world", @@ -265,9 +253,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "a non ascii bucket name", path = s"drs://nonasciibucket£€/hello/world", @@ -280,9 +267,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "an non-absolute path without a host", path = s"drs://blah/", @@ -295,9 +281,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), - GoodPath( description = "an absolute path without a host", path = s"drs://blah", @@ -310,7 +295,7 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), // No spec says this is illegal... so pass it to the DRS Resolver's various GCFs JIC @@ -326,7 +311,7 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), // Sample via: https://docs.google.com/document/d/1Wf4enSGOEXD5_AE-uzLoYqjIp5MnePbZ6kYTVFp1WoM/edit @@ -340,12 +325,11 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers "drs.data.humancellatlas.org/8aca942c-17f7-4e34-b8fd-3c12e50f9291?version=2019-07-04T151444.185805Z", parent = null, getParent = null, - root = - "drs://drs.data.humancellatlas.org/8aca942c-17f7-4e34-b8fd-3c12e50f9291?version=2019-07-04T151444.185805Z", + root = "drs://drs.data.humancellatlas.org/8aca942c-17f7-4e34-b8fd-3c12e50f9291?version=2019-07-04T151444.185805Z", name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, + isAbsolute = false ), // Sample via: https://docs.google.com/document/d/1Wf4enSGOEXD5_AE-uzLoYqjIp5MnePbZ6kYTVFp1WoM/edit @@ -361,8 +345,8 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 1, - isAbsolute = false, - ), + isAbsolute = false + ) ) private def badPaths = Seq( @@ -371,7 +355,7 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers BadPath("a https path", "https://hello/world", "https://hello/world does not have a drs scheme."), BadPath("a file uri path", "file:///hello/world", "file:///hello/world does not have a drs scheme."), BadPath("a relative file path", "hello/world", "hello/world does not have a drs scheme."), - BadPath("an absolute file path", "/hello/world", "/hello/world does not have a drs scheme."), + BadPath("an absolute file path", "/hello/world", "/hello/world does not have a drs scheme.") ) private val drsReadInterpreter: DrsReadInterpreter = (_, _) => @@ -387,7 +371,10 @@ class DrsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers private lazy val fakeCredentials = NoCredentials.getInstance private lazy val drsPathBuilder = DrsPathBuilder( - new DrsCloudNioFileSystemProvider(drsResolverConfig, GoogleOauthDrsCredentials(fakeCredentials, 1.minutes), drsReadInterpreter), - None, + new DrsCloudNioFileSystemProvider(drsResolverConfig, + GoogleOauthDrsCredentials(fakeCredentials, 1.minutes), + drsReadInterpreter + ), + None ) } diff --git a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsReaderSpec.scala b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsReaderSpec.scala index b5bee55b301..c9075333fca 100644 --- a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsReaderSpec.scala +++ b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsReaderSpec.scala @@ -1,6 +1,6 @@ package cromwell.filesystems.drs -import cloud.nio.impl.drs.{AccessUrl, DrsPathResolver, DrsResolverResponse, MockEngineDrsPathResolver} +import cloud.nio.impl.drs.{AccessUrl, DrsPathResolver, DrsResolverResponse, MockDrsPathResolver} import common.assertion.CromwellTimeoutSpec import cromwell.cloudsupport.gcp.auth.MockAuthMode import cromwell.core.WorkflowOptions @@ -26,12 +26,17 @@ class DrsReaderSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with Matche val googleAuthMode = MockAuthMode("unused") val workflowOptions = WorkflowOptions.empty val requesterPaysProjectIdOption = None - val drsPathResolver = new MockEngineDrsPathResolver() + val drsPathResolver = new MockDrsPathResolver() val gsUri = "gs://bucket/object" val googleServiceAccount = None val drsResolverResponse = DrsResolverResponse(gsUri = Option(gsUri), googleServiceAccount = googleServiceAccount) val readerIo = - DrsReader.reader(Option(googleAuthMode), workflowOptions, requesterPaysProjectIdOption, drsPathResolver, drsResolverResponse) + DrsReader.reader(Option(googleAuthMode), + workflowOptions, + requesterPaysProjectIdOption, + drsPathResolver, + drsResolverResponse + ) readerIo.unsafeRunSync() should be( GcsReader(googleAuthMode, workflowOptions, requesterPaysProjectIdOption, gsUri, googleServiceAccount) ) @@ -41,11 +46,16 @@ class DrsReaderSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with Matche val googleAuthMode = MockAuthMode("unused") val workflowOptions = WorkflowOptions.empty val requesterPaysProjectIdOption = None - val drsPathResolver = new MockEngineDrsPathResolver() + val drsPathResolver = new MockDrsPathResolver() val accessUrl = AccessUrl("https://host/object/path", Option(Map("hello" -> "world"))) val drsResolverResponse = DrsResolverResponse(accessUrl = Option(accessUrl)) val readerIo = - DrsReader.reader(Option(googleAuthMode), workflowOptions, requesterPaysProjectIdOption, drsPathResolver, drsResolverResponse) + DrsReader.reader(Option(googleAuthMode), + workflowOptions, + requesterPaysProjectIdOption, + drsPathResolver, + drsResolverResponse + ) readerIo.unsafeRunSync() should be( AccessUrlReader(drsPathResolver, accessUrl) ) @@ -55,10 +65,15 @@ class DrsReaderSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with Matche val googleAuthMode = MockAuthMode("unused") val workflowOptions = WorkflowOptions.empty val requesterPaysProjectIdOption = None - val drsPathResolver = new MockEngineDrsPathResolver() + val drsPathResolver = new MockDrsPathResolver() val drsResolverResponse = DrsResolverResponse() val readerIo = - DrsReader.reader(Option(googleAuthMode), workflowOptions, requesterPaysProjectIdOption, drsPathResolver, drsResolverResponse) + DrsReader.reader(Option(googleAuthMode), + workflowOptions, + requesterPaysProjectIdOption, + drsPathResolver, + drsResolverResponse + ) the[RuntimeException] thrownBy { readerIo.unsafeRunSync() } should have message DrsPathResolver.ExtractUriErrorMsg @@ -75,21 +90,23 @@ class DrsReaderSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with Matche val httpClientBuilder = mock[HttpClientBuilder] httpClientBuilder.build() returns httpClient - val drsPathResolver = new MockEngineDrsPathResolver(httpClientBuilderOverride = Option(httpClientBuilder)) + val drsPathResolver = new MockDrsPathResolver(httpClientBuilderOverride = Option(httpClientBuilder)) val accessUrl = AccessUrl("https://host/object/path", Option(Map("hello" -> "world"))) val drsResolverResponse = DrsResolverResponse(accessUrl = Option(accessUrl)) val channelIo = - DrsReader.readInterpreter(Option(MockAuthMode("unused")), WorkflowOptions.empty, None)(drsPathResolver, drsResolverResponse) + DrsReader.readInterpreter(Option(MockAuthMode("unused")), WorkflowOptions.empty, None)(drsPathResolver, + drsResolverResponse + ) val channel = channelIo.unsafeRunSync() val buffer = ByteBuffer.allocate(exampleBytes.length) - channel.isOpen should be (true) + channel.isOpen should be(true) DrsReaderSpec.readToBuffer(channel, buffer) channel.close() val httpGetCapture = capture[HttpGet] - channel.isOpen should be (false) + channel.isOpen should be(false) buffer.array() should be(exampleBytes) verify(httpClient).execute(httpGetCapture.capture) verify(httpClient).close() @@ -104,7 +121,7 @@ class DrsReaderSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with Matche object DrsReaderSpec { @tailrec - def readToBuffer(input: ReadableByteChannel, buffer: ByteBuffer): Unit = { + def readToBuffer(input: ReadableByteChannel, buffer: ByteBuffer): Unit = if (buffer.remaining() > 0) { if (input.read(buffer) >= 0) { readToBuffer(input, buffer) @@ -112,5 +129,4 @@ object DrsReaderSpec { throw new EOFException(s"input exhausted with ${buffer.remaining()} expected bytes") } } - } } diff --git a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsResolverSpec.scala b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsResolverSpec.scala index 23dcb1bcd66..ac37fe26f59 100644 --- a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsResolverSpec.scala +++ b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsResolverSpec.scala @@ -8,7 +8,6 @@ import org.scalatest.matchers.should.Matchers import scala.jdk.CollectionConverters._ - class DrsResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { private val drsResolverConfig: Config = ConfigFactory.parseMap( @@ -21,37 +20,40 @@ class DrsResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers private val mockFileSystemProvider = new MockDrsCloudNioFileSystemProvider(config = drsResolverConfig) private val drsPathBuilder = DrsPathBuilder(mockFileSystemProvider, None) - behavior of "DrsResolver" it should "find DRS path from a GCS path" in { val drsPath = drsPathBuilder.build(MockDrsPaths.drsPathResolvingGcsPath).get.asInstanceOf[DrsPath] - DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() should be (MockDrsPaths.drsRelativePath) + DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() should be(MockDrsPaths.drsRelativePath) } it should "find DRS path from a path replacing characters" in { val drsPath = drsPathBuilder.build(MockDrsPaths.drsPathWithNonPathChars).get.asInstanceOf[DrsPath] - DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() should be (MockDrsPaths.drsReplacedChar) + DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() should be(MockDrsPaths.drsReplacedChar) } it should "find DRS path from a file name" in { val drsPath = drsPathBuilder.build(MockDrsPaths.drsPathResolvingWithFileName).get.asInstanceOf[DrsPath] - DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() should be (MockDrsPaths.gcsRelativePathWithFileName) + DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() should be(MockDrsPaths.gcsRelativePathWithFileName) } it should "find DRS path from a localization path" in { val drsPath = drsPathBuilder.build(MockDrsPaths.drsPathResolvingWithLocalizationPath).get.asInstanceOf[DrsPath] - DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() should be (MockDrsPaths.gcsRelativePathWithFileNameFromLocalizationPath) + DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() should be( + MockDrsPaths.gcsRelativePathWithFileNameFromLocalizationPath + ) } it should "find DRS path from all the paths" in { val drsPath = drsPathBuilder.build(MockDrsPaths.drsPathResolvingWithAllThePaths).get.asInstanceOf[DrsPath] - DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() should be (MockDrsPaths.gcsRelativePathWithFileNameFromAllThePaths) + DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() should be( + MockDrsPaths.gcsRelativePathWithFileNameFromAllThePaths + ) } it should "throw GcsUrlNotFoundException when DRS path doesn't resolve to at least one GCS url" in { @@ -69,7 +71,7 @@ class DrsResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() } should have message s"Error while resolving DRS path: ${drsPath.pathAsString}. " + - s"Error: RuntimeException: Unexpected response resolving ${drsPath.pathAsString} " + - s"through DRS Resolver url https://drshub-url/drshub_v4. Error: 404 Not Found." + s"Error: RuntimeException: Unexpected response resolving ${drsPath.pathAsString} " + + s"through DRS Resolver url https://drshub-url/drshub_v4. Error: 404 Not Found." } } diff --git a/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/CromwellFtpFileSystems.scala b/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/CromwellFtpFileSystems.scala index 70593c41e68..27fb1698fd7 100644 --- a/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/CromwellFtpFileSystems.scala +++ b/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/CromwellFtpFileSystems.scala @@ -25,19 +25,21 @@ object CromwellFtpFileSystems { } def parseConfig(config: Config): FtpFileSystemsConfiguration = { - val cacheTTL = validate[FiniteDuration] { config.as[FiniteDuration]("cache-ttl") } - val leaseTimeout = validate[Option[FiniteDuration]] { config.getAs[FiniteDuration]("obtain-connection-timeout") } + val cacheTTL = validate[FiniteDuration](config.as[FiniteDuration]("cache-ttl")) + val leaseTimeout = validate[Option[FiniteDuration]](config.getAs[FiniteDuration]("obtain-connection-timeout")) // Cannot be less than 2, otherwise we can't copy files as we need 2 connections to copy a file (one for downstream and one for upstream) - val capacity: ErrorOr[Int] = validate[Int] { config.as[Int]("max-connection-per-server-per-user") } map { c => Math.max(2, c) } - val idleConnectionTimeout = validate[FiniteDuration] { config.as[FiniteDuration]("idle-connection-timeout") } - val connectionPort = validate[Int] { config.as[Int]("connection-port") } - val connectionMode = validate[ConnectionMode] { config.as[ConnectionMode]("connection-mode") } + val capacity: ErrorOr[Int] = validate[Int](config.as[Int]("max-connection-per-server-per-user")) map { c => + Math.max(2, c) + } + val idleConnectionTimeout = validate[FiniteDuration](config.as[FiniteDuration]("idle-connection-timeout")) + val connectionPort = validate[Int](config.as[Int]("connection-port")) + val connectionMode = validate[ConnectionMode](config.as[ConnectionMode]("connection-mode")) (cacheTTL, leaseTimeout, capacity, idleConnectionTimeout, connectionPort, connectionMode) .mapN(FtpFileSystemsConfiguration.apply) .unsafe("Failed to parse FTP configuration") } - + val Default = new CromwellFtpFileSystems(FtpFileSystems.Default) } diff --git a/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/FtpInstanceConfiguration.scala b/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/FtpInstanceConfiguration.scala index 6af3c8863c5..23c8504872f 100644 --- a/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/FtpInstanceConfiguration.scala +++ b/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/FtpInstanceConfiguration.scala @@ -25,12 +25,12 @@ case class FtpInstanceConfiguration(ftpCredentials: FtpCredentials) object FtpInstanceConfiguration { lazy val Default = FtpInstanceConfiguration(FtpAnonymousCredentials) - + def apply(conf: Config): FtpInstanceConfiguration = { val credentials: ErrorOr[FtpCredentials] = conf.getAs[Config]("auth") map { authConfig => - val username = validate { authConfig.as[String]("username") } - val password = validate { authConfig.as[String]("password") } - val account = validate { authConfig.getAs[String]("account") } + val username = validate(authConfig.as[String]("username")) + val password = validate(authConfig.as[String]("password")) + val account = validate(authConfig.getAs[String]("account")) (username, password, account) mapN FtpAuthenticatedCredentials.apply } getOrElse Default.ftpCredentials.validNel diff --git a/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/FtpPathBuilder.scala b/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/FtpPathBuilder.scala index 409a4027437..ed708ef41e0 100644 --- a/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/FtpPathBuilder.scala +++ b/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/FtpPathBuilder.scala @@ -16,7 +16,7 @@ object FtpPathBuilder { case class FtpPathBuilder(fileSystemProvider: CloudNioFileSystemProvider) extends PathBuilder { override def name = "FTP" - override def build(string: String) = { + override def build(string: String) = if (string == "ftp://") Failure(new IllegalArgumentException(s"$string does not have a valid host")) else { Try(URI.create(UrlEscapers.urlFragmentEscaper().escape(string))) flatMap { uri => @@ -31,5 +31,4 @@ case class FtpPathBuilder(fileSystemProvider: CloudNioFileSystemProvider) extend } } } - } } diff --git a/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/FtpPathBuilderFactory.scala b/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/FtpPathBuilderFactory.scala index cd599349d27..da55640d4e1 100644 --- a/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/FtpPathBuilderFactory.scala +++ b/filesystems/ftp/src/main/scala/cromwell/filesystems/ftp/FtpPathBuilderFactory.scala @@ -19,24 +19,37 @@ object FtpPathBuilderFactory { def credentialsFromWorkflowOptions(workflowOptions: WorkflowOptions) = { def getValue(key: String) = workflowOptions.get(key).toOption - (getValue(WorkflowOptions.FtpUsername), getValue(WorkflowOptions.FtpPassword), getValue(WorkflowOptions.FtpAccount)) match { + (getValue(WorkflowOptions.FtpUsername), + getValue(WorkflowOptions.FtpPassword), + getValue(WorkflowOptions.FtpAccount) + ) match { case (Some(username), Some(password), account) => Option(FtpAuthenticatedCredentials(username, password, account)) case _ => None } } } -class FtpPathBuilderFactory(globalConfig: Config, instanceConfig: Config, cromwellFtpFileSystems: CromwellFtpFileSystems) extends PathBuilderFactory { - private [ftp] lazy val configFtpConfiguration = FtpInstanceConfiguration(instanceConfig) - private lazy val defaultFtpProvider = new FtpCloudNioFileSystemProvider(instanceConfig, configFtpConfiguration.ftpCredentials, cromwellFtpFileSystems.ftpFileSystems) - - override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext) = Future.successful { - val provider = credentialsFromWorkflowOptions(options) match { - case Some(overriddenCredentials) => - new FtpCloudNioFileSystemProvider(instanceConfig, overriddenCredentials, cromwellFtpFileSystems.ftpFileSystems) - case _ => defaultFtpProvider +class FtpPathBuilderFactory(globalConfig: Config, + instanceConfig: Config, + cromwellFtpFileSystems: CromwellFtpFileSystems +) extends PathBuilderFactory { + private[ftp] lazy val configFtpConfiguration = FtpInstanceConfiguration(instanceConfig) + private lazy val defaultFtpProvider = new FtpCloudNioFileSystemProvider(instanceConfig, + configFtpConfiguration.ftpCredentials, + cromwellFtpFileSystems.ftpFileSystems + ) + + override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext) = + Future.successful { + val provider = credentialsFromWorkflowOptions(options) match { + case Some(overriddenCredentials) => + new FtpCloudNioFileSystemProvider(instanceConfig, + overriddenCredentials, + cromwellFtpFileSystems.ftpFileSystems + ) + case _ => defaultFtpProvider + } + + FtpPathBuilder(provider) } - - FtpPathBuilder(provider) - } } diff --git a/filesystems/ftp/src/test/scala/cromwell/filesystems/ftp/CromwellFtpFileSystemsSpec.scala b/filesystems/ftp/src/test/scala/cromwell/filesystems/ftp/CromwellFtpFileSystemsSpec.scala index 90229c30cc2..ea1985797e3 100644 --- a/filesystems/ftp/src/test/scala/cromwell/filesystems/ftp/CromwellFtpFileSystemsSpec.scala +++ b/filesystems/ftp/src/test/scala/cromwell/filesystems/ftp/CromwellFtpFileSystemsSpec.scala @@ -13,14 +13,13 @@ class CromwellFtpFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec wi behavior of "CromwellFtpFileSystemsSpec" it should "parse configuration" in { - val config = ConfigFactory.parseString( - """cache-ttl = 10 days - |obtain-connection-timeout = 12 hours - |max-connection-per-server-per-user = 1 - |idle-connection-timeout = 14 hours - |connection-port: 212 - |connection-mode = "active" """.stripMargin) - + val config = ConfigFactory.parseString("""cache-ttl = 10 days + |obtain-connection-timeout = 12 hours + |max-connection-per-server-per-user = 1 + |idle-connection-timeout = 14 hours + |connection-port: 212 + |connection-mode = "active" """.stripMargin) + val fs = new CromwellFtpFileSystems(config) fs.ftpFileSystems.config.cacheTTL shouldBe 10.days fs.ftpFileSystems.config.leaseTimeout shouldBe Some(12.hours) diff --git a/filesystems/ftp/src/test/scala/cromwell/filesystems/ftp/FtpInstanceConfigurationSpec.scala b/filesystems/ftp/src/test/scala/cromwell/filesystems/ftp/FtpInstanceConfigurationSpec.scala index 5fa5de22ea2..09ce6ec863c 100644 --- a/filesystems/ftp/src/test/scala/cromwell/filesystems/ftp/FtpInstanceConfigurationSpec.scala +++ b/filesystems/ftp/src/test/scala/cromwell/filesystems/ftp/FtpInstanceConfigurationSpec.scala @@ -15,21 +15,19 @@ class FtpInstanceConfigurationSpec extends AnyFlatSpec with CromwellTimeoutSpec } it should "parse authenticated credentials" in { - FtpInstanceConfiguration(ConfigFactory.parseString( - """ - |auth { - | username = "me" - | password = "mot de passe" - |} + FtpInstanceConfiguration(ConfigFactory.parseString(""" + |auth { + | username = "me" + | password = "mot de passe" + |} """.stripMargin)).ftpCredentials shouldBe FtpAuthenticatedCredentials("me", "mot de passe", None) - FtpInstanceConfiguration(ConfigFactory.parseString( - """ - |auth { - | username = "me" - | password = "mot de passe" - | account = "account" - |} + FtpInstanceConfiguration(ConfigFactory.parseString(""" + |auth { + | username = "me" + | password = "mot de passe" + | account = "account" + |} """.stripMargin)).ftpCredentials shouldBe FtpAuthenticatedCredentials("me", "mot de passe", Option("account")) } diff --git a/filesystems/ftp/src/test/scala/cromwell/filesystems/ftp/FtpPathSpec.scala b/filesystems/ftp/src/test/scala/cromwell/filesystems/ftp/FtpPathSpec.scala index 9224b4f4124..0c164127514 100644 --- a/filesystems/ftp/src/test/scala/cromwell/filesystems/ftp/FtpPathSpec.scala +++ b/filesystems/ftp/src/test/scala/cromwell/filesystems/ftp/FtpPathSpec.scala @@ -16,9 +16,10 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit behavior of "FtpPathSpec" - val pathBuilderFactory = new FtpPathBuilderFactory(ConfigFactory.empty(), ConfigFactory.empty(), CromwellFtpFileSystems.Default) { - override private [ftp] lazy val configFtpConfiguration = new FtpInstanceConfiguration(FtpAnonymousCredentials) - } + val pathBuilderFactory = + new FtpPathBuilderFactory(ConfigFactory.empty(), ConfigFactory.empty(), CromwellFtpFileSystems.Default) { + override private[ftp] lazy val configFtpConfiguration = new FtpInstanceConfiguration(FtpAnonymousCredentials) + } val pathBuilder = Await.result(pathBuilderFactory.withOptions(WorkflowOptions.empty)(null, null), 1.second) @@ -41,7 +42,10 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit ("ftp://ftp-server.com/path/to/my//dir", "ftp://ftp-server.com/path/to/my/dir/file", "dir/file"), ("ftp://ftp-server.com/path/to/my//dir", "ftp://ftp-server.com/path/to/my/dir//file", "dir//file"), ("ftp://ftp-server.com/path/to/my/dir", "ftp://ftp-server.com/path/./to/my/dir/file", "./to/my/dir/file"), - ("ftp://ftp-server.com/path/to/my/dir/with/file", "ftp://ftp-server.com/path/to/other/dir/with/file", "other/dir/with/file") + ("ftp://ftp-server.com/path/to/my/dir/with/file", + "ftp://ftp-server.com/path/to/other/dir/with/file", + "other/dir/with/file" + ) ) private def goodPaths = Seq( @@ -57,8 +61,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "with spaces", getFileName = s"ftp://ftp-server.com/with spaces", getNameCount = 3, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path with non-ascii", path = s"ftp://ftp-server.com/hello/world/with non ascii £€", @@ -71,8 +75,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "with non ascii £€", getFileName = s"ftp://ftp-server.com/with non ascii £€", getNameCount = 3, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "an ftp uri path with encoded characters", path = s"ftp://ftp-server.com/hello/world/encoded%20spaces", @@ -85,8 +89,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "encoded%20spaces", getFileName = s"ftp://ftp-server.com/encoded%20spaces", getNameCount = 3, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a hostname only path (root path)", path = s"ftp://ftp-server.com", @@ -99,8 +103,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "", getFileName = null, getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a hostname only path ending in a /", path = s"ftp://ftp-server.com/", @@ -113,8 +117,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "", getFileName = null, getNameCount = 0, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a file at the top of the hostname", path = s"ftp://ftp-server.com/hello", @@ -127,8 +131,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "hello", getFileName = s"ftp://ftp-server.com/hello", getNameCount = 1, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path ending in /", path = s"ftp://ftp-server.com/hello/world/", @@ -141,7 +145,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "world", getFileName = s"ftp://ftp-server.com/world", getNameCount = 2, - isAbsolute = true), + isAbsolute = true + ), // Special paths @@ -157,8 +162,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "", getFileName = s"ftp://ftp-server.com/.", getNameCount = 1, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a hostname with a path ..", path = s"ftp://ftp-server.com/..", @@ -171,8 +176,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "", getFileName = s"ftp://ftp-server.com/..", getNameCount = 1, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a bucket including . in the path", path = s"ftp://ftp-server.com/hello/./world", @@ -185,8 +190,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "world", getFileName = s"ftp://ftp-server.com/world", getNameCount = 3, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path including .. in the path", path = s"ftp://ftp-server.com/hello/../world", @@ -199,7 +204,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "world", getFileName = s"ftp://ftp-server.com/world", getNameCount = 3, - isAbsolute = true), + isAbsolute = true + ), // Normalized @@ -215,8 +221,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "", getFileName = null, getNameCount = 0, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path with a normalized path ..", path = s"ftp://ftp-server.com/..", @@ -229,8 +235,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "", getFileName = null, getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a path including . in the normalized path", path = s"ftp://ftp-server.com/hello/./world", @@ -243,8 +249,8 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "world", getFileName = s"ftp://ftp-server.com/world", getNameCount = 2, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path including .. in the normalized path", path = s"ftp://ftp-server.com/hello/../world", @@ -257,14 +263,18 @@ class FtpPathSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit name = "world", getFileName = s"ftp://ftp-server.com/world", getNameCount = 1, - isAbsolute = true), + isAbsolute = true + ) ) private def badPaths = Seq( BadPath("an empty path", "", " does not have an ftp scheme"), BadPath("a hostless path", "ftp://", "ftp:// does not have a valid host"), BadPath("a bucket named .", "ftp://./hello/world", "ftp://./hello/world does not have a valid host"), - BadPath("a non ascii bucket name", "ftp://nonasciibucket£€/hello/world", "ftp://nonasciibucket£€/hello/world does not have a valid host"), + BadPath("a non ascii bucket name", + "ftp://nonasciibucket£€/hello/world", + "ftp://nonasciibucket£€/hello/world does not have a valid host" + ), BadPath("a https path", "https://hello/world", "https://hello/world does not have an ftp scheme"), BadPath("a file uri path", "file:///hello/world", "file:///hello/world does not have an ftp scheme"), BadPath("a relative file path", "hello/world", "hello/world does not have an ftp scheme"), diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsEnhancedRequest.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsEnhancedRequest.scala index 76887905387..638d23e50ec 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsEnhancedRequest.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsEnhancedRequest.scala @@ -11,20 +11,19 @@ import java.io.FileNotFoundException object GcsEnhancedRequest { // If the request fails because no project was passed, recover the request, this time setting the project - def recoverFromProjectNotProvided[A](path: GcsPath, f: Boolean => A) = { - IO(f(false)).handleErrorWith({ - // Only retry with the the project if the error is right - case error: StorageException if isProjectNotProvidedError(error) => - IO(f(true)) - // Use NoSuchFileException for better error reporting - case e: StorageException if e.getCode == StatusCodes.NotFound.intValue => - IO.raiseError(new FileNotFoundException(s"File not found: ${path.pathAsString}")) - case e: GoogleJsonResponseException if isProjectNotProvidedError(e) => - IO(f(true)) - case e: GoogleJsonResponseException if e.getStatusCode == StatusCodes.NotFound.intValue => - IO.raiseError(new FileNotFoundException(s"File not found: ${path.pathAsString}")) - case e => - IO.raiseError(e) - }) - } + def recoverFromProjectNotProvided[A](path: GcsPath, f: Boolean => A) = + IO(f(false)).handleErrorWith { + // Only retry with the the project if the error is right + case error: StorageException if isProjectNotProvidedError(error) => + IO(f(true)) + // Use NoSuchFileException for better error reporting + case e: StorageException if e.getCode == StatusCodes.NotFound.intValue => + IO.raiseError(new FileNotFoundException(s"File not found: ${path.pathAsString}")) + case e: GoogleJsonResponseException if isProjectNotProvidedError(e) => + IO(f(true)) + case e: GoogleJsonResponseException if e.getStatusCode == StatusCodes.NotFound.intValue => + IO.raiseError(new FileNotFoundException(s"File not found: ${path.pathAsString}")) + case e => + IO.raiseError(e) + } } diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala index 00aee93e537..2a618e48aa0 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala @@ -29,11 +29,11 @@ import scala.language.postfixOps import scala.util.{Failure, Try} object GcsPathBuilder { /* - * Provides some level of validation of GCS bucket names - * This is meant to alert the user early if they mistyped a gcs path in their workflow / inputs and not to validate - * exact bucket syntax, which is done by GCS. - * See https://cloud.google.com/storage/docs/naming for full spec - */ + * Provides some level of validation of GCS bucket names + * This is meant to alert the user early if they mistyped a gcs path in their workflow / inputs and not to validate + * exact bucket syntax, which is done by GCS. + * See https://cloud.google.com/storage/docs/naming for full spec + */ private val GcsBucketPattern = """ (?x) # Turn on comments and whitespace insensitivity @@ -57,13 +57,12 @@ object GcsPathBuilder { override def errorMessage = s"Cloud Storage URIs must have 'gs' scheme: $pathString" } final case class InvalidFullGcsPath(pathString: String) extends InvalidGcsPath { - override def errorMessage: String = { + override def errorMessage: String = s""" |The path '$pathString' does not seem to be a valid GCS path. |Please check that it starts with gs:// and that the bucket and object follow GCS naming guidelines at |https://cloud.google.com/storage/docs/naming. """.stripMargin.replace("\n", " ").trim - } } final case class UnparseableGcsPath(pathString: String, throwable: Throwable) extends InvalidGcsPath { override def errorMessage: String = @@ -79,7 +78,7 @@ object GcsPathBuilder { case _ => None } - def validateGcsPath(string: String): GcsPathValidation = { + def validateGcsPath(string: String): GcsPathValidation = Try { val uri = URI.create(UrlEscapers.urlFragmentEscaper().escape(string)) if (uri.getScheme == null) PossiblyValidRelativeGcsPath @@ -89,39 +88,32 @@ object GcsPathBuilder { } else ValidFullGcsPath(uri.getHost, uri.getPath) } else InvalidScheme(string) } recover { case t => UnparseableGcsPath(string, t) } get - } - def isGcsPath(nioPath: NioPath): Boolean = { + def isGcsPath(nioPath: NioPath): Boolean = nioPath.getFileSystem.provider().getScheme.equalsIgnoreCase(CloudStorageFileSystem.URI_SCHEME) - } def fromAuthMode(authMode: GoogleAuthMode, applicationName: String, retrySettings: RetrySettings, cloudStorageConfiguration: CloudStorageConfiguration, options: WorkflowOptions, - defaultProject: Option[String])(implicit as: ActorSystem, ec: ExecutionContext): Future[GcsPathBuilder] = { + defaultProject: Option[String] + )(implicit as: ActorSystem, ec: ExecutionContext): Future[GcsPathBuilder] = authMode.retryCredentials(options, List(StorageScopes.DEVSTORAGE_FULL_CONTROL)) map { credentials => - fromCredentials(credentials, - applicationName, - retrySettings, - cloudStorageConfiguration, - options, - defaultProject - ) + fromCredentials(credentials, applicationName, retrySettings, cloudStorageConfiguration, options, defaultProject) } - } def fromCredentials(credentials: Credentials, applicationName: String, retrySettings: RetrySettings, cloudStorageConfiguration: CloudStorageConfiguration, options: WorkflowOptions, - defaultProject: Option[String]): GcsPathBuilder = { + defaultProject: Option[String] + ): GcsPathBuilder = { // Grab the google project from Workflow Options if specified and set // that to be the project used by the StorageOptions Builder. If it's not // specified use the default project mentioned in config file - val project: Option[String] = options.get("google_project").toOption match { + val project: Option[String] = options.get("google_project").toOption match { case Some(googleProject) => Option(googleProject) case None => defaultProject } @@ -139,8 +131,9 @@ object GcsPathBuilder { class GcsPathBuilder(apiStorage: com.google.api.services.storage.Storage, cloudStorageConfiguration: CloudStorageConfiguration, - storageOptions: StorageOptions) extends PathBuilder { - private [gcs] val projectId = storageOptions.getProjectId + storageOptions: StorageOptions +) extends PathBuilder { + private[gcs] val projectId = storageOptions.getProjectId private lazy val cloudStorage = storageOptions.getService /** @@ -153,7 +146,7 @@ class GcsPathBuilder(apiStorage: com.google.api.services.storage.Storage, * * Also see https://github.com/GoogleCloudPlatform/google-cloud-java/issues/1343 */ - def build(string: String): Try[GcsPath] = { + def build(string: String): Try[GcsPath] = validateGcsPath(string) match { case ValidFullGcsPath(bucket, path) => Try { @@ -161,18 +154,19 @@ class GcsPathBuilder(apiStorage: com.google.api.services.storage.Storage, val cloudStoragePath = fileSystem.getPath(path) GcsPath(cloudStoragePath, apiStorage, cloudStorage, projectId) } - case PossiblyValidRelativeGcsPath => Failure(new IllegalArgumentException(s"""Path "$string" does not have a gcs scheme""")) + case PossiblyValidRelativeGcsPath => + Failure(new IllegalArgumentException(s"""Path "$string" does not have a gcs scheme""")) case invalid: InvalidGcsPath => Failure(new IllegalArgumentException(invalid.errorMessage)) } - } override def name: String = "Google Cloud Storage" } -case class GcsPath private[gcs](nioPath: NioPath, - apiStorage: com.google.api.services.storage.Storage, - cloudStorage: com.google.cloud.storage.Storage, - projectId: String) extends Path { +case class GcsPath private[gcs] (nioPath: NioPath, + apiStorage: com.google.api.services.storage.Storage, + cloudStorage: com.google.cloud.storage.Storage, + projectId: String +) extends Path { lazy val objectBlobId: Try[BlobId] = Try { val bucketName = cloudStoragePath.bucket val objectName = cloudStoragePath.toRealPath().toString @@ -202,13 +196,12 @@ case class GcsPath private[gcs](nioPath: NioPath, BlobId.of(bucketName, objectName) } - lazy val bucketOrObjectBlobId: Try[BlobId] = { + lazy val bucketOrObjectBlobId: Try[BlobId] = Try { val bucketName = cloudStoragePath.bucket val objectName = cloudStoragePath.toRealPath().toString BlobId.of(bucketName, objectName) } - } override protected def newPath(nioPath: NioPath): GcsPath = GcsPath(nioPath, apiStorage, cloudStorage, projectId) @@ -218,9 +211,9 @@ case class GcsPath private[gcs](nioPath: NioPath, s"${CloudStorageFileSystem.URI_SCHEME}://$host/$path" } - override def writeContent(content: String) - (openOptions: OpenOptions, codec: Codec, compressPayload: Boolean) - (implicit ec: ExecutionContext): GcsPath.this.type = { + override def writeContent(content: String)(openOptions: OpenOptions, codec: Codec, compressPayload: Boolean)(implicit + ec: ExecutionContext + ): GcsPath.this.type = { def request(withProject: Boolean) = { val builder = BlobInfo.newBuilder(objectBlobId.get).setContentType(ContentTypes.`text/plain(UTF-8)`.value) if (compressPayload) { @@ -243,14 +236,14 @@ case class GcsPath private[gcs](nioPath: NioPath, } override def mediaInputStream(implicit ec: ExecutionContext): InputStream = { - def request(withProject: Boolean) = { + def request(withProject: Boolean) = // Use apiStorage here instead of cloudStorage, because apiStorage throws now if the bucket has requester pays, // whereas cloudStorage creates the input stream anyway and only throws one `read` is called (which only happens in NioFlow) - apiStorage.objects() + apiStorage + .objects() .get(objectBlobId.get.getBucket, objectBlobId.get.getName) .setUserProject(withProject.option(projectId).orNull) .executeMediaAsInputStream() - } // Since the NIO interface is synchronous we need to run this synchronously here. It is however wrapped in a Future // in the NioFlow so we don't need to worry about exceptions runOnEc(recoverFromProjectNotProvided(this, request)).unsafeRunSync() diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilderFactory.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilderFactory.scala index 74d88b05476..f63ef73b4dd 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilderFactory.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilderFactory.scala @@ -14,8 +14,7 @@ import org.threeten.bp.Duration import scala.concurrent.{ExecutionContext, Future} -final case class GcsPathBuilderFactory(globalConfig: Config, instanceConfig: Config) - extends PathBuilderFactory { +final case class GcsPathBuilderFactory(globalConfig: Config, instanceConfig: Config) extends PathBuilderFactory { import net.ceedubs.ficus.Ficus._ // Parse the configuration and create a GoogleConfiguration val googleConf: GoogleConfiguration = GoogleConfiguration(globalConfig) @@ -30,8 +29,9 @@ final case class GcsPathBuilderFactory(globalConfig: Config, instanceConfig: Con val defaultProject = instanceConfig.as[Option[String]]("project") - lazy val defaultRetrySettings: RetrySettings = { - RetrySettings.newBuilder() + lazy val defaultRetrySettings: RetrySettings = + RetrySettings + .newBuilder() .setMaxAttempts(maxAttempts) .setTotalTimeout(Duration.ofSeconds(30)) .setInitialRetryDelay(Duration.ofMillis(100)) @@ -41,9 +41,8 @@ final case class GcsPathBuilderFactory(globalConfig: Config, instanceConfig: Con .setRpcTimeoutMultiplier(1.1) .setMaxRpcTimeout(Duration.ofSeconds(5)) .build() - } - def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[GcsPathBuilder] = { + def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[GcsPathBuilder] = GcsPathBuilder.fromAuthMode( authMode, applicationName, @@ -52,7 +51,6 @@ final case class GcsPathBuilderFactory(globalConfig: Config, instanceConfig: Con options, defaultProject ) - } } object GcsPathBuilderFactory { diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleUtil.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleUtil.scala index 481a3f36d10..8149c822670 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleUtil.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleUtil.scala @@ -11,43 +11,43 @@ import cromwell.core.{CromwellFatalExceptionMarker, WorkflowOptions} import scala.concurrent.{ExecutionContext, Future} object GoogleUtil { + /** * Extract status code from an exception if it's a com.google.api.client.http.HttpResponseException */ - def extractStatusCode(exception: Throwable): Option[Int] = { + def extractStatusCode(exception: Throwable): Option[Int] = exception match { case t: HttpResponseException => Option(t.getStatusCode) case t: BaseServiceException => Option(t.getCode) case _ => None } - } implicit class EnhancedGoogleAuthMode(val googleAuthMode: GoogleAuthMode) extends AnyVal { + /** * Retries getting the credentials three times. * * There is nothing GCS specific about this method. This package just happens to be the lowest level with access * to core's version of Retry + cloudSupport's implementation of GoogleAuthMode. */ - def retryCredentials(options: WorkflowOptions, scopes: Iterable[String]) - (implicit actorSystem: ActorSystem, executionContext: ExecutionContext): Future[Credentials] = { - def credential(): Credentials = { - - try { + def retryCredentials(options: WorkflowOptions, scopes: Iterable[String])(implicit + actorSystem: ActorSystem, + executionContext: ExecutionContext + ): Future[Credentials] = { + def credential(): Credentials = + try googleAuthMode.credentials(options.get(_).get, scopes) - } catch { + catch { case exception: OptionLookupException => throw new IllegalArgumentException(s"Missing parameters in workflow options: ${exception.key}", exception) with CromwellFatalExceptionMarker } - } - def isFatal(throwable: Throwable): Boolean = { + def isFatal(throwable: Throwable): Boolean = throwable match { case _: IllegalArgumentException => Option(throwable.getCause).exists(isFatal) case _ => GoogleAuthMode.isFatal(throwable) } - } Retry.withRetry( () => Future(credential()), diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/RequesterPaysErrors.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/RequesterPaysErrors.scala index 26ee0868c65..cf693698d18 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/RequesterPaysErrors.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/RequesterPaysErrors.scala @@ -12,7 +12,7 @@ object RequesterPaysErrors { def isProjectNotProvidedError(storageException: StorageException) = storageException.getCode == BucketIsRequesterPaysErrorCode && - StringUtils.contains(storageException.getMessage, BucketIsRequesterPaysErrorMessage) + StringUtils.contains(storageException.getMessage, BucketIsRequesterPaysErrorMessage) def isProjectNotProvidedError(googleJsonError: GoogleJsonError) = googleJsonError.getCode == BucketIsRequesterPaysErrorCode && diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchCommandBuilder.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchCommandBuilder.scala index 5c7b2b724d3..77c15d09a84 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchCommandBuilder.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchCommandBuilder.scala @@ -7,32 +7,32 @@ import cromwell.filesystems.gcs.GcsPath import scala.util.Try private case object PartialGcsBatchCommandBuilder extends PartialIoCommandBuilder { - override def sizeCommand: PartialFunction[Path, Try[GcsBatchSizeCommand]] = { - case gcsPath: GcsPath => GcsBatchSizeCommand.forPath(gcsPath) + override def sizeCommand: PartialFunction[Path, Try[GcsBatchSizeCommand]] = { case gcsPath: GcsPath => + GcsBatchSizeCommand.forPath(gcsPath) } - + override def deleteCommand: PartialFunction[(Path, Boolean), Try[GcsBatchDeleteCommand]] = { case (gcsPath: GcsPath, swallowIoExceptions) => GcsBatchDeleteCommand.forPath(gcsPath, swallowIoExceptions) } - + override def copyCommand: PartialFunction[(Path, Path), Try[GcsBatchCopyCommand]] = { case (gcsSrc: GcsPath, gcsDest: GcsPath) => GcsBatchCopyCommand.forPaths(gcsSrc, gcsDest) } - - override def hashCommand: PartialFunction[Path, Try[GcsBatchCrc32Command]] = { - case gcsPath: GcsPath => GcsBatchCrc32Command.forPath(gcsPath) + + override def hashCommand: PartialFunction[Path, Try[GcsBatchCrc32Command]] = { case gcsPath: GcsPath => + GcsBatchCrc32Command.forPath(gcsPath) } - override def touchCommand: PartialFunction[Path, Try[GcsBatchTouchCommand]] = { - case gcsPath: GcsPath => GcsBatchTouchCommand.forPath(gcsPath) + override def touchCommand: PartialFunction[Path, Try[GcsBatchTouchCommand]] = { case gcsPath: GcsPath => + GcsBatchTouchCommand.forPath(gcsPath) } - override def existsCommand: PartialFunction[Path, Try[GcsBatchExistsCommand]] = { - case gcsPath: GcsPath => GcsBatchExistsCommand.forPath(gcsPath) + override def existsCommand: PartialFunction[Path, Try[GcsBatchExistsCommand]] = { case gcsPath: GcsPath => + GcsBatchExistsCommand.forPath(gcsPath) } - override def isDirectoryCommand: PartialFunction[Path, Try[GcsBatchIsDirectoryCommand]] = { - case gcsPath: GcsPath => GcsBatchIsDirectoryCommand.forPath(gcsPath) + override def isDirectoryCommand: PartialFunction[Path, Try[GcsBatchIsDirectoryCommand]] = { case gcsPath: GcsPath => + GcsBatchIsDirectoryCommand.forPath(gcsPath) } } diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchIoCommand.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchIoCommand.scala index ed07fdb3605..f8d1239992a 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchIoCommand.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchIoCommand.scala @@ -24,6 +24,7 @@ import scala.util.Try * @tparam U Return type of the Google response */ sealed trait GcsBatchIoCommand[T, U] extends IoCommand[T] { + /** * StorageRequest operation to be executed by this command */ @@ -50,7 +51,9 @@ sealed trait GcsBatchIoCommand[T, U] extends IoCommand[T] { /** * Override to handle a failure differently and potentially return a successful response. */ - def onFailure(googleJsonError: GoogleJsonError, httpHeaders: HttpHeaders): Option[Either[T, GcsBatchIoCommand[T, U]]] = None + def onFailure(googleJsonError: GoogleJsonError, + httpHeaders: HttpHeaders + ): Option[Either[T, GcsBatchIoCommand[T, U]]] = None /** * Use to signal that the request has failed because the user project was not set and that it can be retried with it. @@ -60,25 +63,26 @@ sealed trait GcsBatchIoCommand[T, U] extends IoCommand[T] { sealed trait SingleFileGcsBatchIoCommand[T, U] extends GcsBatchIoCommand[T, U] with SingleFileIoCommand[T] { override def file: GcsPath - //noinspection MutatorLikeMethodIsParameterless + // noinspection MutatorLikeMethodIsParameterless def setUserProject: Boolean def userProject: String = setUserProject.option(file.projectId).orNull } case class GcsBatchCopyCommand( - override val source: GcsPath, - sourceBlob: BlobId, - override val destination: GcsPath, - destinationBlob: BlobId, - rewriteToken: Option[String] = None, - setUserProject: Boolean = false - ) - extends IoCopyCommand(source, destination) with GcsBatchIoCommand[Unit, RewriteResponse] { + override val source: GcsPath, + sourceBlob: BlobId, + override val destination: GcsPath, + destinationBlob: BlobId, + rewriteToken: Option[String] = None, + setUserProject: Boolean = false +) extends IoCopyCommand(source, destination) + with GcsBatchIoCommand[Unit, RewriteResponse] { override def commandDescription: String = s"GcsBatchCopyCommand source '$source' destination '$destination' " + s"setUserProject '$setUserProject' rewriteToken '$rewriteToken'" override def operation: StorageRequest[RewriteResponse] = { - val rewriteOperation = source.apiStorage.objects() + val rewriteOperation = source.apiStorage + .objects() .rewrite(sourceBlob.getBucket, sourceBlob.getName, destinationBlob.getBucket, destinationBlob.getName, null) .setUserProject(setUserProject.option(source.projectId).orNull) @@ -92,13 +96,14 @@ case class GcsBatchCopyCommand( */ def withRewriteToken(rewriteToken: String): GcsBatchCopyCommand = copy(rewriteToken = Option(rewriteToken)) - override def onSuccess(response: RewriteResponse, httpHeaders: HttpHeaders): ErrorOr[Either[Unit, GcsBatchCopyCommand]] = { + override def onSuccess(response: RewriteResponse, + httpHeaders: HttpHeaders + ): ErrorOr[Either[Unit, GcsBatchCopyCommand]] = if (response.getDone) { mapGoogleResponse(response) map Left.apply } else { Right(withRewriteToken(response.getRewriteToken)).validNel } - } override def mapGoogleResponse(response: RewriteResponse): ErrorOr[Unit] = ().validNel @@ -106,31 +111,29 @@ case class GcsBatchCopyCommand( } object GcsBatchCopyCommand { - def forPaths(source: GcsPath, destination: GcsPath): Try[GcsBatchCopyCommand] = { + def forPaths(source: GcsPath, destination: GcsPath): Try[GcsBatchCopyCommand] = for { sourceBlob <- source.objectBlobId destinationBlob <- destination.objectBlobId } yield GcsBatchCopyCommand(source, sourceBlob, destination, destinationBlob) - } } case class GcsBatchDeleteCommand( - override val file: GcsPath, - blob: BlobId, - override val swallowIOExceptions: Boolean, - setUserProject: Boolean = false - ) extends IoDeleteCommand(file, swallowIOExceptions) with SingleFileGcsBatchIoCommand[Unit, Void] { - override def operation: StorageRequest[Void] = { + override val file: GcsPath, + blob: BlobId, + override val swallowIOExceptions: Boolean, + setUserProject: Boolean = false +) extends IoDeleteCommand(file, swallowIOExceptions) + with SingleFileGcsBatchIoCommand[Unit, Void] { + override def operation: StorageRequest[Void] = file.apiStorage.objects().delete(blob.getBucket, blob.getName).setUserProject(userProject) - } override def mapGoogleResponse(response: Void): ErrorOr[Unit] = ().validNel override def onFailure(googleJsonError: GoogleJsonError, - httpHeaders: HttpHeaders, - ): Option[Either[Unit, GcsBatchDeleteCommand]] = { + httpHeaders: HttpHeaders + ): Option[Either[Unit, GcsBatchDeleteCommand]] = if (swallowIOExceptions) Option(Left(())) else None - } override def withUserProject: GcsBatchDeleteCommand = this.copy(setUserProject = true) override def commandDescription: String = s"GcsBatchDeleteCommand file '$file' swallowIOExceptions " + @@ -138,9 +141,8 @@ case class GcsBatchDeleteCommand( } object GcsBatchDeleteCommand { - def forPath(file: GcsPath, swallowIOExceptions: Boolean): Try[GcsBatchDeleteCommand] = { + def forPath(file: GcsPath, swallowIOExceptions: Boolean): Try[GcsBatchDeleteCommand] = file.objectBlobId.map(GcsBatchDeleteCommand(file, _, swallowIOExceptions)) - } } /** @@ -149,21 +151,18 @@ object GcsBatchDeleteCommand { sealed trait GcsBatchGetCommand[T] extends SingleFileGcsBatchIoCommand[T, StorageObject] { def file: GcsPath def blob: BlobId - override def operation: StorageRequest[StorageObject] = { + override def operation: StorageRequest[StorageObject] = file.apiStorage.objects().get(blob.getBucket, blob.getName).setUserProject(userProject) - } } -case class GcsBatchSizeCommand(override val file: GcsPath, - override val blob: BlobId, - setUserProject: Boolean = false, - ) extends IoSizeCommand(file) with GcsBatchGetCommand[Long] { - override def mapGoogleResponse(response: StorageObject): ErrorOr[Long] = { +case class GcsBatchSizeCommand(override val file: GcsPath, override val blob: BlobId, setUserProject: Boolean = false) + extends IoSizeCommand(file) + with GcsBatchGetCommand[Long] { + override def mapGoogleResponse(response: StorageObject): ErrorOr[Long] = Option(response.getSize) match { case None => s"'${file.pathAsString}' in project '${file.projectId}' returned null size".invalidNel case Some(size) => size.longValue().validNel } - } override def withUserProject: GcsBatchSizeCommand = this.copy(setUserProject = true) @@ -171,21 +170,18 @@ case class GcsBatchSizeCommand(override val file: GcsPath, } object GcsBatchSizeCommand { - def forPath(file: GcsPath): Try[GcsBatchSizeCommand] = { + def forPath(file: GcsPath): Try[GcsBatchSizeCommand] = file.objectBlobId.map(GcsBatchSizeCommand(file, _)) - } } -case class GcsBatchCrc32Command(override val file: GcsPath, - override val blob: BlobId, - setUserProject: Boolean = false, - ) extends IoHashCommand(file) with GcsBatchGetCommand[String] { - override def mapGoogleResponse(response: StorageObject): ErrorOr[String] = { +case class GcsBatchCrc32Command(override val file: GcsPath, override val blob: BlobId, setUserProject: Boolean = false) + extends IoHashCommand(file) + with GcsBatchGetCommand[String] { + override def mapGoogleResponse(response: StorageObject): ErrorOr[String] = Option(response.getCrc32c) match { case None => s"'${file.pathAsString}' in project '${file.projectId}' returned null CRC32C checksum".invalidNel case Some(crc32c) => crc32c.validNel } - } override def withUserProject: GcsBatchCrc32Command = this.copy(setUserProject = true) @@ -193,15 +189,13 @@ case class GcsBatchCrc32Command(override val file: GcsPath, } object GcsBatchCrc32Command { - def forPath(file: GcsPath): Try[GcsBatchCrc32Command] = { + def forPath(file: GcsPath): Try[GcsBatchCrc32Command] = file.objectBlobId.map(GcsBatchCrc32Command(file, _)) - } } -case class GcsBatchTouchCommand(override val file: GcsPath, - override val blob: BlobId, - setUserProject: Boolean = false, - ) extends IoTouchCommand(file) with GcsBatchGetCommand[Unit] { +case class GcsBatchTouchCommand(override val file: GcsPath, override val blob: BlobId, setUserProject: Boolean = false) + extends IoTouchCommand(file) + with GcsBatchGetCommand[Unit] { override def mapGoogleResponse(response: StorageObject): ErrorOr[Unit] = ().validNel override def withUserProject: GcsBatchTouchCommand = this.copy(setUserProject = true) @@ -210,9 +204,8 @@ case class GcsBatchTouchCommand(override val file: GcsPath, } object GcsBatchTouchCommand { - def forPath(file: GcsPath): Try[GcsBatchTouchCommand] = { + def forPath(file: GcsPath): Try[GcsBatchTouchCommand] = file.objectBlobId.map(GcsBatchTouchCommand(file, _)) - } } /* @@ -220,47 +213,46 @@ object GcsBatchTouchCommand { * Specifically, list objects that have this path as a prefix. Since we don't really care about what's inside here, * set max results to 1 to avoid unnecessary payload. */ -case class GcsBatchIsDirectoryCommand(override val file: GcsPath, - blob: BlobId, - setUserProject: Boolean = false, - ) - extends IoIsDirectoryCommand(file) with SingleFileGcsBatchIoCommand[Boolean, Objects] { - override def operation: StorageRequest[Objects] = { - file.apiStorage.objects().list(blob.getBucket).setPrefix(blob.getName.ensureSlashed).setMaxResults(1L).setUserProject(userProject) - } - - override def mapGoogleResponse(response: Objects): ErrorOr[Boolean] = { +case class GcsBatchIsDirectoryCommand(override val file: GcsPath, blob: BlobId, setUserProject: Boolean = false) + extends IoIsDirectoryCommand(file) + with SingleFileGcsBatchIoCommand[Boolean, Objects] { + override def operation: StorageRequest[Objects] = + file.apiStorage + .objects() + .list(blob.getBucket) + .setPrefix(blob.getName.ensureSlashed) + .setMaxResults(1L) + .setUserProject(userProject) + + override def mapGoogleResponse(response: Objects): ErrorOr[Boolean] = // Wrap in an Option because getItems can (always ?) return null if there are no objects Option(response.getItems).map(_.asScala).exists(_.nonEmpty).validNel - } override def withUserProject: GcsBatchIsDirectoryCommand = this.copy(setUserProject = true) override def commandDescription: String = s"GcsBatchIsDirectoryCommand file '$file' setUserProject '$setUserProject'" } object GcsBatchIsDirectoryCommand { - def forPath(file: GcsPath): Try[GcsBatchIsDirectoryCommand] = { + def forPath(file: GcsPath): Try[GcsBatchIsDirectoryCommand] = file.bucketOrObjectBlobId.map(GcsBatchIsDirectoryCommand(file, _)) - } } -case class GcsBatchExistsCommand(override val file: GcsPath, - override val blob: BlobId, - setUserProject: Boolean = false, - ) extends IoExistsCommand(file) with GcsBatchGetCommand[Boolean] { +case class GcsBatchExistsCommand(override val file: GcsPath, override val blob: BlobId, setUserProject: Boolean = false) + extends IoExistsCommand(file) + with GcsBatchGetCommand[Boolean] { override def mapGoogleResponse(response: StorageObject): ErrorOr[Boolean] = true.validNel - override def onFailure(googleJsonError: GoogleJsonError, httpHeaders: HttpHeaders): Option[Either[Boolean, GcsBatchIoCommand[Boolean, StorageObject]]] = { + override def onFailure(googleJsonError: GoogleJsonError, + httpHeaders: HttpHeaders + ): Option[Either[Boolean, GcsBatchIoCommand[Boolean, StorageObject]]] = // If the object can't be found, don't fail the request but just return false as we were testing for existence if (googleJsonError.getCode == 404) Option(Left(false)) else None - } override def withUserProject: GcsBatchExistsCommand = this.copy(setUserProject = true) override def commandDescription: String = s"GcsBatchExistsCommand file '$file' setUserProject '$setUserProject'" } object GcsBatchExistsCommand { - def forPath(file: GcsPath): Try[GcsBatchExistsCommand] = { + def forPath(file: GcsPath): Try[GcsBatchExistsCommand] = file.objectBlobId.map(GcsBatchExistsCommand(file, _)) - } } /** A GcsBatchIoCommand for use in tests. */ diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsEnhancedRequestSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsEnhancedRequestSpec.scala index 01a75e21ee5..be91888a610 100644 --- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsEnhancedRequestSpec.scala +++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsEnhancedRequestSpec.scala @@ -22,9 +22,11 @@ class GcsEnhancedRequestSpec extends AnyFlatSpec with CromwellTimeoutSpec with M CloudStorageFileSystem.forBucket("bucket").getPath("test"), mock[com.google.api.services.storage.Storage], mock[com.google.cloud.storage.Storage], - "GcsEnhancedRequest-project", + "GcsEnhancedRequest-project" ) - val requesterPaysException = new StorageException(BucketIsRequesterPaysErrorCode, "Bucket is a requester pays bucket but no user project provided.") + val requesterPaysException = new StorageException(BucketIsRequesterPaysErrorCode, + "Bucket is a requester pays bucket but no user project provided." + ) it should "attempt first without project, and not retry if the requests succeeds" in { val testFunction = mock[Boolean => String] @@ -54,7 +56,9 @@ class GcsEnhancedRequestSpec extends AnyFlatSpec with CromwellTimeoutSpec with M when(testFunction.apply(false)).thenThrow(requesterPaysException) // We expect it to be called a second time with withProject = true this time, and fail for another reason when(testFunction.apply(true)).thenThrow(new RuntimeException("it really doesn't work")) - a[RuntimeException] should be thrownBy GcsEnhancedRequest.recoverFromProjectNotProvided(path, testFunction).unsafeRunSync() + a[RuntimeException] should be thrownBy GcsEnhancedRequest + .recoverFromProjectNotProvided(path, testFunction) + .unsafeRunSync() } it should "not retry requests if the error does not match" in { @@ -62,7 +66,9 @@ class GcsEnhancedRequestSpec extends AnyFlatSpec with CromwellTimeoutSpec with M // Throw an unrelated exception, should only be called once when(testFunction.apply(false)).thenThrow(new RuntimeException("it really doesn't work")) - a[RuntimeException] should be thrownBy GcsEnhancedRequest.recoverFromProjectNotProvided(path, testFunction).unsafeRunSync() + a[RuntimeException] should be thrownBy GcsEnhancedRequest + .recoverFromProjectNotProvided(path, testFunction) + .unsafeRunSync() verify(testFunction).apply(false) verify(testFunction).apply(anyBoolean) } @@ -72,7 +78,9 @@ class GcsEnhancedRequestSpec extends AnyFlatSpec with CromwellTimeoutSpec with M // Throw an unrelated exception, should only be called once when(testFunction.apply(false)).thenThrow(new StorageException(404, "gs://does/not/exist")) - a[FileNotFoundException] should be thrownBy GcsEnhancedRequest.recoverFromProjectNotProvided(path, testFunction).unsafeRunSync() + a[FileNotFoundException] should be thrownBy GcsEnhancedRequest + .recoverFromProjectNotProvided(path, testFunction) + .unsafeRunSync() verify(testFunction).apply(false) verify(testFunction).apply(anyBoolean) } @@ -86,7 +94,9 @@ class GcsEnhancedRequestSpec extends AnyFlatSpec with CromwellTimeoutSpec with M // Throw an unrelated exception, should only be called once when(testFunction.apply(false)).thenAnswer(_ => throw new GoogleJsonResponseException(builder, error)) - a[FileNotFoundException] should be thrownBy GcsEnhancedRequest.recoverFromProjectNotProvided(path, testFunction).unsafeRunSync() + a[FileNotFoundException] should be thrownBy GcsEnhancedRequest + .recoverFromProjectNotProvided(path, testFunction) + .unsafeRunSync() verify(testFunction).apply(false) verify(testFunction).apply(anyBoolean) } diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala index 22e63ea0f32..a275d163e4a 100644 --- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala +++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala @@ -15,7 +15,12 @@ import org.scalatest.prop.Tables.Table import java.io.ByteArrayInputStream -class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with PathBuilderSpecUtils with ServiceAccountTestSupport { +class GcsPathBuilderSpec + extends TestKitSuite + with AnyFlatSpecLike + with Matchers + with PathBuilderSpecUtils + with ServiceAccountTestSupport { behavior of "GcsPathBuilder" @@ -64,8 +69,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "with spaces", getFileName = s"gs://$bucket/with spaces", getNameCount = 3, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path with non-ascii", path = s"gs://$bucket/hello/world/with non ascii £€", @@ -78,8 +83,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "with non ascii £€", getFileName = s"gs://$bucket/with non ascii £€", getNameCount = 3, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a gs uri path with encoded characters", path = s"gs://$bucket/hello/world/encoded%20spaces", @@ -92,8 +97,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "encoded%20spaces", getFileName = s"gs://$bucket/encoded%20spaces", getNameCount = 3, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a bucket only path", path = s"gs://$bucket", @@ -106,8 +111,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = s"gs://$bucket/", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a bucket only path ending in a /", path = s"gs://$bucket/", @@ -120,8 +125,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 0, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a file at the top of the bucket", path = s"gs://$bucket/hello", @@ -134,8 +139,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "hello", getFileName = s"gs://$bucket/hello", getNameCount = 1, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path ending in /", path = s"gs://$bucket/hello/world/", @@ -148,7 +153,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "world", getFileName = s"gs://$bucket/world", getNameCount = 2, - isAbsolute = true), + isAbsolute = true + ), // Special paths @@ -164,8 +170,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = s"gs://$bucket/.", getNameCount = 1, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a bucket with a path ..", path = s"gs://$bucket/..", @@ -178,8 +184,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = s"gs://$bucket/..", getNameCount = 1, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a bucket including . in the path", path = s"gs://$bucket/hello/./world", @@ -192,8 +198,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "world", getFileName = s"gs://$bucket/world", getNameCount = 3, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a bucket including .. in the path", path = s"gs://$bucket/hello/../world", @@ -206,7 +212,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "world", getFileName = s"gs://$bucket/world", getNameCount = 3, - isAbsolute = true), + isAbsolute = true + ), // Normalized @@ -222,8 +229,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 0, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a bucket with a normalized path ..", path = s"gs://$bucket/..", @@ -236,8 +243,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = s"gs://$bucket/", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a bucket including . in the normalized path", path = s"gs://$bucket/hello/./world", @@ -250,8 +257,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "world", getFileName = s"gs://$bucket/world", getNameCount = 2, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a bucket including .. in the normalized path", path = s"gs://$bucket/hello/../world", @@ -264,8 +271,8 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "world", getFileName = s"gs://$bucket/world", getNameCount = 1, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a bucket with an underscore", path = s"gs://hello_underscore/world", @@ -278,24 +285,34 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "world", getFileName = s"gs://hello_underscore/world", getNameCount = 1, - isAbsolute = true) + isAbsolute = true + ) ) private def badPaths = Seq( BadPath("an empty path", "", "Path \"\" does not have a gcs scheme"), - BadPath("an bucketless path", "gs://", "The specified GCS path 'gs://' does not parse as a URI.\nExpected authority at index 5: gs://"), - BadPath("a bucket named .", "gs://./hello/world", "The path 'gs://./hello/world' does not seem to be a valid GCS path. Please check that it starts with gs:// and that the bucket and object follow GCS naming guidelines at https://cloud.google.com/storage/docs/naming."), - BadPath("a non ascii bucket name", "gs://nonasciibucket£€/hello/world", - "The path 'gs://nonasciibucket£€/hello/world' does not seem to be a valid GCS path. Please check that it starts with gs:// and that the bucket and object follow GCS naming guidelines at https://cloud.google.com/storage/docs/naming."), + BadPath("an bucketless path", + "gs://", + "The specified GCS path 'gs://' does not parse as a URI.\nExpected authority at index 5: gs://" + ), + BadPath( + "a bucket named .", + "gs://./hello/world", + "The path 'gs://./hello/world' does not seem to be a valid GCS path. Please check that it starts with gs:// and that the bucket and object follow GCS naming guidelines at https://cloud.google.com/storage/docs/naming." + ), + BadPath( + "a non ascii bucket name", + "gs://nonasciibucket£€/hello/world", + "The path 'gs://nonasciibucket£€/hello/world' does not seem to be a valid GCS path. Please check that it starts with gs:// and that the bucket and object follow GCS naming guidelines at https://cloud.google.com/storage/docs/naming." + ), BadPath("a https path", "https://hello/world", "Cloud Storage URIs must have 'gs' scheme: https://hello/world"), BadPath("a file uri path", "file:///hello/world", "Cloud Storage URIs must have 'gs' scheme: file:///hello/world"), BadPath("a relative file path", "hello/world", "Path \"hello/world\" does not have a gcs scheme"), BadPath("an absolute file path", "/hello/world", "Path \"/hello/world\" does not have a gcs scheme") ) - private lazy val pathBuilder = { + private lazy val pathBuilder = MockGcsPathBuilder.instance - } it should "not mix up credentials" in { def retrySettings: RetrySettings = RetrySettings.newBuilder().build() @@ -304,22 +321,25 @@ class GcsPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers val noCredentials = NoCredentials.getInstance() val noCredentialsPathBuilder: GcsPathBuilder = { val noCredentialsStorage = GcsStorage.gcsStorage("no-credentials", noCredentials, retrySettings) - val noCredentialsStorageOptions = GcsStorage.gcsStorageOptions(noCredentials, retrySettings, Option("proj-no-credentials")) + val noCredentialsStorageOptions = + GcsStorage.gcsStorageOptions(noCredentials, retrySettings, Option("proj-no-credentials")) new GcsPathBuilder(noCredentialsStorage, cloudStorageConfig, noCredentialsStorageOptions) } - val serviceAccountCredentials = ServiceAccountCredentials.fromStream( - new ByteArrayInputStream(serviceAccountJsonContents.getBytes)) + val serviceAccountCredentials = + ServiceAccountCredentials.fromStream(new ByteArrayInputStream(serviceAccountJsonContents.getBytes)) val serviceAccountPathBuilder: GcsPathBuilder = { val serviceAccountStorage = GcsStorage.gcsStorage("service-account", serviceAccountCredentials, retrySettings) - val serviceAccountStorageOptions = GcsStorage.gcsStorageOptions(serviceAccountCredentials, retrySettings, Option("proj-service-account")) + val serviceAccountStorageOptions = + GcsStorage.gcsStorageOptions(serviceAccountCredentials, retrySettings, Option("proj-service-account")) new GcsPathBuilder(serviceAccountStorage, cloudStorageConfig, serviceAccountStorageOptions) } def credentialsForPath(gcsPath: GcsPath): Credentials = { - val cloudFilesystemProvider = gcsPath.nioPath.getFileSystem.provider().asInstanceOf[CloudStorageFileSystemProvider] + val cloudFilesystemProvider = + gcsPath.nioPath.getFileSystem.provider().asInstanceOf[CloudStorageFileSystemProvider] val storageOptionsField = cloudFilesystemProvider.getClass.getDeclaredField("storageOptions") storageOptionsField.setAccessible(true) val storageOptions = storageOptionsField.get(cloudFilesystemProvider) diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/MockGcsPathBuilder.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/MockGcsPathBuilder.scala index c42b7b324d0..8834d4c6476 100644 --- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/MockGcsPathBuilder.scala +++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/MockGcsPathBuilder.scala @@ -12,12 +12,13 @@ import scala.concurrent.ExecutionContext object MockGcsPathBuilder { implicit val ec = ExecutionContext.fromExecutor(Executors.newSingleThreadExecutor()) - private def makeStorageOptions(project: Option[String] = Option("cromwell-test")) = GcsStorage.gcsStorageOptions(NoCredentials.getInstance(), RetrySettings.newBuilder().build(), project) + private def makeStorageOptions(project: Option[String] = Option("cromwell-test")) = + GcsStorage.gcsStorageOptions(NoCredentials.getInstance(), RetrySettings.newBuilder().build(), project) private val storageOptions = makeStorageOptions() private val apiStorage = GcsStorage.gcsStorage("cromwell-test-app", storageOptions) lazy val instance = new GcsPathBuilder(apiStorage, CloudStorageConfiguration.DEFAULT, storageOptions) - + def withOptions(workflowOptions: WorkflowOptions) = { val customStorageOptions = makeStorageOptions(workflowOptions.get("google_project").toOption) new GcsPathBuilder(apiStorage, GcsStorage.DefaultCloudStorageConfiguration, customStorageOptions) diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/batch/GcsBatchIoCommandSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/batch/GcsBatchIoCommandSpec.scala index d54662f3535..a4c936a463d 100644 --- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/batch/GcsBatchIoCommandSpec.scala +++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/batch/GcsBatchIoCommandSpec.scala @@ -182,7 +182,9 @@ class GcsBatchIoCommandSpec extends AnyFlatSpec with Matchers with BeforeAndAfte response.setDone(false) response.setRewriteToken("token") - command.onSuccess(response, new HttpHeaders()).toEither.toOption.get.toOption.get.rewriteToken should be(Option("token")) + command.onSuccess(response, new HttpHeaders()).toEither.toOption.get.toOption.get.rewriteToken should be( + Option("token") + ) command.onFailure(new GoogleJsonError(), new HttpHeaders()) should be(None) } diff --git a/filesystems/http/src/main/scala/cromwell/filesystems/http/HttpPathBuilder.scala b/filesystems/http/src/main/scala/cromwell/filesystems/http/HttpPathBuilder.scala index 89bb22021df..29910cd43ac 100644 --- a/filesystems/http/src/main/scala/cromwell/filesystems/http/HttpPathBuilder.scala +++ b/filesystems/http/src/main/scala/cromwell/filesystems/http/HttpPathBuilder.scala @@ -16,19 +16,20 @@ object HttpPathBuilder { def accepts(url: String): Boolean = url.matches("^http[s]?://.*") } - class HttpPathBuilder extends PathBuilder { override def name: String = "HTTP" - override def build(pathAsString: String): Try[Path] = { + override def build(pathAsString: String): Try[Path] = if (HttpPathBuilder.accepts(pathAsString)) Try { HttpPath(Paths.get(pathAsString)) - } else { + } + else { Failure(new IllegalArgumentException(s"$pathAsString does not have an http or https scheme")) } - } - def content(url: String)(implicit actorContext: ActorContext, actorMaterializer: ActorMaterializer): Future[NioPath] = { + def content( + url: String + )(implicit actorContext: ActorContext, actorMaterializer: ActorMaterializer): Future[NioPath] = { implicit val actorSystem: ActorSystem = actorContext.system implicit val executionContext: ExecutionContext = actorContext.dispatcher @@ -54,19 +55,21 @@ case class HttpPath(nioPath: NioPath) extends Path { override def pathWithoutScheme: String = pathAsString.replaceFirst("http[s]?://", "") - def fetchSize(implicit executionContext: ExecutionContext, actorSystem: ActorSystem): Future[Long] = { + def pathWithoutSchemeOrQueryOrFragment: String = pathWithoutScheme.split("[?#]").head + + def fetchSize(implicit executionContext: ExecutionContext, actorSystem: ActorSystem): Future[Long] = Http().singleRequest(HttpRequest(uri = pathAsString, method = HttpMethods.HEAD)).map { response => response.discardEntityBytes() - val length = if (response.status.isSuccess()) - response.entity.contentLengthOption - else - None + val length = + if (response.status.isSuccess()) + response.entity.contentLengthOption + else + None length.getOrElse( throw new RuntimeException( s"Couldn't fetch size for $pathAsString, missing Content-Length header or path doesn't exist (HTTP ${response.status.toString()})." ) ) } - } } diff --git a/filesystems/http/src/main/scala/cromwell/filesystems/http/HttpPathBuilderFactory.scala b/filesystems/http/src/main/scala/cromwell/filesystems/http/HttpPathBuilderFactory.scala index 7a42c113921..bf65ed34a31 100644 --- a/filesystems/http/src/main/scala/cromwell/filesystems/http/HttpPathBuilderFactory.scala +++ b/filesystems/http/src/main/scala/cromwell/filesystems/http/HttpPathBuilderFactory.scala @@ -8,10 +8,10 @@ import cromwell.core.path.{PathBuilder, PathBuilderFactory} import scala.concurrent.{ExecutionContext, Future} class HttpPathBuilderFactory(globalConfig: Config, instanceConfig: Config) extends PathBuilderFactory { - override def withOptions(options: WorkflowOptions) - (implicit as: ActorSystem, ec: ExecutionContext): Future[PathBuilder] = { + override def withOptions( + options: WorkflowOptions + )(implicit as: ActorSystem, ec: ExecutionContext): Future[PathBuilder] = Future { new HttpPathBuilder } - } } diff --git a/filesystems/s3/src/main/scala/cromwell/filesystems/s3/S3PathBuilder.scala b/filesystems/s3/src/main/scala/cromwell/filesystems/s3/S3PathBuilder.scala index 0713afa1a71..4de28288cd6 100644 --- a/filesystems/s3/src/main/scala/cromwell/filesystems/s3/S3PathBuilder.scala +++ b/filesystems/s3/src/main/scala/cromwell/filesystems/s3/S3PathBuilder.scala @@ -76,13 +76,12 @@ object S3PathBuilder { override def errorMessage: String = s"S3 URIs must have 's3' scheme: $pathString" } final case class InvalidFullS3Path(pathString: String) extends InvalidS3Path { - override def errorMessage: String = { + override def errorMessage: String = s""" |The path '$pathString' does not seem to be a valid S3 path. |Please check that it starts with s3:// and that the bucket and object follow S3 naming guidelines at |https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html """.stripMargin.replace("\n", " ").trim - } } final case class UnparseableS3Path(pathString: String, throwable: Throwable) extends InvalidS3Path { override def errorMessage: String = @@ -98,7 +97,7 @@ object S3PathBuilder { def pathToUri(string: String): URI = URI.create(UrlEscapers.urlFragmentEscaper.escape(string)) - def validatePath(string: String): S3PathValidation = { + def validatePath(string: String): S3PathValidation = Try { val uri = pathToUri(string) if (uri.getScheme == null) { PossiblyValidRelativeS3Path } @@ -108,56 +107,46 @@ object S3PathBuilder { } else { ValidFullS3Path(uri.getHost, uri.getPath) } } else { InvalidScheme(string) } } recover { case t => UnparseableS3Path(string, t) } get - } def fromAuthMode(authMode: AwsAuthMode, configuration: S3Configuration, options: WorkflowOptions, - storageRegion: Option[Region])(implicit ec: ExecutionContext): Future[S3PathBuilder] = { + storageRegion: Option[Region] + )(implicit ec: ExecutionContext): Future[S3PathBuilder] = { val provider = authMode.provider() // Other backends needed retry here. In case we need retry, we'll return // a future. This will allow us to add capability without changing signature - Future(fromProvider(provider, - configuration, - options, - storageRegion - )) + Future(fromProvider(provider, configuration, options, storageRegion)) } def fromProvider(provider: AwsCredentialsProvider, configuration: S3Configuration, options: WorkflowOptions, - storageRegion: Option[Region]): S3PathBuilder = { + storageRegion: Option[Region] + ): S3PathBuilder = new S3PathBuilder(configuration) - } } -class S3PathBuilder(configuration: S3Configuration - ) extends PathBuilder { +class S3PathBuilder(configuration: S3Configuration) extends PathBuilder { // Tries to create a new S3Path from a String representing an absolute s3 path: s3://[/]. - def build(string: String): Try[S3Path] = { + def build(string: String): Try[S3Path] = validatePath(string) match { case ValidFullS3Path(bucket, path) => Try { val s3Path = new S3FileSystemProvider() .getFileSystem(URI.create("s3:////"), System.getenv) .getPath(s"""/$bucket/$path""") - S3Path(s3Path, bucket, - new AmazonS3ClientFactory().getS3Client(URI.create("s3:////"), System.getProperties)) + S3Path(s3Path, bucket, new AmazonS3ClientFactory().getS3Client(URI.create("s3:////"), System.getProperties)) } case PossiblyValidRelativeS3Path => Failure(new IllegalArgumentException(s"$string does not have a s3 scheme")) case invalid: InvalidS3Path => Failure(new IllegalArgumentException(invalid.errorMessage)) } - } override def name: String = "s3" } -case class S3Path private[s3](nioPath: NioPath, - bucket: String, - client: S3Client - ) extends Path { +case class S3Path private[s3] (nioPath: NioPath, bucket: String, client: S3Client) extends Path { override protected def newPath(nioPath: NioPath): S3Path = S3Path(nioPath, bucket, client) override def pathAsString: String = s"s3://$pathWithoutScheme" @@ -188,7 +177,7 @@ case class S3Path private[s3](nioPath: NioPath, val originalPath = s3Path.toString if (originalPath.startsWith("s3")) return s3Path.toAbsolutePath.toString originalPath.charAt(0) match { - case '/' => s3Path.toAbsolutePath.toString + case '/' => s3Path.toAbsolutePath.toString case _ => s3Path.resolve(s"/$bucket/$originalPath").toAbsolutePath.toString } } diff --git a/filesystems/s3/src/main/scala/cromwell/filesystems/s3/S3PathBuilderFactory.scala b/filesystems/s3/src/main/scala/cromwell/filesystems/s3/S3PathBuilderFactory.scala index 880b0b984c1..ad505a35929 100644 --- a/filesystems/s3/src/main/scala/cromwell/filesystems/s3/S3PathBuilderFactory.scala +++ b/filesystems/s3/src/main/scala/cromwell/filesystems/s3/S3PathBuilderFactory.scala @@ -46,8 +46,8 @@ import scala.concurrent.{ExecutionContext, Future} // The constructor of this class is required to be Config, Config by cromwell // So, we need to take this config and get the AuthMode out of it -final case class S3PathBuilderFactory private(globalConfig: Config, instanceConfig: Config) - extends PathBuilderFactory { +final case class S3PathBuilderFactory private (globalConfig: Config, instanceConfig: Config) + extends PathBuilderFactory { // Grab the authMode out of configuration val conf: AwsConfiguration = AwsConfiguration(globalConfig) @@ -55,19 +55,16 @@ final case class S3PathBuilderFactory private(globalConfig: Config, instanceConf val authModeValidation: ErrorOr[AwsAuthMode] = conf.auth(authModeAsString) val authMode = authModeValidation.unsafe(s"Failed to get authentication mode for $authModeAsString") - def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[S3PathBuilder] = { - S3PathBuilder.fromAuthMode(authMode, S3Storage.DefaultConfiguration, options, conf.region) - } + def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[S3PathBuilder] = + S3PathBuilder.fromAuthMode(authMode, S3Storage.DefaultConfiguration, options, conf.region) // Ignores the authMode and creates an S3PathBuilder using the passed credentials directly. // Can be used when the Credentials are already available. - def fromProvider(options: WorkflowOptions, provider: AwsCredentialsProvider): S3PathBuilder = { + def fromProvider(options: WorkflowOptions, provider: AwsCredentialsProvider): S3PathBuilder = S3PathBuilder.fromProvider(provider, S3Storage.DefaultConfiguration, options, conf.region) - } } object S3PathBuilderFactory { - def apply(globalConfig: Config, instanceConfig: Config): S3PathBuilderFactory = { + def apply(globalConfig: Config, instanceConfig: Config): S3PathBuilderFactory = new S3PathBuilderFactory(globalConfig, instanceConfig) - } } diff --git a/filesystems/s3/src/main/scala/cromwell/filesystems/s3/batch/S3BatchCommandBuilder.scala b/filesystems/s3/src/main/scala/cromwell/filesystems/s3/batch/S3BatchCommandBuilder.scala index 8058d897975..74ec4e1b6b6 100644 --- a/filesystems/s3/src/main/scala/cromwell/filesystems/s3/batch/S3BatchCommandBuilder.scala +++ b/filesystems/s3/src/main/scala/cromwell/filesystems/s3/batch/S3BatchCommandBuilder.scala @@ -40,8 +40,8 @@ import scala.util.Try * Generates commands for IO operations on S3 */ private case object PartialS3BatchCommandBuilder extends PartialIoCommandBuilder { - override def sizeCommand: PartialFunction[Path, Try[S3BatchSizeCommand]] = { - case path: S3Path => Try(S3BatchSizeCommand(path)) + override def sizeCommand: PartialFunction[Path, Try[S3BatchSizeCommand]] = { case path: S3Path => + Try(S3BatchSizeCommand(path)) } override def deleteCommand: PartialFunction[(Path, Boolean), Try[S3BatchDeleteCommand]] = { @@ -52,16 +52,16 @@ private case object PartialS3BatchCommandBuilder extends PartialIoCommandBuilder case (src: S3Path, dest: S3Path) => Try(S3BatchCopyCommand(src, dest)) } - override def hashCommand: PartialFunction[Path, Try[S3BatchEtagCommand]] = { - case path: S3Path => Try(S3BatchEtagCommand(path)) + override def hashCommand: PartialFunction[Path, Try[S3BatchEtagCommand]] = { case path: S3Path => + Try(S3BatchEtagCommand(path)) } - override def touchCommand: PartialFunction[Path, Try[S3BatchTouchCommand]] = { - case path: S3Path => Try(S3BatchTouchCommand(path)) + override def touchCommand: PartialFunction[Path, Try[S3BatchTouchCommand]] = { case path: S3Path => + Try(S3BatchTouchCommand(path)) } - override def existsCommand: PartialFunction[Path, Try[S3BatchExistsCommand]] = { - case path: S3Path => Try(S3BatchExistsCommand(path)) + override def existsCommand: PartialFunction[Path, Try[S3BatchExistsCommand]] = { case path: S3Path => + Try(S3BatchExistsCommand(path)) } } diff --git a/filesystems/s3/src/main/scala/cromwell/filesystems/s3/batch/S3BatchIoCommand.scala b/filesystems/s3/src/main/scala/cromwell/filesystems/s3/batch/S3BatchIoCommand.scala index 75c224f6418..696c4d0240c 100644 --- a/filesystems/s3/src/main/scala/cromwell/filesystems/s3/batch/S3BatchIoCommand.scala +++ b/filesystems/s3/src/main/scala/cromwell/filesystems/s3/batch/S3BatchIoCommand.scala @@ -32,14 +32,16 @@ package cromwell.filesystems.s3.batch import software.amazon.awssdk.core.exception.SdkException -import software.amazon.awssdk.services.s3.model.{HeadObjectResponse, CopyObjectResponse, NoSuchKeyException} -import cromwell.core.io.{IoCommand, - IoDeleteCommand, - IoSizeCommand, - IoHashCommand, - IoTouchCommand, - IoExistsCommand, - IoCopyCommand} +import software.amazon.awssdk.services.s3.model.{CopyObjectResponse, HeadObjectResponse, NoSuchKeyException} +import cromwell.core.io.{ + IoCommand, + IoCopyCommand, + IoDeleteCommand, + IoExistsCommand, + IoHashCommand, + IoSizeCommand, + IoTouchCommand +} import cromwell.filesystems.s3.S3Path @@ -49,6 +51,7 @@ import cromwell.filesystems.s3.S3Path * @tparam U Return type of the response */ sealed trait S3BatchIoCommand[T, U] extends IoCommand[T] { + /** * Maps the response of type U to the Cromwell Io response of type T */ @@ -61,9 +64,8 @@ sealed trait S3BatchIoCommand[T, U] extends IoCommand[T] { * Right(newCommand) means the command is not complete and needs another request to be executed. * Most commands will reply with Left(value). */ - def onSuccess(response: U): Either[T, S3BatchIoCommand[T, U]] = { + def onSuccess(response: U): Either[T, S3BatchIoCommand[T, U]] = Left(mapResponse(response)) - } /** * Override to handle a failure differently and potentially return a successful response. @@ -72,19 +74,22 @@ sealed trait S3BatchIoCommand[T, U] extends IoCommand[T] { } case class S3BatchCopyCommand( - override val source: S3Path, - override val destination: S3Path, - ) extends IoCopyCommand(source, destination) with S3BatchIoCommand[Unit, CopyObjectResponse] { + override val source: S3Path, + override val destination: S3Path +) extends IoCopyCommand(source, destination) + with S3BatchIoCommand[Unit, CopyObjectResponse] { override def mapResponse(response: CopyObjectResponse): Unit = () override def commandDescription: String = s"S3BatchCopyCommand source '$source' destination '$destination'" } case class S3BatchDeleteCommand( - override val file: S3Path, - override val swallowIOExceptions: Boolean - ) extends IoDeleteCommand(file, swallowIOExceptions) with S3BatchIoCommand[Unit, Void] { + override val file: S3Path, + override val swallowIOExceptions: Boolean +) extends IoDeleteCommand(file, swallowIOExceptions) + with S3BatchIoCommand[Unit, Void] { override protected def mapResponse(response: Void): Unit = () - override def commandDescription: String = s"S3BatchDeleteCommand file '$file' swallowIOExceptions '$swallowIOExceptions'" + override def commandDescription: String = + s"S3BatchDeleteCommand file '$file' swallowIOExceptions '$swallowIOExceptions'" } /** @@ -126,14 +131,15 @@ case class S3BatchTouchCommand(override val file: S3Path) extends IoTouchCommand * `IoCommand` to determine the existence of an object in S3 * @param file the path to the object */ -case class S3BatchExistsCommand(override val file: S3Path) extends IoExistsCommand(file) with S3BatchHeadCommand[Boolean] { +case class S3BatchExistsCommand(override val file: S3Path) + extends IoExistsCommand(file) + with S3BatchHeadCommand[Boolean] { override def mapResponse(response: HeadObjectResponse): Boolean = true - override def onFailure(error: SdkException): Option[Left[Boolean, Nothing]] = { + override def onFailure(error: SdkException): Option[Left[Boolean, Nothing]] = // If the object can't be found, don't fail the request but just return false as we were testing for existence error match { - case _ : NoSuchKeyException => Option(Left(false)) + case _: NoSuchKeyException => Option(Left(false)) case _ => None } - } override def commandDescription: String = s"S3BatchExistsCommand file '$file'" } diff --git a/filesystems/s3/src/test/scala/cromwell/filesystems/s3/S3PathBuilderSpec.scala b/filesystems/s3/src/test/scala/cromwell/filesystems/s3/S3PathBuilderSpec.scala index 262ad2cc7de..05703f548d2 100644 --- a/filesystems/s3/src/test/scala/cromwell/filesystems/s3/S3PathBuilderSpec.scala +++ b/filesystems/s3/src/test/scala/cromwell/filesystems/s3/S3PathBuilderSpec.scala @@ -107,7 +107,8 @@ class S3PathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "with spaces", getFileName = s"s3://$bucket/with spaces", getNameCount = 3, - isAbsolute = true), + isAbsolute = true + ), // GoodPath( // description = "a path with non-ascii", @@ -134,7 +135,8 @@ class S3PathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "with non ascii £€", getFileName = s"s3://$bucket/with non ascii £€", getNameCount = 3, - isAbsolute = true), + isAbsolute = true + ), // GoodPath( // description = "a s3 uri path with encoded characters", @@ -162,7 +164,8 @@ class S3PathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "encoded paces", getFileName = s"s3://$bucket/encoded paces", getNameCount = 3, - isAbsolute = true), + isAbsolute = true + ), // TODO: In order for this to pass tests, S3Path needs to implement the // Path trait directly and cannot inherit. We will work on this later @@ -192,8 +195,8 @@ class S3PathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = null, getNameCount = 0, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a file at the top of the bucket", path = s"s3://$bucket/hello", @@ -206,7 +209,8 @@ class S3PathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "hello", getFileName = s"s3://$bucket/hello", getNameCount = 1, - isAbsolute = true), + isAbsolute = true + ), // parent/getParent do not end in a "/". // TODO: Determine if this is critcal. Note @@ -236,7 +240,8 @@ class S3PathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "world", getFileName = s"s3://$bucket/world", getNameCount = 2, - isAbsolute = true), + isAbsolute = true + ), // Special paths @@ -252,7 +257,8 @@ class S3PathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "", getFileName = s"s3://$bucket/.", getNameCount = 1, - isAbsolute = true), + isAbsolute = true + ), // GoodPath( // description = "a bucket with a path ..", @@ -352,8 +358,8 @@ class S3PathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "world", getFileName = s"s3://$bucket/world", getNameCount = 1, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a bucket with an underscore", path = s"s3://hello_underscore/world", @@ -366,27 +372,37 @@ class S3PathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers name = "world", getFileName = s"s3://hello_underscore/world", getNameCount = 1, - isAbsolute = true) + isAbsolute = true + ) ) private def badPaths = Seq( BadPath("an empty path", "", " does not have a s3 scheme"), - BadPath("a bucketless path", "s3://", "The specified S3 path 's3://' does not parse as a URI.\nExpected authority at index 5: s3://"), - BadPath("a bucket named .", "s3://./hello/world", "The path 's3://./hello/world' does not seem to be a valid S3 path. Please check that it starts with s3:// and that the bucket and object follow S3 naming guidelines at https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html"), - BadPath("a non ascii bucket name", "s3://nonasciibucket£€/hello/world", - "The path 's3://nonasciibucket£€/hello/world' does not seem to be a valid S3 path. Please check that it starts with s3:// and that the bucket and object follow S3 naming guidelines at https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html"), + BadPath("a bucketless path", + "s3://", + "The specified S3 path 's3://' does not parse as a URI.\nExpected authority at index 5: s3://" + ), + BadPath( + "a bucket named .", + "s3://./hello/world", + "The path 's3://./hello/world' does not seem to be a valid S3 path. Please check that it starts with s3:// and that the bucket and object follow S3 naming guidelines at https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html" + ), + BadPath( + "a non ascii bucket name", + "s3://nonasciibucket£€/hello/world", + "The path 's3://nonasciibucket£€/hello/world' does not seem to be a valid S3 path. Please check that it starts with s3:// and that the bucket and object follow S3 naming guidelines at https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html" + ), BadPath("a https path", "https://hello/world", "S3 URIs must have 's3' scheme: https://hello/world"), BadPath("a file uri path", "file:///hello/world", "S3 URIs must have 's3' scheme: file:///hello/world"), BadPath("a relative file path", "hello/world", "hello/world does not have a s3 scheme"), BadPath("an absolute file path", "/hello/world", "/hello/world does not have a s3 scheme") ) - private lazy val pathBuilder = { + private lazy val pathBuilder = S3PathBuilder.fromProvider( AnonymousCredentialsProvider.create, S3Storage.s3Configuration(), WorkflowOptions.empty, Option(Region.US_EAST_1) ) - } } diff --git a/filesystems/sra/src/main/scala/cromwell/filesystems/sra/SraPathBuilder.scala b/filesystems/sra/src/main/scala/cromwell/filesystems/sra/SraPathBuilder.scala index 7b539709079..1455a8898ea 100644 --- a/filesystems/sra/src/main/scala/cromwell/filesystems/sra/SraPathBuilder.scala +++ b/filesystems/sra/src/main/scala/cromwell/filesystems/sra/SraPathBuilder.scala @@ -27,6 +27,8 @@ case class SraPath(accession: String, path: String) extends Path { override def pathAsString: String = SraPath.Scheme + pathWithoutScheme override def pathWithoutScheme: String = accession + "/" + path - protected def newPath(nioPath: NioPath): Path = throw new UnsupportedOperationException("'newPath' not implemented for SraPath") + protected def newPath(nioPath: NioPath): Path = throw new UnsupportedOperationException( + "'newPath' not implemented for SraPath" + ) def nioPath: NioPath = throw new UnsupportedOperationException("'nioPath' not implemented for SraPath") } diff --git a/filesystems/sra/src/main/scala/cromwell/filesystems/sra/SraPathBuilderFactory.scala b/filesystems/sra/src/main/scala/cromwell/filesystems/sra/SraPathBuilderFactory.scala index bc487f1dd73..c3855aa4635 100644 --- a/filesystems/sra/src/main/scala/cromwell/filesystems/sra/SraPathBuilderFactory.scala +++ b/filesystems/sra/src/main/scala/cromwell/filesystems/sra/SraPathBuilderFactory.scala @@ -7,9 +7,9 @@ import cromwell.core.path.PathBuilderFactory import scala.concurrent.{ExecutionContext, Future} -final case class SraPathBuilderFactory(globalConfig: Config, instanceConfig: Config) - extends PathBuilderFactory { - def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[SraPathBuilder] = Future.successful(SraPathBuilderFactory.pathBuilder) +final case class SraPathBuilderFactory(globalConfig: Config, instanceConfig: Config) extends PathBuilderFactory { + def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[SraPathBuilder] = + Future.successful(SraPathBuilderFactory.pathBuilder) } object SraPathBuilderFactory { diff --git a/filesystems/sra/src/test/scala/cromwell/filesystems/sra/SraPathBuilderSpec.scala b/filesystems/sra/src/test/scala/cromwell/filesystems/sra/SraPathBuilderSpec.scala index bbc44b55793..193e684c0ae 100644 --- a/filesystems/sra/src/test/scala/cromwell/filesystems/sra/SraPathBuilderSpec.scala +++ b/filesystems/sra/src/test/scala/cromwell/filesystems/sra/SraPathBuilderSpec.scala @@ -8,12 +8,12 @@ import org.scalatest.matchers.should.Matchers class SraPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with PathBuilderSpecUtils { behavior of "SraPathBuilder" - goodPaths foreach { - good => it should behave like buildGood(good) + goodPaths foreach { good => + it should behave like buildGood(good) } - badPaths foreach { - bad => it should behave like buildBad(bad) + badPaths foreach { bad => + it should behave like buildBad(bad) } private def buildGood(good: Good): Unit = { @@ -28,11 +28,11 @@ class SraPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers val sraPath = path.asInstanceOf[SraPath] it should "match expected accession" in { - sraPath.accession should be (good.accession) + sraPath.accession should be(good.accession) } it should "match expected path" in { - sraPath.path should be (good.path) + sraPath.path should be(good.path) } } @@ -40,50 +40,47 @@ class SraPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers behavior of s"Building ${bad.description}" it should "fail to build an SraPath" in { - pathBuilder.build(bad.in).isSuccess should be (false) + pathBuilder.build(bad.in).isSuccess should be(false) } } private lazy val pathBuilder = new SraPathBuilder - private case class Good(description: String, - in: String, - accession: String, - path: String) + private case class Good(description: String, in: String, accession: String, path: String) private def goodPaths = Seq( Good( description = "well-formed path", in = "sra://SXP000001/asdf.bam", accession = "SXP000001", - path = "asdf.bam", + path = "asdf.bam" ), Good( description = "nested path", in = "sra://SRA42424242/first/second/third.bam.bai", accession = "SRA42424242", - path = "first/second/third.bam.bai", + path = "first/second/third.bam.bai" ), Good( description = "path with spaces", in = "sra://SXP111111/top level/nested level/file name.bz2", accession = "SXP111111", - path = "top level/nested level/file name.bz2", - ), + path = "top level/nested level/file name.bz2" + ) ) private case class Bad(description: String, in: String) private def badPaths = Seq( Bad( description = "not an SRA path", - in = "gcs://some/gcs/path/thing.txt", + in = "gcs://some/gcs/path/thing.txt" ), Bad( description = "missing accession", - in = "sra://", + in = "sra://" ), Bad( description = "missing path within accession", - in = "sra://SRA00001", - ), + in = "sra://SRA00001" + ) ) } diff --git a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/LanguageFactory.scala b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/LanguageFactory.scala index f16054f15a7..e2f1d48e9f4 100644 --- a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/LanguageFactory.scala +++ b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/LanguageFactory.scala @@ -23,27 +23,31 @@ trait LanguageFactory { import net.ceedubs.ficus.Ficus._ lazy val enabled = !config.as[Option[Boolean]]("enabled").contains(false) - lazy val enabledCheck: Checked[Unit] = if (enabled) ().validNelCheck else - s"The language factory for $languageName ($languageVersionName) is not currently enabled in this Cromwell".invalidNelCheck - + lazy val enabledCheck: Checked[Unit] = + if (enabled) ().validNelCheck + else + s"The language factory for $languageName ($languageVersionName) is not currently enabled in this Cromwell".invalidNelCheck lazy val strictValidation: Boolean = !config.as[Option[Boolean]]("strict-validation").contains(false) - lazy val womOutputRuntimeExtractor: Checked[Option[WomOutputRuntimeExtractor]] = config.getAs[Config]("output-runtime-extractor") match { - case Some(c) => WomOutputRuntimeExtractor.fromConfig(c).map(Option.apply).toEither - case _ => None.validNelCheck - } + lazy val womOutputRuntimeExtractor: Checked[Option[WomOutputRuntimeExtractor]] = + config.getAs[Config]("output-runtime-extractor") match { + case Some(c) => WomOutputRuntimeExtractor.fromConfig(c).map(Option.apply).toEither + case _ => None.validNelCheck + } def getWomBundle(workflowSource: WorkflowSource, workflowSourceOrigin: Option[ResolvedImportRecord], workflowOptionsJson: WorkflowOptionsJson, importResolvers: List[ImportResolver], languageFactories: List[LanguageFactory], - convertNestedScatterToSubworkflow : Boolean = true): Checked[WomBundle] + convertNestedScatterToSubworkflow: Boolean = true + ): Checked[WomBundle] def createExecutable(womBundle: WomBundle, inputs: WorkflowJson, - ioFunctions: IoFunctionSet): Checked[ValidatedWomNamespace] + ioFunctions: IoFunctionSet + ): Checked[ValidatedWomNamespace] def validateNamespace(source: WorkflowSourceFilesCollection, workflowSource: WorkflowSource, @@ -51,7 +55,8 @@ trait LanguageFactory { importLocalFilesystem: Boolean, workflowIdForLogging: WorkflowId, ioFunctions: IoFunctionSet, - importResolvers: List[ImportResolver]): IOChecked[ValidatedWomNamespace] + importResolvers: List[ImportResolver] + ): IOChecked[ValidatedWomNamespace] /** * In case no version is specified: does this language factory feel like it might be suitable for this file? diff --git a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/ValidatedWomNamespace.scala b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/ValidatedWomNamespace.scala index 306a442672b..828b9c2bbc0 100644 --- a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/ValidatedWomNamespace.scala +++ b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/ValidatedWomNamespace.scala @@ -11,4 +11,5 @@ import wom.values.WomValue */ final case class ValidatedWomNamespace(executable: Executable, womValueInputs: Map[OutputPort, WomValue], - importedFileContent: Map[String, String]) + importedFileContent: Map[String, String] +) diff --git a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/config/CromwellLanguages.scala b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/config/CromwellLanguages.scala index b7c4460df98..e05b48d991b 100644 --- a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/config/CromwellLanguages.scala +++ b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/config/CromwellLanguages.scala @@ -5,10 +5,11 @@ import cromwell.languages.config.CromwellLanguages.{CromwellLanguageName, Cromwe import cromwell.languages.LanguageFactory // Construct a singleton instance of this class using 'initLanguages' below. -final case class CromwellLanguages private(languageConfig: LanguagesConfiguration) { +final case class CromwellLanguages private (languageConfig: LanguagesConfiguration) { val languages: Map[CromwellLanguageName, LanguageVersions] = makeLanguages - val default: LanguageVersions = languages.find(lang => languageConfig.default.contains(lang._1)).getOrElse(languages.head)._2 + val default: LanguageVersions = + languages.find(lang => languageConfig.default.contains(lang._1)).getOrElse(languages.head)._2 private def makeLanguages: Map[CromwellLanguageName, LanguageVersions] = (languageConfig.languages map { lc => val versions = lc.versions map { case (languageVersion, languageConfigEntryFields) => @@ -19,18 +20,20 @@ final case class CromwellLanguages private(languageConfig: LanguagesConfiguratio lc.name.toUpperCase -> LanguageVersions(versions, default) }).toMap - private def makeLanguageFactory(className: String, config: Config) = { - Class.forName(className) + private def makeLanguageFactory(className: String, config: Config) = + Class + .forName(className) .getConstructor(classOf[Config]) .newInstance(config) .asInstanceOf[LanguageFactory] - } } /** * Holds all the registered versions of a language. */ -final case class LanguageVersions private(allVersions: Map[CromwellLanguageVersion, LanguageFactory], default: LanguageFactory) +final case class LanguageVersions private (allVersions: Map[CromwellLanguageVersion, LanguageFactory], + default: LanguageFactory +) object CromwellLanguages { type CromwellLanguageName = String @@ -39,7 +42,6 @@ object CromwellLanguages { private var _instance: CromwellLanguages = _ lazy val instance: CromwellLanguages = _instance - def initLanguages(backendEntries: LanguagesConfiguration): Unit = { + def initLanguages(backendEntries: LanguagesConfiguration): Unit = _instance = CromwellLanguages(backendEntries) - } } diff --git a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/config/LanguageConfiguration.scala b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/config/LanguageConfiguration.scala index 2cb09dd8a3d..153bdfd029c 100644 --- a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/config/LanguageConfiguration.scala +++ b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/config/LanguageConfiguration.scala @@ -8,18 +8,22 @@ import cromwell.languages.config.CromwellLanguages.{CromwellLanguageName, Cromwe import scala.jdk.CollectionConverters._ final case class LanguagesConfiguration(languages: List[LanguageVersionConfigurationEntry], default: Option[String]) -final case class LanguageVersionConfigurationEntry(name: CromwellLanguageName, versions: Map[CromwellLanguageVersion, LanguageVersionConfig], default: Option[String]) +final case class LanguageVersionConfigurationEntry(name: CromwellLanguageName, + versions: Map[CromwellLanguageVersion, LanguageVersionConfig], + default: Option[String] +) final case class LanguageVersionConfig(className: String, config: Config) object LanguageConfiguration { private val LanguagesConfig = ConfigFactory.load.getConfig("languages") - private val DefaultLanguageName: Option[String] = if (LanguagesConfig.hasPath("default")) Option(LanguagesConfig.getString("default")) else None + private val DefaultLanguageName: Option[String] = + if (LanguagesConfig.hasPath("default")) Option(LanguagesConfig.getString("default")) else None - private val LanguageNames: Set[String] = LanguagesConfig.entrySet().asScala.map(findFirstKey).filterNot(_ == "default").toSet + private val LanguageNames: Set[String] = + LanguagesConfig.entrySet().asScala.map(findFirstKey).filterNot(_ == "default").toSet val AllLanguageEntries: LanguagesConfiguration = { val languages = LanguageNames.toList map { languageName => - val languageConfig = LanguagesConfig.getConfig(languageName) val versionSet = languageConfig.getConfig("versions") val allLanguageVersionNames: Set[String] = versionSet.entrySet().asScala.map(findFirstKey).toSet @@ -29,7 +33,8 @@ object LanguageConfiguration { val versions = (languageVersionNames.toList map { languageVersionName => val configEntry = versionSet.getConfig(s""""$languageVersionName"""") val className: String = configEntry.getString("language-factory") - val factoryConfig: Config = if (configEntry.hasPath("config")) configEntry.getConfig("config") else ConfigFactory.empty() + val factoryConfig: Config = + if (configEntry.hasPath("config")) configEntry.getConfig("config") else ConfigFactory.empty() val fields = LanguageVersionConfig(className, factoryConfig) languageVersionName -> fields }).toMap diff --git a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/ImportResolver.scala b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/ImportResolver.scala index f7fd70cfc56..bdcd0c8b311 100644 --- a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/ImportResolver.scala +++ b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/ImportResolver.scala @@ -10,6 +10,7 @@ import cats.syntax.validated._ import com.softwaremill.sttp._ import com.softwaremill.sttp.asynchttpclient.cats.AsyncHttpClientCatsBackend import com.typesafe.config.ConfigFactory +import com.typesafe.scalalogging.StrictLogging import common.Checked import common.transforms.CheckedAtoB import common.validation.ErrorOr._ @@ -22,23 +23,37 @@ import java.nio.file.{Path => NioPath} import java.security.MessageDigest import cromwell.core.WorkflowId import wom.ResolvedImportRecord -import wom.core.WorkflowSource +import wom.core.{WorkflowSource, WorkflowUrl} import wom.values._ import scala.concurrent.duration._ -import scala.concurrent.{Await, ExecutionContext} +import scala.concurrent.{Await, ExecutionContext, Future} import scala.util.{Failure, Success, Try} object ImportResolver { case class ImportResolutionRequest(toResolve: String, currentResolvers: List[ImportResolver]) - case class ResolvedImportBundle(source: WorkflowSource, newResolvers: List[ImportResolver], resolvedImportRecord: ResolvedImportRecord) + case class ResolvedImportBundle(source: WorkflowSource, + newResolvers: List[ImportResolver], + resolvedImportRecord: ResolvedImportRecord + ) + + trait ImportAuthProvider { + def validHosts: List[String] + def authHeader(): Future[Map[String, String]] + } + + trait GithubImportAuthProvider extends ImportAuthProvider { + override def validHosts: List[String] = List("github.com", "githubusercontent.com", "raw.githubusercontent.com") + } trait ImportResolver { def name: String protected def innerResolver(path: String, currentResolvers: List[ImportResolver]): Checked[ResolvedImportBundle] def resolver: CheckedAtoB[ImportResolutionRequest, ResolvedImportBundle] = CheckedAtoB.fromCheck { request => - innerResolver(request.toResolve, request.currentResolvers).contextualizeErrors(s"resolve '${request.toResolve}' using resolver: '$name'") + innerResolver(request.toResolve, request.currentResolvers).contextualizeErrors( + s"resolve '${request.toResolve}' using resolver: '$name'" + ) } def cleanupIfNecessary(): ErrorOr[Unit] @@ -48,7 +63,10 @@ object ImportResolver { } object DirectoryResolver { - private def apply(directory: Path, allowEscapingDirectory: Boolean, customName: Option[String]): DirectoryResolver = { + private def apply(directory: Path, + allowEscapingDirectory: Boolean, + customName: Option[String] + ): DirectoryResolver = { val dontEscapeFrom = if (allowEscapingDirectory) None else Option(directory.toJava.getCanonicalPath) DirectoryResolver(directory, dontEscapeFrom, customName, deleteOnClose = false, directoryHash = None) } @@ -77,19 +95,23 @@ object ImportResolver { dontEscapeFrom: Option[String] = None, customName: Option[String], deleteOnClose: Boolean, - directoryHash: Option[String]) extends ImportResolver { + directoryHash: Option[String] + ) extends ImportResolver { lazy val absolutePathToDirectory: String = directory.toJava.getCanonicalPath override def innerResolver(path: String, currentResolvers: List[ImportResolver]): Checked[ResolvedImportBundle] = { - def updatedResolverSet(oldRootDirectory: Path, newRootDirectory: Path, current: List[ImportResolver]): List[ImportResolver] = { + def updatedResolverSet(oldRootDirectory: Path, + newRootDirectory: Path, + current: List[ImportResolver] + ): List[ImportResolver] = current map { - case d if d == this => DirectoryResolver(newRootDirectory, dontEscapeFrom, customName, deleteOnClose = false, directoryHash = None) + case d if d == this => + DirectoryResolver(newRootDirectory, dontEscapeFrom, customName, deleteOnClose = false, directoryHash = None) case other => other } - } - def fetchContentFromAbsolutePath(absolutePathToFile: NioPath): ErrorOr[String] = { + def fetchContentFromAbsolutePath(absolutePathToFile: NioPath): ErrorOr[String] = checkLocation(absolutePathToFile, path) flatMap { _ => val file = File(absolutePathToFile) if (file.exists) { @@ -98,7 +120,6 @@ object ImportResolver { s"File not found: $path".invalidNel } } - } val errorOr = for { resolvedPath <- resolvePath(path) @@ -111,7 +132,9 @@ object ImportResolver { } private def resolvePath(path: String): ErrorOr[Path] = Try(directory.resolve(path)).toErrorOr - private def makeAbsolute(resolvedPath: Path): ErrorOr[NioPath] = Try(Paths.get(resolvedPath.toFile.getCanonicalPath)).toErrorOr + private def makeAbsolute(resolvedPath: Path): ErrorOr[NioPath] = Try( + Paths.get(resolvedPath.toFile.getCanonicalPath) + ).toErrorOr private def checkLocation(absoluteNioPath: NioPath, reportedPathIfBad: String): ErrorOr[Unit] = if (dontEscapeFrom.forall(absoluteNioPath.startsWith)) ().validNel @@ -135,7 +158,8 @@ object ImportResolver { s"relative to directory $relativePathToDirectory (without escaping $relativePathToDontEscapeFrom)" case (None, None) => - val shortPathToDirectory = Paths.get(absolutePathToDirectory).toFile.getCanonicalFile.toPath.getFileName.toString + val shortPathToDirectory = + Paths.get(absolutePathToDirectory).toFile.getCanonicalFile.toPath.getFileName.toString s"relative to directory [...]/$shortPathToDirectory (escaping allowed)" } @@ -148,32 +172,40 @@ object ImportResolver { else ().validNel - override def hashKey: ErrorOr[String] = directoryHash.map(_.validNel).getOrElse("No hashKey available for directory importer".invalidNel) + override def hashKey: ErrorOr[String] = + directoryHash.map(_.validNel).getOrElse("No hashKey available for directory importer".invalidNel) } def zippedImportResolver(zippedImports: Array[Byte], workflowId: WorkflowId): ErrorOr[DirectoryResolver] = { val zipHash = new String(MessageDigest.getInstance("MD5").digest(zippedImports)) LanguageFactoryUtil.createImportsDirectory(zippedImports, workflowId) map { dir => - DirectoryResolver(dir, Option(dir.toJava.getCanonicalPath), None, deleteOnClose = true, directoryHash = Option(zipHash)) + DirectoryResolver(dir, + Option(dir.toJava.getCanonicalPath), + None, + deleteOnClose = true, + directoryHash = Option(zipHash) + ) } } case class HttpResolver(relativeTo: Option[String], headers: Map[String, String], - hostAllowlist: Option[List[String]]) extends ImportResolver { + hostAllowlist: Option[List[String]], + authProviders: List[ImportAuthProvider] + ) extends ImportResolver + with StrictLogging { import HttpResolver._ override def name: String = relativeTo match { case Some(relativeToPath) => s"http importer (relative to $relativeToPath)" case None => "http importer (no 'relative-to' origin)" - } def newResolverList(newRoot: String): List[ImportResolver] = { val rootWithoutFilename = newRoot.split('/').init.mkString("", "/", "/") List( - HttpResolver(relativeTo = Some(canonicalize(rootWithoutFilename)), headers, hostAllowlist) + HttpResolver(relativeTo = Some(canonicalize(rootWithoutFilename)), headers, hostAllowlist, authProviders) ) } @@ -192,8 +224,13 @@ object ImportResolver { case None => true } - override def innerResolver(str: String, currentResolvers: List[ImportResolver]): Checked[ResolvedImportBundle] = { - pathToLookup(str) flatMap { toLookup: WorkflowSource => + def fetchAuthHeaders(uri: Uri): Future[Map[String, String]] = + authProviders collectFirst { + case provider if provider.validHosts.contains(uri.host) => provider.authHeader() + } getOrElse Future.successful(Map.empty[String, String]) + + override def innerResolver(str: String, currentResolvers: List[ImportResolver]): Checked[ResolvedImportBundle] = + pathToLookup(str) flatMap { toLookup: WorkflowUrl => (Try { val uri: Uri = uri"$toLookup" @@ -207,21 +244,38 @@ object ImportResolver { case Failure(e) => s"HTTP resolver with headers had an unexpected error (${e.getMessage})".invalidNelCheck }).contextualizeErrors(s"download $toLookup") } - } - - private def getUri(toLookup: WorkflowSource): Either[NonEmptyList[WorkflowSource], ResolvedImportBundle] = { - implicit val sttpBackend = HttpResolver.sttpBackend() - val responseIO: IO[Response[WorkflowSource]] = sttp.get(uri"$toLookup").headers(headers).send() - // temporary situation to get functionality working before + private def getUri(toLookup: WorkflowUrl): Checked[ResolvedImportBundle] = { + // Temporary situation to get functionality working before // starting in on async-ifying the entire WdlNamespace flow - val result: Checked[WorkflowSource] = Await.result(responseIO.unsafeToFuture(), 15.seconds).body.leftMap { e => NonEmptyList(e.toString.trim, List.empty) } + // Note: this will cause the calling thread to block for up to 20 seconds + // (5 for the auth header lookup, 15 for the http request) + val unauthedAttempt = getUriInner(toLookup, Map.empty) + val result = if (unauthedAttempt.code == StatusCodes.NotFound) { + val authHeaders = Await.result(fetchAuthHeaders(uri"$toLookup"), 5.seconds) + if (authHeaders.nonEmpty) { + getUriInner(toLookup, authHeaders) + } else { + unauthedAttempt + } + } else { + unauthedAttempt + } - result map { + result.body.leftMap { e => + NonEmptyList(e.trim, List.empty) + } map { ResolvedImportBundle(_, newResolverList(toLookup), ResolvedImportRecord(toLookup)) } } + protected def getUriInner(toLookup: WorkflowUrl, authHeaders: Map[String, String]): Response[WorkflowSource] = { + implicit val sttpBackend = HttpResolver.sttpBackend() + + val responseIO: IO[Response[WorkflowSource]] = sttp.get(uri"$toLookup").headers(headers ++ authHeaders).send() + Await.result(responseIO.unsafeToFuture(), 15.seconds) + } + override def cleanupIfNecessary(): ErrorOr[Unit] = ().validNel override def hashKey: ErrorOr[String] = relativeTo.toString.md5Sum.validNel @@ -233,7 +287,9 @@ object ImportResolver { import common.util.IntrospectableLazy._ def apply(relativeTo: Option[String] = None, - headers: Map[String, String] = Map.empty): HttpResolver = { + headers: Map[String, String] = Map.empty, + authProviders: List[ImportAuthProvider] = List.empty + ): HttpResolver = { val config = ConfigFactory.load().getConfig("languages.WDL.http-allow-list") val allowListEnabled = config.as[Option[Boolean]]("enabled").getOrElse(false) val allowList: Option[List[String]] = @@ -241,7 +297,7 @@ object ImportResolver { config.as[Option[List[String]]]("allowed-http-hosts") else None - new HttpResolver(relativeTo, headers, allowList) + new HttpResolver(relativeTo, headers, allowList, authProviders) } val sttpBackend: IntrospectableLazy[SttpBackend[IO, Nothing]] = lazily { diff --git a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/LanguageFactoryUtil.scala b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/LanguageFactoryUtil.scala index da0860c3a86..8fabb445bac 100644 --- a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/LanguageFactoryUtil.scala +++ b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/LanguageFactoryUtil.scala @@ -32,7 +32,9 @@ object LanguageFactoryUtil { def createImportsDirectory(zipContents: Array[Byte], workflowId: WorkflowId): ErrorOr[Path] = { def makeZipFile: Try[Path] = Try { - DefaultPathBuilder.createTempFile(s"imports_workflow_${workflowId}_", ".zip").writeByteArray(zipContents)(OpenOptions.default) + DefaultPathBuilder + .createTempFile(s"imports_workflow_${workflowId}_", ".zip") + .writeByteArray(zipContents)(OpenOptions.default) } def unZipFile(f: Path) = Try(f.unzip()) @@ -49,25 +51,28 @@ object LanguageFactoryUtil { } } - def validateWomNamespace(womExecutable: Executable, ioFunctions: IoFunctionSet): Checked[ValidatedWomNamespace] = for { - evaluatedInputs <- validateExecutableInputs(womExecutable.resolvedExecutableInputs, ioFunctions).toEither - validatedWomNamespace = ValidatedWomNamespace(womExecutable, evaluatedInputs, Map.empty) - _ <- validateWdlFiles(validatedWomNamespace.womValueInputs) - } yield validatedWomNamespace + def validateWomNamespace(womExecutable: Executable, ioFunctions: IoFunctionSet): Checked[ValidatedWomNamespace] = + for { + evaluatedInputs <- validateExecutableInputs(womExecutable.resolvedExecutableInputs, ioFunctions).toEither + validatedWomNamespace = ValidatedWomNamespace(womExecutable, evaluatedInputs, Map.empty) + _ <- validateWdlFiles(validatedWomNamespace.womValueInputs) + } yield validatedWomNamespace /* - * At this point input values are either a WomValue (if it was provided through the input file) - * or a WomExpression (if we fell back to the default). - * We assume that default expressions do NOT reference any "variables" (other inputs, call outputs ...) - * Should this assumption prove not sufficient InstantiatedExpressions or ExpressionNodes would need to be provided - * instead so that they can be evaluated JIT. - * Note that the ioFunctions use engine level pathBuilders. This means that their credentials come from the engine section - * of the configuration, and are not backend specific. + * At this point input values are either a WomValue (if it was provided through the input file) + * or a WomExpression (if we fell back to the default). + * We assume that default expressions do NOT reference any "variables" (other inputs, call outputs ...) + * Should this assumption prove not sufficient InstantiatedExpressions or ExpressionNodes would need to be provided + * instead so that they can be evaluated JIT. + * Note that the ioFunctions use engine level pathBuilders. This means that their credentials come from the engine section + * of the configuration, and are not backend specific. */ - private def validateExecutableInputs(inputs: ResolvedExecutableInputs, ioFunctions: IoFunctionSet): ErrorOr[Map[OutputPort, WomValue]] = { + private def validateExecutableInputs(inputs: ResolvedExecutableInputs, + ioFunctions: IoFunctionSet + ): ErrorOr[Map[OutputPort, WomValue]] = { import common.validation.ErrorOr.MapTraversal - inputs.traverse { - case (key, value) => value.fold(ResolvedExecutableInputsPoly).apply(ioFunctions) map { key -> _ } + inputs.traverse { case (key, value) => + value.fold(ResolvedExecutableInputsPoly).apply(ioFunctions) map { key -> _ } } } @@ -88,16 +93,18 @@ object LanguageFactoryUtil { } } - def simpleLooksParseable(startsWithOptions: List[String], commentIndicators: List[String])(content: String): Boolean = { + def simpleLooksParseable(startsWithOptions: List[String], commentIndicators: List[String])( + content: String + ): Boolean = { val fileWithoutInitialWhitespace = content.linesIterator.toList.dropWhile { l => l.forall(_.isWhitespace) || commentIndicators.exists(l.dropWhile(_.isWhitespace).startsWith(_)) } val firstCodeLine = fileWithoutInitialWhitespace.headOption.map(_.dropWhile(_.isWhitespace)) - firstCodeLine.exists { line => startsWithOptions.contains(line.trim) } + firstCodeLine.exists(line => startsWithOptions.contains(line.trim)) } - def chooseFactory(workflowSource: WorkflowSource, wsfc: WorkflowSourceFilesCollection): ErrorOr[LanguageFactory] = { + def chooseFactory(workflowSource: WorkflowSource, wsfc: WorkflowSourceFilesCollection): ErrorOr[LanguageFactory] = wsfc.workflowType match { case Some(languageName) if CromwellLanguages.instance.languages.contains(languageName.toUpperCase) => val language = CromwellLanguages.instance.languages(languageName.toUpperCase) @@ -110,21 +117,25 @@ object LanguageFactoryUtil { case Some(other) => s"Unknown workflow type: $other".invalidNel[LanguageFactory] case None => val allFactories = CromwellLanguages.instance.languages.values.flatMap(_.allVersions.values) - allFactories.find(_.looksParsable(workflowSource)).getOrElse(CromwellLanguages.instance.default.default).validNel + allFactories + .find(_.looksParsable(workflowSource)) + .getOrElse(CromwellLanguages.instance.default.default) + .validNel } - } def findWorkflowSource(workflowSource: Option[WorkflowSource], workflowUrl: Option[WorkflowUrl], - resolvers: List[ImportResolver]): Checked[(WorkflowSource, List[ImportResolver])] = { + resolvers: List[ImportResolver] + ): Checked[(WorkflowSource, List[ImportResolver])] = (workflowSource, workflowUrl) match { case (Some(source), None) => (source, resolvers).validNelCheck case (None, Some(url)) => - val compoundImportResolver: CheckedAtoB[ImportResolutionRequest, ResolvedImportBundle] = CheckedAtoB.firstSuccess(resolvers.map(_.resolver), s"resolve workflowUrl '$url'") - val wfSourceAndResolvers: Checked[ResolvedImportBundle] = compoundImportResolver.run(ImportResolutionRequest(url, resolvers)) + val compoundImportResolver: CheckedAtoB[ImportResolutionRequest, ResolvedImportBundle] = + CheckedAtoB.firstSuccess(resolvers.map(_.resolver), s"resolve workflowUrl '$url'") + val wfSourceAndResolvers: Checked[ResolvedImportBundle] = + compoundImportResolver.run(ImportResolutionRequest(url, resolvers)) wfSourceAndResolvers map { v => (v.source, v.newResolvers) } case (Some(_), Some(_)) => "Both workflow source and url can't be supplied".invalidNelCheck case (None, None) => "Either workflow source or url has to be supplied".invalidNelCheck } - } } diff --git a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/ParserCache.scala b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/ParserCache.scala index 5d6feda9882..839c4e855f8 100644 --- a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/ParserCache.scala +++ b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/ParserCache.scala @@ -22,23 +22,27 @@ import scala.concurrent.duration._ trait ParserCache[A] extends StrictLogging { this: LanguageFactory => - def retrieveOrCalculate(cacheInputs: ParserCacheInputs, - calculationCallable: Callable[ErrorOr[A]]): ErrorOr[A] = { - + def retrieveOrCalculate(cacheInputs: ParserCacheInputs, calculationCallable: Callable[ErrorOr[A]]): ErrorOr[A] = (cache map { c: Cache[String, ErrorOr[A]] => - workflowHashKey(cacheInputs.workflowSource, cacheInputs.workflowUrl, cacheInputs.workflowRoot, cacheInputs.importResolvers) match { + workflowHashKey(cacheInputs.workflowSource, + cacheInputs.workflowUrl, + cacheInputs.workflowRoot, + cacheInputs.importResolvers + ) match { case Valid(hashKey) => c.get(hashKey, calculationCallable) case Invalid(errors) => - logger.info(s"Failed to calculate hash key for 'workflow source to WOM' cache: {}", errors.toList.mkString(", ")) + logger.info(s"Failed to calculate hash key for 'workflow source to WOM' cache: {}", + errors.toList.mkString(", ") + ) calculationCallable.call } }).getOrElse(calculationCallable.call()) - } private[this] def workflowHashKey(workflowSource: Option[WorkflowSource], workflowUrl: Option[WorkflowUrl], workflowRoot: Option[String], - importResolvers: List[ImportResolver]): ErrorOr[String] = { + importResolvers: List[ImportResolver] + ): ErrorOr[String] = { def stringOptionToHash(opt: Option[String]): String = opt map { _.md5Sum } getOrElse "" val importResolversToHash: ErrorOr[String] = importResolvers.traverse(_.hashKey).map(_.mkString(":")) @@ -48,17 +52,21 @@ trait ParserCache[A] extends StrictLogging { this: LanguageFactory => } } - private[this] lazy val cacheConfig: Option[CacheConfig] = { + private[this] lazy val cacheConfig: Option[CacheConfig] = // Caching is an opt-in activity: for { _ <- enabled.option(()) cachingConfigSection <- config.as[Option[Config]]("caching") - cc <- CacheConfig.optionalConfig(cachingConfigSection, defaultConcurrency = 2, defaultSize = 1000L, defaultTtl = 20.minutes) + cc <- CacheConfig.optionalConfig(cachingConfigSection, + defaultConcurrency = 2, + defaultSize = 1000L, + defaultTtl = 20.minutes + ) } yield cc - } private[this] lazy val cache: Option[Cache[String, ErrorOr[A]]] = cacheConfig map { c => - CacheBuilder.newBuilder() + CacheBuilder + .newBuilder() .concurrencyLevel(c.concurrency) .expireAfterAccess(c.ttl.length, c.ttl.unit) .maximumSize(c.size) @@ -70,5 +78,6 @@ object ParserCache { final case class ParserCacheInputs(workflowSource: Option[WorkflowSource], workflowUrl: Option[WorkflowUrl], workflowRoot: Option[String], - importResolvers: List[ImportResolver]) + importResolvers: List[ImportResolver] + ) } diff --git a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/ResolvedExecutableInputsPoly.scala b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/ResolvedExecutableInputsPoly.scala index dce4da0df38..888be25cfb5 100644 --- a/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/ResolvedExecutableInputsPoly.scala +++ b/languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/ResolvedExecutableInputsPoly.scala @@ -8,10 +8,13 @@ import wom.expression.{IoFunctionSet, WomExpression} import wom.values.WomValue object ResolvedExecutableInputsPoly extends Poly1 { - implicit def fromWdlValue: Case.Aux[WomValue, IoFunctionSet => ErrorOr[WomValue]] = at[WomValue] { wdlValue => - _: IoFunctionSet => wdlValue.validNel + implicit def fromWdlValue: Case.Aux[WomValue, IoFunctionSet => ErrorOr[WomValue]] = at[WomValue] { + wdlValue => _: IoFunctionSet => wdlValue.validNel } - implicit def fromWomExpression: Case.Aux[WomExpression, IoFunctionSet => ErrorOr[WomValue]] = at[WomExpression] { womExpression => - ioFunctions: IoFunctionSet => womExpression.evaluateValue(Map.empty, ioFunctions).contextualizeErrors(s"evaluate expression '${womExpression.sourceString}'") + implicit def fromWomExpression: Case.Aux[WomExpression, IoFunctionSet => ErrorOr[WomValue]] = at[WomExpression] { + womExpression => ioFunctions: IoFunctionSet => + womExpression + .evaluateValue(Map.empty, ioFunctions) + .contextualizeErrors(s"evaluate expression '${womExpression.sourceString}'") } } diff --git a/languageFactories/language-factory-core/src/test/scala/cromwell/languages/util/ImportResolverSpec.scala b/languageFactories/language-factory-core/src/test/scala/cromwell/languages/util/ImportResolverSpec.scala index 813a688a573..a451d0c8df6 100644 --- a/languageFactories/language-factory-core/src/test/scala/cromwell/languages/util/ImportResolverSpec.scala +++ b/languageFactories/language-factory-core/src/test/scala/cromwell/languages/util/ImportResolverSpec.scala @@ -7,10 +7,17 @@ import common.assertion.CromwellTimeoutSpec import common.assertion.ErrorOrAssertions._ import cromwell.core.WorkflowId import cromwell.core.path.DefaultPath -import cromwell.languages.util.ImportResolver.{DirectoryResolver, HttpResolver} +import cromwell.languages.util.ImportResolver.{ + DirectoryResolver, + GithubImportAuthProvider, + HttpResolver, + ImportAuthProvider +} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import wom.core.{WorkflowSource, WorkflowUrl} +import scala.concurrent.Future class ImportResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "HttpResolver" @@ -42,14 +49,15 @@ class ImportResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match } it should "resolve a path from no initial root" in { - val resolver = HttpResolver(None, Map.empty, None) + val resolver = HttpResolver(None, Map.empty, None, List.empty) val toResolve = resolver.pathToLookup("http://abc.com:8000/blah1/blah2.wdl") toResolve shouldBeValid "http://abc.com:8000/blah1/blah2.wdl" } it should "resolve a path and store the import in ResolvedImportRecord" in { - val resolver = HttpResolver(None, Map.empty, None) - val importUri = "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/centaur/src/main/resources/standardTestCases/hello/hello.wdl" + val resolver = HttpResolver(None, Map.empty, None, List.empty) + val importUri = + "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/centaur/src/main/resources/standardTestCases/hello/hello.wdl" val resolvedBundle = resolver.innerResolver(importUri, List(resolver)) resolvedBundle.map(_.resolvedImportRecord) match { @@ -64,14 +72,14 @@ class ImportResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match val pathEnd = "bob/loblaw/blah/blah.wdl" it should "allow any import when there is no allow list" in { - val resolver = HttpResolver(None, Map.empty, None) + val resolver = HttpResolver(None, Map.empty, None, List.empty) resolver.isAllowed(uri"https://my.favorite.wdls.com/$pathEnd") shouldBe true resolver.isAllowed(uri"http://some-garbage.whatever.eu/$pathEnd") shouldBe true resolver.isAllowed(uri"localhost:8080/my/secrets") shouldBe true } it should "allow any import that's on the allow list" in { - val resolver = HttpResolver(None, Map.empty, allowList) + val resolver = HttpResolver(None, Map.empty, allowList, List.empty) resolver.isAllowed(uri"https://my.favorite.wdls.com/$pathEnd") shouldBe true resolver.isAllowed(uri"http://anotherwdlsite.org/$pathEnd") shouldBe true resolver.isAllowed(uri"https://yetanotherwdlsite.org/$pathEnd") shouldBe false @@ -81,7 +89,7 @@ class ImportResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match } it should "allow nothing with an empty allow list" in { - val resolver = HttpResolver(None, Map.empty, Option(List.empty)) + val resolver = HttpResolver(None, Map.empty, Option(List.empty), List.empty) resolver.isAllowed(uri"https://my.favorite.wdls.com/$pathEnd") shouldBe false resolver.isAllowed(uri"http://anotherwdlsite.org/$pathEnd") shouldBe false resolver.isAllowed(uri"https://yetanotherwdlsite.org/$pathEnd") shouldBe false @@ -92,8 +100,9 @@ class ImportResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match behavior of "HttpResolver with a 'relativeTo' value" - val relativeHttpResolver = HttpResolver(relativeTo = Some("http://abc.com:8000/blah1/blah2/"), Map.empty, None) - val relativeToGithubHttpResolver = HttpResolver(relativeTo = Some(relativeToGithubRoot), Map.empty, None) + val relativeHttpResolver = + HttpResolver(relativeTo = Some("http://abc.com:8000/blah1/blah2/"), Map.empty, None, List.empty) + val relativeToGithubHttpResolver = HttpResolver(relativeTo = Some(relativeToGithubRoot), Map.empty, None, List.empty) it should "resolve an absolute path from a different initial root" in { val pathToLookup = relativeHttpResolver.pathToLookup("http://def.org:8080/blah3.wdl") @@ -155,10 +164,123 @@ class ImportResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match } } + behavior of "HttpResolver with an ImportAuthProvider" + + class RecordingHttpResolver(unauthedResponse: Response[String], + authedResponse: Response[String], + authProvider: ImportAuthProvider + ) extends HttpResolver(None, Map.empty, None, List(authProvider)) { + + case class RequestRecord(url: WorkflowUrl, headers: Map[String, String], response: Response[String]) + var requestRecords: List[RequestRecord] = List.empty + + override protected def getUriInner(toLookup: WorkflowUrl, + authHeaders: Map[String, String] + ): Response[WorkflowSource] = { + val result = + if ( + Uri.parse(toLookup).get.host == "raw.githubusercontent.com" && authHeaders + .get("Authorization") + .contains("Bearer 1234") + ) { + authedResponse + } else if (headers.contains("Authorization")) { + throw new RuntimeException(s"Unexpected auth header applied") + } else { + unauthedResponse + } + + requestRecords = requestRecords :+ RequestRecord(toLookup, authHeaders, result) + result + } + } + + it should "lookup headers from auth provider after a 404 for valid host" in { + val unauthedResponse = new Response[String](Left("Not found or no permissions".getBytes), + StatusCodes.NotFound, + "NotFound", + Nil, + List.empty + ) + val authedResponse = new Response[String](Right("Hello World"), 200, "OK", Nil, List.empty) + val authProvider = new GithubImportAuthProvider { + override def authHeader() = Future.successful(Map("Authorization" -> "Bearer 1234")) + } + val resolver = new RecordingHttpResolver(unauthedResponse, authedResponse, authProvider) + val importUri = + "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/centaur/src/main/resources/standardTestCases/hello/hello.wdl" + val resolvedBundle = resolver.innerResolver(importUri, List(resolver)) + + resolvedBundle match { + case Left(e) => fail(s"Expected ResolvedImportBundle but got $e") + case Right(bundle) => + bundle.resolvedImportRecord.importPath shouldBe importUri + bundle.source shouldBe "Hello World" + + resolver.requestRecords.size shouldBe 2 + resolver.requestRecords.head.url shouldBe importUri + resolver.requestRecords.head.headers shouldBe Map.empty + resolver.requestRecords(1).url shouldBe importUri + resolver.requestRecords(1).headers should be(Map(("Authorization", "Bearer 1234"))) + } + } + + it should "not lookup headers for urls which require no auth" in { + val unauthedResponse = new Response[String](Right("Hello World"), 200, "OK", Nil, List.empty) + val authedResponse = new Response[String](Left("Shouldn't be authed".getBytes), 500, "BAD", Nil, List.empty) + val authProvider = new GithubImportAuthProvider { + override def authHeader() = Future.failed(new Exception("Should not be called")) + } + val resolver = new RecordingHttpResolver(unauthedResponse, authedResponse, authProvider) + val importUri = + "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/centaur/src/main/resources/standardTestCases/hello/hello.wdl" + val resolvedBundle = resolver.innerResolver(importUri, List(resolver)) + + resolvedBundle match { + case Left(e) => fail(s"Expected ResolvedImportBundle but got $e") + case Right(bundle) => + bundle.resolvedImportRecord.importPath shouldBe importUri + bundle.source shouldBe "Hello World" + + resolver.requestRecords.size shouldBe 1 + resolver.requestRecords.head.url shouldBe importUri + resolver.requestRecords.head.headers shouldBe Map.empty + } + } + + it should "not lookup headers for urls which failed with other errors" in { + val unauthedResponse = new Response[String](Left("Some other error".getBytes), + StatusCodes.ServiceUnavailable, + "ServiceUnavailable", + Nil, + List.empty + ) + val authedResponse = new Response[String](Left("Shouldn't be authed".getBytes), 500, "BAD", Nil, List.empty) + val authProvider = new GithubImportAuthProvider { + override def authHeader() = Future.failed(new Exception("Should not be called")) + } + val resolver = new RecordingHttpResolver(unauthedResponse, authedResponse, authProvider) + val importUri = + "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/centaur/src/main/resources/standardTestCases/hello/hello.wdl" + val resolvedBundle = resolver.innerResolver(importUri, List(resolver)) + + resolvedBundle match { + case Left(e) => + e.length should be(1) + e.head.contains("Some other error") should be(true) + resolver.requestRecords.size shouldBe 1 + resolver.requestRecords.head.url shouldBe importUri + resolver.requestRecords.head.headers shouldBe Map.empty + case Right(bundle) => + fail(s"Expected an error but got $bundle") + } + } + behavior of "directory resolver from root" val workingDirectory = sys.props("user.dir") - val rootDirectoryResolver = DirectoryResolver(DefaultPath(Paths.get("/")), customName = None, deleteOnClose = false, directoryHash = None) + val rootDirectoryResolver = + DirectoryResolver(DefaultPath(Paths.get("/")), customName = None, deleteOnClose = false, directoryHash = None) it should "resolve a random path" in { val pathToLookup = rootDirectoryResolver.resolveAndMakeAbsolute("/path/to/file.wdl") @@ -177,10 +299,18 @@ class ImportResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match behavior of "unprotected relative directory resolver" - val relativeDirectoryResolver = DirectoryResolver(DefaultPath(Paths.get("/path/to/imports/")), customName = None, deleteOnClose = false, directoryHash = None) + val relativeDirectoryResolver = DirectoryResolver(DefaultPath(Paths.get("/path/to/imports/")), + customName = None, + deleteOnClose = false, + directoryHash = None + ) val relativeDirForSampleWf = s"$workingDirectory/languageFactories/language-factory-core/src/test/" - val relativeDirResolverForSampleWf = DirectoryResolver(DefaultPath(Paths.get(relativeDirForSampleWf)), customName = None, deleteOnClose = false, directoryHash = None) + val relativeDirResolverForSampleWf = DirectoryResolver(DefaultPath(Paths.get(relativeDirForSampleWf)), + customName = None, + deleteOnClose = false, + directoryHash = None + ) it should "resolve an absolute path" in { val pathToLookup = relativeDirectoryResolver.resolveAndMakeAbsolute("/path/to/file.wdl") @@ -208,14 +338,24 @@ class ImportResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match resolvedBundle.map(_.resolvedImportRecord) match { case Left(e) => fail(s"Expected ResolvedImportBundle but got $e") - case Right(resolvedImport) => resolvedImport.importPath shouldBe(relativeDirForSampleWf + path) + case Right(resolvedImport) => resolvedImport.importPath shouldBe (relativeDirForSampleWf + path) } } behavior of "protected relative directory resolver" - val protectedRelativeDirectoryResolver = DirectoryResolver(DefaultPath(Paths.get("/path/to/imports/")), Some("/path/to/imports/"), customName = None, deleteOnClose = false, directoryHash = None) - val protectedRelativeDirResolverForSampleWf = DirectoryResolver(DefaultPath(Paths.get(relativeDirForSampleWf)), Some(relativeDirForSampleWf), customName = None, deleteOnClose = false, directoryHash = None) + val protectedRelativeDirectoryResolver = DirectoryResolver(DefaultPath(Paths.get("/path/to/imports/")), + Some("/path/to/imports/"), + customName = None, + deleteOnClose = false, + directoryHash = None + ) + val protectedRelativeDirResolverForSampleWf = DirectoryResolver(DefaultPath(Paths.get(relativeDirForSampleWf)), + Some(relativeDirForSampleWf), + customName = None, + deleteOnClose = false, + directoryHash = None + ) it should "resolve a good relative path" in { val pathToLookup = protectedRelativeDirectoryResolver.resolveAndMakeAbsolute("path/to/file.wdl") @@ -224,11 +364,12 @@ class ImportResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match it should "resolve a good relative path to sampleWorkflow" in { val path = "resources/sampleWorkflow.wdl" - val resolvedBundle = protectedRelativeDirResolverForSampleWf.innerResolver(path, List(protectedRelativeDirResolverForSampleWf)) + val resolvedBundle = + protectedRelativeDirResolverForSampleWf.innerResolver(path, List(protectedRelativeDirResolverForSampleWf)) resolvedBundle.map(_.resolvedImportRecord) match { case Left(e) => fail(s"Expected ResolvedImportBundle but got $e") - case Right(resolvedImport) => resolvedImport.importPath shouldBe(relativeDirForSampleWf + path) + case Right(resolvedImport) => resolvedImport.importPath shouldBe (relativeDirForSampleWf + path) } } @@ -252,7 +393,7 @@ class ImportResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match resolver.resolveAndMakeAbsolute("QC.wdl").map(Files.exists(_)).toOption shouldBe Some(true) resolver.resolveAndMakeAbsolute("tasks/cutadapt.wdl").map(Files.exists(_)).toOption shouldBe Some(true) resolver.resolveAndMakeAbsolute("tasks/fastqc.wdl").map(Files.exists(_)).toOption shouldBe Some(true) - // Make sure above testing is correct by testing for a non-existent wdl. + // Make sure above testing is correct by testing for a non-existent wdl. resolver.resolveAndMakeAbsolute("machine_learning_skynet.wdl").map(Files.exists(_)).toOption shouldBe Some(false) } diff --git a/languageFactories/wdl-biscayne/src/main/scala/languages/wdl/biscayne/WdlBiscayneLanguageFactory.scala b/languageFactories/wdl-biscayne/src/main/scala/languages/wdl/biscayne/WdlBiscayneLanguageFactory.scala index 355a637439b..eace0c91f9b 100644 --- a/languageFactories/wdl-biscayne/src/main/scala/languages/wdl/biscayne/WdlBiscayneLanguageFactory.scala +++ b/languageFactories/wdl-biscayne/src/main/scala/languages/wdl/biscayne/WdlBiscayneLanguageFactory.scala @@ -38,13 +38,19 @@ class WdlBiscayneLanguageFactory(override val config: Config) extends LanguageFa importLocalFilesystem: Boolean, workflowIdForLogging: WorkflowId, ioFunctions: IoFunctionSet, - importResolvers: List[ImportResolver]): IOChecked[ValidatedWomNamespace] = { + importResolvers: List[ImportResolver] + ): IOChecked[ValidatedWomNamespace] = { val factories: List[LanguageFactory] = List(this) val checked: Checked[ValidatedWomNamespace] = for { _ <- enabledCheck - bundle <- getWomBundle(workflowSource, workflowSourceOrigin = None, source.workflowOptions.asPrettyJson, importResolvers, factories) + bundle <- getWomBundle(workflowSource, + workflowSourceOrigin = None, + source.workflowOptions.asPrettyJson, + importResolvers, + factories + ) executable <- createExecutable(bundle, source.inputsJson, ioFunctions) } yield executable @@ -57,17 +63,31 @@ class WdlBiscayneLanguageFactory(override val config: Config) extends LanguageFa workflowOptionsJson: WorkflowOptionsJson, importResolvers: List[ImportResolver], languageFactories: List[LanguageFactory], - convertNestedScatterToSubworkflow : Boolean = true): Checked[WomBundle] = { - - val converter: CheckedAtoB[FileStringParserInput, WomBundle] = stringToAst andThen wrapAst andThen astToFileElement.map(FileElementToWomBundleInputs(_, workflowOptionsJson, convertNestedScatterToSubworkflow, importResolvers, languageFactories, workflowDefinitionElementToWomWorkflowDefinition, taskDefinitionElementToWomTaskDefinition)) andThen fileElementToWomBundle + convertNestedScatterToSubworkflow: Boolean = true + ): Checked[WomBundle] = { + + val converter: CheckedAtoB[FileStringParserInput, WomBundle] = + stringToAst andThen wrapAst andThen astToFileElement.map( + FileElementToWomBundleInputs( + _, + workflowOptionsJson, + convertNestedScatterToSubworkflow, + importResolvers, + languageFactories, + workflowDefinitionElementToWomWorkflowDefinition, + taskDefinitionElementToWomTaskDefinition + ) + ) andThen fileElementToWomBundle lazy val validationCallable = new Callable[ErrorOr[WomBundle]] { def call: ErrorOr[WomBundle] = converter .run(FileStringParserInput(workflowSource, workflowSourceOrigin.map(_.importPath).getOrElse("input.wdl"))) - .map(b => b.copyResolvedImportRecord(b, workflowSourceOrigin)).toValidated + .map(b => b.copyResolvedImportRecord(b, workflowSourceOrigin)) + .toValidated } - lazy val parserCacheInputs = ParserCacheInputs(Option(workflowSource), workflowSourceOrigin.map(_.importPath), None, importResolvers) + lazy val parserCacheInputs = + ParserCacheInputs(Option(workflowSource), workflowSourceOrigin.map(_.importPath), None, importResolvers) for { _ <- enabledCheck @@ -75,13 +95,16 @@ class WdlBiscayneLanguageFactory(override val config: Config) extends LanguageFa } yield womBundle } - override def createExecutable(womBundle: WomBundle, inputsJson: WorkflowJson, ioFunctions: IoFunctionSet): Checked[ValidatedWomNamespace] = { + override def createExecutable(womBundle: WomBundle, + inputsJson: WorkflowJson, + ioFunctions: IoFunctionSet + ): Checked[ValidatedWomNamespace] = for { _ <- enabledCheck executable <- womBundle.toWomExecutable(Option(inputsJson), ioFunctions, strictValidation) validated <- LanguageFactoryUtil.validateWomNamespace(executable, ioFunctions) } yield validated - } - override def looksParsable(content: String): Boolean = LanguageFactoryUtil.simpleLooksParseable(List("version development-1.1"), List("#"))(content) + override def looksParsable(content: String): Boolean = + LanguageFactoryUtil.simpleLooksParseable(List("version development-1.1"), List("#"))(content) } diff --git a/languageFactories/wdl-cascades/src/main/scala/languages/wdl/cascades/WdlCascadesLanguageFactory.scala b/languageFactories/wdl-cascades/src/main/scala/languages/wdl/cascades/WdlCascadesLanguageFactory.scala index d7757e83e71..5954f964fc5 100644 --- a/languageFactories/wdl-cascades/src/main/scala/languages/wdl/cascades/WdlCascadesLanguageFactory.scala +++ b/languageFactories/wdl-cascades/src/main/scala/languages/wdl/cascades/WdlCascadesLanguageFactory.scala @@ -38,13 +38,19 @@ class WdlCascadesLanguageFactory(override val config: Config) extends LanguageFa importLocalFilesystem: Boolean, workflowIdForLogging: WorkflowId, ioFunctions: IoFunctionSet, - importResolvers: List[ImportResolver]): IOChecked[ValidatedWomNamespace] = { + importResolvers: List[ImportResolver] + ): IOChecked[ValidatedWomNamespace] = { val factories: List[LanguageFactory] = List(this) val checked: Checked[ValidatedWomNamespace] = for { _ <- enabledCheck - bundle <- getWomBundle(workflowSource, workflowSourceOrigin = None, source.workflowOptions.asPrettyJson, importResolvers, factories) + bundle <- getWomBundle(workflowSource, + workflowSourceOrigin = None, + source.workflowOptions.asPrettyJson, + importResolvers, + factories + ) executable <- createExecutable(bundle, source.inputsJson, ioFunctions) } yield executable @@ -57,17 +63,31 @@ class WdlCascadesLanguageFactory(override val config: Config) extends LanguageFa workflowOptionsJson: WorkflowOptionsJson, importResolvers: List[ImportResolver], languageFactories: List[LanguageFactory], - convertNestedScatterToSubworkflow : Boolean = true): Checked[WomBundle] = { - - val converter: CheckedAtoB[FileStringParserInput, WomBundle] = stringToAst andThen wrapAst andThen astToFileElement.map(FileElementToWomBundleInputs(_, workflowOptionsJson, convertNestedScatterToSubworkflow, importResolvers, languageFactories, workflowDefinitionElementToWomWorkflowDefinition, taskDefinitionElementToWomTaskDefinition)) andThen fileElementToWomBundle + convertNestedScatterToSubworkflow: Boolean = true + ): Checked[WomBundle] = { + + val converter: CheckedAtoB[FileStringParserInput, WomBundle] = + stringToAst andThen wrapAst andThen astToFileElement.map( + FileElementToWomBundleInputs( + _, + workflowOptionsJson, + convertNestedScatterToSubworkflow, + importResolvers, + languageFactories, + workflowDefinitionElementToWomWorkflowDefinition, + taskDefinitionElementToWomTaskDefinition + ) + ) andThen fileElementToWomBundle lazy val validationCallable = new Callable[ErrorOr[WomBundle]] { def call: ErrorOr[WomBundle] = converter .run(FileStringParserInput(workflowSource, workflowSourceOrigin.map(_.importPath).getOrElse("input.wdl"))) - .map(b => b.copyResolvedImportRecord(b, workflowSourceOrigin)).toValidated + .map(b => b.copyResolvedImportRecord(b, workflowSourceOrigin)) + .toValidated } - lazy val parserCacheInputs = ParserCacheInputs(Option(workflowSource), workflowSourceOrigin.map(_.importPath), None, importResolvers) + lazy val parserCacheInputs = + ParserCacheInputs(Option(workflowSource), workflowSourceOrigin.map(_.importPath), None, importResolvers) for { _ <- enabledCheck @@ -75,13 +95,16 @@ class WdlCascadesLanguageFactory(override val config: Config) extends LanguageFa } yield womBundle } - override def createExecutable(womBundle: WomBundle, inputsJson: WorkflowJson, ioFunctions: IoFunctionSet): Checked[ValidatedWomNamespace] = { + override def createExecutable(womBundle: WomBundle, + inputsJson: WorkflowJson, + ioFunctions: IoFunctionSet + ): Checked[ValidatedWomNamespace] = for { _ <- enabledCheck executable <- womBundle.toWomExecutable(Option(inputsJson), ioFunctions, strictValidation) validated <- LanguageFactoryUtil.validateWomNamespace(executable, ioFunctions) } yield validated - } - override def looksParsable(content: String): Boolean = LanguageFactoryUtil.simpleLooksParseable(List("version development"), List("#"))(content) + override def looksParsable(content: String): Boolean = + LanguageFactoryUtil.simpleLooksParseable(List("version development"), List("#"))(content) } diff --git a/languageFactories/wdl-draft2/src/main/scala/languages/wdl/draft2/WdlDraft2LanguageFactory.scala b/languageFactories/wdl-draft2/src/main/scala/languages/wdl/draft2/WdlDraft2LanguageFactory.scala index a2cbc5e90f4..a1b4927517a 100644 --- a/languageFactories/wdl-draft2/src/main/scala/languages/wdl/draft2/WdlDraft2LanguageFactory.scala +++ b/languageFactories/wdl-draft2/src/main/scala/languages/wdl/draft2/WdlDraft2LanguageFactory.scala @@ -24,7 +24,7 @@ import languages.wdl.draft2.WdlDraft2LanguageFactory._ import mouse.all._ import net.ceedubs.ficus.Ficus._ import wdl.draft2.Draft2ResolvedImportBundle -import wdl.draft2.model.{Draft2ImportResolver, WdlNamespace, WdlNamespaceWithWorkflow, WdlNamespaceWithoutWorkflow} +import wdl.draft2.model.{Draft2ImportResolver, WdlNamespace, WdlNamespaceWithoutWorkflow, WdlNamespaceWithWorkflow} import wdl.shared.transforms.wdlom2wom.WdlSharedInputParsing import wdl.transforms.draft2.wdlom2wom.WdlDraft2WomBundleMakers._ import wdl.transforms.draft2.wdlom2wom.WdlDraft2WomExecutableMakers._ @@ -40,7 +40,10 @@ import wom.values._ import scala.concurrent.duration._ import scala.language.postfixOps -class WdlDraft2LanguageFactory(override val config: Config) extends LanguageFactory with ParserCache[WdlNamespace] with StrictLogging { +class WdlDraft2LanguageFactory(override val config: Config) + extends LanguageFactory + with ParserCache[WdlNamespace] + with StrictLogging { override val languageName: String = "WDL" override val languageVersionName: String = "draft-2" @@ -51,19 +54,22 @@ class WdlDraft2LanguageFactory(override val config: Config) extends LanguageFact importLocalFilesystem: Boolean, workflowIdForLogging: WorkflowId, ioFunctions: IoFunctionSet, - importResolvers: List[ImportResolver]): IOChecked[ValidatedWomNamespace] = { + importResolvers: List[ImportResolver] + ): IOChecked[ValidatedWomNamespace] = { def checkTypes(namespace: WdlNamespace, inputs: Map[OutputPort, WomValue]): Checked[Unit] = namespace match { case namespaceWithWorkflow: WdlNamespaceWithWorkflow => - val allDeclarations = namespaceWithWorkflow.workflow.declarations ++ namespaceWithWorkflow.workflow.calls.flatMap(_.declarations) - val list: List[Checked[Unit]] = inputs.map({ case (k, v) => + val allDeclarations = + namespaceWithWorkflow.workflow.declarations ++ namespaceWithWorkflow.workflow.calls.flatMap(_.declarations) + val list: List[Checked[Unit]] = inputs.map { case (k, v) => allDeclarations.find(_.fullyQualifiedName == k) match { case Some(decl) if decl.womType.coerceRawValue(v).isFailure => - s"Invalid right-side type of '$k'. Expecting ${decl.womType.stableName}, got ${v.womType.stableName}".invalidNelCheck[Unit] + s"Invalid right-side type of '$k'. Expecting ${decl.womType.stableName}, got ${v.womType.stableName}" + .invalidNelCheck[Unit] case _ => ().validNelCheck } - }).toList + }.toList list.sequence[Checked, Unit].void @@ -73,14 +79,16 @@ class WdlDraft2LanguageFactory(override val config: Config) extends LanguageFact } def validationCallable = new Callable[ErrorOr[WdlNamespace]] { - def call: ErrorOr[WdlNamespace] = WdlNamespaceWithWorkflow.load(workflowSource, importResolvers map resolverConverter).toErrorOr + def call: ErrorOr[WdlNamespace] = + WdlNamespaceWithWorkflow.load(workflowSource, importResolvers map resolverConverter).toErrorOr } - lazy val wdlNamespaceValidation: ErrorOr[WdlNamespace] = retrieveOrCalculate(ParserCacheInputs(Option(workflowSource), None, None, importResolvers), validationCallable) + lazy val wdlNamespaceValidation: ErrorOr[WdlNamespace] = + retrieveOrCalculate(ParserCacheInputs(Option(workflowSource), None, None, importResolvers), validationCallable) def evaluateImports(wdlNamespace: WdlNamespace): Map[String, String] = { // Descend the namespace looking for imports and construct `MetadataEvent`s for them. - def collectImportEvents: Map[String, String] = { + def collectImportEvents: Map[String, String] = (wdlNamespace.allNamespacesRecursively flatMap { ns => ns.importUri.toList collect { // Do not publish import events for URIs which correspond to literal strings as these are the top-level @@ -88,7 +96,6 @@ class WdlDraft2LanguageFactory(override val config: Config) extends LanguageFact case uri if uri != WdlNamespace.WorkflowResourceString => uri -> ns.sourceString } }).toMap - } collectImportEvents } @@ -108,7 +115,8 @@ class WdlDraft2LanguageFactory(override val config: Config) extends LanguageFact private def validateWorkflowNameLengths(namespace: WdlNamespace): Checked[Unit] = { import common.validation.Checked._ - def allWorkflowNames(n: WdlNamespace): Seq[String] = n.workflows.map(_.unqualifiedName) ++ n.namespaces.flatMap(allWorkflowNames) + def allWorkflowNames(n: WdlNamespace): Seq[String] = + n.workflows.map(_.unqualifiedName) ++ n.namespaces.flatMap(allWorkflowNames) val tooLong = allWorkflowNames(namespace).filter(_.length >= 100) if (tooLong.nonEmpty) { ("Workflow names must be shorter than 100 characters: " + tooLong.mkString(" ")).invalidNelCheck @@ -122,12 +130,15 @@ class WdlDraft2LanguageFactory(override val config: Config) extends LanguageFact workflowOptionsJson: WorkflowOptionsJson, importResolvers: List[ImportResolver], languageFactories: List[LanguageFactory], - convertNestedScatterToSubworkflow : Boolean = true): Checked[WomBundle] = { + convertNestedScatterToSubworkflow: Boolean = true + ): Checked[WomBundle] = { lazy val validationCallable = new Callable[ErrorOr[WdlNamespace]] { - def call: ErrorOr[WdlNamespace] = WdlNamespace.loadUsingSource(workflowSource, None, Some(importResolvers map resolverConverter)).toErrorOr + def call: ErrorOr[WdlNamespace] = + WdlNamespace.loadUsingSource(workflowSource, None, Some(importResolvers map resolverConverter)).toErrorOr } - lazy val parserCacheInputs = ParserCacheInputs(Option(workflowSource), workflowSourceOrigin.map(_.importPath), None, importResolvers) + lazy val parserCacheInputs = + ParserCacheInputs(Option(workflowSource), workflowSourceOrigin.map(_.importPath), None, importResolvers) for { _ <- enabledCheck @@ -136,7 +147,10 @@ class WdlDraft2LanguageFactory(override val config: Config) extends LanguageFact } yield womBundle.copyResolvedImportRecord(womBundle, workflowSourceOrigin) } - override def createExecutable(womBundle: WomBundle, inputs: WorkflowJson, ioFunctions: IoFunctionSet): Checked[ValidatedWomNamespace] = for { + override def createExecutable(womBundle: WomBundle, + inputs: WorkflowJson, + ioFunctions: IoFunctionSet + ): Checked[ValidatedWomNamespace] = for { _ <- enabledCheck executable <- WdlSharedInputParsing.buildWomExecutable(womBundle, Option(inputs), ioFunctions, strictValidation) validatedNamespace <- LanguageFactoryUtil.validateWomNamespace(executable, ioFunctions) @@ -145,22 +159,25 @@ class WdlDraft2LanguageFactory(override val config: Config) extends LanguageFact // Commentary: we'll set this as the default in the reference.conf, so most people will get WDL draft 2 if nothing else looks parsable. override def looksParsable(content: String): Boolean = false - private[draft2] lazy val cacheConfig: Option[CacheConfig] = { + private[draft2] lazy val cacheConfig: Option[CacheConfig] = // WDL version 2 namespace caching is now opt-in. for { _ <- enabled.option(()) caching <- config.as[Option[Config]]("caching") cc <- CacheConfig.optionalConfig(caching, defaultConcurrency = 2, defaultSize = 1000L, defaultTtl = 20 minutes) } yield cc - } } object WdlDraft2LanguageFactory { - private def resolverConverter(importResolver: ImportResolver): Draft2ImportResolver = str => importResolver.resolver.run(ImportResolutionRequest(str, List.empty)) match { - case Right(imported) => Draft2ResolvedImportBundle(imported.source, imported.resolvedImportRecord) - case Left(errors) => throw new RuntimeException(s"Bad import $str: ${errors.toList.mkString(System.lineSeparator)}") - } + private def resolverConverter(importResolver: ImportResolver): Draft2ImportResolver = str => + importResolver.resolver.run(ImportResolutionRequest(str, List.empty)) match { + case Right(imported) => Draft2ResolvedImportBundle(imported.source, imported.resolvedImportRecord) + case Left(errors) => + throw new RuntimeException(s"Bad import $str: ${errors.toList.mkString(System.lineSeparator)}") + } val httpResolver = resolverConverter(ImportResolver.HttpResolver()) - def httpResolverWithHeaders(headers: Map[String, String]) = resolverConverter(ImportResolver.HttpResolver(headers = headers)) + def httpResolverWithHeaders(headers: Map[String, String]) = resolverConverter( + ImportResolver.HttpResolver(headers = headers) + ) } diff --git a/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/ArrayCoercionsSpec.scala b/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/ArrayCoercionsSpec.scala index 08cd91f3432..a44f34f5af1 100644 --- a/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/ArrayCoercionsSpec.scala +++ b/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/ArrayCoercionsSpec.scala @@ -14,22 +14,25 @@ import wom.expression.EmptyIoFunctionSet import wom.types.{WomArrayType, WomSingleFileType, WomStringType} import wom.values.{WomArray, WomSingleFile, WomString} - class ArrayCoercionsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { var factory: WdlDraft2LanguageFactory = new WdlDraft2LanguageFactory(ConfigFactory.parseString(ConfigString)) - val arrayLiteralNamespace: WdlNamespaceWithWorkflow = WdlNamespaceWithWorkflow.load(ArrayDeclarationWorkflow, List.empty).get + val arrayLiteralNamespace: WdlNamespaceWithWorkflow = + WdlNamespaceWithWorkflow.load(ArrayDeclarationWorkflow, List.empty).get "A static Array[File] declaration" should "be a valid declaration" in { - val declaration = arrayLiteralNamespace.workflow.declarations.find {_.unqualifiedName == "arr"}.getOrElse { + val declaration = arrayLiteralNamespace.workflow.declarations.find(_.unqualifiedName == "arr").getOrElse { fail("Expected declaration 'arr' to be found") } val expression = declaration.expression.getOrElse { fail("Expected an expression for declaration 'arr'") } - expression.evaluate((_: String) => fail("No lookups"), NoFunctions).toChecked.shouldBeValid( - WomArray(WomArrayType(WomStringType), Seq(WomString("f1"), WomString("f2"), WomString("f3"))) - ) + expression + .evaluate((_: String) => fail("No lookups"), NoFunctions) + .toChecked + .shouldBeValid( + WomArray(WomArrayType(WomStringType), Seq(WomString("f1"), WomString("f2"), WomString("f3"))) + ) } "An Array[File]" should "be usable as an input" in { @@ -40,9 +43,10 @@ class ArrayCoercionsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match val catTask = arrayLiteralNamespace.findTask("cat").getOrElse { fail("Expected to find task 'cat'") } - val command = catTask.instantiateCommand(catTask.inputsFromMap(Map("cat.files" -> expectedArray)), NoFunctions).getOrElse { - fail("Expected instantiation to work") - } + val command = + catTask.instantiateCommand(catTask.inputsFromMap(Map("cat.files" -> expectedArray)), NoFunctions).getOrElse { + fail("Expected instantiation to work") + } command.head.commandString shouldEqual "cat -s f1 f2 f3" } diff --git a/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/MapWorkflowSpec.scala b/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/MapWorkflowSpec.scala index 981ad1380c4..dbba026fcbb 100644 --- a/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/MapWorkflowSpec.scala +++ b/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/MapWorkflowSpec.scala @@ -11,21 +11,25 @@ import wom.values.{WomMap, WomSingleFile, WomString, WomValue} import scala.util.{Success, Try} - class MapWorkflowSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { val namespace = WdlNamespaceWithWorkflow.load(WorkflowSource, Seq.empty[Draft2ImportResolver]).get - val expectedMap = WomMap(WomMapType(WomSingleFileType, WomStringType), Map( - WomSingleFile("f1") -> WomString("alice"), - WomSingleFile("f2") -> WomString("bob"), - WomSingleFile("f3") -> WomString("chuck") - )) + val expectedMap = WomMap( + WomMapType(WomSingleFileType, WomStringType), + Map( + WomSingleFile("f1") -> WomString("alice"), + WomSingleFile("f2") -> WomString("bob"), + WomSingleFile("f3") -> WomString("chuck") + ) + ) "A static Map[File, String] declaration" should "be a valid declaration" in { - val declaration = namespace.workflow.declarations.find { - _.unqualifiedName == "map" - }.getOrElse { - fail("Expected declaration 'map' to be found") - } + val declaration = namespace.workflow.declarations + .find { + _.unqualifiedName == "map" + } + .getOrElse { + fail("Expected declaration 'map' to be found") + } val expression = declaration.expression.getOrElse { fail("Expected an expression for declaration 'map'") } @@ -47,9 +51,11 @@ class MapWorkflowSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers case _ => throw new UnsupportedOperationException("Only write_map should be called") } } - val command = writeMapTask.instantiateCommand(writeMapTask.inputsFromMap(Map("file_to_name" -> expectedMap)), new CannedFunctions).getOrElse { - fail("Expected instantiation to work") - } + val command = writeMapTask + .instantiateCommand(writeMapTask.inputsFromMap(Map("file_to_name" -> expectedMap)), new CannedFunctions) + .getOrElse { + fail("Expected instantiation to work") + } command.head.commandString shouldEqual "cat /test/map/path" } } diff --git a/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/NamespaceCacheSpec.scala b/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/NamespaceCacheSpec.scala index b96cfa5c2f6..c50fb0df33a 100644 --- a/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/NamespaceCacheSpec.scala +++ b/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/NamespaceCacheSpec.scala @@ -61,7 +61,7 @@ class NamespaceCacheSpec extends AnyFlatSpec with CromwellTimeoutSpec with Befor ) var lookupCount = 0 - val countingResolver = new HttpResolver(None, Map.empty, None) { + val countingResolver = new HttpResolver(None, Map.empty, None, List.empty) { override def pathToLookup(str: String): Checked[String] = { lookupCount = lookupCount + 1 super.pathToLookup(str) @@ -69,14 +69,18 @@ class NamespaceCacheSpec extends AnyFlatSpec with CromwellTimeoutSpec with Befor } def validate = { - val futureNamespace = factory.validateNamespace( - source = collection, - workflowSource = ThreeStep, - workflowOptions = WorkflowOptions(new spray.json.JsObject(Map.empty)), - importLocalFilesystem = false, - workflowIdForLogging = WorkflowId.randomId(), - ioFunctions = NoIoFunctionSet, - importResolvers = List(countingResolver)).value.unsafeToFuture() + val futureNamespace = factory + .validateNamespace( + source = collection, + workflowSource = ThreeStep, + workflowOptions = WorkflowOptions(new spray.json.JsObject(Map.empty)), + importLocalFilesystem = false, + workflowIdForLogging = WorkflowId.randomId(), + ioFunctions = NoIoFunctionSet, + importResolvers = List(countingResolver) + ) + .value + .unsafeToFuture() Await.result(futureNamespace, Duration.Inf).toOption.get } diff --git a/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/WdlWorkflowHttpImportSpec.scala b/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/WdlWorkflowHttpImportSpec.scala index a31df438f3a..1d1d96c07e8 100644 --- a/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/WdlWorkflowHttpImportSpec.scala +++ b/languageFactories/wdl-draft2/src/test/scala/languages.wdl.draft2/WdlWorkflowHttpImportSpec.scala @@ -11,8 +11,7 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import wdl.draft2.model._ - -class WdlWorkflowHttpImportSpec extends AnyFlatSpec with CromwellTimeoutSpec with BeforeAndAfterAll with Matchers { +class WdlWorkflowHttpImportSpec extends AnyFlatSpec with CromwellTimeoutSpec with BeforeAndAfterAll with Matchers { val tinyImport = s""" |task hello { @@ -27,7 +26,7 @@ class WdlWorkflowHttpImportSpec extends AnyFlatSpec with CromwellTimeoutSpec wit | """.stripMargin - def tinyWorkflow(imp:String) = + def tinyWorkflow(imp: String) = s""" | import "$imp" as imp | @@ -45,29 +44,35 @@ class WdlWorkflowHttpImportSpec extends AnyFlatSpec with CromwellTimeoutSpec wit mockServer = startClientAndServer(PortFactory.findFreePort()) host = "http://localhost:" + mockServer.getLocalPort - mockServer.when( - request().withPath("/hello.wdl") - ).respond( - response().withStatusCode(200).withBody(tinyImport) - ) - - mockServer.when( - request().withPath("/protected.wdl").withHeader("Authorization", "Bearer my-token-value") - ). - respond( - response().withStatusCode(200).withBody(tinyImport) - ) - - mockServer.when( - request().withPath("/redirect.wdl") - ).respond( - response().withStatusCode(301).withHeader("Location","/hello.wdl")) - - mockServer.when( - request().withPath("/none.wdl") - ).respond( - response().withStatusCode(404) - ) + mockServer + .when( + request().withPath("/hello.wdl") + ) + .respond( + response().withStatusCode(200).withBody(tinyImport) + ) + + mockServer + .when( + request().withPath("/protected.wdl").withHeader("Authorization", "Bearer my-token-value") + ) + .respond( + response().withStatusCode(200).withBody(tinyImport) + ) + + mockServer + .when( + request().withPath("/redirect.wdl") + ) + .respond(response().withStatusCode(301).withHeader("Location", "/hello.wdl")) + + mockServer + .when( + request().withPath("/none.wdl") + ) + .respond( + response().withStatusCode(404) + ) () // explicitly return unit } @@ -84,28 +89,28 @@ class WdlWorkflowHttpImportSpec extends AnyFlatSpec with CromwellTimeoutSpec wit } it should "resolve an http URL" in { - val wf = tinyWorkflow( s"$host/hello.wdl") + val wf = tinyWorkflow(s"$host/hello.wdl") val ns = WdlNamespaceWithWorkflow.load(wf, httpResolver) ns.isFailure shouldBe false } it should "fail with a 404" in { - val wf = tinyWorkflow( s"$host/none.wdl") + val wf = tinyWorkflow(s"$host/none.wdl") val ns = WdlNamespaceWithWorkflow.load(wf, httpResolver) ns.isFailure shouldBe true } it should "follow a redirect" in { - val wf = tinyWorkflow( s"$host/redirect.wdl") + val wf = tinyWorkflow(s"$host/redirect.wdl") val ns = WdlNamespaceWithWorkflow.load(wf, httpResolver) ns.isFailure shouldBe false } it should "be able to supply a bearer token to a protected resource" in { val auth = Map("Authorization" -> "Bearer my-token-value") - val authHttpResolver : Seq[Draft2ImportResolver] = Seq(WdlDraft2LanguageFactory.httpResolverWithHeaders(auth)) + val authHttpResolver: Seq[Draft2ImportResolver] = Seq(WdlDraft2LanguageFactory.httpResolverWithHeaders(auth)) - val wf = tinyWorkflow( s"$host/protected.wdl") + val wf = tinyWorkflow(s"$host/protected.wdl") val ns = WdlNamespaceWithWorkflow.load(wf, authHttpResolver) ns.isFailure shouldBe false } diff --git a/languageFactories/wdl-draft3/src/main/scala/languages/wdl/draft3/WdlDraft3LanguageFactory.scala b/languageFactories/wdl-draft3/src/main/scala/languages/wdl/draft3/WdlDraft3LanguageFactory.scala index 56337b0010e..d6eb4ab056d 100644 --- a/languageFactories/wdl-draft3/src/main/scala/languages/wdl/draft3/WdlDraft3LanguageFactory.scala +++ b/languageFactories/wdl-draft3/src/main/scala/languages/wdl/draft3/WdlDraft3LanguageFactory.scala @@ -32,40 +32,57 @@ class WdlDraft3LanguageFactory(override val config: Config) extends LanguageFact override val languageName: String = "WDL" override val languageVersionName: String = "1.0" - override def validateNamespace(source: WorkflowSourceFilesCollection, workflowSource: WorkflowSource, workflowOptions: WorkflowOptions, importLocalFilesystem: Boolean, workflowIdForLogging: WorkflowId, ioFunctions: IoFunctionSet, - importResolvers: List[ImportResolver]): IOChecked[ValidatedWomNamespace] = { + importResolvers: List[ImportResolver] + ): IOChecked[ValidatedWomNamespace] = { val factories: List[LanguageFactory] = List(this) val checked: Checked[ValidatedWomNamespace] = for { _ <- enabledCheck - bundle <- getWomBundle(workflowSource, workflowSourceOrigin = None, source.workflowOptions.asPrettyJson, importResolvers, factories) + bundle <- getWomBundle(workflowSource, + workflowSourceOrigin = None, + source.workflowOptions.asPrettyJson, + importResolvers, + factories + ) executable <- createExecutable(bundle, source.inputsJson, ioFunctions) } yield executable fromEither[IO](checked) } - // The only reason this isn't a sub-def inside 'getWomBundle' is that it gets overridden in test cases: protected def makeWomBundle(workflowSource: WorkflowSource, - workflowSourceOrigin: Option[ResolvedImportRecord], - workflowOptionsJson: WorkflowOptionsJson, - importResolvers: List[ImportResolver], - languageFactories: List[LanguageFactory], - convertNestedScatterToSubworkflow : Boolean = true): ErrorOr[WomBundle] = { - - val converter: CheckedAtoB[FileStringParserInput, WomBundle] = stringToAst andThen wrapAst andThen astToFileElement.map(FileElementToWomBundleInputs(_, workflowOptionsJson, convertNestedScatterToSubworkflow, importResolvers, languageFactories, workflowDefinitionElementToWomWorkflowDefinition, taskDefinitionElementToWomTaskDefinition)) andThen fileElementToWomBundle + workflowSourceOrigin: Option[ResolvedImportRecord], + workflowOptionsJson: WorkflowOptionsJson, + importResolvers: List[ImportResolver], + languageFactories: List[LanguageFactory], + convertNestedScatterToSubworkflow: Boolean = true + ): ErrorOr[WomBundle] = { + + val converter: CheckedAtoB[FileStringParserInput, WomBundle] = + stringToAst andThen wrapAst andThen astToFileElement.map( + FileElementToWomBundleInputs( + _, + workflowOptionsJson, + convertNestedScatterToSubworkflow, + importResolvers, + languageFactories, + workflowDefinitionElementToWomWorkflowDefinition, + taskDefinitionElementToWomTaskDefinition + ) + ) andThen fileElementToWomBundle converter .run(FileStringParserInput(workflowSource, workflowSourceOrigin.map(_.importPath).getOrElse("input.wdl"))) - .map(b => b.copyResolvedImportRecord(b, workflowSourceOrigin)).toValidated + .map(b => b.copyResolvedImportRecord(b, workflowSourceOrigin)) + .toValidated } override def getWomBundle(workflowSource: WorkflowSource, @@ -73,13 +90,21 @@ class WdlDraft3LanguageFactory(override val config: Config) extends LanguageFact workflowOptionsJson: WorkflowOptionsJson, importResolvers: List[ImportResolver], languageFactories: List[LanguageFactory], - convertNestedScatterToSubworkflow : Boolean = true): Checked[WomBundle] = { + convertNestedScatterToSubworkflow: Boolean = true + ): Checked[WomBundle] = { lazy val validationCallable = new Callable[ErrorOr[WomBundle]] { - def call: ErrorOr[WomBundle] = makeWomBundle(workflowSource, workflowSourceOrigin, workflowOptionsJson, importResolvers, languageFactories, convertNestedScatterToSubworkflow) + def call: ErrorOr[WomBundle] = makeWomBundle(workflowSource, + workflowSourceOrigin, + workflowOptionsJson, + importResolvers, + languageFactories, + convertNestedScatterToSubworkflow + ) } - lazy val parserCacheInputs = ParserCacheInputs(Option(workflowSource), workflowSourceOrigin.map(_.importPath), None, importResolvers) + lazy val parserCacheInputs = + ParserCacheInputs(Option(workflowSource), workflowSourceOrigin.map(_.importPath), None, importResolvers) for { _ <- enabledCheck @@ -87,13 +112,16 @@ class WdlDraft3LanguageFactory(override val config: Config) extends LanguageFact } yield womBundle } - override def createExecutable(womBundle: WomBundle, inputsJson: WorkflowJson, ioFunctions: IoFunctionSet): Checked[ValidatedWomNamespace] = { + override def createExecutable(womBundle: WomBundle, + inputsJson: WorkflowJson, + ioFunctions: IoFunctionSet + ): Checked[ValidatedWomNamespace] = for { _ <- enabledCheck executable <- womBundle.toWomExecutable(Option(inputsJson), ioFunctions, strictValidation) validated <- LanguageFactoryUtil.validateWomNamespace(executable, ioFunctions) } yield validated - } - override def looksParsable(content: String): Boolean = LanguageFactoryUtil.simpleLooksParseable(List("version 1.0"), List("#"))(content) + override def looksParsable(content: String): Boolean = + LanguageFactoryUtil.simpleLooksParseable(List("version 1.0"), List("#"))(content) } diff --git a/languageFactories/wdl-draft3/src/test/scala/languages/wdl/draft3/WdlDraft3CachingSpec.scala b/languageFactories/wdl-draft3/src/test/scala/languages/wdl/draft3/WdlDraft3CachingSpec.scala index bca467e89c6..9ed41458025 100644 --- a/languageFactories/wdl-draft3/src/test/scala/languages/wdl/draft3/WdlDraft3CachingSpec.scala +++ b/languageFactories/wdl-draft3/src/test/scala/languages/wdl/draft3/WdlDraft3CachingSpec.scala @@ -14,7 +14,6 @@ import wom.core.{WorkflowOptionsJson, WorkflowSource} import wom.executable.WomBundle import wom.expression.NoIoFunctionSet - class WdlDraft3CachingSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { val languageConfig = ConfigFactory.parseString( @@ -29,11 +28,8 @@ class WdlDraft3CachingSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat | } |} |""".stripMargin - ) - - it should "only evaluate files once" in { val invalidWorkflowSource = """ @@ -124,9 +120,21 @@ object WdlDraft3CachingSpec { var evaluationCount = 0 - override protected def makeWomBundle(workflowSource: WorkflowSource, workflowSourceOrigin: Option[ResolvedImportRecord], workflowOptionsJson: WorkflowOptionsJson, importResolvers: List[ImportResolver.ImportResolver], languageFactories: List[LanguageFactory], convertNestedScatterToSubworkflow: Boolean): ErrorOr[WomBundle] = { + override protected def makeWomBundle(workflowSource: WorkflowSource, + workflowSourceOrigin: Option[ResolvedImportRecord], + workflowOptionsJson: WorkflowOptionsJson, + importResolvers: List[ImportResolver.ImportResolver], + languageFactories: List[LanguageFactory], + convertNestedScatterToSubworkflow: Boolean + ): ErrorOr[WomBundle] = { evaluationCount = evaluationCount + 1 - super.makeWomBundle(workflowSource, workflowSourceOrigin, workflowOptionsJson, importResolvers, languageFactories, convertNestedScatterToSubworkflow) + super.makeWomBundle(workflowSource, + workflowSourceOrigin, + workflowOptionsJson, + importResolvers, + languageFactories, + convertNestedScatterToSubworkflow + ) } } diff --git a/mkdocs.yml b/mkdocs.yml index 4ea08cf6fa8..87d48d3171e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -110,5 +110,5 @@ extra_css: extra_javascript: - js/extra.js -copyright: Copyright © 2023 Broad Institute -# google_analytics: ['identifierID', 'URL.org'] +hooks: + - docs/set_copyright.py diff --git a/pact4s/README.md b/pact4s/README.md index d33447f7356..053307943cf 100644 --- a/pact4s/README.md +++ b/pact4s/README.md @@ -8,6 +8,7 @@ pact4s is used for contract testing. val pact4sDependencies = Seq( pact4sScalaTest, pact4sCirce, + pact4sSpray http4sEmberClient, http4sDsl, http4sEmberServer, @@ -46,6 +47,6 @@ docker run --rm -v $PWD:/working \ ``` The generated contracts can be found in the `./target/pacts` folder -- `cromwell-consumer-drshub-provider.json` -- `cromwell-consumer-fake-provider.json` +- `cromwell-drshub.json` +- `cromwell-cbas.json` diff --git a/pact4s/src/main/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/DrsHubClient.scala b/pact4s/src/main/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/DrsHubClient.scala index 3fa11515118..9590b9f405f 100644 --- a/pact4s/src/main/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/DrsHubClient.scala +++ b/pact4s/src/main/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/DrsHubClient.scala @@ -57,19 +57,19 @@ class DrsHubClientImpl[F[_]: Concurrent](client: Client[F], baseUrl: Uri) extend "localizationPath", "bondProvider" )(ResourceMetadata.apply) - implicit val resourceMetadataRequestEncoder: Encoder[ResourceMetadataRequest] = Encoder.forProduct2("url", "fields")(x => - (x.url, x.fields) - ) - implicit val resourceMetadataRequestEntityEncoder: EntityEncoder[F, ResourceMetadataRequest] = circeEntityEncoder[F, ResourceMetadataRequest] + implicit val resourceMetadataRequestEncoder: Encoder[ResourceMetadataRequest] = + Encoder.forProduct2("url", "fields")(x => (x.url, x.fields)) + implicit val resourceMetadataRequestEntityEncoder: EntityEncoder[F, ResourceMetadataRequest] = + circeEntityEncoder[F, ResourceMetadataRequest] override def fetchSystemStatus(): F[Boolean] = { val request = Request[F](uri = baseUrl / "status").withHeaders( org.http4s.headers.Accept(MediaType.application.json) ) client.run(request).use { resp => resp.status match { - case Status.Ok => true.pure[F] + case Status.Ok => true.pure[F] case Status.InternalServerError => false.pure[F] - case _ => UnknownError.raiseError + case _ => UnknownError.raiseError } } } @@ -77,17 +77,17 @@ class DrsHubClientImpl[F[_]: Concurrent](client: Client[F], baseUrl: Uri) extend override def resolveDrsObject(drsPath: String, fields: List[String]): F[ResourceMetadata] = { val body = ResourceMetadataRequest(url = drsPath, fields = fields) val entityBody: EntityBody[F] = EntityEncoder[F, ResourceMetadataRequest].toEntity(body).body - val request = Request[F](uri = baseUrl / "api" / apiVersion / "drs" / "resolve", method=Method.POST, body=entityBody).withHeaders( - org.http4s.headers.`Content-Type`(MediaType.application.json) - ) + val request = + Request[F](uri = baseUrl / "api" / apiVersion / "drs" / "resolve", method = Method.POST, body = entityBody) + .withHeaders( + org.http4s.headers.`Content-Type`(MediaType.application.json) + ) client.run(request).use { resp => resp.status match { - case Status.Ok => resp.as[ResourceMetadata] - case _ => UnknownError.raiseError + case Status.Ok => resp.as[ResourceMetadata] + case _ => UnknownError.raiseError } } } } - - diff --git a/pact4s/src/main/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/Helper.scala b/pact4s/src/main/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/Helper.scala index ceeefc7479f..1405ec20f87 100644 --- a/pact4s/src/main/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/Helper.scala +++ b/pact4s/src/main/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/Helper.scala @@ -14,7 +14,7 @@ object PactHelper { status: Int, responseHeaders: Seq[(String, String)], body: DslPart - ): PactDslResponse = + ): PactDslResponse = builder .`given`(state) .uponReceiving(uponReceiving) @@ -33,8 +33,8 @@ object PactHelper { path: String, requestHeaders: Seq[(String, String)], status: Int, - responseHeaders: Seq[(String, String)], - ): PactDslResponse = + responseHeaders: Seq[(String, String)] + ): PactDslResponse = builder .`given`(state) .uponReceiving(uponReceiving) @@ -52,7 +52,7 @@ object PactHelper { path: String, requestHeaders: Seq[(String, String)], status: Int - ): PactDslResponse = + ): PactDslResponse = builder .`given`(state) .uponReceiving(uponReceiving) @@ -68,7 +68,7 @@ object PactHelper { path: String, requestHeaders: Seq[(String, String)], status: Int - ): PactDslResponse = + ): PactDslResponse = builder .uponReceiving(uponReceiving) .method(method) @@ -86,7 +86,7 @@ object PactHelper { status: Int, responseHeaders: Seq[(String, String)], body: DslPart - ): PactDslResponse = + ): PactDslResponse = builder .`given`(state) .uponReceiving(uponReceiving) @@ -108,8 +108,8 @@ object PactHelper { requestBody: DslPart, status: Int, responseHeaders: Seq[(String, String)], - responsBody: DslPart - ): PactDslResponse = + responseBody: DslPart + ): PactDslResponse = builder .`given`(state, scala.jdk.CollectionConverters.MapHasAsJava(stateParams).asJava) .uponReceiving(uponReceiving) @@ -120,7 +120,26 @@ object PactHelper { .willRespondWith() .status(status) .headers(scala.jdk.CollectionConverters.MapHasAsJava(responseHeaders.toMap).asJava) - .body(responsBody) + .body(responseBody) + + def buildInteraction[A](builder: PactDslWithProvider, + state: String, + uponReceiving: String, + method: String, + path: String, + requestHeaders: Seq[(String, String)], + requestBody: A, + status: Int + )(implicit ev: PactBodyJsonEncoder[A]): PactDslResponse = + builder + .`given`(state) + .uponReceiving(uponReceiving) + .method(method) + .path(path) + .headers(scala.jdk.CollectionConverters.MapHasAsJava(requestHeaders.toMap).asJava) + .body(ev.toJsonString(requestBody)) + .willRespondWith() + .status(status) def buildInteraction(builder: PactDslResponse, state: String, @@ -132,7 +151,7 @@ object PactHelper { status: Int, responseHeaders: Seq[(String, String)], body: DslPart - ): PactDslResponse = + ): PactDslResponse = builder .`given`(state, scala.jdk.CollectionConverters.MapHasAsJava(stateParams).asJava) .uponReceiving(uponReceiving) @@ -154,7 +173,7 @@ object PactHelper { status: Int, responseHeaders: Seq[(String, String)], body: A - )(implicit ev: PactBodyJsonEncoder[A]): PactDslResponse = + )(implicit ev: PactBodyJsonEncoder[A]): PactDslResponse = builder .`given`(state, scala.jdk.CollectionConverters.MapHasAsJava(stateParams).asJava) .uponReceiving(uponReceiving) @@ -174,7 +193,7 @@ object PactHelper { requestHeaders: Seq[(String, String)], requestBody: A, status: Int - )(implicit ev: PactBodyJsonEncoder[A]): PactDslResponse = + )(implicit ev: PactBodyJsonEncoder[A]): PactDslResponse = builder .`given`(state) .uponReceiving(uponReceiving) @@ -194,7 +213,7 @@ object PactHelper { requestHeaders: Seq[(String, String)], requestBody: A, status: Int - )(implicit ev: PactBodyJsonEncoder[A]): PactDslResponse = + )(implicit ev: PactBodyJsonEncoder[A]): PactDslResponse = builder .`given`(state, scala.jdk.CollectionConverters.MapHasAsJava(stateParams).asJava) .uponReceiving(uponReceiving) diff --git a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/BlobFileSystemContractSpec.scala b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/BlobFileSystemContractSpec.scala index f023257fef9..0e003b5d3fe 100644 --- a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/BlobFileSystemContractSpec.scala +++ b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/BlobFileSystemContractSpec.scala @@ -13,6 +13,7 @@ class BlobFileSystemContractSpec extends AnyFlatSpec with Matchers with RequestR val resourceId = ""; val workspaceId = ""; + /** * we can define the folder that the pact contracts get written to upon completion of this test suite. */ @@ -27,24 +28,27 @@ class BlobFileSystemContractSpec extends AnyFlatSpec with Matchers with RequestR * scala tests on build and if the tests pass when run a pact file will be generated locally */ override def pact: RequestResponsePact = ConsumerPactBuilder - .consumer("cromwell-blob-filesystem-consumer") - .hasPactWith("wsm-provider") - .`given`( - "resource exists", - Map("id" -> resourceId.asJson, "value" -> 123.asJson) // we can use parameters to specify details about the provider state - ) - .`given`( - "workspace exists", - Map("id" -> workspaceId, "value" -> 123) // we can use parameters to specify details about the provider state - ) - .uponReceiving("Request to fetch SAS Token") - .method("POST") - .path(s"/api/workspaces/v1/${workspaceId}/resources/controlled/azure/storageContainer/${resourceId}/getSasToken") - .headers("Authorization" -> "sampleToken") - .willRespondWith() - .status(200) - .body( - Json.obj("id" -> "".asJson, "value" -> 123.asJson) - ).toPact() + .consumer("cromwell-blob-filesystem-consumer") + .hasPactWith("wsm-provider") + .`given`( + "resource exists", + Map("id" -> resourceId.asJson, + "value" -> 123.asJson + ) // we can use parameters to specify details about the provider state + ) + .`given`( + "workspace exists", + Map("id" -> workspaceId, "value" -> 123) // we can use parameters to specify details about the provider state + ) + .uponReceiving("Request to fetch SAS Token") + .method("POST") + .path(s"/api/workspaces/v1/${workspaceId}/resources/controlled/azure/storageContainer/${resourceId}/getSasToken") + .headers("Authorization" -> "sampleToken") + .willRespondWith() + .status(200) + .body( + Json.obj("id" -> "".asJson, "value" -> 123.asJson) + ) + .toPact() } diff --git a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/CbasCallbackSpec.scala b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/CbasCallbackSpec.scala new file mode 100644 index 00000000000..e2b353dd70f --- /dev/null +++ b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/CbasCallbackSpec.scala @@ -0,0 +1,118 @@ +package org.broadinstitute.dsde.workbench.cromwell.consumer + +import akka.testkit._ +import au.com.dius.pact.consumer.dsl._ +import au.com.dius.pact.consumer.{ConsumerPactBuilder, PactTestExecutionContext} +import au.com.dius.pact.core.model.RequestResponsePact +import cromwell.core.retry.SimpleExponentialBackoff +import cromwell.core.{CallOutputs, TestKitSuite, WorkflowId, WorkflowSucceeded} +import cromwell.engine.workflow.lifecycle.finalization.WorkflowCallbackActor.PerformCallbackCommand +import cromwell.engine.workflow.lifecycle.finalization.WorkflowCallbackConfig.StaticTokenAuth +import cromwell.engine.workflow.lifecycle.finalization.WorkflowCallbackJsonSupport._ +import cromwell.engine.workflow.lifecycle.finalization.{CallbackMessage, WorkflowCallbackActor, WorkflowCallbackConfig} +import cromwell.services.metadata.MetadataKey +import cromwell.services.metadata.MetadataService.PutMetadataAction +import cromwell.util.GracefulShutdownHelper +import org.broadinstitute.dsde.workbench.cromwell.consumer.PactHelper._ +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import pact4s.scalatest.RequestResponsePactForger +import pact4s.sprayjson.implicits._ +import wom.graph.GraphNodePort.GraphNodeOutputPort +import wom.graph.WomIdentifier +import wom.types.WomStringType +import wom.values._ + +import java.net.URI +import java.util.UUID +import scala.concurrent.ExecutionContextExecutor +import scala.concurrent.duration._ + +class CbasCallbackSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with RequestResponsePactForger { + + implicit val ec: ExecutionContextExecutor = system.dispatcher + + // Akka test setup + private val msgWait = 10.second.dilated + private val serviceRegistryActor = TestProbe("testServiceRegistryActor") + private val deathWatch = TestProbe("deathWatch") + + private val callbackEndpoint = "/api/batch/v1/runs/results" + private val bearerToken = "my-token" + private val workflowId = WorkflowId(UUID.fromString("12345678-1234-1234-1111-111111111111")) + private val basicOutputs = CallOutputs( + Map( + GraphNodeOutputPort(WomIdentifier("foo", "wf.foo"), WomStringType, null) -> WomString("bar"), + GraphNodeOutputPort(WomIdentifier("hello", "wf.hello.hello"), WomStringType, null) -> WomString("Hello") + ) + ) + + // This is the message that we expect Cromwell to send CBAS in this test + private val expectedCallbackMessage = CallbackMessage( + workflowId.toString, + "Succeeded", + Map(("wf.foo", WomString("bar")), ("wf.hello.hello", WomString("Hello"))), + List.empty + ) + + // Define the folder that the pact contracts get written to upon completion of this test suite. + override val pactTestExecutionContext: PactTestExecutionContext = + new PactTestExecutionContext( + "./target/pacts" + ) + + val consumerPactBuilder: ConsumerPactBuilder = ConsumerPactBuilder + .consumer("cromwell") + + val pactProvider: PactDslWithProvider = consumerPactBuilder + .hasPactWith("cbas") + + var pactUpdateCompletedRunDslResponse: PactDslResponse = buildInteraction( + pactProvider, + state = "post completed workflow results", + uponReceiving = "Request to post workflow results", + method = "POST", + path = callbackEndpoint, + requestHeaders = Seq("Authorization" -> "Bearer %s".format(bearerToken), "Content-type" -> "application/json"), + requestBody = expectedCallbackMessage, + status = 200 + ) + override val pact: RequestResponsePact = pactUpdateCompletedRunDslResponse.toPact + + it should "send the right callback to the right URI" in { + // Create actor + val callbackConfig = WorkflowCallbackConfig.empty + .copy(enabled = true) + .copy(retryBackoff = SimpleExponentialBackoff(100.millis, 200.millis, 1.1)) + .copy(authMethod = Option(StaticTokenAuth(bearerToken))) + .copy(defaultUri = Option(new URI(mockServer.getUrl + callbackEndpoint))) + + val props = WorkflowCallbackActor.props( + serviceRegistryActor.ref, + callbackConfig + ) + val workflowCallbackActor = system.actorOf(props, "testWorkflowCallbackActorPact") + + // Send a command to trigger callback + val cmd = PerformCallbackCommand( + workflowId = workflowId, + uri = None, + terminalState = WorkflowSucceeded, + workflowOutputs = basicOutputs, + List.empty + ) + workflowCallbackActor ! cmd + + // Confirm the callback was successful + serviceRegistryActor.expectMsgPF(msgWait) { + case PutMetadataAction(List(resultEvent, _, _), _) => + resultEvent.key shouldBe MetadataKey(workflowId, None, "workflowCallback", "successful") + case _ => + } + + // Shut the actor down + deathWatch.watch(workflowCallbackActor) + workflowCallbackActor ! GracefulShutdownHelper.ShutdownCommand + deathWatch.expectTerminated(workflowCallbackActor, msgWait) + } +} diff --git a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/DrsHubClientSpec.scala b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/DrsHubClientSpec.scala index a4ebe56d76a..bc0155020f1 100644 --- a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/DrsHubClientSpec.scala +++ b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/DrsHubClientSpec.scala @@ -28,19 +28,19 @@ class DrsHubClientSpec extends AnyFlatSpec with Matchers with RequestResponsePac ) private val requestFields = List( - "bucket", - "accessUrl", - "googleServiceAccount", - "fileName", - "hashes", - "localizationPath", - "bondProvider", - "name", - "size", - "timeCreated", - "timeUpdated", - "gsUri", - "contentType", + "bucket", + "accessUrl", + "googleServiceAccount", + "fileName", + "hashes", + "localizationPath", + "bondProvider", + "name", + "size", + "timeCreated", + "timeUpdated", + "gsUri", + "contentType" ) val filesize = 123L @@ -51,7 +51,6 @@ class DrsHubClientSpec extends AnyFlatSpec with Matchers with RequestResponsePac val fileHash = "a2317edbd2eb6cf6b0ee49cb81e3a556" val accessUrl = f"gs://${bucket}/${filename}" - val drsResourceResponsePlaceholder: ResourceMetadata = ResourceMetadata( "application/octet-stream", filesize, @@ -70,27 +69,33 @@ class DrsHubClientSpec extends AnyFlatSpec with Matchers with RequestResponsePac val resourceMetadataResponseDsl: DslPart = newJsonBody { o => o.stringType("contentType", "application/octet-stream") - o.numberType("size", filesize) - o.stringType("timeCreated", timeCreated) - o.stringType("timeUpdated", timeCreated) - o.nullValue("gsUri") - o.nullValue("googleServiceAccount") - o.nullValue("fileName") - o.`object`("accessUrl" , { a => - a.stringType("url", accessUrl) - a.`array`("headers", { h => - h.stringType("Header") - h.stringType("Example") - () - }) - () - }) - o.`object`("hashes", { o => - o.stringType("md5", fileHash) - () - }) - o.nullValue("localizationPath") - o.stringType("bondProvider", bondProvider) + o.numberType("size", filesize) + o.stringType("timeCreated", timeCreated) + o.stringType("timeUpdated", timeCreated) + o.nullValue("gsUri") + o.nullValue("googleServiceAccount") + o.nullValue("fileName") + o.`object`("accessUrl", + { a => + a.stringType("url", accessUrl) + a.`array`("headers", + { h => + h.stringType("Header") + h.stringType("Example") + () + } + ) + () + } + ) + o.`object`("hashes", + { o => + o.stringType("md5", fileHash) + () + } + ) + o.nullValue("localizationPath") + o.stringType("bondProvider", bondProvider) () }.build @@ -98,18 +103,20 @@ class DrsHubClientSpec extends AnyFlatSpec with Matchers with RequestResponsePac val resourceRequestDsl = newJsonBody { o => o.stringType("url", f"drs://test.theanvil.io/${fileId}") - o.array("fields", { a => - requestFields.map(a.stringType) - () - }) + o.array("fields", + { a => + requestFields.map(a.stringType) + () + } + ) () }.build val consumerPactBuilder: ConsumerPactBuilder = ConsumerPactBuilder - .consumer("cromwell-consumer") + .consumer("cromwell") val pactProvider: PactDslWithProvider = consumerPactBuilder - .hasPactWith("drshub-provider") + .hasPactWith("drshub") var pactDslResponse: PactDslResponse = buildInteraction( pactProvider, @@ -127,11 +134,11 @@ class DrsHubClientSpec extends AnyFlatSpec with Matchers with RequestResponsePac uponReceiving = "Request to resolve drs url", method = "POST", path = "/api/v4/drs/resolve", - requestHeaders = Seq("Content-type" -> "application/json"), + requestHeaders = Seq("Content-Type" -> "application/json"), requestBody = resourceRequestDsl, status = 200, responseHeaders = Seq(), - responsBody = resourceMetadataResponseDsl + responseBody = resourceMetadataResponseDsl ) pactDslResponse = buildInteraction( @@ -147,9 +154,8 @@ class DrsHubClientSpec extends AnyFlatSpec with Matchers with RequestResponsePac override val pact: RequestResponsePact = pactDslResponse.toPact - val client: Client[IO] = { + val client: Client[IO] = BlazeClientBuilder[IO](ExecutionContext.global).resource.allocated.unsafeRunSync()._1 - } /* we should use these tests to ensure that our client class correctly handles responses from the provider - i.e. decoding, error mapping, validation diff --git a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/EcmServiceContractSpec.scala b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/EcmServiceContractSpec.scala new file mode 100644 index 00000000000..e8101f66927 --- /dev/null +++ b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/cromwell/consumer/EcmServiceContractSpec.scala @@ -0,0 +1,63 @@ +package org.broadinstitute.dsde.workbench.cromwell.consumer + +import au.com.dius.pact.consumer.dsl._ +import au.com.dius.pact.consumer.{ConsumerPactBuilder, PactTestExecutionContext} +import au.com.dius.pact.core.model.RequestResponsePact +import cromwell.core.TestKitSuite +import cromwell.services.auth.GithubAuthVending.{GithubToken, TerraToken} +import cromwell.services.auth.ecm.EcmService +import org.scalatest.flatspec.AnyFlatSpecLike + +import scala.concurrent.{Await, ExecutionContextExecutor} +import org.scalatest.matchers.should.Matchers +import pact4s.scalatest.RequestResponsePactForger + +import scala.concurrent.duration.DurationInt + +class EcmServiceContractSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with RequestResponsePactForger { + + implicit val ec: ExecutionContextExecutor = system.dispatcher + + /* + Define the folder that the pact contracts get written to upon completion of this test suite. + */ + override val pactTestExecutionContext: PactTestExecutionContext = + new PactTestExecutionContext( + "./target/pacts" + ) + + val consumerPactBuilder: ConsumerPactBuilder = ConsumerPactBuilder + .consumer("cromwell") + + val pactProvider: PactDslWithProvider = consumerPactBuilder + .hasPactWith("ecm") + + var testUser: String = "cromwell_test_user@test.com" + var testBearerToken: String = "cromwellBearerToken" + var testGithubToken: String = "githubToken" + + var pactDslResponse: PactDslResponse = pactProvider + .`given`("a user is registered", + Map[String, String]( + "userEmail" -> testUser, + "bearerToken" -> testBearerToken + ) + ) + .uponReceiving("a github token request") + .method("GET") + .path("/api/oauth/v1/github/access-token") + .headers(Map[String, String]("Authorization" -> s"Bearer $testBearerToken")) + .willRespondWith() + .status(200) + .bodyMatchingContentType("text/plain", testGithubToken) + + override val pact: RequestResponsePact = pactDslResponse.toPact + + it should "get a github token" in { + Await.result( + new EcmService(mockServer.getUrl) + .getGithubAccessToken(TerraToken(testBearerToken)), + 10.seconds + ) shouldBe GithubToken(testGithubToken) + } +} diff --git a/perf/src/main/resources/logback.xml b/perf/src/main/resources/logback.xml deleted file mode 100644 index 8a90f3ba0ab..00000000000 --- a/perf/src/main/resources/logback.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - INFO - ACCEPT - DENY - - - %msg%n - - - - - - ERROR - - - %red(%msg%n) - - - - - - - - - diff --git a/perf/src/main/scala/cromwell/perf/Call.scala b/perf/src/main/scala/cromwell/perf/Call.scala deleted file mode 100644 index 3849cb4093d..00000000000 --- a/perf/src/main/scala/cromwell/perf/Call.scala +++ /dev/null @@ -1,85 +0,0 @@ -package cromwell.perf - -import java.time.{Duration, OffsetDateTime} - -import io.circe.JsonObject - -case class CallCaching(hit: Option[Boolean], - result: Option[String], - hitFailures: Option[Seq[Map[String, Seq[JsonObject]]]]) - - -case class ExecutionEvents(startTime: OffsetDateTime, - description: String, - endTime: OffsetDateTime) - - -case class Call(shardIndex: Int, - start: OffsetDateTime, - end: OffsetDateTime, - callCaching: Option[CallCaching], - executionEvents: Seq[ExecutionEvents]) { - val callCachingEventStates = List("CheckingCallCache", "FetchingCachedOutputsFromDatabase", "BackendIsCopyingCachedOutputs") - val jobPreparationEventStates = List("Pending", "RequestingExecutionToken", "WaitingForValueStore", "PreparingJob", "CheckingJobStore") - val cacheCopyingEventStates = List("FetchingCachedOutputsFromDatabase", "BackendIsCopyingCachedOutputs") - - /** - * @return Cache copy retries before getting a successful hit or running the job - */ - val cacheCopyRetries: Int = { - val numFailures: Option[Int] = for { - cachingObject <- callCaching - failuresMap <- cachingObject.hitFailures - } yield failuresMap.size - - numFailures.getOrElse(0) - } - - /*** - * @return Time (in Duration) job spent in just CheckingCallCache state - */ - val timeInCheckingCallCacheState: Duration = { - val eventsRelatedToCC = executionEvents.collect { - case event if event.description.equalsIgnoreCase("CheckingCallCache") => Duration.between(event.startTime, event.endTime) - } - - if(eventsRelatedToCC.nonEmpty) eventsRelatedToCC.reduce(_ plus _) else Duration.ZERO - } - - /** - * @return Time (in Duration) job spent in Call Caching states - */ - val timeInCallCachingState: Duration = { - val eventsRelatedToCC = executionEvents.filter(event => callCachingEventStates.exists(state => state.equalsIgnoreCase(event.description))) - - if (eventsRelatedToCC.nonEmpty) { - val eventsSortedByStartTime = eventsRelatedToCC.sortBy(_.startTime) - Duration.between(eventsSortedByStartTime.head.startTime, eventsSortedByStartTime.last.endTime) - } - else Duration.ZERO - } - - /** - * @return Time (in Duration) job spent in job preparation state i.e time between job submission and running it - * (this doesn't consider the time it spent in call caching states - */ - val timeInJobPreparation: Duration = { - val durationOfEventsInPreparationState = executionEvents.collect { - case event if jobPreparationEventStates.exists(_.equalsIgnoreCase(event.description)) => Duration.between(event.startTime, event.endTime) - } - - if(durationOfEventsInPreparationState.nonEmpty) durationOfEventsInPreparationState.reduce(_ plus _) else Duration.ZERO - } - - /** - * - * @return Time (in Duration) job spent in fetching and copying cache hit(s) - */ - val timeForFetchingAndCopyingCacheHit: Duration = { - val durationOfEventsInCacheCopyingState = executionEvents.collect { - case event if cacheCopyingEventStates.exists(_.equalsIgnoreCase(event.description)) => Duration.between(event.startTime, event.endTime) - } - - if(durationOfEventsInCacheCopyingState.nonEmpty) durationOfEventsInCacheCopyingState.reduce(_ plus _) else Duration.ZERO - } -} diff --git a/perf/src/main/scala/cromwell/perf/CompareMetadata.scala b/perf/src/main/scala/cromwell/perf/CompareMetadata.scala deleted file mode 100644 index 600bd38aba0..00000000000 --- a/perf/src/main/scala/cromwell/perf/CompareMetadata.scala +++ /dev/null @@ -1,154 +0,0 @@ -package cromwell.perf - -import java.io.FileInputStream -import java.time.{Duration, OffsetDateTime} - -import better.files.File -import cats.data.Validated.{Invalid, Valid} -import cats.syntax.all._ -import cats.instances.list._ -import com.google.auth.oauth2.GoogleCredentials -import com.google.cloud.storage.StorageOptions -import com.typesafe.scalalogging.StrictLogging -import common.validation.ErrorOr._ -import io.circe._ -import io.circe.generic.semiauto._ -import io.circe.parser._ - -object CompareMetadata extends App with StrictLogging{ - private val REGRESSION_CONST = 1.1 - - // If removed, IntelliJ (Oct '18) thinks the import isn't used. - // Later the compiler fails to find a decoder for OffsetDateTime. - lazy val anchorOffsetDateTime: Decoder[OffsetDateTime] = implicitly - implicit lazy val decodeCall: Decoder[Call] = deriveDecoder - implicit lazy val decodeCallCaching: Decoder[CallCaching] = deriveDecoder - implicit lazy val decodeExecutionEvents: Decoder[ExecutionEvents] = deriveDecoder - implicit lazy val decodeMetadata: Decoder[Metadata] = deriveDecoder - - def parseMetadataFromLocalFile(filePath: String): Either[Error, Metadata] = { - val metadataFile = File(filePath) - val metadataFileContent = metadataFile.contentAsString - - decode[Metadata](metadataFileContent) - } - - - def parseMetadataFromGcsFile(gcsUrl: String, pathToServiceAccount: String): Either[Error, Metadata] = { - val gcsUrlArray = gcsUrl.replace("gs://", "").split("/", 2) - val Array(gcsBucket, fileToBeLocalized) = gcsUrlArray - - val credentials = GoogleCredentials.fromStream(new FileInputStream(pathToServiceAccount)) - val storage = StorageOptions.newBuilder().setCredentials(credentials).build().getService - val blob = storage.get(gcsBucket, fileToBeLocalized) - val metadataFileContent = blob.getContent().map(_.toChar).mkString - - decode[Metadata](metadataFileContent) - } - - - def parseMetadata(inputFile: String, pathToServiceAccount: String): Either[Error, Metadata] = { - if (inputFile.startsWith("gs://")) - parseMetadataFromGcsFile(inputFile, pathToServiceAccount) - else parseMetadataFromLocalFile(inputFile) - } - - - def displayComputedMetrics(metadata: Metadata, displayMsg: String): Unit = { - logger.info(displayMsg) - - logger.info(s"Workflow started after: ${metadata.workflowStartedAfter}") - logger.info(s"Workflow Running time: ${metadata.workflowRunningTime}") - logger.info(s"Total jobs per root workflow: ${metadata.totalJobsPerRootWf}") - logger.info(s"Avg cache copy retries: ${metadata.avgCacheRetries}") - logger.info(s"Avg time job spent in Call Caching state: ${metadata.avgTimeInCallCachingState}") - logger.info(s"Avg time job spent in just CheckingCallCache state: ${metadata.avgTimeInCheckingCallCacheState}") - logger.info(s"Avg time job spent in Job Preparation state: ${metadata.avgTimeInJobPreparation}") - logger.info(s"Avg time job spent in fetching and copying cache hit(s) state: ${metadata.avgTimeForFetchingAndCopyingCacheHit}") - } - - - /*** - * Compares the metrics in Metadata which have type 'Duration' (mostly the ones related to time metrics) - */ - def compareDurationMetrics(metadataOld: Metadata, metadataNew: Metadata, metricFunc: Metadata => Duration, metricName: String): ErrorOr[String] = { - if(metricFunc(metadataNew).toMillis > REGRESSION_CONST * metricFunc(metadataOld).toMillis){ - (s"$metricName of new metadata is greater than 10% of old metadata. " + - s"New metric value: ${metricFunc(metadataNew)}. " + - s"Old metric value: ${metricFunc(metadataOld)}.").invalidNel[String] - } - else s"$metricName hasn't regressed yet".valid - } - - - /*** - * Compares the metrics in Metadata which have type 'Int' (mostly the ones related to size metrics) - */ - def compareIntMetrics(metadataOld: Metadata, metadataNew: Metadata, metricFunc: Metadata => Int, metricName: String): ErrorOr[String] = { - if(metricFunc(metadataNew) > REGRESSION_CONST * metricFunc(metadataOld)){ - (s"$metricName of new metadata is greater than 10% of old metadata. " + - s"New metric value: ${metricFunc(metadataNew)}. " + - s"Old metric value: ${metricFunc(metadataOld)}.").invalidNel[String] - } - else s"$metricName hasn't regressed yet".valid - } - - - def compareMetadataMetrics(metadataOld: Metadata, metadataNew: Metadata): ErrorOr[List[String]] = { - val durationFuncList: List[(Metadata => Duration, String)] = List((_.workflowStartedAfter, "workflowStartedAfter"), - (_.workflowRunningTime, "workflowRunningTime"), - (_.avgTimeInCallCachingState, "avgTimeInCallCachingState"), - (_.avgTimeInCheckingCallCacheState, "avgTimeInCheckingCallCacheState"), - (_.avgTimeInJobPreparation, "avgTimeInJobPreparation"), - (_.avgTimeForFetchingAndCopyingCacheHit, "avgTimeForFetchingAndCopyingCacheHit")) - - val intFuncList: List[(Metadata => Int, String)] = List((_.totalJobsPerRootWf, "totalJobsPerRootWf"), (_.avgCacheRetries, "avgCacheRetries")) - - val durationMetricsComp = durationFuncList.map(x => compareDurationMetrics(metadataOld, metadataNew, x._1, x._2)) - val intMetricsComp = intFuncList.map(x => compareIntMetrics(metadataOld, metadataNew, x._1, x._2)) - (durationMetricsComp ::: intMetricsComp).sequence[ErrorOr, String] - } - - - def printParseErrorToConsoleAndExit(metadataFile: String, error: Error, systemExit: Boolean): Unit = { - logger.error(s"Something went wrong while parsing $metadataFile. Error: ${error.getLocalizedMessage}") - if (systemExit) System.exit(1) - } - - - def generateAndCompareMetrics(metadataOldEither: Either[Error, Metadata], metadataNewEither: Either[Error, Metadata]): Unit = { - (metadataOldEither, metadataNewEither) match { - case (Right(metadataOld), Right(metadataNew)) => - val metadataOldMsg = s"Metrics for metadata generated from ${args(0)}" - val metadataNewMsg = s"\nMetrics for metadata generated from ${args(1)}" - displayComputedMetrics(metadataOld, metadataOldMsg) - displayComputedMetrics(metadataNew, metadataNewMsg) - compareMetadataMetrics(metadataOld, metadataNew) match { - case Valid(_) => logger.info("\nYAY!! Metrics from new metadata json haven't regressed!") - case Invalid(listOfErrors) => - logger.error("\nBelow metadata metrics have regressed:") - logger.error(listOfErrors.toList.mkString("\n")) - System.exit(1) - } - case (Right(_), Left(e)) => printParseErrorToConsoleAndExit(args(1), e, systemExit = true) - case (Left(e), Right(_)) => printParseErrorToConsoleAndExit(args(0), e, systemExit = true) - case (Left(e1), Left(e2)) => - printParseErrorToConsoleAndExit(args(0), e1, systemExit = false) - printParseErrorToConsoleAndExit(args(1), e2, systemExit = true) - } - } - - - args.length match { - case 2 => - if (args(0).startsWith("gs://") || args(1).startsWith("gs://")) { - logger.error("Path to service account is needed to download GCS file. Please pass it as 3rd argument.") - System.exit(1) - } - else generateAndCompareMetrics(parseMetadataFromLocalFile(args(0)), parseMetadataFromLocalFile(args(1))) - case 3 => generateAndCompareMetrics(parseMetadata(args(0), args(2)), parseMetadata(args(1), args(2))) - case _ => - logger.error("Please pass in 2 file paths!") - System.exit(1) - } -} diff --git a/perf/src/main/scala/cromwell/perf/Metadata.scala b/perf/src/main/scala/cromwell/perf/Metadata.scala deleted file mode 100644 index 190063e48b3..00000000000 --- a/perf/src/main/scala/cromwell/perf/Metadata.scala +++ /dev/null @@ -1,74 +0,0 @@ -package cromwell.perf - -import java.time.{Duration, OffsetDateTime} - -case class Metadata(id: String, - workflowName: String, - submission: OffsetDateTime, - start: OffsetDateTime, - end: OffsetDateTime, - status: String, - calls: Option[Map[String, Seq[Call]]]) { - - private def addInt(x: Int, y: Int): Int = x + y - - private def addDuration(x: Duration, y: Duration): Duration = x.plus(y) - - private def sumElementsInOptionSeq[A](listOption: Option[Iterable[A]], op: (A, A) => A, default: A): A = { - listOption match { - case Some(list) => list.reduce[A]((a, b) => op(a,b)) - case None => default - } - } - - /** - * @return Time between the submission and start of workflow - */ - val workflowStartedAfter: Duration = Duration.between(submission, start) - - val workflowRunningTime: Duration = Duration.between(start, end) - - val totalJobsPerRootWf: Int = sumElementsInOptionSeq(calls.map(taskMap => taskMap.map(callsPerTask => callsPerTask._2.size)), addInt, 0) - - val avgCacheRetries: Int = { - if (totalJobsPerRootWf > 0) { - val cacheRetriesList = calls.map(taskMap => taskMap.flatMap(callsPerTask => callsPerTask._2.map(call => call.cacheCopyRetries))) - sumElementsInOptionSeq(cacheRetriesList, addInt, 0) / totalJobsPerRootWf - } - else 0 - } - - val avgTimeInCallCachingState: Duration = { - if (totalJobsPerRootWf > 0) { - val timeInCallCachingStateList = calls.map(taskMap => taskMap.flatMap(callsPerTask => callsPerTask._2.map(call => call.timeInCallCachingState))) - sumElementsInOptionSeq(timeInCallCachingStateList, addDuration, Duration.ZERO).dividedBy(totalJobsPerRootWf.toLong) - } - else Duration.ZERO - } - - val avgTimeInCheckingCallCacheState: Duration = { - if (totalJobsPerRootWf > 0) { - val timeInCheckingCallCacheStateList = calls.map(taskMap => taskMap.flatMap(callsPerTask => callsPerTask._2.map(call => call.timeInCheckingCallCacheState))) - sumElementsInOptionSeq(timeInCheckingCallCacheStateList, addDuration, Duration.ZERO).dividedBy(totalJobsPerRootWf.toLong) - } - else Duration.ZERO - } - - val avgTimeInJobPreparation: Duration = { - if (totalJobsPerRootWf > 0) { - val timeInJobPreparationList = calls.map(taskMap => taskMap.flatMap(callsPerTask => callsPerTask._2.map(call => call.timeInJobPreparation))) - sumElementsInOptionSeq(timeInJobPreparationList, addDuration, Duration.ZERO).dividedBy(totalJobsPerRootWf.toLong) - } - else Duration.ZERO - } - - val avgTimeForFetchingAndCopyingCacheHit: Duration = { - if (totalJobsPerRootWf > 0) { - val timeInCopyingList = calls.map(taskMap => taskMap.flatMap(callsPerTask => callsPerTask._2.map(call => call.timeForFetchingAndCopyingCacheHit))) - sumElementsInOptionSeq(timeInCopyingList, addDuration, Duration.ZERO).dividedBy(totalJobsPerRootWf.toLong) - } - else Duration.ZERO - } -} - - diff --git a/processes/release_processes/README.MD b/processes/release_processes/README.MD index 146c2479620..0d2f4e9475e 100644 --- a/processes/release_processes/README.MD +++ b/processes/release_processes/README.MD @@ -90,6 +90,13 @@ The workflow outputs its status to the console. * Announce release in `#dsp-workflows`, set expectations about when the new version will be available in Terra. * **One business day later,** confirm that [the Homebrew package](https://formulae.brew.sh/formula/cromwell) has the latest version. If it doesn't, start investigation by looking at [Homebrew PR's](https://github.com/Homebrew/homebrew-core/pulls?q=is%3Apr+cromwell). +### Publish Docker Image +* If the release workflow went well, it's time to also publish Docker images for this release. +* `git checkout` the Cromwell hash that was just published (i.e. the one directly BEFORE the "Update Cromwell version from x to x+1" commit that the publish WDL makes). It's important that the image being built uses the exact same code as the .jar files published to github. +* Run `sbt -Dproject.isSnapshot=false -Dproject.isRelease=true dockerBuildAndPush` from your local Cromwell directory. +* Grab a cup of coffee, and verify that all of the new images were pushed successfully. For example, you should now be able to do `docker pull broadinstitute/cromwell:{new version #}` + * The list of images is `cromwell`, `cromiam`, `cromwell-drs-localizer`, and `womtool` + ### How to Deploy Cromwell in CaaS staging and CaaS prod CaaS is "Cromwell as a Service". It is used by a couple of Broad teams (Pipelines and Epigenomics), though the long-term plan is for those teams to migrate to using Terra. diff --git a/project/ContinuousIntegration.scala b/project/ContinuousIntegration.scala index e2b61994c22..aa171a72338 100644 --- a/project/ContinuousIntegration.scala +++ b/project/ContinuousIntegration.scala @@ -9,6 +9,8 @@ object ContinuousIntegration { lazy val ciSettings: Seq[Setting[_]] = List( srcCiResources := sourceDirectory.value / "ci" / "resources", targetCiResources := target.value / "ci" / "resources", + envFile := srcCiResources.value / "env.temp", // generated by resources/acquire_b2c_token.sh + vaultToken := userHome / ".vault-token", copyCiResources := { IO.copyDirectory(srcCiResources.value, targetCiResources.value) @@ -26,17 +28,34 @@ object ContinuousIntegration { if (vaultToken.value.isDirectory) { sys.error(s"""The vault token file "${vaultToken.value}" should not be a directory.""") } - val cmd = List( - "docker", - "run", - "--rm", - "-v", s"${vaultToken.value}:/root/.vault-token", - "-v", s"${srcCiResources.value}:${srcCiResources.value}", - "-v", s"${targetCiResources.value}:${targetCiResources.value}", - "-e", "ENVIRONMENT=not_used", - "-e", s"INPUT_PATH=${srcCiResources.value}", - "-e", s"OUT_PATH=${targetCiResources.value}", - "broadinstitute/dsde-toolbox:dev", "render-templates.sh" + + // Only include the local file argument if the file exists (local development w/ acquire_b2c_token.sh) + // Don't include it otherwise (CI/CD and other development) + val localEnvFileArgs = if (envFile.value.exists()) List("-e", s"ENV_FILE=${envFile.value}") else List() + + val cmd: List[String] = List.concat( + List( + "docker", + "run", + "--rm", + "-v", + s"${vaultToken.value}:/root/.vault-token", + "-v", + s"${srcCiResources.value}:${srcCiResources.value}", + "-v", + s"${targetCiResources.value}:${targetCiResources.value}" + ), + localEnvFileArgs, + List( + "-e", + "ENVIRONMENT=not_used", + "-e", + s"INPUT_PATH=${srcCiResources.value}", + "-e", + s"OUT_PATH=${targetCiResources.value}", + "broadinstitute/dsde-toolbox:dev", + "render-templates.sh" + ) ) val result = cmd ! log if (result != 0) { @@ -45,7 +64,7 @@ object ContinuousIntegration { "https://hub.docker.com/r/broadinstitute/dsde-toolbox/" ) } - }, + } ) def aggregateSettings(rootProject: Project): Seq[Setting[_]] = List( @@ -54,7 +73,7 @@ object ContinuousIntegration { streams.value.log // make sure logger is loaded validateAggregatedProjects(rootProject, state.value) (Compile / compile).value - }, + } ) private val copyCiResources: TaskKey[Unit] = taskKey[Unit](s"Copy CI resources.") @@ -63,6 +82,8 @@ object ContinuousIntegration { private val srcCiResources: SettingKey[File] = settingKey[File]("Source directory for CI resources") private val targetCiResources: SettingKey[File] = settingKey[File]("Target directory for CI resources") private val vaultToken: SettingKey[File] = settingKey[File]("File with the vault token") + private val envFile: SettingKey[File] = + settingKey[File]("File with the environment variables needed to render CI resources.") /** * For "reasons" these projects are excluded from the root aggregation in build.sbt. @@ -74,9 +95,9 @@ object ContinuousIntegration { */ private def getBuildSbtNames(rootProject: Project, state: State): Set[String] = { val extracted = Project.extract(state) - extracted.structure.units.flatMap({ - case (_, loadedBuildUnit) => loadedBuildUnit.defined.keys - }).toSet - rootProject.id + extracted.structure.units.flatMap { case (_, loadedBuildUnit) => + loadedBuildUnit.defined.keys + }.toSet - rootProject.id } /** @@ -85,8 +106,8 @@ object ContinuousIntegration { private def validateAggregatedProjects(rootProject: Project, state: State): Unit = { // Get the list of projects explicitly aggregated val projectReferences: Seq[ProjectReference] = rootProject.aggregate - val localProjectReferences = projectReferences collect { - case localProject: LocalProject => localProject + val localProjectReferences = projectReferences collect { case localProject: LocalProject => + localProject } val aggregatedNames = localProjectReferences.map(_.project).toSet @@ -98,7 +119,7 @@ object ContinuousIntegration { val falseNames = unaggregatedProjects.filterKeys(aggregatedNames.contains) if (falseNames.nonEmpty) { - val reasons = falseNames.map({case (name, reason) => s" ${name}: ${reason}"}).mkString("\n") + val reasons = falseNames.map { case (name, reason) => s" ${name}: ${reason}" }.mkString("\n") sys.error(s"There are projects aggregated in build.sbt that shouldn't be:\n$reasons") } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b2e94cc7470..fc9beae1f12 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -78,7 +78,7 @@ object Dependencies { private val kindProjectorV = "0.13.2" private val kittensV = "2.3.2" private val liquibaseV = "4.8.0" - private val logbackV = "1.2.11" + private val logbackV = "1.2.13" private val lz4JavaV = "1.8.0" private val mariadbV = "2.7.4" /* @@ -100,9 +100,10 @@ object Dependencies { private val nettyV = "4.1.72.Final" private val owlApiV = "5.1.19" private val pact4sV = "0.9.0" - private val postgresV = "42.4.1" + private val postgresV = "42.4.4" private val pprintV = "0.7.3" private val rdf4jV = "3.7.1" + private val re2jV = "1.6" private val refinedV = "0.10.1" private val rhinoV = "1.7.14" @@ -499,7 +500,7 @@ object Dependencies { List("scalatest", "mysql", "mariadb", "postgresql") .map(name => "com.dimafeng" %% s"testcontainers-scala-$name" % testContainersScalaV % Test) - val blobFileSystemDependencies: List[ModuleID] = azureDependencies ++ wsmDependencies + val blobFileSystemDependencies: List[ModuleID] = azureDependencies ++ wsmDependencies ++ akkaHttpDependencies val s3FileSystemDependencies: List[ModuleID] = junitDependencies @@ -517,7 +518,8 @@ object Dependencies { val wdlDependencies: List[ModuleID] = List( "commons-io" % "commons-io" % commonsIoV, "org.scala-graph" %% "graph-core" % scalaGraphV, - "com.chuusai" %% "shapeless" % shapelessV + "com.chuusai" %% "shapeless" % shapelessV, + "com.google.re2j" % "re2j" % re2jV, ) ++ betterFilesDependencies val languageFactoryDependencies = List( @@ -591,7 +593,7 @@ object Dependencies { val servicesDependencies: List[ModuleID] = List( "com.google.api" % "gax-grpc" % googleGaxGrpcV, "org.apache.commons" % "commons-csv" % commonsCsvV, - ) ++ testDatabaseDependencies + ) ++ testDatabaseDependencies ++ akkaHttpDependencies val serverDependencies: List[ModuleID] = slf4jBindingDependencies @@ -628,7 +630,7 @@ object Dependencies { "org.lz4" % "lz4-java" % lz4JavaV ) val scalaTest = "org.scalatest" %% "scalatest" % scalatestV - + val testDependencies: List[ModuleID] = List( "org.scalatest" %% "scalatest" % scalatestV, // Use mockito Java DSL directly instead of the numerous and often hard to keep updated Scala DSLs. @@ -641,9 +643,6 @@ object Dependencies { // Version of the swagger UI to write into config files val swaggerUiVersion: String = swaggerUiV - val perfDependencies: List[ModuleID] = circeDependencies ++ betterFilesDependencies ++ commonDependencies ++ - googleApiClientDependencies ++ googleCloudDependencies - val drsLocalizerDependencies: List[ModuleID] = List( "com.google.auth" % "google-auth-library-oauth2-http" % googleOauth2V, "com.google.cloud" % "google-cloud-storage" % googleCloudStorageV, @@ -652,6 +651,7 @@ object Dependencies { "com.softwaremill.sttp" %% "circe" % sttpV, "com.github.scopt" %% "scopt" % scoptV, "org.apache.commons" % "commons-csv" % commonsCsvV, + "io.spray" %% "spray-json" % sprayJsonV, ) ++ circeDependencies ++ catsDependencies ++ slf4jBindingDependencies ++ languageFactoryDependencies ++ azureDependencies val allProjectDependencies: List[ModuleID] = @@ -672,7 +672,6 @@ object Dependencies { implDrsDependencies ++ implFtpDependencies ++ languageFactoryDependencies ++ - perfDependencies ++ serverDependencies ++ sfsBackendDependencies ++ spiDependencies ++ @@ -838,14 +837,16 @@ object Dependencies { val http4sCirce = "org.http4s" %% "http4s-circe" % http4sV val pact4sScalaTest = "io.github.jbwheatley" %% "pact4s-scalatest" % pact4sV % Test val pact4sCirce = "io.github.jbwheatley" %% "pact4s-circe" % pact4sV + val pact4sSpray = "io.github.jbwheatley" %% "pact4s-spray-json" % pact4sV val pact4sDependencies = Seq( pact4sScalaTest, pact4sCirce, + pact4sSpray, http4sEmberClient, http4sDsl, http4sEmberServer, http4sCirce, scalaTest, - ) + ) ++ akkaDependencies } diff --git a/project/GenerateRestApiDocs.scala b/project/GenerateRestApiDocs.scala index 9173a25c3c5..3e0297a09f5 100644 --- a/project/GenerateRestApiDocs.scala +++ b/project/GenerateRestApiDocs.scala @@ -65,7 +65,7 @@ object GenerateRestApiDocs { * @param content The original contents of the RESTAPI.md. * @return The contents with updated paths. */ - private def replacePaths(content: String): String = { + private def replacePaths(content: String): String = content match { case PathsRegex(start, paths, end) => val replacedPaths = paths.linesWithSeparators map { @@ -73,12 +73,13 @@ object GenerateRestApiDocs { case other => other } replacedPaths.mkString(start, "", end) - case _ => throw new IllegalArgumentException( - "Content did not match expected regex. " + - "Did the swagger2markdown format change significantly? " + - "If so, a new regex may be required.") + case _ => + throw new IllegalArgumentException( + "Content did not match expected regex. " + + "Did the swagger2markdown format change significantly? " + + "If so, a new regex may be required." + ) } - } /** * Replaces generic strings in the generated RESTAPI.md. @@ -86,9 +87,8 @@ object GenerateRestApiDocs { * @param content The contents of the RESTAPI.md. * @return The contents with generic replacements. */ - private def replaceGenerics(content: String): String = { + private def replaceGenerics(content: String): String = GenericReplacements.foldRight(content)(replaceGeneric) - } /** * Replaces a single generic string in the generated RESTAPI.md. @@ -118,12 +118,11 @@ object GenerateRestApiDocs { try { currentThread.setContextClassLoader(classUtilsClassLoader) block - } finally { + } finally currentThread.setContextClassLoader(originalThreadClassLoader) - } } - private def getModifiedMarkdown: String = { + private def getModifiedMarkdown: String = withPatchedClassLoader { val config = new Swagger2MarkupConfigBuilder() .withMarkupLanguage(MarkupLanguage.MARKDOWN) @@ -135,7 +134,6 @@ object GenerateRestApiDocs { val contents = converter.toString replaceGenerics(replacePaths(contents)) } - } /** * Generates the markdown from the swagger YAML, with some Cromwell customizations. @@ -164,6 +162,6 @@ object GenerateRestApiDocs { // Returns a settings including the `generateRestApiDocs` task. val generateRestApiDocsSettings: Seq[Setting[_]] = List( generateRestApiDocs := writeModifiedMarkdown(), - checkRestApiDocs := checkModifiedMarkdown(streams.value.log), + checkRestApiDocs := checkModifiedMarkdown(streams.value.log) ) } diff --git a/project/Merging.scala b/project/Merging.scala index 91b135e59eb..4ed133b82ac 100644 --- a/project/Merging.scala +++ b/project/Merging.scala @@ -4,23 +4,30 @@ import sbtassembly.{MergeStrategy, PathList} object Merging { val customMergeStrategy: Def.Initialize[String => MergeStrategy] = Def.setting { - case PathList(ps@_*) if Set("project.properties", "execution.interceptors").contains(ps.last) => + case PathList(ps @ _*) if Set("project.properties", "execution.interceptors").contains(ps.last) => // Merge/Filter files from AWS/Google jars that otherwise collide at merge time. MergeStrategy.filterDistinctLines - case PathList(ps@_*) if ps.last == "logback.xml" => + case PathList(ps @ _*) if ps.last == "logback.xml" => MergeStrategy.first // Merge mozilla/public-suffix-list.txt if duplicated - case PathList(ps@_*) if ps.last == "public-suffix-list.txt" => + case PathList(ps @ _*) if ps.last == "public-suffix-list.txt" => MergeStrategy.last // Merge kotlin modules if duplicated - case PathList(ps@_*) if ps.last == "kotlin-stdlib-common.kotlin_module" => + case PathList(ps @ _*) if ps.last == "kotlin-stdlib-common.kotlin_module" => MergeStrategy.last - case PathList(ps@_*) if ps.last == "kotlin-stdlib.kotlin_module" => + case PathList(ps @ _*) if ps.last == "kotlin-stdlib.kotlin_module" => MergeStrategy.last // AWS SDK v2 configuration files - can be discarded - case PathList(ps@_*) if Set("codegen.config" , "service-2.json" , "waiters-2.json" , "customization.config" , "examples-1.json" , "paginators-1.json").contains(ps.last) => + case PathList(ps @ _*) + if Set("codegen.config", + "service-2.json", + "waiters-2.json", + "customization.config", + "examples-1.json", + "paginators-1.json" + ).contains(ps.last) => MergeStrategy.discard - case x@PathList("META-INF", path@_*) => + case x @ PathList("META-INF", path @ _*) => path map { _.toLowerCase } match { @@ -51,7 +58,7 @@ object Merging { val oldStrategy = (assembly / assemblyMergeStrategy).value oldStrategy(x) } - case x@PathList("OSGI-INF", path@_*) => + case x @ PathList("OSGI-INF", path @ _*) => path map { _.toLowerCase } match { @@ -61,10 +68,11 @@ object Merging { val oldStrategy = (assembly / assemblyMergeStrategy).value oldStrategy(x) } - case "asm-license.txt" | "module-info.class" | "overview.html" | "cobertura.properties" | "grammar.hgr" | "CHANGELOG.txt" => + case "asm-license.txt" | "module-info.class" | "overview.html" | "cobertura.properties" | "grammar.hgr" | + "CHANGELOG.txt" => MergeStrategy.discard // inspired by https://github.com/ergoplatform/explorer-backend/blob/7364ecfdeabeb691f0f25525e577d6c48240c672/build.sbt#L14-L15 - case other if other.contains("scala/annotation/nowarn.class") => MergeStrategy.discard + case other if other.contains("scala/annotation/nowarn.class") => MergeStrategy.discard case other if other.contains("scala/annotation/nowarn$.class") => MergeStrategy.discard case PathList("mime.types") => MergeStrategy.last diff --git a/project/Publishing.scala b/project/Publishing.scala index 1d4143fc13e..4abc5cc3258 100644 --- a/project/Publishing.scala +++ b/project/Publishing.scala @@ -1,4 +1,4 @@ -import Version.cromwellVersion +import Version.{Debug, Release, Snapshot, Standard, cromwellVersion} import org.apache.ivy.Ivy import org.apache.ivy.core.IvyPatternHelper import org.apache.ivy.core.module.descriptor.{DefaultModuleDescriptor, MDArtifact} @@ -34,22 +34,28 @@ object Publishing { `CROMWELL_SBT_DOCKER_TAGS=dev,develop sbt 'show docker::imageNames'` returns: ArrayBuffer(broadinstitute/womtool:dev, broadinstitute/womtool:develop) ArrayBuffer(broadinstitute/cromwell:dev, broadinstitute/cromwell:develop) - */ + */ dockerTags := { - val versionsCsv = if (Version.isSnapshot) { - // Tag looks like `85-443a6fc-SNAP` - version.value - } else { - if (Version.isRelease) { - // Tags look like `85`, `85-443a6fc` - s"$cromwellVersion,${version.value}" - } else { - // Tag looks like `85-443a6fc` - version.value - } + val tags = Version.buildType match { + case Snapshot => + // Ordinary local build + // Looks like `85-443a6fc-SNAP` + Seq(version.value) + case Release => + // Looks like `85`, `85-443a6fc` + Seq(cromwellVersion, version.value) + case Debug => + // Ordinary local build with debug stuff + // Looks like `85-443a6fc-DEBUG` + Seq(version.value) + case Standard => + // Merge to `develop` + // Looks like `85-443a6fc`, `latest`, `develop` + // TODO: once we automate releases, `latest` should move to `Release` + Seq(version.value, "latest", "develop") } - // Travis applies (as of 10/22) the `dev` and `develop` tags on merge to `develop` + val versionsCsv = tags.mkString(",") sys.env.getOrElse("CROMWELL_SBT_DOCKER_TAGS", versionsCsv).split(",") }, docker / imageNames := dockerTags.value map { tag => @@ -68,6 +74,11 @@ object Publishing { add(artifact, artifactTargetPath) runRaw(s"ln -s $artifactTargetPath /app/$projectName.jar") + // Extra tools in debug mode only + if (Version.buildType == Debug) { + addInstruction(installDebugFacilities) + } + /* If you use the 'exec' form for an entry point, shell processing is not performed and environment variable substitution does not occur. Thus we have to /bin/bash here @@ -113,19 +124,57 @@ object Publishing { docker / buildOptions := BuildOptions( cache = false, removeIntermediateContainers = BuildOptions.Remove.Always - ), + ) ) - def dockerPushSettings(pushEnabled: Boolean): Seq[Setting[_]] = { + /** + * Install packages needed for debugging when shelled in to a running image. + * + * This includes: + * - The JDK, which includes tools like `jstack` not present in the JRE + * - The YourKit Java Profiler + * - Various Linux system & development utilities + * + * @return Instruction to run in the build + */ + def installDebugFacilities: Instruction = { + import sbtdocker.Instructions + + // It is optimal to use a single `Run` instruction to minimize the number of layers in the image. + // Do not be tempted to install the default JDK in the repositories, it's from Oracle. + // + // Documentation: + // - https://www.yourkit.com/docs/java-profiler/2024.3/help/docker_broker.jsp#setup + // - https://adoptium.net/installation/linux/#_deb_installation_on_debian_or_ubuntu + Instructions.Run( + """apt-get update -qq && \ + |apt-get install -qq --no-install-recommends file gpg htop jq less nload unzip vim && \ + |wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public | gpg --dearmor | \ + | tee /etc/apt/trusted.gpg.d/adoptium.gpg > /dev/null && \ + |echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | \ + | tee /etc/apt/sources.list.d/adoptium.list && \ + |apt-get update -qq && \ + |apt-get install -qq temurin-11-jdk && \ + |rm -rf /var/lib/apt/lists/* && \ + |wget -q https://www.yourkit.com/download/docker/YourKit-JavaProfiler-2024.3-docker.zip -P /tmp/docker-build-cache/ && \ + |unzip /tmp/docker-build-cache/YourKit-JavaProfiler-2024.3-docker.zip -d /tmp/docker-build-cache && \ + |mkdir -p /usr/local/YourKit-JavaProfiler-2024.3/bin/ && \ + |cp -R /tmp/docker-build-cache/YourKit-JavaProfiler-2024.3/bin/linux-x86-64/ /usr/local/YourKit-JavaProfiler-2024.3/bin/linux-x86-64/ && \ + |rm -rf /tmp/docker-build-cache + |""".stripMargin + ) + } + + def dockerPushSettings(pushEnabled: Boolean): Seq[Setting[_]] = if (pushEnabled) { List( dockerPushCheck := { val projectName = name.value val repositoryName = s"broadinstitute/$projectName" val repositoryUrl = s"https://registry.hub.docker.com/v2/repositories/$repositoryName/" - try { + try url(repositoryUrl).cat.lineStream - } catch { + catch { case exception: Exception => throw new IllegalStateException( s"""|Verify that public repository https://hub.docker.com/r/$repositoryName exists. @@ -144,7 +193,6 @@ object Publishing { } ) } - } private val broadArtifactoryResolver: Resolver = "Broad Artifactory" at @@ -170,12 +218,11 @@ object Publishing { private val artifactoryCredentialsFile = file("target/ci/resources/artifactory_credentials.properties").getAbsoluteFile - private val artifactoryCredentials: Seq[Credentials] = { + private val artifactoryCredentials: Seq[Credentials] = if (artifactoryCredentialsFile.exists) List(Credentials(artifactoryCredentialsFile)) else Nil - } // BT-250 Check if publishing will fail due to already published artifacts val checkAlreadyPublished = taskKey[Boolean]("Verifies if publishing has already occurred") @@ -183,7 +230,8 @@ object Publishing { private case class CromwellMDArtifactType(artifactType: String, artifactExtension: String, - classifierOption: Option[String]) + classifierOption: Option[String] + ) /** * The types of MDArtifacts published by this sbt build. @@ -201,18 +249,18 @@ object Publishing { /** * Retrieve the IBiblioResolver from sbt's Ivy setup. */ - private def getIBiblioResolver(ivy: Ivy): IBiblioResolver = { + private def getIBiblioResolver(ivy: Ivy): IBiblioResolver = ivy.getSettings.getResolver(broadArtifactoryResolver.name) match { case iBiblioResolver: IBiblioResolver => iBiblioResolver case other => sys.error(s"Expected an IBiblioResolver, got $other") } - } /** * Maps an sbt artifact to the Apache Ivy artifact type. */ - private def makeMDArtifact(moduleDescriptor: DefaultModuleDescriptor) - (cromwellMDArtifactType: CromwellMDArtifactType): MDArtifact = { + private def makeMDArtifact(moduleDescriptor: DefaultModuleDescriptor)( + cromwellMDArtifactType: CromwellMDArtifactType + ): MDArtifact = new MDArtifact( moduleDescriptor, moduleDescriptor.getModuleRevisionId.getName, @@ -221,7 +269,6 @@ object Publishing { null, cromwellMDArtifactType.classifierOption.map("classifier" -> _).toMap.asJava ) - } /** * Returns true and prints out an error if an artifact already exists. @@ -243,20 +290,19 @@ object Publishing { val module = ivyModule.value val log = streams.value.log - module.withModule(log) { - case (ivy, moduleDescriptor, _) => - val resolver = getIBiblioResolver(ivy) - cromwellMDArtifactTypes - .map(makeMDArtifact(moduleDescriptor)) - .map(existsMDArtifact(resolver, log)) - .exists(identity) + module.withModule(log) { case (ivy, moduleDescriptor, _) => + val resolver = getIBiblioResolver(ivy) + cromwellMDArtifactTypes + .map(makeMDArtifact(moduleDescriptor)) + .map(existsMDArtifact(resolver, log)) + .exists(identity) } }, errorIfAlreadyPublished := { if (checkAlreadyPublished.value) { sys.error( s"Some ${version.value} artifacts were already published and will need to be manually deleted. " + - "See the errors above for the list of published artifacts." + "See the errors above for the list of published artifacts." ) } } diff --git a/project/Settings.scala b/project/Settings.scala index ea775ed2bf5..628a77db4c6 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -77,12 +77,13 @@ object Settings { organization := "org.broadinstitute", scalaVersion := ScalaVersion, resolvers ++= additionalResolvers, - // Don't run tasks in parallel, especially helps in low CPU environments like Travis - Global / parallelExecution := false, + Global / parallelExecution := true, + // Seems to cause race conditions in tests, that are not pertinent to what's being tested + Test / parallelExecution := false, Global / concurrentRestrictions ++= List( - // Don't run any other tasks while running tests, especially helps in low CPU environments like Travis + // Don't run any other tasks while running tests Tags.exclusive(Tags.Test), - // Only run tests on one sub-project at a time, especially helps in low CPU environments like Travis + // Only run tests on one sub-project at a time Tags.limit(Tags.Test, 1) ), dependencyOverrides ++= cromwellDependencyOverrides, @@ -100,7 +101,6 @@ object Settings { val pact4sSettings = sharedSettings ++ List( libraryDependencies ++= pact4sDependencies, - /** * Invoking pact tests from root project (sbt "project pact" test) * will launch tests in a separate JVM context that ensures contracts @@ -110,9 +110,6 @@ object Settings { Test / fork := true ) ++ assemblySettings - lazy val pact4s = project.in(file("pact4s")) - .settings(pact4sSettings) - /* Docker instructions to install Google Cloud SDK image in docker image. It also installs `crcmod` which is needed while downloading large files using `gsutil`. diff --git a/project/Testing.scala b/project/Testing.scala index 54ddc458353..6997e31960c 100644 --- a/project/Testing.scala +++ b/project/Testing.scala @@ -62,7 +62,7 @@ object Testing { spanScaleFactor, "-W", "300", - "300", + "300" ) /** Run minnie-kenny only once per sbt invocation. */ @@ -71,7 +71,7 @@ object Testing { private var resultOption: Option[Int] = None /** Run using the logger, throwing an exception only on the first failure. */ - def runOnce(log: Logger, args: Seq[String]): Unit = { + def runOnce(log: Logger, args: Seq[String]): Unit = mutex synchronized { if (resultOption.isEmpty) { log.debug(s"Running minnie-kenny.sh${args.mkString(" ", " ", "")}") @@ -83,7 +83,6 @@ object Testing { sys.error("Running minnie-kenny.sh failed. Please double check for errors above.") } } - } } // Only run one minnie-kenny.sh at a time! @@ -116,24 +115,24 @@ object Testing { Test / test := { minnieKenny.toTask("").value (Test / test).value - }, + } ) private val integrationTestSettings = List( libraryDependencies ++= testDependencies.map(_ % IntegrationTest) ) ++ itSettings - def addTestSettings(project: Project) = { + def addTestSettings(project: Project) = project .settings(testSettings) - .configs(AllTests).settings(inConfig(AllTests)(Defaults.testTasks): _*) - .configs(CromwellBenchmarkTest).settings(inConfig(CromwellBenchmarkTest)(Defaults.testTasks): _*) - } + .configs(AllTests) + .settings(inConfig(AllTests)(Defaults.testTasks): _*) + .configs(CromwellBenchmarkTest) + .settings(inConfig(CromwellBenchmarkTest)(Defaults.testTasks): _*) - def addIntegrationTestSettings(project: Project) = { + def addIntegrationTestSettings(project: Project) = project .settings(integrationTestSettings) .configs(IntegrationTest) - } } diff --git a/project/Version.scala b/project/Version.scala index cfdd3352dc2..26b0057eb10 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -6,7 +6,20 @@ import com.github.sbt.git.SbtGit object Version { // Upcoming release, or current if we're on a master / hotfix branch - val cromwellVersion = "86" + val cromwellVersion = "87" + + sealed trait BuildType + case object Snapshot extends BuildType + case object Debug extends BuildType + case object Release extends BuildType + case object Standard extends BuildType + + def buildType: BuildType = { + if (isDebug) Debug + else if (isRelease) Release + else if (isSnapshot) Snapshot + else Standard + } /** * Returns true if this project should be considered a snapshot. @@ -14,18 +27,25 @@ object Version { * The value is read in directly from the system property `project.isSnapshot` as there were confusing issues with * the multi-project and sbt.Keys#isSnapshot(). * - * Default `true`. + * This is the default if no arguments are provided. */ - val isSnapshot: Boolean = sys.props.get("project.isSnapshot").forall(_.toBoolean) + private lazy val isSnapshot: Boolean = getPropOrDefault("project.isSnapshot", default = true) /** - * Returns `true` if this project should tag a release like `85` in addition to a hash like `85-443a6fc`. - * - * Has no effect when `isSnapshot` is `true`. + * Returns true if this project should be built in the debugging configuration. * - * Default `true`. + * Note that this image is much larger than the default build! */ - val isRelease: Boolean = sys.props.get("project.isRelease").forall(_.toBoolean) + private lazy val isDebug: Boolean = getPropOrDefault("project.isDebug") + + /** + * Returns `true` if this project should tag a release like `85` in addition to a hash like `85-443a6fc`. + */ + private lazy val isRelease: Boolean = getPropOrDefault("project.isRelease") + + private def getPropOrDefault(prop: String, default: Boolean = false): Boolean = { + sys.props.get(prop).map(_.toBoolean).getOrElse(default) + } // Adapted from SbtGit.versionWithGit def cromwellVersionWithGit: Seq[Setting[_]] = @@ -33,10 +53,10 @@ object Version { ThisBuild / git.versionProperty := "project.version", ThisBuild / git.baseVersion := cromwellVersion, ThisBuild / version := - makeVersion( - versionProperty = git.versionProperty.value, - baseVersion = git.baseVersion.?.value, - headCommit = git.gitHeadCommit.value), + makeVersion(versionProperty = git.versionProperty.value, + baseVersion = git.baseVersion.?.value, + headCommit = git.gitHeadCommit.value + ), ThisBuild / shellPrompt := { state => "%s| %s> ".format(GitCommand.prompt.apply(state), cromwellVersion) } ) @@ -74,9 +94,7 @@ object Version { List(file) } - private def makeVersion(versionProperty: String, - baseVersion: Option[String], - headCommit: Option[String]): String = { + private def makeVersion(versionProperty: String, baseVersion: Option[String], headCommit: Option[String]): String = { // The version string passed in via command line settings, if desired. def overrideVersion = Option(sys.props(versionProperty)) @@ -88,10 +106,15 @@ object Version { // Version string fallback. val unknownVersion = basePrefix + "unknown" - //Now we fall through the potential version numbers... + // Now we fall through the potential version numbers... val version = overrideVersion orElse commitVersion getOrElse unknownVersion // For now, obfuscate SNAPSHOTs from sbt's developers: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 - if (isSnapshot) s"$version-SNAP" else version + // (by calling it `SNAP` instead of `SNAPSHOT`) + buildType match { + case Snapshot => s"$version-SNAP" + case Debug => s"$version-DEBUG" + case _ => version + } } } diff --git a/project/plugins.sbt b/project/plugins.sbt index bdb54f675bf..7a0203e08ea 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,4 +3,5 @@ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.1") addSbtPlugin("com.github.sbt" % "sbt-git" % "2.0.0") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.4") addSbtPlugin("com.github.cb372" % "sbt-explicit-dependencies" % "0.2.16") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2") addDependencyTreePlugin diff --git a/runConfigurations/Repo template_ Cromwell server TES Postgres.run.xml b/runConfigurations/Repo template_ Cromwell server TES Postgres.run.xml new file mode 100644 index 00000000000..8cf24e70daa --- /dev/null +++ b/runConfigurations/Repo template_ Cromwell server TES Postgres.run.xml @@ -0,0 +1,21 @@ + + + + \ No newline at end of file diff --git a/runConfigurations/Repo template_ Cromwell server TES.run.xml b/runConfigurations/Repo template_ Cromwell server TES.run.xml index 32127027ecc..192b73735b8 100644 --- a/runConfigurations/Repo template_ Cromwell server TES.run.xml +++ b/runConfigurations/Repo template_ Cromwell server TES.run.xml @@ -19,4 +19,24 @@