diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml new file mode 100644 index 000000000..988ae1ee7 --- /dev/null +++ b/.github/workflows/pull-request.yml @@ -0,0 +1,44 @@ +name: Actions on pull requests +on: + pull_request: + branches: + - develop + - master + +jobs: + pull-request-job: + runs-on: ubuntu-latest + steps: + - name: Step 1 - Checkout repository code + uses: actions/checkout@v4 + + - name: Step 2 - Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: 'maven' + + - name: Step 3 - Cache Docker images. + uses: ScribeMD/docker-cache@0.3.7 + with: + key: docker-${{ runner.os }} + + - name: Step 4 - Build & Test + run: mvn clean verify -ntp + + - name: Step 5 - Upload coverage unittests reports to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + gcov_ignore: '!*datasafe-business*' + flags: unittests + verbose: true + + - name: Step 6 - Upload coverage e2e tests reports to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + gcov_include: '*datasafe-business*' + flags: e2e_tests + verbose: true \ No newline at end of file diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml deleted file mode 100644 index 0891234c0..000000000 --- a/.github/workflows/pull-requests.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Builds Pull-requests to key branches -name: Pull request CI - -on: - pull_request: - branches: - - develop - - master - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - java-version: 17 - distribution: 'adopt' - - - name: Build project and run default test suite - run: mvn clean verify -ntp diff --git a/.github/workflows/push-develop.yml b/.github/workflows/push-develop.yml new file mode 100644 index 000000000..14dc23fc3 --- /dev/null +++ b/.github/workflows/push-develop.yml @@ -0,0 +1,53 @@ +name: Develop branch build +on: + push: + branches: + - develop + +jobs: + develop-build-job: + runs-on: ubuntu-latest + steps: + - name: Step 1 - Checkout repository code + uses: actions/checkout@v4 + + - name: Step 2 - Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: 'maven' + + - name: Step 3 - Cache Docker images. + uses: ScribeMD/docker-cache@0.3.7 + with: + key: docker-${{ runner.os }} + + - name: Step 4 - Build & Test + run: mvn clean verify -ntp + + - name: Step 5 - Deploy Snapshot + run: ./scripts/mvn_deploy.sh + env: + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + GPG_EXECUTABLE: gpg + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + GPG_SECRET_KEY: ${{ secrets.GPG_SECRET_KEY }} + GPG_OWNERTRUST: ${{ secrets.GPG_OWNERTRUST }} + + - name: Step 6 - Upload coverage unittests reports to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + gcov_ignore: '!*datasafe-business*' + flags: unittests + verbose: true + + - name: Step 7 - Upload coverage e2e tests reports to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + gcov_include: '*datasafe-business*' + flags: e2e_tests + verbose: true \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..2ecc29728 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,37 @@ +name: Release to Maven Central +on: + push: + tags: + - v* + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Step 1 - Checkout repository code + uses: actions/checkout@v4 + + - name: Step 2 - Setup JDK + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: 'maven' + + - name: Step 3 - Cache Docker images. + uses: ScribeMD/docker-cache@0.3.7 + with: + key: docker-${{ runner.os }} + + - name: Step 4 - Maven deploy release + run: ./scripts/mvn_deploy.sh + env: + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + GPG_EXECUTABLE: gpg + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + GPG_SECRET_KEY: ${{ secrets.GPG_SECRET_KEY }} + GPG_OWNERTRUST: ${{ secrets.GPG_OWNERTRUST }} + +# - name: Step 4 - Push JavaDoc to GitHub Pages +# run: ./scripts/push-javadoc-to-gh-pages.sh diff --git a/.travis/codecov_bash.sh b/.travis/codecov_bash.sh deleted file mode 100644 index a11db3d6a..000000000 --- a/.travis/codecov_bash.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -BRANCH="$TRAVIS_BRANCH" -if [[ -n "$TRAVIS_PULL_REQUEST_BRANCH" ]]; then - BRANCH="$TRAVIS_PULL_REQUEST_BRANCH" - echo "Pull request branch identified: $TRAVIS_PULL_REQUEST_BRANCH" -fi - -echo "Sending test results to codecov using $BRANCH" -bash <(curl -s https://codecov.io/bash) -f '!*datasafe-business*' -F unittests -B "$BRANCH"; -bash <(curl -s https://codecov.io/bash) -s '*datasafe-business*' -F e2e_tests -B "$BRANCH"; \ No newline at end of file diff --git a/.travis/deploy.sh b/.travis/deploy.sh deleted file mode 100644 index 3f1052e59..000000000 --- a/.travis/deploy.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -echo "$GPG_SECRET_KEY" | base64 --decode | "$GPG_EXECUTABLE" --import -echo "$GPG_OWNERTRUST" | base64 --decode | "$GPG_EXECUTABLE" --import-ownertrust - -set -e - -mvn --settings .travis/settings.xml package gpg:sign deploy -Prelease -DskipTests -B -U; diff --git a/README.md b/README.md index 374713738..6a867c696 100644 --- a/README.md +++ b/README.md @@ -2,852 +2,85 @@ [![codecov](https://codecov.io/gh/adorsys/datasafe/branch/develop/graph/badge.svg)](https://codecov.io/gh/adorsys/datasafe) [![Maintainability](https://api.codeclimate.com/v1/badges/06ae7d4cafc3012cee85/maintainability)](https://codeclimate.com/github/adorsys/datasafe/maintainability) +# Secure, Encrypted and Versioned Data Storage Library -# Licensing model change to dual license: _AGPL v.3_ or _commercial license_ +## Overview +Datasafe is a robust library tailored for developers and enterprises, offering encrypted and versioned data storage. It enhances the security of data-sensitive applications, making it ideal for mission-critical usage. -**Attention: this open-source project will change its licensing model as of _01.01.2022_!** +### Key Features +- **Enhanced Security**: Each user gets unique key encryption keys, reducing the risk of widespread data breaches. +- **Full Confidentiality**: Offers full or partial file path encryption. +- **Ransomware Protection**: Versioned storage safeguards against ransomware attacks. +- **Secure File Exchange**: Features asynchronous inboxes for safe user-to-user file transfers. +- **Zero Trust Environments**: Ideal as a data encryption layer in critical data processing backends. -Constantly evolving and extending scope, production traffic and support in open banking world call for high maintenance -and service investments on our part. +## Technical Specifications +Datasafe uses AES-GCM (and Chacha-Poly for large files) for encryption, with CMS-envelopes ([RFC 5652](https://www.rfc-editor.org/rfc/rfc8933#RFC5652)) for encrypted content wrapping. The library's flexible design allows seamless integration and customization. For more details, refer to our [Security Whitepaper](SECURITY.WHITEPAPER.md). -Henceforth, Adorsys will offer all versions higher than v1.0.0.1 of Datasafe under a dual-license model. -Thus, this repository will be available either under Affero GNU General Public License v.3 (AGPL v.3) or -alternatively under a commercial license agreement. +### Highlights +- **Configurability**: Dagger2 for dependency injection and modular design. +- **Storage Compatibility**: Tested with Amazon S3, Minio, CEPH, and local filesystems. +- **User Privacy**: Encrypts both the document and its path in each user's private space. +- **Versioning Support**: Provides application layer versioning for systems lacking native versioning support. -We would like to thank all our users for their trust so far and are convinced that we will be able to provide an even -better service going forward. +## Getting Started -For more information, advice for your implementation project or if your use case requires more time to adapt this -change, please contact us at [sales@adorsys.com](mailto:sales@adorsys.com). -For additional details please see the section [FAQ on Licensing Change](#faq-on-licensing-change). - -# General information -Datasafe is a cross-platform library that allows sharing and storing data and documents securely. It encrypts -your data using **AES-GCM** algorithm and uses **CMS-envelopes** as encrypted content wrapper. CMS-envelope -wraps and encrypts document encryption key using key encryption key that provides additional level of security. -For user private files, Datasafe uses CMS-envelope with symmetric encryption of data encryption key. For files -that are shared with other users (sent to their INBOX folder), Datasafe uses asymmetric encryption for -data encryption key, so only recipient (or multiple recipients) can read it. - -Datasafe is built with the idea to be as configurable as possible - it uses Dagger2 for dependency injection and modular -architecture to combine everything into the business layer, so the user can override any aspect he wants - i.e. to change -encryption algorithm or to turn path encryption off. Each module is as independent as it is possible - to be used separately. - -- Each user has private space that can reside on Amazon S3, minio, filesystem or anything else with proper adapter. -In his private space, each document and its path is encrypted. -- For document sharing user has inbox space, that can be accessed from outside. Another user can write the document he -wants to share into users' inbox space using the recipients' public key so that only inbox owner can read it. -- For storage systems that do not support file versioning natively (i.e. minio) this library provides versioning -capability too. - -Details about used encryption algorithms can be found in [security whitepaper](SECURITY.WHITEPAPER.md). - -## Features - -- Proprietary software **friendly license** -- **Flexibility** - you can easily change encryption and configure or customize other aspects of library -- AES encryption using **CMS-envelopes** for increased security and interoperability with other languages -- Secure file sharing with other users -- **Extra protection layer** - encryption using securely generated keys that are completely unrelated to your password -- **Client side encryption** - you own your data -- Works with filesystem and Amazon S3 compatible storage - S3, minio, CEPH, etc. -- File names are encrypted -- Thorough testing - -## Performance - -Datasafe was tested for performance in Amazon cloud. -In short, on m5.xlarge amazon instance with Datasafe library can have write throughput of 50 MiB/s and 80 MiB/s of -read throughput, when using **Amazon S3 bucket** as backing storage (performance is CPU-bound and network-bound). - -Detailed performance report is here: -[Datasafe performance results](datasafe-long-run-tests/README.md) - -## Quick demo -### Datasafe-CLI -You can try Datasafe as a CLI (command-line-interface) executable for encryption of your own sensitive files. -Your encrypted files can be saved either in S3 bucket or local filesystem safely, because encryption will happen -locally - on your machine (See [CLI-README](datasafe-cli/README.md) for details). - -**Download CLI executable**: - -1. [MacOS native executable](https://github.com/adorsys/datasafe/releases/download/v0.7.0/datasafe-cli-osx-x64) -1. [Linux native executable](https://github.com/adorsys/datasafe/releases/download/v0.7.0/datasafe-cli-linux-x64) -1. Windows executable (N/A yet), please use java version below -1. [Java-based jar](https://github.com/adorsys/datasafe/releases/download/v0.7.0/datasafe-cli.jar), requires JRE (1.8+), use `java -jar datasafe-cli.jar` to execute - -#### Example actions: -##### Download application and create new user: - -
New profile animation transcript - -- Download CLI application (MacOS url) - -```bash -curl -L https://github.com/adorsys/datasafe/releases/download/v0.6.0/datasafe-cli-osx-x64 > datasafe-cli && chmod +x datasafe-cli -``` -- Create file with your credentials (they also can be passed through commandline) - -```bash -echo '{"username": "john", "password": "Doe", "systemPassword": "password"}' > john.credentials -``` -- Create your new user profile (credentials come from john.credentials). You can enter value or click enter to accept -the default value when prompted. - -```bash -./datasafe-cli -c john.credentials profile create -``` -
- -![new_profile](docs/demo/new_profile.gif) - -**Note**: Instead of creating file with credentials you can provide credentials directly into terminal (this is less -secure than having credentials file, but is fine for demo purposes): -```bash -./datasafe-cli -u=MeHappyUser -p=MyCoolPassword -sp=greatSystemPassword private cat secret.txt -``` -Command above will show private file `secret.txt` content for user `MeHappyUser` who has password `MyCoolPassword` and -system password `greatSystemPassword` - -##### Encrypt and decrypt some secret data for our user: - -
Encrypting/decrypting data animation transcript - -- Create some unencrypted content - -```bash -echo "Hello world" > unencrypted.txt -``` -- Encrypt and store file from above in privatespace. In privatespace it will have decrypted name `secret.txt` -```bash -./datasafe-cli -c john.credentials private cp unencrypted.txt secret.txt -``` -- Show that filename is encrypted in privatespace: - -```bash -ls private -``` - -- Show that file content is encrypted too: - -```bash -cat private/encrypted_file_name_from_above -``` - -- Decrypt file content: - -```bash -./datasafe-cli -c john.credentials private cat secret.txt -``` -
- -![encrypt_decrypt_file](docs/demo/encrypt_decrypt_file.gif) - -##### You can always list available actions in context: - -
List actions animation transcript - -- Show top-level commands - -```bash -./datasafe-cli -c john.credentials -``` - -- Show commands for privatespace - -```bash -./datasafe-cli -c john.credentials private -``` -
- -![list_actions](docs/demo/list_actions.gif) - -### REST based demo -[Here](datasafe-rest-impl/DEMO.md) you can find quick docker-based demo of project capabilities with -instructions of how to use it (REST-api based to show how to deploy as encryption server). - - -## Building project +### Building the Project Without tests: ```bash mvn clean install -DskipTests=true ``` -Full build: + +Full Build: ```bash mvn clean install ``` -## Adding to your project +### Adding Datasafe to Your Project +Include Datasafe in your Maven project: -Datasafe is available from maven-central repository, you can add it to your project using: ```xml + de.adorsys datasafe-business - 0.5.0 + {datasafe.version} -``` -To add filesystem storage provider: -```xml + de.adorsys - datasafe-storage-impl-fs - 0.5.0 + datasafe-storage-impl-s3 + {datasafe.version} -``` -To add S3 storage provider: -```xml + de.adorsys - datasafe-storage-impl-s3 - 0.5.0 + datasafe-storage-impl-fs + {datasafe.version} + test ``` +## In-depth Insights +- **Performance Analysis**: Tested on AWS, Datasafe achieves up to 50 MiB/s write and 80 MiB/s read throughput with Amazon S3. [See full report](datasafe-long-run-tests/README.md). +- **Quick Demo**: Explore Datasafe's capabilities through our [quick demo](./docs/readme/Demo.md). +- **Deployment Models**: Followings are among others [possible deployment models](./docs/readme/DeploymentModels.md). +- **How It Works**: Get a look at [detailed integration information](./docs/readme/HowItWorks.md). -# Project overview -In short, Datasafe [core logic](datasafe-business/src/main/java/de/adorsys/datasafe/business/impl/service/DefaultDatasafeServices.java) -provides these key services: -* [Privatespace service](datasafe-privatestore/datasafe-privatestore-impl/src/main/java/de/adorsys/datasafe/privatestore/impl/PrivateSpaceServiceImpl.java) -that securely stores private files by encrypting them using users' secret key. -* [Inbox service](datasafe-inbox/datasafe-inbox-impl/src/main/java/de/adorsys/datasafe/inbox/impl/InboxServiceImpl.java) -that allows a user to share files with someone so that the only inbox owner can read files that are -shared with him using private key. -* [User profile service](datasafe-directory/datasafe-directory-impl/src/main/java/de/adorsys/datasafe/directory/impl/profile/operations/DFSBasedProfileStorageImpl.java) -that provides user metadata, such as where is user privatespace, his keystore, etc. - -These services are automatically built from -[modules](datasafe-business/src/main/java/de/adorsys/datasafe/business/impl) -and the only thing needed from a user is to provide storage adapter - by using -[predefined](datasafe-storage) adapters, -or by implementing his own using -[this interface](datasafe-storage/datasafe-storage-api/src/main/java/de/adorsys/datasafe/storage/api/StorageService.java). - -These services have interfaces that resemble actions that you can do with file or folder on your local file system - -list,write,read,delete file or folder. So, one can think that Datasafe provides mount-points for -inbox and private space virtual folders - you get similar actions available from Datasafe service. - -Additionally, for file versioning purposes like reading only last file version, there is [versioned privatespace](datasafe-business/src/main/java/de/adorsys/datasafe/business/impl/service/VersionedDatasafeServices.java) -that supports versioned and encrypted private file storage (for storage providers that do not support versioning). - -# How it works - -## Library modules -![Modules map](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/modules_map.puml&fmt=svg&vvv=1&sanitize=true) - -## Users' files - where are they? - -Whenever user wants to store or read file at some location - be it inbox or his private space, following things do happen: -1. System resolves his profile location -1. His profile is read from some storage (and typically cached, then direct cache access happens) -1. Based on his profile content, root folder where data should be read/written is deduced -1. If data is going to private space - request path is encrypted -1. Root path is prepended to request path -1. Encryption/decryption of data happens -1. Credentials required to access the storage are added ([BucketAccessService](datasafe-directory/datasafe-directory-api/src/main/java/de/adorsys/datasafe/directory/api/profile/dfs/BucketAccessService.java)) -1. Data stream with path is sent to storage adapter -1. Optionally, storage adapter analyzes based on protocol which storage service to use -1. Storage adapter stores the data - -This diagram shows path resolution flow for private space with more details. It is mostly same both for private and -inbox files, with the only difference that private files have relative path (relative to private space location) -additionally encrypted. - -![Path resolution](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/profiles/locate_profile.puml&fmt=svg&vvv=2&sanitize=true) - -## Storing private files - -Private files are always encrypted using users' secret symmetric key. Additionally their path is encrypted too, but -this encryption is very special in the sense that it has form of a/b/c encrypted as -encrypted(a)/encrypted(b)/encrypted(c), so that folder traversal operations are efficient. - -![How privatespace diagram](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/high-level/how_it_works_private.puml&fmt=svg&vvv=2&sanitize=true) - -| Reading files from private space | Writing files to private space | -|---|---| -| ![Read modules](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/high-level/how_it_works_private_read_modules.puml&fmt=svg&vvv=1&sanitize=true) |![Write modules](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/high-level/how_it_works_private_write_modules.puml&fmt=svg&vvv=1&sanitize=true) | - -[Details](datasafe-privatestore) - -## Sharing files with another user - -Shared files are protected using asymmetrical cryptography, so that sender encrypts file with recipients' public key -and only recipient can read it using his private key. Paths are kept unencrypted for inbox. - -![How inbox diagram](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/high-level/how_it_works_inbox.puml&fmt=svg&vvv=1&sanitize=true) - -| Reading files from inbox | Writing files to inbox | -|---|---| -| ![Read modules](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/high-level/how_it_works_inbox_read_modules.puml&fmt=svg&vvv=1&sanitize=true) |![Write modules](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/high-level/how_it_works_inbox_write_modules.puml&fmt=svg&vvv=1&sanitize=true) | - -[Details](datasafe-inbox) - -# Examples of how to use the library - - -## Generic Datasafe usage -First, you want to create Datasafe services. This snippet provides you Datasafe that uses filesystem storage adapter: -[Example:Create Datasafe services](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L46-L52) -```groovy -// this will create all Datasafe files and user documents under -defaultDatasafeServices = DaggerDefaultDatasafeServices.builder() - .config(new DefaultDFSConfig(root.toAbsolutePath().toUri(), "secret"::toCharArray)) - .storage(new FileSystemStorageService(root)) - .build(); -``` - -Second you want to add new users: -[Example:Create new user](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L60-L67) -```groovy -// Creating new user with username 'user' and private/secret key password 'passwrd': -/* -IMPORTANT: For cases when user profile is stored on S3 without object locks, this requires some global -synchronization due to eventual consistency or you need to supply globally unique username on registration -*/ -defaultDatasafeServices.userProfile().registerUsingDefaults(new UserIDAuth("user", "passwrd"::toCharArray)); -``` - -After you have a user, he wants to store some data or document securely in his privatespace: -[Example:Store file in privatespace](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L78-L88) -```groovy -// creating new user -UserIDAuth user = registerUser("john"); - -// writing string "Hello" to my/own/file.txt: -// note that both resulting file content and its path are encrypted: -try (OutputStream os = defaultDatasafeServices.privateService() - .write(WriteRequest.forDefaultPrivate(user, "my/own/file.txt"))) { - os.write("Hello".getBytes(StandardCharsets.UTF_8)); -} -``` - -Now user wants to read again his secured file: -[Example:Read file from privatespace](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L99-L112) -```groovy -// creating new user -UserIDAuth user = registerUser("jane"); - -// writing string "Hello Jane" to my/secret.txt into users' Jane privatespace: -writeToPrivate(user, "my/secret.txt", "Hello Jane"); - -byte[] helloJane; -// reading encrypted data from my/secret.txt, note that path is also encrypted -try (InputStream is = defaultDatasafeServices.privateService() - .read(ReadRequest.forDefaultPrivate(user, "my/secret.txt"))) { - helloJane = ByteStreams.toByteArray(is); -} -``` - -But he doesn't remember the name of file he stored, so he will list all files in privatespace and read first: -[Example:Read file from privatespace using list](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L246-L260) -```groovy -// creating new user -UserIDAuth user = registerUser("john"); - -// let's create 1 file: -writeToPrivate(user, "home/my/secret.txt", "secret"); - -List> johnsPrivateFilesInMy = defaultDatasafeServices.privateService() - .list(ListRequest.forDefaultPrivate(user, "home/my")).collect(Collectors.toList()); - -// we have successfully read that file -assertThat(defaultDatasafeServices.privateService().read( - ReadRequest.forPrivate(user, johnsPrivateFilesInMy.get(0).getResource().asPrivate())) -).hasContent("secret"); -``` - -Now he wants to share some data with another user: -[Example:Send file to INBOX](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L124-L134) -```groovy -// create Jane, so her INBOX does exist -UserIDAuth jane = registerUser("jane"); -UserID janeUsername = new UserID("jane"); - -// We send message "Hello John" to John just by his username -try (OutputStream os = defaultDatasafeServices.inboxService() - .write(WriteRequest.forDefaultPublic(Collections.singleton(janeUsername), "hello.txt"))) { - os.write("Hello Jane".getBytes(StandardCharsets.UTF_8)); -} -``` - -Now he wants to share some data with couple of users, so that it will be encrypted once and they both -could read the file using each using own private key: -[Example:Send file to INBOX - multiple users](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L146-L158) -```groovy -// create Jane, so her INBOX does exist -UserIDAuth jane = registerUser("jane"); -// create Jamie, so his INBOX does exist -UserIDAuth jamie = registerUser("jamie"); - -// We send message to both users by using their username: -try (OutputStream os = defaultDatasafeServices.inboxService().write( - WriteRequest.forDefaultPublic(ImmutableSet.of(jane.getUserID(), jamie.getUserID()), "hello.txt")) -) { - os.write("Hello Jane and Jamie".getBytes(StandardCharsets.UTF_8)); -} -``` - -And finally it is time to read data that was shared with you: -[Example:Read file from INBOX](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L268-L284) -```groovy -// creating new user -UserIDAuth user = registerUser("john"); -UserID johnUsername = new UserID("john"); - -// let's create 1 file: -shareMessage(johnUsername, "home/my/shared.txt", "shared message"); - -// Lets list our INBOX -List> johnsInboxFilesInMy = defaultDatasafeServices.inboxService() - .list(ListRequest.forDefaultPrivate(user, "")).collect(Collectors.toList()); - -// we have successfully read that file -assertThat(defaultDatasafeServices.inboxService().read( - ReadRequest.forPrivate(user, johnsInboxFilesInMy.get(0).getResource().asPrivate())) -).hasContent("shared message"); -``` - -## Datasafe with file versioning -Suppose we need to preserve file history, so accidental file removal won't destroy everything. In such case -we can use storage provider that supports versioning. But if we have storage provider does not support versions -(i.e. minio) we can turn-on software versioning, here is its usage examples; - -First, we will obtain versioned Datasafe services that uses filesystem storage adapter: -[Example:Create versioned Datasafe services](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithVersionedDatasafeTest.java#L47-L53) -```groovy -// this will create all Datasafe files and user documents under -versionedServices = DaggerVersionedDatasafeServices.builder() - .config(new DefaultDFSConfig(root.toAbsolutePath().toUri(), "secret"::toCharArray)) - .storage(new FileSystemStorageService(root)) - .build(); -``` - -Next we will create user, this is same as in non-versioned services: -[Example:Creating user for versioned services looks same](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithVersionedDatasafeTest.java#L61-L68) -```groovy -// Creating new user: -/* -IMPORTANT: For cases when user profile is stored on S3 without object locks, this requires some global -synchronization due to eventual consistency or you need to supply globally unique username on registration -*/ -versionedServices.userProfile().registerUsingDefaults(new UserIDAuth("user", "passwrd"::toCharArray)); -``` - -This is how file versioning works when saving file multiple times: -[Example:Saving file couple of times - versioned](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithVersionedDatasafeTest.java#L80-L104) -```groovy -// creating new user -UserIDAuth user = registerUser("john"); - -// writing string "Hello " + index to my/own/file.txt 3 times: -// note that both resulting file content and its path are encrypted: -for (int i = 1; i <= 3; ++i) { - try (OutputStream os = versionedServices.latestPrivate() - .write(WriteRequest.forDefaultPrivate(user, "my/own/file.txt"))) { - os.write(("Hello " + i).getBytes(StandardCharsets.UTF_8)); - Thread.sleep(1000L); // this will change file modified dates - } -} - -// and still we read only latest file -assertThat(versionedServices.latestPrivate() - .read(ReadRequest.forDefaultPrivate(user, "my/own/file.txt")) -).hasContent("Hello 3"); -// but there are 3 versions of file stored physically in users' privatespace: -assertThat(versionedServices.privateService().list( - ListRequest.forDefaultPrivate(user, "my/own/file.txt")) -).hasSize(3); -// and still only one file visible on latest view -assertThat(versionedServices.latestPrivate().list(ListRequest.forDefaultPrivate(user, ""))).hasSize(1); -``` - -And we can work with file versions too, of course, everything is encrypted: -[Example:Lets check how to read oldest file version](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithVersionedDatasafeTest.java#L106-L122) -```groovy -// so lets collect all versions -List, PrivateResource, DFSVersion>> withVersions = - versionedServices.versionInfo().versionsOf( - ListRequest.forDefaultPrivate(user, "my/own/file.txt") - ).collect(Collectors.toList()); -// so that we can find oldest -Versioned, PrivateResource, DFSVersion> oldest = - withVersions.stream() - .sorted(Comparator.comparing(it -> it.absolute().getResource().getModifiedAt())) - .collect(Collectors.toList()) - .get(0); -// and read oldest content -assertThat(versionedServices.privateService() - .read(ReadRequest.forPrivate(user, oldest.absolute().getResource().asPrivate())) -).hasContent("Hello 1"); -``` - -Another important case to mention is how to determine if file has changed on storage compared to some copy we have: -[Example:Check if we have latest file locally](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithVersionedDatasafeTest.java#L132-L163) -```groovy -// creating new user -UserIDAuth user = registerUser("john"); - -// First lets store some file, for example John stored it from mobile phone -try (OutputStream os = versionedServices.latestPrivate() - .write(WriteRequest.forDefaultPrivate(user, "my/own/file.txt"))) { - os.write(("Hello old version").getBytes(StandardCharsets.UTF_8)); -} - -// Application on mobile phone caches file content to improve performance, so it should cache timestamp too -Instant savedOnMobile = versionedServices.latestPrivate() - .list(ListRequest.forDefaultPrivate(user, "my/own/file.txt")) - .findAny().get().getResource().getModifiedAt(); - -// Now John uses PC to write data to my/own/file.txt with some updated data -Thread.sleep(1000L); // it took some time for him to get to PC -try (OutputStream os = versionedServices.latestPrivate() - .write(WriteRequest.forDefaultPrivate(user, "my/own/file.txt"))) { - os.write(("Hello new version").getBytes(StandardCharsets.UTF_8)); -} - -// John takes his mobile phone and application checks if it needs to sync content -Instant savedOnPC = versionedServices.latestPrivate() - .list(ListRequest.forDefaultPrivate(user, "my/own/file.txt")) - .findAny().get().getResource().getModifiedAt(); - -// This indicates that we need to update our cache on mobile phone -// Modified date of saved file has changed and it is newer that our cached date -// So mobile application should download latest file version -assertThat(savedOnPC).isAfter(savedOnMobile); -``` - -## Datasafe on versioned storage -If you have storage for user files on **versioned S3 bucket** and want to get object version when you write -object or to read some older version encrypted object, you can follow this example of how to do that: -[Example:Versioned storage support - writing file and reading back](datasafe-examples/datasafe-examples-versioned-s3/src/test/java/de/adorsys/datasafe/examples/business/s3/BaseUserOperationsWithDefaultDatasafeOnVersionedStorageTest.java#L138-L172) -```groovy -// creating new user -UserIDAuth user = registerUser("john"); - -// writing data to my/own/file.txt 3 times with different content: -// 1st time, writing into my/own/file.txt: -// Expanded snippet of how to capture file version when writing object: -AtomicReference version = new AtomicReference<>(); -try (OutputStream os = defaultDatasafeServices.privateService() - .write(WriteRequest.forDefaultPrivate(user, MY_OWN_FILE_TXT) - .toBuilder() - .callback((PhysicalVersionCallback) version::set) - .build()) -) { - // Initial version will contain "Hello 1": - os.write("Hello 1".getBytes(StandardCharsets.UTF_8)); -} -// this variable has our initial file version: -String version1 = version.get(); - -// Write 2 more times different data to same file - my/own/file.txt: -String version2 = writeToPrivate(user, MY_OWN_FILE_TXT, "Hello 2"); -// Last version will contain "Hello 3": -String version3 = writeToPrivate(user, MY_OWN_FILE_TXT, "Hello 3"); - -// now, when we read file without specifying version - we see latest file content: -assertThat(defaultDatasafeServices.privateService().read( - ReadRequest.forDefaultPrivate(user, MY_OWN_FILE_TXT)) -).hasContent("Hello 3"); - -// but if we specify file version - we get content for it: -assertThat(defaultDatasafeServices.privateService().read( - ReadRequest.forDefaultPrivateWithVersion(user, MY_OWN_FILE_TXT, new StorageVersion(version1))) -).hasContent("Hello 1"); -``` -Removing old file version can be done by [bucket policy](https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#non-current-days-calculations) -or manually, using this snippet: -[Example:Versioned storage support - removing specific version](datasafe-examples/datasafe-examples-versioned-s3/src/test/java/de/adorsys/datasafe/examples/business/s3/BaseUserOperationsWithDefaultDatasafeOnVersionedStorageTest.java#L188-L215) -```groovy -// creating new user -UserIDAuth user = registerUser("john"); - -// writing data to my/own/file.txt 2 times with different content: -String versionId = writeToPrivate(user, MY_OWN_FILE_TXT, "Hello 1"); -writeToPrivate(user, MY_OWN_FILE_TXT, "Hello 2"); - -// now, we read old file version -assertThat(defaultDatasafeServices.privateService().read( - ReadRequest.forDefaultPrivateWithVersion(user, MY_OWN_FILE_TXT, new StorageVersion(versionId))) -).hasContent("Hello 1"); - -// now, we remove old file version -defaultDatasafeServices.privateService().remove( - RemoveRequest.forDefaultPrivateWithVersion(user, MY_OWN_FILE_TXT, new StorageVersion(versionId)) -); - -// it is removed from storage, so when we read it we get exception -assertThrows(AmazonS3Exception.class, () -> defaultDatasafeServices.privateService().read( - ReadRequest.forDefaultPrivateWithVersion(user, MY_OWN_FILE_TXT, new StorageVersion(versionId))) -); - -// but latest file version is still available -assertThat(defaultDatasafeServices.privateService().read( - ReadRequest.forDefaultPrivate(user, MY_OWN_FILE_TXT)) -).hasContent("Hello 2"); -``` - -## Overriding Datasafe functionality -Whenever you want to have some custom functionality of Datasafe, instead of default ones, there are -two possible ways to achieve this: -- using OverridesRegistry without project recompilation. -- using Dagger2 to build a customized version of Datasafe. - -### Overriding functionality without recompilation -This approach is for classes annotated with -[@RuntimeDelegate](datasafe-types-api/src/main/java/de/adorsys/datasafe/types/api/context/annotations/RuntimeDelegate.java) -and it works by putting the custom implementation of a class to be overridden into -[OverridesRegistry](datasafe-types-api/src/main/java/de/adorsys/datasafe/types/api/context/overrides/OverridesRegistry.java). -During runtime, when accessing desired functionality, the library will look into OverridesRegistry for -custom class implementation and use it if present. This one has the advantage of not requiring recompilation of -Datasafe library, but has a limitation of working on static dependency graph - you can't rebuild it. -[Example:Create overridable Datasafe services without recompilation](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/RuntimeOverrideOperationsTest.java#L31-L53) -```groovy -// This shows how to override path encryption service, in particular we are going to disable it -OverridesRegistry registry = new BaseOverridesRegistry(); - -// PathEncryptionImpl now will have completely different functionality -// instead of calling PathEncryptionImpl methods we will call PathEncryptionImplOverridden methods -PathEncryptionImplRuntimeDelegatable.overrideWith(registry, PathEncryptionImplOverridden::new); - -// Customized service, without creating complete module and building it: -DefaultDatasafeServices datasafeServices = DaggerDefaultDatasafeServices.builder() - .config(new DefaultDFSConfig(root.toAbsolutePath().toUri(), "secret"::toCharArray)) - .storage(new FileSystemStorageService(root)) - .overridesRegistry(registry) - .build(); - -// registering user -UserIDAuth user = new UserIDAuth("user", "passwrd"::toCharArray); -datasafeServices.userProfile().registerUsingDefaults(user); -// writing into user privatespace, note that with default implementation `file.txt` would be encrypted -datasafeServices.privateService().write(WriteRequest.forDefaultPrivate(user, "file.txt")); -// but we see raw filename here: -assertThat(Files.walk(root)).asString().contains("file.txt"); -``` - -### Overriding functionality by building custom Datasafe library -This is actually the preferred way to override something or to customize Datasafe. It has no limitations because -you can compose any Datasafe service you want using Dagger2 for dependency injection. Its major drawback is that -you need to add a dependency to Dagger2 into your project and compile this custom library version. Because of -compile-time dependency injection and modular structure it is a comparatively error-free approach. -To create custom Datasafe service we need to follow these 3 steps: -1. Create your own custom module (or modules) - see [CustomPathEncryptionModule](datasafe-examples/datasafe-examples-customize-dagger/src/main/java/de/adorsys/datasafe/examples/business/filesystem/CustomPathEncryptionModule.java) -1. Create custom Datasafe with custom module list - see [CustomlyBuiltDatasafeServices](datasafe-examples/datasafe-examples-customize-dagger/src/main/java/de/adorsys/datasafe/examples/business/filesystem/CustomlyBuiltDatasafeServices.java) -1. Use custom-built Datasafe as shown here: -[Example:Create custom-built Datasafe service](datasafe-examples/datasafe-examples-customize-dagger/src/test/java/de/adorsys/datasafe/examples/business/filesystem/CustomlyBuiltDatasafeServiceTest.java#L25-L39) -```groovy -// Customized service, we create required module using compile time DI provided by Dagger: -CustomlyBuiltDatasafeServices datasafeServices = DaggerCustomlyBuiltDatasafeServices.builder() - .config(new DefaultDFSConfig(root.toAbsolutePath().toUri(), "secret"::toCharArray)) - .storage(new FileSystemStorageService(root)) - .build(); - -// registering user -UserIDAuth user = new UserIDAuth("user", "password"::toCharArray); -datasafeServices.userProfile().registerUsingDefaults(user); -// writing into user privatespace, note that with default implementation `file.txt` would be encrypted -datasafeServices.privateService().write(WriteRequest.forDefaultPrivate(user, "file.txt")); -// but we see raw filename here: -assertThat(walk(root)).asString().contains("file.txt"); -``` - -### Customizing Datasafe to store dynamic and user-provided credentials -In case user wants to register storage credentials himself or place keystore within credentials-protected -location one can use this example: -[Example:Datasafe with multi-dfs setup](datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleTest.java#L106-L220) -```groovy -String directoryBucketS3Uri = "s3://" + DIRECTORY_BUCKET.getBucketName() + "/"; -// static client that will be used to access `directory` bucket: -StorageService directoryStorage = new S3StorageService( - directoryClient, - DIRECTORY_BUCKET.getBucketName(), - EXECUTOR -); - -OverridesRegistry registry = new BaseOverridesRegistry(); -DefaultDatasafeServices multiDfsDatasafe = DaggerDefaultDatasafeServices - .builder() - .config(new DFSConfigWithStorageCreds(directoryBucketS3Uri, "PAZZWORT"::toCharArray)) - // This storage service will route requests to proper bucket based on URI content: - // URI with directoryBucket to `directoryStorage` - // URI with filesBucketOne will get dynamically generated S3Storage - // URI with filesBucketTwo will get dynamically generated S3Storage - .storage( - new RegexDelegatingStorage( - ImmutableMap.builder() - // bind URI that contains `directoryBucket` to directoryStorage - .put(Pattern.compile(directoryBucketS3Uri + ".+"), directoryStorage) - .put( - Pattern.compile(getDockerUri("http://127.0.0.1") + ".+"), - // Dynamically creates S3 client with bucket name equal to host value - new UriBasedAuthStorageService( - acc -> new S3StorageService( - S3ClientFactory.getClient( - acc.getEndpoint(), - acc.getRegion(), - acc.getAccessKey(), - acc.getSecretKey() - ), - // Bucket name is encoded in first path segment - acc.getBucketName(), - EXECUTOR - ) - ) - ).build() - ) - ) - .overridesRegistry(registry) - .build(); -// Instead of default BucketAccessService we will use service that reads storage access credentials from -// keystore -BucketAccessServiceImplRuntimeDelegatable.overrideWith( - registry, args -> new WithCredentialProvider(args.getStorageKeyStoreOperations()) -); - -// John will have all his private files stored on `filesBucketOne` and `filesBucketOne`. -// Depending on path of file - filesBucketOne or filesBucketTwo - requests will be routed to proper bucket. -// I.e. path filesBucketOne/path/to/file will end up in `filesBucketOne` with key path/to/file -// his profile and access credentials for `filesBucketOne` will be in `configBucket` -UserIDAuth john = new UserIDAuth("john", "secret"::toCharArray); -// Here, nothing expects John has own storage credentials: -multiDfsDatasafe.userProfile().registerUsingDefaults(john); - -// Tell system that John will use his own storage credentials - regex match: -StorageIdentifier bucketOne = new StorageIdentifier(endpointsByHost.get(FILES_BUCKET_ONE) + ".+"); -StorageIdentifier bucketTwo = new StorageIdentifier(endpointsByHost.get(FILES_BUCKET_TWO) + ".+"); -// Set location for John's credentials keystore and put storage credentials into it: -UserPrivateProfile profile = multiDfsDatasafe.userProfile().privateProfile(john); -profile.getPrivateStorage().put( - bucketOne, - new AbsoluteLocation<>(BasePrivateResource.forPrivate(endpointsByHost.get(FILES_BUCKET_ONE) + "/")) -); -profile.getPrivateStorage().put( - bucketTwo, - new AbsoluteLocation<>(BasePrivateResource.forPrivate(endpointsByHost.get(FILES_BUCKET_TWO) + "/")) -); -multiDfsDatasafe.userProfile().updatePrivateProfile(john, profile); - -// register John's DFS access for `filesBucketOne` minio bucket -multiDfsDatasafe.userProfile().registerStorageCredentials( - john, - bucketOne, - new StorageCredentials( - FILES_BUCKET_ONE.getAccessKey(), - FILES_BUCKET_ONE.getSecretKey() - ) -); -// register John's DFS access for `filesBucketTwo` minio bucket -multiDfsDatasafe.userProfile().registerStorageCredentials( - john, - bucketTwo, - new StorageCredentials( - FILES_BUCKET_TWO.getAccessKey(), - FILES_BUCKET_TWO.getSecretKey() - ) -); - -// Configuring multi-storage is done, user can use his multi-storage: - -// store this file on `filesBucketOne` -try (OutputStream os = multiDfsDatasafe.privateService() - .write(WriteRequest.forPrivate(john, bucketOne, "my/file.txt"))) { - os.write("Content on bucket number ONE".getBytes(StandardCharsets.UTF_8)); -} - -// store this file on `filesBucketTwo` -try (OutputStream os = multiDfsDatasafe.privateService() - .write(WriteRequest.forPrivate(john, bucketTwo, "my/file.txt"))) { - os.write("Content on bucket number TWO".getBytes(StandardCharsets.UTF_8)); -} - -// read file from `filesBucketOne` -assertThat(multiDfsDatasafe.privateService() - .read(ReadRequest.forPrivate(john, bucketOne, "my/file.txt")) -).hasContent("Content on bucket number ONE"); - -// read file from `filesBucketTwo` -assertThat(multiDfsDatasafe.privateService() - .read(ReadRequest.forPrivate(john, bucketTwo, "my/file.txt")) -).hasContent("Content on bucket number TWO"); -``` - -You can visit the **[project homepage](https://adorsys.github.io/datasafe)** for additional information. - -# JavaDoc -You can read JavaDoc [here](https://adorsys.github.io/datasafe/javadoc/latest/index.html) - -# Contributing -* [CodingRules](docs/codingrules/CodingRules.md) -* [Branching and commiting](docs/branching/branch-and-commit.md) -* [Deployment to maven central](docs/general/deployment_maven_central.md) - - -# FAQ on Licensing Change - -## What is a dual-licensing model? -Under a dual-licensing model, our product is available under two licenses -- [The Affero GNU General Public License v3 (AGPL v3)](https://www.gnu.org/licenses/agpl-3.0.en.html) -- A proprietary commercial license - -If you are a developer or business that would like to review our products in detail, test and implement in your -open-source projects and share the changes back to the community, the product repository is freely available under AGPL v3. - -If you are a business that would like to implement our products in a commercial setting and would like to protect your -individual changes, we offer the option to license our products under a commercial license. - -This change will still allow free access and ensure openness under AGPL v3 but with assurance of committing any -alterations or extensions back to the project and preventing redistribution of such implementations under commercial license. - -## Will there be any differences between the open-source and commercially licensed versions of your products? -Our public release frequency will be reduced as our focus shifts towards the continuous maintenance of the commercial version. -Nevertheless, we are committed to also provide open-source releases of our products on a regular basis as per our release policy. - -For customers with a commercial license, we will offer new intermediate releases in a more frequent pace. - -## Does this mean that this product is no longer open source? -No, the product will still be published and available on GitHub under an OSI-approved open-source license (AGPL v3). - -## What about adorsys’ commitment to open source? Will adorsys provide future product releases on GitHub? -We at adorsys are committed to continue actively participating in the open-source community. Our products remain -licensed under OSI-approved open-source licenses, and we are looking forward to expanding our product portfolio on GitHub even further. - -## How does the change impact me if I already use the open-source edition of your product? -All currently published versions until v1.0.0.1 will remain under their current Apache 2.0 license and its respective -requirements and you may continue using it as-is. To upgrade to future versions, you will be required to either abide -by the requirements of AGPL v3, including documenting and sharing your implemented changes to the product when distributing, -or alternatively approach us to obtain a commercial license. - -## What if I cannot adjust to the new licensing model until 01.01.2022? Can I extend the deadline? -We understand that adjustment to licensing changes can take time and therefore are open to discuss extension options -on an individual basis. For inquiries please contact us as [psd2@adorsys.com](mailto:sales@adorsys.com). - -## Which versions of the product are affected? -All versions of Datasafe after v1.0.0.1 will be affected by the licensing changes and move to a dual-licensing model. - -## What will happen to older, Apache 2.0 licensed product versions? -All older Apache 2.0 licensed versions prior and including v1.0.0.1 will remain available under their existing license. +## Additional Resources +- **Project Homepage**: Visit [adorsys.github.io/datasafe](https://adorsys.github.io/datasafe) for more information. +- **JavaDoc**: Access our detailed [JavaDoc here](https://adorsys.github.io/datasafe/javadoc/latest/index.html). -## What open-source products from Adorsys are affected by the licensing change? -The following products are affected: -- [XS2A Core](https://github.com/adorsys/xs2a) -- [XS2A Sandbox & ModelBank](https://github.com/adorsys/XS2A-Sandbox) -- [Open Banking Gateway](https://github.com/adorsys/open-banking-gateway) incl. [XS2A Adapters](https://github.com/adorsys/xs2a-adapter) -- [SmartAnalytics](https://github.com/adorsys/smartanalytics) -- [Datasafe](https://github.com/adorsys/datasafe) +## Contributing to Datasafe +- **Contributor License**: See [Developer Certificate of Origin (DCO) Enforcement](https://github.com/adorsys/datasafe/discussions/253) +- **Coding Guidelines**: [CodingRules](docs/codingrules/CodingRules.md) +- **Branching and Committing Practices**: [Branching Guide](docs/branching/branch-and-commit.md) +- **Deployment Process**: [Maven Central Deployment](docs/general/deployment_maven_central.md) +- **Feature Requests and Comments**: [Open a Discussion Ticket](https://github.com/adorsys/datasafe/discussions) -## I’m using one of these products indirectly via some software integrator. How does the licensing change affect me? -The licensing change does not affect you as user, but it is relevant to your provider who has used our product in their -solution implementation. In case of uncertainty please contact your service provider or approach us at [sales@adorsys.com](mailto:sales@adorsys.com). +## Dual-Licensing +Datasafe is available under two licenses: +- **Open-Source Projects**: [AGPL v3 License](https://www.gnu.org/licenses/agpl-3.0.en.html) +- **Commercial License**: Proprietary license available. Contact [sales@adorsys.com](mailto:sales@adorsys.com) for details. diff --git a/datasafe-business/pom.xml b/datasafe-business/pom.xml index 4d72dfef4..53a4d3248 100644 --- a/datasafe-business/pom.xml +++ b/datasafe-business/pom.xml @@ -5,7 +5,7 @@ datasafe de.adorsys - 2.0.0 + 2.0.1 4.0.0 @@ -113,6 +113,7 @@ de.adorsys datasafe-test-storages ${project.version} + test-jar test diff --git a/datasafe-cli/pom.xml b/datasafe-cli/pom.xml index dd92f3ec2..bdb4c22c2 100644 --- a/datasafe-cli/pom.xml +++ b/datasafe-cli/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe - 2.0.0 + 2.0.1 datasafe-cli diff --git a/datasafe-directory/datasafe-directory-api/pom.xml b/datasafe-directory/datasafe-directory-api/pom.xml index e47d87e34..1729b407f 100644 --- a/datasafe-directory/datasafe-directory-api/pom.xml +++ b/datasafe-directory/datasafe-directory-api/pom.xml @@ -3,7 +3,7 @@ de.adorsys datasafe-directory - 2.0.0 + 2.0.1 datasafe-directory-api diff --git a/datasafe-directory/datasafe-directory-impl/pom.xml b/datasafe-directory/datasafe-directory-impl/pom.xml index ac56f72f5..b237067df 100644 --- a/datasafe-directory/datasafe-directory-impl/pom.xml +++ b/datasafe-directory/datasafe-directory-impl/pom.xml @@ -3,7 +3,7 @@ de.adorsys datasafe-directory - 2.0.0 + 2.0.1 datasafe-directory-impl diff --git a/datasafe-directory/pom.xml b/datasafe-directory/pom.xml index d40ff6dfd..0e574a805 100644 --- a/datasafe-directory/pom.xml +++ b/datasafe-directory/pom.xml @@ -5,7 +5,7 @@ datasafe de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-encryption/datasafe-encryption-api/pom.xml b/datasafe-encryption/datasafe-encryption-api/pom.xml index 0460c9ffd..4bd72cee4 100644 --- a/datasafe-encryption/datasafe-encryption-api/pom.xml +++ b/datasafe-encryption/datasafe-encryption-api/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe-encryption - 2.0.0 + 2.0.1 datasafe-encryption-api diff --git a/datasafe-encryption/datasafe-encryption-impl/pom.xml b/datasafe-encryption/datasafe-encryption-impl/pom.xml index 0036b7e51..fb463cd25 100644 --- a/datasafe-encryption/datasafe-encryption-impl/pom.xml +++ b/datasafe-encryption/datasafe-encryption-impl/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe-encryption - 2.0.0 + 2.0.1 datasafe-encryption-impl diff --git a/datasafe-encryption/datasafe-encryption-impl/src/main/java/de/adorsys/datasafe/encrypiton/impl/cmsencryption/CMSEncryptionServiceImpl.java b/datasafe-encryption/datasafe-encryption-impl/src/main/java/de/adorsys/datasafe/encrypiton/impl/cmsencryption/CMSEncryptionServiceImpl.java index 4d8dc886d..abca2737d 100644 --- a/datasafe-encryption/datasafe-encryption-impl/src/main/java/de/adorsys/datasafe/encrypiton/impl/cmsencryption/CMSEncryptionServiceImpl.java +++ b/datasafe-encryption/datasafe-encryption-impl/src/main/java/de/adorsys/datasafe/encrypiton/impl/cmsencryption/CMSEncryptionServiceImpl.java @@ -36,7 +36,7 @@ /** * Cryptographic message syntax document encoder/decoder - see * - * @see CMS + * @see CMS * wiki */ @Slf4j diff --git a/datasafe-encryption/datasafe-encryption-impl/src/main/java/de/adorsys/datasafe/encrypiton/impl/pathencryption/IntegrityPreservingUriEncryption.java b/datasafe-encryption/datasafe-encryption-impl/src/main/java/de/adorsys/datasafe/encrypiton/impl/pathencryption/IntegrityPreservingUriEncryption.java index 928a358d7..f24d11f7f 100644 --- a/datasafe-encryption/datasafe-encryption-impl/src/main/java/de/adorsys/datasafe/encrypiton/impl/pathencryption/IntegrityPreservingUriEncryption.java +++ b/datasafe-encryption/datasafe-encryption-impl/src/main/java/de/adorsys/datasafe/encrypiton/impl/pathencryption/IntegrityPreservingUriEncryption.java @@ -24,7 +24,7 @@ * It means that path/to/file is encrypted to cipher(path)/cipher(to)/cipher(file) and each invocation of example: * cipher(path) will yield same string, but cipher(path)/cipher(path) will not yield same segments - * it will be more like abc/cde and not like abc/abc. - * Additionally each segment is authenticated against its parent path hash, so attacker can't + * Additionally, each segment is authenticated against its parent path hash, so attacker can't * move a/file to b/file without being detected. */ @Slf4j diff --git a/datasafe-encryption/pom.xml b/datasafe-encryption/pom.xml index 90c6f4422..edae50b9c 100644 --- a/datasafe-encryption/pom.xml +++ b/datasafe-encryption/pom.xml @@ -5,7 +5,7 @@ datasafe de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-examples/datasafe-examples-business/pom.xml b/datasafe-examples/datasafe-examples-business/pom.xml index 16bffc4bc..f8cdb51ae 100644 --- a/datasafe-examples/datasafe-examples-business/pom.xml +++ b/datasafe-examples/datasafe-examples-business/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe-examples - 2.0.0 + 2.0.1 datasafe-examples-business diff --git a/datasafe-examples/datasafe-examples-customize-dagger/pom.xml b/datasafe-examples/datasafe-examples-customize-dagger/pom.xml index d2ee6ca21..a4067f9ec 100644 --- a/datasafe-examples/datasafe-examples-customize-dagger/pom.xml +++ b/datasafe-examples/datasafe-examples-customize-dagger/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe-examples - 2.0.0 + 2.0.1 datasafe-examples-customize-dagger diff --git a/datasafe-examples/datasafe-examples-customize-dagger/src/main/java/de/adorsys/datasafe/examples/business/filesystem/CustomlyBuiltDatasafeServices.java b/datasafe-examples/datasafe-examples-customize-dagger/src/main/java/de/adorsys/datasafe/examples/business/filesystem/CustomlyBuiltDatasafeServices.java index 898b265c7..215083b39 100644 --- a/datasafe-examples/datasafe-examples-customize-dagger/src/main/java/de/adorsys/datasafe/examples/business/filesystem/CustomlyBuiltDatasafeServices.java +++ b/datasafe-examples/datasafe-examples-customize-dagger/src/main/java/de/adorsys/datasafe/examples/business/filesystem/CustomlyBuiltDatasafeServices.java @@ -23,7 +23,7 @@ /** * This is Datasafe services customized implementation. - * Note, that despite is has {@code @Singleton} annotation, it is not real singleton, the only shared thing + * Note, that despite it has {@code @Singleton} annotation, it is not real singleton, the only shared thing * across all services instantiated using build() is bindings with {@code Singleton} in its Module. */ @Singleton diff --git a/datasafe-examples/datasafe-examples-multidfs/pom.xml b/datasafe-examples/datasafe-examples-multidfs/pom.xml index 17feb9450..c6ec4d139 100644 --- a/datasafe-examples/datasafe-examples-multidfs/pom.xml +++ b/datasafe-examples/datasafe-examples-multidfs/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe-examples - 2.0.0 + 2.0.1 datasafe-examples-multidfs diff --git a/datasafe-examples/datasafe-examples-versioned-s3/pom.xml b/datasafe-examples/datasafe-examples-versioned-s3/pom.xml index af6dcdf69..c869cb813 100644 --- a/datasafe-examples/datasafe-examples-versioned-s3/pom.xml +++ b/datasafe-examples/datasafe-examples-versioned-s3/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe-examples - 2.0.0 + 2.0.1 datasafe-examples-versioned-s3 diff --git a/datasafe-examples/pom.xml b/datasafe-examples/pom.xml index 8f7af5a3f..f3994a10a 100644 --- a/datasafe-examples/pom.xml +++ b/datasafe-examples/pom.xml @@ -5,7 +5,7 @@ datasafe de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-inbox/datasafe-inbox-api/pom.xml b/datasafe-inbox/datasafe-inbox-api/pom.xml index 04c70b749..253bc7577 100644 --- a/datasafe-inbox/datasafe-inbox-api/pom.xml +++ b/datasafe-inbox/datasafe-inbox-api/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe-inbox - 2.0.0 + 2.0.1 datasafe-inbox-api diff --git a/datasafe-inbox/datasafe-inbox-impl/pom.xml b/datasafe-inbox/datasafe-inbox-impl/pom.xml index 69390da78..cb416cea9 100644 --- a/datasafe-inbox/datasafe-inbox-impl/pom.xml +++ b/datasafe-inbox/datasafe-inbox-impl/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe-inbox - 2.0.0 + 2.0.1 datasafe-inbox-impl diff --git a/datasafe-inbox/pom.xml b/datasafe-inbox/pom.xml index 979cc6224..7c8f01f32 100644 --- a/datasafe-inbox/pom.xml +++ b/datasafe-inbox/pom.xml @@ -5,7 +5,7 @@ datasafe de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-long-run-tests/datasafe-business-tests-random-actions/pom.xml b/datasafe-long-run-tests/datasafe-business-tests-random-actions/pom.xml index 68e292184..8d838d7a5 100644 --- a/datasafe-long-run-tests/datasafe-business-tests-random-actions/pom.xml +++ b/datasafe-long-run-tests/datasafe-business-tests-random-actions/pom.xml @@ -5,16 +5,12 @@ datasafe-long-run-tests de.adorsys - 2.0.0 + 2.0.1 4.0.0 datasafe-business-tests-random-actions - - 2.22.2 - - de.adorsys @@ -110,6 +106,7 @@ de.adorsys datasafe-test-storages ${project.version} + test-jar test @@ -185,9 +182,8 @@ org.apache.maven.plugins maven-surefire-plugin - ${maven.surefire.plugin.version} + ${surefire.version} - once ${testArgs} diff --git a/datasafe-long-run-tests/pom.xml b/datasafe-long-run-tests/pom.xml index 0c8b29271..5b098fa87 100644 --- a/datasafe-long-run-tests/pom.xml +++ b/datasafe-long-run-tests/pom.xml @@ -5,7 +5,7 @@ datasafe de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-metainfo/datasafe-metainfo-version-api/pom.xml b/datasafe-metainfo/datasafe-metainfo-version-api/pom.xml index 8c9ea5867..d911534e0 100644 --- a/datasafe-metainfo/datasafe-metainfo-version-api/pom.xml +++ b/datasafe-metainfo/datasafe-metainfo-version-api/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe-metainfo - 2.0.0 + 2.0.1 datasafe-metainfo-version-api diff --git a/datasafe-metainfo/datasafe-metainfo-version-impl/pom.xml b/datasafe-metainfo/datasafe-metainfo-version-impl/pom.xml index 22620b071..d9eaa8867 100644 --- a/datasafe-metainfo/datasafe-metainfo-version-impl/pom.xml +++ b/datasafe-metainfo/datasafe-metainfo-version-impl/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe-metainfo - 2.0.0 + 2.0.1 datasafe-metainfo-version-impl diff --git a/datasafe-metainfo/pom.xml b/datasafe-metainfo/pom.xml index 7663fb743..34ce68e9c 100644 --- a/datasafe-metainfo/pom.xml +++ b/datasafe-metainfo/pom.xml @@ -5,7 +5,7 @@ datasafe de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-privatestore/datasafe-privatestore-api/pom.xml b/datasafe-privatestore/datasafe-privatestore-api/pom.xml index bfb824bb2..bb699901a 100644 --- a/datasafe-privatestore/datasafe-privatestore-api/pom.xml +++ b/datasafe-privatestore/datasafe-privatestore-api/pom.xml @@ -5,7 +5,7 @@ datasafe-privatestore de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-privatestore/datasafe-privatestore-impl/pom.xml b/datasafe-privatestore/datasafe-privatestore-impl/pom.xml index cba7aaf0d..f53e09367 100644 --- a/datasafe-privatestore/datasafe-privatestore-impl/pom.xml +++ b/datasafe-privatestore/datasafe-privatestore-impl/pom.xml @@ -5,7 +5,7 @@ datasafe-privatestore de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-privatestore/pom.xml b/datasafe-privatestore/pom.xml index a6ed1f8b7..fbf863855 100644 --- a/datasafe-privatestore/pom.xml +++ b/datasafe-privatestore/pom.xml @@ -5,7 +5,7 @@ datasafe de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-rest-impl/pom.xml b/datasafe-rest-impl/pom.xml index dd53fb7d0..c1395a3c1 100644 --- a/datasafe-rest-impl/pom.xml +++ b/datasafe-rest-impl/pom.xml @@ -5,11 +5,10 @@ datasafe de.adorsys - 2.0.0 + 2.0.1 datasafe-rest-impl - 2.0.0 datasafe-rest-impl Spring Boot DataSafe Application diff --git a/datasafe-runtime-delegate/pom.xml b/datasafe-runtime-delegate/pom.xml index e03d0fb4d..f83c9197b 100644 --- a/datasafe-runtime-delegate/pom.xml +++ b/datasafe-runtime-delegate/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe - 2.0.0 + 2.0.1 datasafe-runtime-delegate diff --git a/datasafe-simple-adapter/datasafe-simple-adapter-api/pom.xml b/datasafe-simple-adapter/datasafe-simple-adapter-api/pom.xml index cc5f4a7f1..87d84c37a 100644 --- a/datasafe-simple-adapter/datasafe-simple-adapter-api/pom.xml +++ b/datasafe-simple-adapter/datasafe-simple-adapter-api/pom.xml @@ -5,7 +5,7 @@ datasafe-simple-adapter de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-simple-adapter/datasafe-simple-adapter-impl/pom.xml b/datasafe-simple-adapter/datasafe-simple-adapter-impl/pom.xml index 1c3cc9822..17b269f6f 100644 --- a/datasafe-simple-adapter/datasafe-simple-adapter-impl/pom.xml +++ b/datasafe-simple-adapter/datasafe-simple-adapter-impl/pom.xml @@ -5,7 +5,7 @@ datasafe-simple-adapter de.adorsys - 2.0.0 + 2.0.1 4.0.0 @@ -45,6 +45,7 @@ de.adorsys datasafe-test-storages ${project.version} + test-jar test diff --git a/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/test/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeAdapterTest.java b/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/test/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeAdapterTest.java index a7ce59715..2fb8aa818 100644 --- a/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/test/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeAdapterTest.java +++ b/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/test/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeAdapterTest.java @@ -1,7 +1,6 @@ package de.adorsys.datasafe.simple.adapter.impl; import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.sun.management.UnixOperatingSystemMXBean; import de.adorsys.datasafe.encrypiton.api.types.UserID; import de.adorsys.datasafe.encrypiton.api.types.UserIDAuth; import de.adorsys.datasafe.encrypiton.api.types.encryption.MutableEncryptionConfig; @@ -24,17 +23,13 @@ import de.adorsys.datasafe.types.api.utils.ReadKeyPasswordTestFactory; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import net.bytebuddy.implementation.bytecode.Throw; import org.bouncycastle.util.io.Streams; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import java.io.ByteArrayInputStream; import java.io.OutputStream; -import java.lang.management.ManagementFactory; -import java.lang.management.OperatingSystemMXBean; import java.nio.file.NoSuchFileException; import java.security.UnrecoverableKeyException; import java.util.ArrayList; diff --git a/datasafe-simple-adapter/datasafe-simple-adapter-spring/pom.xml b/datasafe-simple-adapter/datasafe-simple-adapter-spring/pom.xml index b5131525b..1bc1c5feb 100644 --- a/datasafe-simple-adapter/datasafe-simple-adapter-spring/pom.xml +++ b/datasafe-simple-adapter/datasafe-simple-adapter-spring/pom.xml @@ -5,7 +5,7 @@ datasafe-simple-adapter de.adorsys - 2.0.0 + 2.0.1 4.0.0 @@ -126,6 +126,7 @@ de.adorsys datasafe-test-storages ${project.version} + test-jar test diff --git a/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionTest.java b/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionTest.java index e230609f1..e10f39431 100644 --- a/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionTest.java +++ b/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionTest.java @@ -21,7 +21,6 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.Nullable; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.test.context.SpringBootTest; diff --git a/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionWithoutPathEncryptionForFilesystemTest.java b/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionWithoutPathEncryptionForFilesystemTest.java index 969653f13..52b6636ea 100644 --- a/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionWithoutPathEncryptionForFilesystemTest.java +++ b/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionWithoutPathEncryptionForFilesystemTest.java @@ -11,9 +11,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringBootConfiguration; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; import java.util.stream.Stream; diff --git a/datasafe-simple-adapter/pom.xml b/datasafe-simple-adapter/pom.xml index a6d5053b1..d28c3af93 100644 --- a/datasafe-simple-adapter/pom.xml +++ b/datasafe-simple-adapter/pom.xml @@ -5,7 +5,7 @@ datasafe de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-storage/datasafe-storage-api/pom.xml b/datasafe-storage/datasafe-storage-api/pom.xml index 9ea0879c3..f90294668 100644 --- a/datasafe-storage/datasafe-storage-api/pom.xml +++ b/datasafe-storage/datasafe-storage-api/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe-storage - 2.0.0 + 2.0.1 datasafe-storage-api diff --git a/datasafe-storage/datasafe-storage-impl-db/pom.xml b/datasafe-storage/datasafe-storage-impl-db/pom.xml index 5ac2af5c2..764825140 100644 --- a/datasafe-storage/datasafe-storage-impl-db/pom.xml +++ b/datasafe-storage/datasafe-storage-impl-db/pom.xml @@ -5,7 +5,7 @@ datasafe-storage de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-storage/datasafe-storage-impl-fs/pom.xml b/datasafe-storage/datasafe-storage-impl-fs/pom.xml index fbf20109c..8a31094fc 100644 --- a/datasafe-storage/datasafe-storage-impl-fs/pom.xml +++ b/datasafe-storage/datasafe-storage-impl-fs/pom.xml @@ -5,7 +5,7 @@ datasafe-storage de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-storage/datasafe-storage-impl-s3/pom.xml b/datasafe-storage/datasafe-storage-impl-s3/pom.xml index 7335d35e7..341810d84 100644 --- a/datasafe-storage/datasafe-storage-impl-s3/pom.xml +++ b/datasafe-storage/datasafe-storage-impl-s3/pom.xml @@ -5,7 +5,7 @@ datasafe-storage de.adorsys - 2.0.0 + 2.0.1 4.0.0 @@ -24,7 +24,7 @@ de.adorsys datasafe-types-api ${project.version} - + com.amazonaws aws-java-sdk-s3 diff --git a/datasafe-storage/pom.xml b/datasafe-storage/pom.xml index 6802e49cc..7547515f2 100644 --- a/datasafe-storage/pom.xml +++ b/datasafe-storage/pom.xml @@ -5,7 +5,7 @@ datasafe de.adorsys - 2.0.0 + 2.0.1 4.0.0 diff --git a/datasafe-test-storages/pom.xml b/datasafe-test-storages/pom.xml index e4b819f4e..c961573ed 100644 --- a/datasafe-test-storages/pom.xml +++ b/datasafe-test-storages/pom.xml @@ -5,7 +5,7 @@ datasafe de.adorsys - 2.0.0 + 2.0.1 4.0.0 @@ -73,4 +73,22 @@ + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + test-jar + + test-compile + + + + + diff --git a/datasafe-test-storages/src/main/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java b/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java similarity index 99% rename from datasafe-test-storages/src/main/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java rename to datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java index 2b5684e13..ba03f52c4 100644 --- a/datasafe-test-storages/src/main/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java +++ b/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java @@ -18,7 +18,6 @@ import de.adorsys.datasafe.types.api.resource.Uri; import de.adorsys.datasafe.types.api.shared.BaseMockitoTest; import de.adorsys.datasafe.types.api.utils.ExecutorServiceUtil; -import lombok.AllArgsConstructor; import lombok.Getter; import lombok.SneakyThrows; import lombok.ToString; diff --git a/datasafe-types-api/pom.xml b/datasafe-types-api/pom.xml index c05645df0..2450208b9 100644 --- a/datasafe-types-api/pom.xml +++ b/datasafe-types-api/pom.xml @@ -5,7 +5,7 @@ de.adorsys datasafe - 2.0.0 + 2.0.1 datasafe-types-api diff --git a/docs/demo/deployment-model.png b/docs/demo/deployment-model.png new file mode 100644 index 000000000..485fd49c5 Binary files /dev/null and b/docs/demo/deployment-model.png differ diff --git a/docs/readme/Demo.md b/docs/readme/Demo.md new file mode 100644 index 000000000..5ce58abc1 --- /dev/null +++ b/docs/readme/Demo.md @@ -0,0 +1,102 @@ +## Quick demo +### Datasafe-CLI +You can try datasafe as a CLI (command-line-interface) executable for encryption of your own sensitive files. +Your encrypted files can be saved either in S3 bucket or local filesystem safely, because encryption will happen +locally - on your machine (See [CLI-README](datasafe-cli/README.md) for details). + +**Download CLI executable**: + +1. [MacOS native executable](https://github.com/adorsys/datasafe/releases/download/v0.7.0/datasafe-cli-osx-x64) +1. [Linux native executable](https://github.com/adorsys/datasafe/releases/download/v0.7.0/datasafe-cli-linux-x64) +1. Windows executable (N/A yet), please use java version below +1. [Java-based jar](https://github.com/adorsys/datasafe/releases/download/v0.7.0/datasafe-cli.jar), requires JRE (1.8+), use `java -jar datasafe-cli.jar` to execute + +#### Example actions: +##### Download application and create new user: + +
New profile animation transcript + +- Download CLI application (MacOS url) + +```bash +curl -L https://github.com/adorsys/datasafe/releases/download/v0.6.0/datasafe-cli-osx-x64 > datasafe-cli && chmod +x datasafe-cli +``` +- Create file with your credentials (they also can be passed through command line) + +```bash +echo '{"username": "john", "password": "Doe", "systemPassword": "password"}' > john.credentials +``` +- Create your new user profile (credentials come from john.credentials). You can enter value or click enter to accept +the default value when prompted. + +```bash +./datasafe-cli -c john.credentials profile create +``` +
+ +![new_profile](../demo/new_profile.gif) + +**Note**: Instead of creating file with credentials you can provide credentials directly into terminal (this is less +secure than having credentials file, but is fine for demo purposes): +```bash +./datasafe-cli -u=MeHappyUser -p=MyCoolPassword -sp=greatSystemPassword private cat secret.txt +``` +Command above will show private file `secret.txt` content for user `MeHappyUser` who has password `MyCoolPassword` and +system password `greatSystemPassword` + +##### Encrypt and decrypt some secret data for our user: + +
Encrypting/decrypting data animation transcript + +- Create some unencrypted content + +```bash +echo "Hello world" > unencrypted.txt +``` +- Encrypt and store file from above in privatespace. In privatespace it will have decrypted name `secret.txt` +```bash +./datasafe-cli -c john.credentials private cp unencrypted.txt secret.txt +``` +- Show that filename is encrypted in privatespace: + +```bash +ls private +``` + +- Show that file content is encrypted too: + +```bash +cat private/encrypted_file_name_from_above +``` + +- Decrypt file content: + +```bash +./datasafe-cli -c john.credentials private cat secret.txt +``` +
+ +![encrypt_decrypt_file](../demo/encrypt_decrypt_file.gif) + +##### You can always list available actions in context: + +
List actions animation transcript + +- Show top-level commands + +```bash +./datasafe-cli -c john.credentials +``` + +- Show commands for privatespace + +```bash +./datasafe-cli -c john.credentials private +``` +
+ +![list_actions](../demo/list_actions.gif) + +### REST API demo +[Here](../../datasafe-rest-impl/DEMO.md) you can find quick docker-based demo of project capabilities with +instructions of how to use it (REST-api based to show how to deploy as encryption server). diff --git a/docs/readme/DeploymentModels.md b/docs/readme/DeploymentModels.md new file mode 100644 index 000000000..2106fbb7c --- /dev/null +++ b/docs/readme/DeploymentModels.md @@ -0,0 +1,3 @@ +## Deployment Models +Followings are among others possible deployment models of the datasafe application. +![Datasafe deployment model](../demo/deployment-model.png) diff --git a/docs/readme/HowItWorks.md b/docs/readme/HowItWorks.md new file mode 100644 index 000000000..20d793f1a --- /dev/null +++ b/docs/readme/HowItWorks.md @@ -0,0 +1,578 @@ +# Integrating Datasafe + +## Project overview +In short, datasafe [core logic](datasafe-business/src/main/java/de/adorsys/datasafe/business/impl/service/DefaultDatasafeServices.java) +provides these key services: +* [Privatespace service](datasafe-privatestore/datasafe-privatestore-impl/src/main/java/de/adorsys/datasafe/privatestore/impl/PrivateSpaceServiceImpl.java) +that securely stores private files by encrypting them using users' secret key. +* [Inbox service](datasafe-inbox/datasafe-inbox-impl/src/main/java/de/adorsys/datasafe/inbox/impl/InboxServiceImpl.java) +that allows a user to share files with someone so that the only inbox owner can read files that are +shared with him using private key. +* [User profile service](datasafe-directory/datasafe-directory-impl/src/main/java/de/adorsys/datasafe/directory/impl/profile/operations/DFSBasedProfileStorageImpl.java) +that provides user metadata, such as where is user privatespace, his keystore, etc. + +These services are automatically built from +[modules](datasafe-business/src/main/java/de/adorsys/datasafe/business/impl) +and the only thing needed from a user is to provide storage adapter - by using +[predefined](datasafe-storage) adapters, +or by implementing his own using +[this interface](datasafe-storage/datasafe-storage-api/src/main/java/de/adorsys/datasafe/storage/api/StorageService.java). + +These services have interfaces that resemble actions that you can do with file or folder on your local file system - +list,write,read,delete file or folder. So, one can think that Datasafe provides mount-points for +inbox and private space virtual folders - you get similar actions available from Datasafe service. + +Additionally, for file versioning purposes like reading only last file version, there is [versioned privatespace](datasafe-business/src/main/java/de/adorsys/datasafe/business/impl/service/VersionedDatasafeServices.java) +that supports versioned and encrypted private file storage (for storage providers that do not support versioning). + +# How it Works + +## Library Modules +![Modules map](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/modules_map.puml&fmt=svg&vvv=1&sanitize=true) + +## Users' Files - where are they? + +Whenever user wants to store or read file at some location - be it inbox or his private space, following things do happen: +1. System resolves his profile location +1. His profile is read from some storage (and typically cached, then direct cache access happens) +1. Based on his profile content, root folder where data should be read/written is deduced +1. If data is going to private space - request path is encrypted +1. Root path is prepended to request path +1. Encryption/decryption of data happens +1. Credentials required to access the storage are added ([BucketAccessService](datasafe-directory/datasafe-directory-api/src/main/java/de/adorsys/datasafe/directory/api/profile/dfs/BucketAccessService.java)) +1. Data stream with path is sent to storage adapter +1. Optionally, storage adapter analyzes based on protocol which storage service to use +1. Storage adapter stores the data + +This diagram shows path resolution flow for private space with more details. It is mostly same both for private and +inbox files, with the only difference that private files have relative path (relative to private space location) +additionally encrypted. + +![Path resolution](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/profiles/locate_profile.puml&fmt=svg&vvv=2&sanitize=true) + +## Storing Private Files + +Private files are always encrypted using users' secret symmetric key. Additionally their path is encrypted too, but +this encryption is very special in the sense that it has form of a/b/c encrypted as +encrypted(a)/encrypted(b)/encrypted(c), so that folder traversal operations are efficient. + +![How privatespace diagram](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/high-level/how_it_works_private.puml&fmt=svg&vvv=2&sanitize=true) + +| Reading files from private space | Writing files to private space | +|---|---| +| ![Read modules](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/high-level/how_it_works_private_read_modules.puml&fmt=svg&vvv=1&sanitize=true) |![Write modules](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/high-level/how_it_works_private_write_modules.puml&fmt=svg&vvv=1&sanitize=true) | + +[Details](datasafe-privatestore) + +## Sharing files with another User + +Shared files are protected using asymmetrical cryptography, so that sender encrypts file with recipient's public key +and only recipient can read it using his private key. Paths are kept unencrypted for inbox. + +![How inbox diagram](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/high-level/how_it_works_inbox.puml&fmt=svg&vvv=1&sanitize=true) + +| Reading files from inbox | Writing files to inbox | +|---|---| +| ![Read modules](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/high-level/how_it_works_inbox_read_modules.puml&fmt=svg&vvv=1&sanitize=true) |![Write modules](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/adorsys/datasafe/develop/docs/diagrams/high-level/how_it_works_inbox_write_modules.puml&fmt=svg&vvv=1&sanitize=true) | + +[Details](datasafe-inbox) + +# Examples of how to Use the Library + + +## Generic Datasafe Usage +First, you want to create datasafe services. This snippet provides a datasafe that uses filesystem storage adapter: +[Example:Create Datasafe services](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L46-L52) +```groovy +// this will create all Datasafe files and user documents under +defaultDatasafeServices = DaggerDefaultDatasafeServices.builder() + .config(new DefaultDFSConfig(root.toAbsolutePath().toUri(), "secret"::toCharArray)) + .storage(new FileSystemStorageService(root)) + .build(); +``` + +Second you want to add new users: +[Example:Create new user](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L60-L67) +```groovy +// Creating new user with username 'user' and private/secret key password 'passwrd': +/* +IMPORTANT: For cases when user profile is stored on S3 without object locks, this requires some global +synchronization due to eventual consistency or you need to supply globally unique username on registration +*/ +defaultDatasafeServices.userProfile().registerUsingDefaults(new UserIDAuth("user", "passwrd"::toCharArray)); +``` + +After you have a user, he wants to store some data or document securely in his privatespace: +[Example:Store file in privatespace](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L78-L88) +```groovy +// creating new user +UserIDAuth user = registerUser("john"); + +// writing string "Hello" to my/own/file.txt: +// note that both resulting file content and its path are encrypted: +try (OutputStream os = defaultDatasafeServices.privateService() + .write(WriteRequest.forDefaultPrivate(user, "my/own/file.txt"))) { + os.write("Hello".getBytes(StandardCharsets.UTF_8)); +} +``` + +Now user wants to read again his secured file: +[Example:Read file from privatespace](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L99-L112) +```groovy +// creating new user +UserIDAuth user = registerUser("jane"); + +// writing string "Hello Jane" to my/secret.txt into users' Jane privatespace: +writeToPrivate(user, "my/secret.txt", "Hello Jane"); + +byte[] helloJane; +// reading encrypted data from my/secret.txt, note that path is also encrypted +try (InputStream is = defaultDatasafeServices.privateService() + .read(ReadRequest.forDefaultPrivate(user, "my/secret.txt"))) { + helloJane = ByteStreams.toByteArray(is); +} +``` + +But he doesn't remember the name of file he stored, so he will list all files in privatespace and read first: +[Example:Read file from privatespace using list](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L246-L260) +```groovy +// creating new user +UserIDAuth user = registerUser("john"); + +// let's create 1 file: +writeToPrivate(user, "home/my/secret.txt", "secret"); + +List> johnsPrivateFilesInMy = defaultDatasafeServices.privateService() + .list(ListRequest.forDefaultPrivate(user, "home/my")).collect(Collectors.toList()); + +// we have successfully read that file +assertThat(defaultDatasafeServices.privateService().read( + ReadRequest.forPrivate(user, johnsPrivateFilesInMy.get(0).getResource().asPrivate())) +).hasContent("secret"); +``` + +Now he wants to share some data with another user: +[Example:Send file to INBOX](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L124-L134) +```groovy +// create Jane, so her INBOX does exist +UserIDAuth jane = registerUser("jane"); +UserID janeUsername = new UserID("jane"); + +// We send message "Hello John" to John just by his username +try (OutputStream os = defaultDatasafeServices.inboxService() + .write(WriteRequest.forDefaultPublic(Collections.singleton(janeUsername), "hello.txt"))) { + os.write("Hello Jane".getBytes(StandardCharsets.UTF_8)); +} +``` + +Now he wants to share some data with couple of users, so that it will be encrypted once and they both +could read the file using each using own private key: +[Example:Send file to INBOX - multiple users](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L146-L158) +```groovy +// create Jane, so her INBOX does exist +UserIDAuth jane = registerUser("jane"); +// create Jamie, so his INBOX does exist +UserIDAuth jamie = registerUser("jamie"); + +// We send message to both users by using their username: +try (OutputStream os = defaultDatasafeServices.inboxService().write( + WriteRequest.forDefaultPublic(ImmutableSet.of(jane.getUserID(), jamie.getUserID()), "hello.txt")) +) { + os.write("Hello Jane and Jamie".getBytes(StandardCharsets.UTF_8)); +} +``` + +And finally it is time to read data that was shared with you: +[Example:Read file from INBOX](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithDefaultDatasafeTest.java#L268-L284) +```groovy +// creating new user +UserIDAuth user = registerUser("john"); +UserID johnUsername = new UserID("john"); + +// let's create 1 file: +shareMessage(johnUsername, "home/my/shared.txt", "shared message"); + +// Lets list our INBOX +List> johnsInboxFilesInMy = defaultDatasafeServices.inboxService() + .list(ListRequest.forDefaultPrivate(user, "")).collect(Collectors.toList()); + +// we have successfully read that file +assertThat(defaultDatasafeServices.inboxService().read( + ReadRequest.forPrivate(user, johnsInboxFilesInMy.get(0).getResource().asPrivate())) +).hasContent("shared message"); +``` + +## Datasafe with file Versioning +Suppose we need to preserve file history, so accidental file removal won't destroy everything. In such case +we can use storage provider that supports versioning. But if we have storage provider does not support versions +(i.e. minio) we can turn-on software versioning, here is its usage examples; + +First, we will obtain versioned datasafe services that uses filesystem storage adapter: +[Example:Create versioned datasafe services](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithVersionedDatasafeTest.java#L47-L53) +```groovy +// this will create all datasafe files and user documents under +versionedServices = DaggerVersionedDatasafeServices.builder() + .config(new DefaultDFSConfig(root.toAbsolutePath().toUri(), "secret"::toCharArray)) + .storage(new FileSystemStorageService(root)) + .build(); +``` + +Next we will create user, this is same as in non-versioned services: +[Example:Creating user for versioned services looks same](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithVersionedDatasafeTest.java#L61-L68) +```groovy +// Creating new user: +/* +IMPORTANT: For cases when user profile is stored on S3 without object locks, this requires some global +synchronization due to eventual consistency or you need to supply globally unique username on registration +*/ +versionedServices.userProfile().registerUsingDefaults(new UserIDAuth("user", "passwrd"::toCharArray)); +``` + +This is how file versioning works when saving file multiple times: +[Example:Saving file couple of times - versioned](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithVersionedDatasafeTest.java#L80-L104) +```groovy +// creating new user +UserIDAuth user = registerUser("john"); + +// writing string "Hello " + index to my/own/file.txt 3 times: +// note that both resulting file content and its path are encrypted: +for (int i = 1; i <= 3; ++i) { + try (OutputStream os = versionedServices.latestPrivate() + .write(WriteRequest.forDefaultPrivate(user, "my/own/file.txt"))) { + os.write(("Hello " + i).getBytes(StandardCharsets.UTF_8)); + Thread.sleep(1000L); // this will change file modified dates + } +} + +// and still we read only latest file +assertThat(versionedServices.latestPrivate() + .read(ReadRequest.forDefaultPrivate(user, "my/own/file.txt")) +).hasContent("Hello 3"); +// but there are 3 versions of file stored physically in users' privatespace: +assertThat(versionedServices.privateService().list( + ListRequest.forDefaultPrivate(user, "my/own/file.txt")) +).hasSize(3); +// and still only one file visible on latest view +assertThat(versionedServices.latestPrivate().list(ListRequest.forDefaultPrivate(user, ""))).hasSize(1); +``` + +And we can work with file versions too, of course, everything is encrypted: +[Example:Lets check how to read oldest file version](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithVersionedDatasafeTest.java#L106-L122) +```groovy +// so lets collect all versions +List, PrivateResource, DFSVersion>> withVersions = + versionedServices.versionInfo().versionsOf( + ListRequest.forDefaultPrivate(user, "my/own/file.txt") + ).collect(Collectors.toList()); +// so that we can find oldest +Versioned, PrivateResource, DFSVersion> oldest = + withVersions.stream() + .sorted(Comparator.comparing(it -> it.absolute().getResource().getModifiedAt())) + .collect(Collectors.toList()) + .get(0); +// and read oldest content +assertThat(versionedServices.privateService() + .read(ReadRequest.forPrivate(user, oldest.absolute().getResource().asPrivate())) +).hasContent("Hello 1"); +``` + +Another important case to mention is how to determine if file has changed on storage compared to some copy we have: +[Example:Check if we have latest file locally](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/BaseUserOperationsTestWithVersionedDatasafeTest.java#L132-L163) +```groovy +// creating new user +UserIDAuth user = registerUser("john"); + +// First lets store some file, for example John stored it from mobile phone +try (OutputStream os = versionedServices.latestPrivate() + .write(WriteRequest.forDefaultPrivate(user, "my/own/file.txt"))) { + os.write(("Hello old version").getBytes(StandardCharsets.UTF_8)); +} + +// Application on mobile phone caches file content to improve performance, so it should cache timestamp too +Instant savedOnMobile = versionedServices.latestPrivate() + .list(ListRequest.forDefaultPrivate(user, "my/own/file.txt")) + .findAny().get().getResource().getModifiedAt(); + +// Now John uses PC to write data to my/own/file.txt with some updated data +Thread.sleep(1000L); // it took some time for him to get to PC +try (OutputStream os = versionedServices.latestPrivate() + .write(WriteRequest.forDefaultPrivate(user, "my/own/file.txt"))) { + os.write(("Hello new version").getBytes(StandardCharsets.UTF_8)); +} + +// John takes his mobile phone and application checks if it needs to sync content +Instant savedOnPC = versionedServices.latestPrivate() + .list(ListRequest.forDefaultPrivate(user, "my/own/file.txt")) + .findAny().get().getResource().getModifiedAt(); + +// This indicates that we need to update our cache on mobile phone +// Modified date of saved file has changed and it is newer that our cached date +// So mobile application should download latest file version +assertThat(savedOnPC).isAfter(savedOnMobile); +``` + +## Datasafe on Versioned Storage +If you have storage for user files on **versioned S3 bucket** and want to get object version when you write +object or to read some older version encrypted object, you can follow this example of how to do that: +[Example:Versioned storage support - writing file and reading back](datasafe-examples/datasafe-examples-versioned-s3/src/test/java/de/adorsys/datasafe/examples/business/s3/BaseUserOperationsWithDefaultDatasafeOnVersionedStorageTest.java#L138-L172) +```groovy +// creating new user +UserIDAuth user = registerUser("john"); + +// writing data to my/own/file.txt 3 times with different content: +// 1st time, writing into my/own/file.txt: +// Expanded snippet of how to capture file version when writing object: +AtomicReference version = new AtomicReference<>(); +try (OutputStream os = defaultDatasafeServices.privateService() + .write(WriteRequest.forDefaultPrivate(user, MY_OWN_FILE_TXT) + .toBuilder() + .callback((PhysicalVersionCallback) version::set) + .build()) +) { + // Initial version will contain "Hello 1": + os.write("Hello 1".getBytes(StandardCharsets.UTF_8)); +} +// this variable has our initial file version: +String version1 = version.get(); + +// Write 2 more times different data to same file - my/own/file.txt: +String version2 = writeToPrivate(user, MY_OWN_FILE_TXT, "Hello 2"); +// Last version will contain "Hello 3": +String version3 = writeToPrivate(user, MY_OWN_FILE_TXT, "Hello 3"); + +// now, when we read file without specifying version - we see latest file content: +assertThat(defaultDatasafeServices.privateService().read( + ReadRequest.forDefaultPrivate(user, MY_OWN_FILE_TXT)) +).hasContent("Hello 3"); + +// but if we specify file version - we get content for it: +assertThat(defaultDatasafeServices.privateService().read( + ReadRequest.forDefaultPrivateWithVersion(user, MY_OWN_FILE_TXT, new StorageVersion(version1))) +).hasContent("Hello 1"); +``` +Removing old file version can be done by [bucket policy](https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#non-current-days-calculations) +or manually, using this snippet: +[Example:Versioned storage support - removing specific version](datasafe-examples/datasafe-examples-versioned-s3/src/test/java/de/adorsys/datasafe/examples/business/s3/BaseUserOperationsWithDefaultDatasafeOnVersionedStorageTest.java#L188-L215) +```groovy +// creating new user +UserIDAuth user = registerUser("john"); + +// writing data to my/own/file.txt 2 times with different content: +String versionId = writeToPrivate(user, MY_OWN_FILE_TXT, "Hello 1"); +writeToPrivate(user, MY_OWN_FILE_TXT, "Hello 2"); + +// now, we read old file version +assertThat(defaultDatasafeServices.privateService().read( + ReadRequest.forDefaultPrivateWithVersion(user, MY_OWN_FILE_TXT, new StorageVersion(versionId))) +).hasContent("Hello 1"); + +// now, we remove old file version +defaultDatasafeServices.privateService().remove( + RemoveRequest.forDefaultPrivateWithVersion(user, MY_OWN_FILE_TXT, new StorageVersion(versionId)) +); + +// it is removed from storage, so when we read it we get exception +assertThrows(AmazonS3Exception.class, () -> defaultDatasafeServices.privateService().read( + ReadRequest.forDefaultPrivateWithVersion(user, MY_OWN_FILE_TXT, new StorageVersion(versionId))) +); + +// but latest file version is still available +assertThat(defaultDatasafeServices.privateService().read( + ReadRequest.forDefaultPrivate(user, MY_OWN_FILE_TXT)) +).hasContent("Hello 2"); +``` + +## Overriding Datasafe Functionality +Whenever you want to have some custom functionality of datasafe, instead of default ones, there are +two possible ways to achieve this: +- using OverridesRegistry without project recompilation. +- using Dagger2 to build a customized version of datasafe. + +### Overriding Functionality without Recompilation +This approach is for classes annotated with +[@RuntimeDelegate](datasafe-types-api/src/main/java/de/adorsys/datasafe/types/api/context/annotations/RuntimeDelegate.java) +and it works by putting the custom implementation of a class to be overridden into +[OverridesRegistry](datasafe-types-api/src/main/java/de/adorsys/datasafe/types/api/context/overrides/OverridesRegistry.java). +During runtime, when accessing desired functionality, the library will look into OverridesRegistry for +custom class implementation and use it if present. This one has the advantage of not requiring recompilation of +Datasafe library, but has a limitation of working on static dependency graph - you can't rebuild it. +[Example:Create overridable datasafe services without recompilation](datasafe-examples/datasafe-examples-business/src/test/java/de/adorsys/datasafe/examples/business/filesystem/RuntimeOverrideOperationsTest.java#L31-L53) +```groovy +// This shows how to override path encryption service, in particular we are going to disable it +OverridesRegistry registry = new BaseOverridesRegistry(); + +// PathEncryptionImpl now will have completely different functionality +// instead of calling PathEncryptionImpl methods we will call PathEncryptionImplOverridden methods +PathEncryptionImplRuntimeDelegatable.overrideWith(registry, PathEncryptionImplOverridden::new); + +// Customized service, without creating complete module and building it: +DefaultDatasafeServices datasafeServices = DaggerDefaultDatasafeServices.builder() + .config(new DefaultDFSConfig(root.toAbsolutePath().toUri(), "secret"::toCharArray)) + .storage(new FileSystemStorageService(root)) + .overridesRegistry(registry) + .build(); + +// registering user +UserIDAuth user = new UserIDAuth("user", "passwrd"::toCharArray); +datasafeServices.userProfile().registerUsingDefaults(user); +// writing into user privatespace, note that with default implementation `file.txt` would be encrypted +datasafeServices.privateService().write(WriteRequest.forDefaultPrivate(user, "file.txt")); +// but we see raw filename here: +assertThat(Files.walk(root)).asString().contains("file.txt"); +``` + +### Overriding Functionality by Building custom Datasafe Library +This is actually the preferred way to override something or to customize datasafe. It has no limitations because +you can compose any datasafe service you want using Dagger2 for dependency injection. Its major drawback is that +you need to add a dependency to Dagger2 into your project and compile this custom library version. Because of +compile-time dependency injection and modular structure it is a comparatively error-free approach. +To create custom datasafe service we need to follow these 3 steps: +1. Create your own custom module (or modules) - see [CustomPathEncryptionModule](datasafe-examples/datasafe-examples-customize-dagger/src/main/java/de/adorsys/datasafe/examples/business/filesystem/CustomPathEncryptionModule.java) +1. Create custom Datasafe with custom module list - see [CustomlyBuiltDatasafeServices](datasafe-examples/datasafe-examples-customize-dagger/src/main/java/de/adorsys/datasafe/examples/business/filesystem/CustomlyBuiltDatasafeServices.java) +1. Use custom-built Datasafe as shown here: +[Example:Create custom-built Datasafe service](datasafe-examples/datasafe-examples-customize-dagger/src/test/java/de/adorsys/datasafe/examples/business/filesystem/CustomlyBuiltDatasafeServiceTest.java#L25-L39) +```groovy +// Customized service, we create required module using compile time DI provided by Dagger: +CustomlyBuiltDatasafeServices datasafeServices = DaggerCustomlyBuiltDatasafeServices.builder() + .config(new DefaultDFSConfig(root.toAbsolutePath().toUri(), "secret"::toCharArray)) + .storage(new FileSystemStorageService(root)) + .build(); + +// registering user +UserIDAuth user = new UserIDAuth("user", "password"::toCharArray); +datasafeServices.userProfile().registerUsingDefaults(user); +// writing into user privatespace, note that with default implementation `file.txt` would be encrypted +datasafeServices.privateService().write(WriteRequest.forDefaultPrivate(user, "file.txt")); +// but we see raw filename here: +assertThat(walk(root)).asString().contains("file.txt"); +``` + +### Customizing Datasafe to store dynamic and user-provided Credentials +In case user wants to register storage credentials himself or place keystore within credentials-protected +location one can use this example: +[Example:Datasafe with multi-dfs setup](datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleTest.java#L106-L220) +```groovy +String directoryBucketS3Uri = "s3://" + DIRECTORY_BUCKET.getBucketName() + "/"; +// static client that will be used to access `directory` bucket: +StorageService directoryStorage = new S3StorageService( + directoryClient, + DIRECTORY_BUCKET.getBucketName(), + EXECUTOR +); + +OverridesRegistry registry = new BaseOverridesRegistry(); +DefaultDatasafeServices multiDfsDatasafe = DaggerDefaultDatasafeServices + .builder() + .config(new DFSConfigWithStorageCreds(directoryBucketS3Uri, "PAZZWORT"::toCharArray)) + // This storage service will route requests to proper bucket based on URI content: + // URI with directoryBucket to `directoryStorage` + // URI with filesBucketOne will get dynamically generated S3Storage + // URI with filesBucketTwo will get dynamically generated S3Storage + .storage( + new RegexDelegatingStorage( + ImmutableMap.builder() + // bind URI that contains `directoryBucket` to directoryStorage + .put(Pattern.compile(directoryBucketS3Uri + ".+"), directoryStorage) + .put( + Pattern.compile(getDockerUri("http://127.0.0.1") + ".+"), + // Dynamically creates S3 client with bucket name equal to host value + new UriBasedAuthStorageService( + acc -> new S3StorageService( + S3ClientFactory.getClient( + acc.getEndpoint(), + acc.getRegion(), + acc.getAccessKey(), + acc.getSecretKey() + ), + // Bucket name is encoded in first path segment + acc.getBucketName(), + EXECUTOR + ) + ) + ).build() + ) + ) + .overridesRegistry(registry) + .build(); +// Instead of default BucketAccessService we will use service that reads storage access credentials from +// keystore +BucketAccessServiceImplRuntimeDelegatable.overrideWith( + registry, args -> new WithCredentialProvider(args.getStorageKeyStoreOperations()) +); + +// John will have all his private files stored on `filesBucketOne` and `filesBucketOne`. +// Depending on path of file - filesBucketOne or filesBucketTwo - requests will be routed to proper bucket. +// I.e. path filesBucketOne/path/to/file will end up in `filesBucketOne` with key path/to/file +// his profile and access credentials for `filesBucketOne` will be in `configBucket` +UserIDAuth john = new UserIDAuth("john", "secret"::toCharArray); +// Here, nothing expects John has own storage credentials: +multiDfsDatasafe.userProfile().registerUsingDefaults(john); + +// Tell system that John will use his own storage credentials - regex match: +StorageIdentifier bucketOne = new StorageIdentifier(endpointsByHost.get(FILES_BUCKET_ONE) + ".+"); +StorageIdentifier bucketTwo = new StorageIdentifier(endpointsByHost.get(FILES_BUCKET_TWO) + ".+"); +// Set location for John's credentials keystore and put storage credentials into it: +UserPrivateProfile profile = multiDfsDatasafe.userProfile().privateProfile(john); +profile.getPrivateStorage().put( + bucketOne, + new AbsoluteLocation<>(BasePrivateResource.forPrivate(endpointsByHost.get(FILES_BUCKET_ONE) + "/")) +); +profile.getPrivateStorage().put( + bucketTwo, + new AbsoluteLocation<>(BasePrivateResource.forPrivate(endpointsByHost.get(FILES_BUCKET_TWO) + "/")) +); +multiDfsDatasafe.userProfile().updatePrivateProfile(john, profile); + +// register John's DFS access for `filesBucketOne` minio bucket +multiDfsDatasafe.userProfile().registerStorageCredentials( + john, + bucketOne, + new StorageCredentials( + FILES_BUCKET_ONE.getAccessKey(), + FILES_BUCKET_ONE.getSecretKey() + ) +); +// register John's DFS access for `filesBucketTwo` minio bucket +multiDfsDatasafe.userProfile().registerStorageCredentials( + john, + bucketTwo, + new StorageCredentials( + FILES_BUCKET_TWO.getAccessKey(), + FILES_BUCKET_TWO.getSecretKey() + ) +); + +// Configuring multi-storage is done, user can use his multi-storage: + +// store this file on `filesBucketOne` +try (OutputStream os = multiDfsDatasafe.privateService() + .write(WriteRequest.forPrivate(john, bucketOne, "my/file.txt"))) { + os.write("Content on bucket number ONE".getBytes(StandardCharsets.UTF_8)); +} + +// store this file on `filesBucketTwo` +try (OutputStream os = multiDfsDatasafe.privateService() + .write(WriteRequest.forPrivate(john, bucketTwo, "my/file.txt"))) { + os.write("Content on bucket number TWO".getBytes(StandardCharsets.UTF_8)); +} + +// read file from `filesBucketOne` +assertThat(multiDfsDatasafe.privateService() + .read(ReadRequest.forPrivate(john, bucketOne, "my/file.txt")) +).hasContent("Content on bucket number ONE"); + +// read file from `filesBucketTwo` +assertThat(multiDfsDatasafe.privateService() + .read(ReadRequest.forPrivate(john, bucketTwo, "my/file.txt")) +).hasContent("Content on bucket number TWO"); +``` \ No newline at end of file diff --git a/last-module-codecoverage-check/pom.xml b/last-module-codecoverage-check/pom.xml index 5ebeee0ae..4022225ac 100644 --- a/last-module-codecoverage-check/pom.xml +++ b/last-module-codecoverage-check/pom.xml @@ -5,12 +5,16 @@ datasafe de.adorsys - 2.0.0 + 2.0.1 4.0.0 last-module-codecoverage-check + + 3.3.1 + + de.adorsys @@ -144,7 +148,7 @@ maven-resources-plugin - 3.3.1 + ${maven-resources-plugin.version} package diff --git a/pom.xml b/pom.xml index 172da127a..227ad2ae0 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ de.adorsys datasafe - 2.0.0 + 2.0.1 datasafe Datasafe https://github.com/adorsys/datasafe @@ -73,17 +73,12 @@ none src/main/java src/test/java - - - ${basedir}/../../target/jacoco.exec - ${basedir}/../target/jacoco-e2e.exec - 19.2.0 3.11.0 17 17 - 3.6.0 - 1.18.28 + 3.6.1 + 1.18.30 2.8.9 2.46.1 32.1.1-jre @@ -91,19 +86,19 @@ 5.10.0 3.12.2 5.5.0 - 2.22.1 + 3.1.2 1.21 UTF-8 false 1.18.3 - 0.8.10 + 0.8.11 2.5 2.0.7 9.44.0.Final 1.18.20.0 3.0.0 1.0.2 - 1.12.402 + 1.12.646 3.0.0 1 1.9.0 @@ -119,9 +114,18 @@ 1.4.4 2.12.7 2.12.7.1 - 0.0.7 + 0.0.9 2.1.1 2.3.1 + 2.8.1 + 1.3 + 3.3.0 + 2.5.2 + 2.7 + 3.6.2 + 3.1.0 + 3.0.1 + 1.6.13 @@ -371,25 +375,14 @@ ${surefireArgLine}, - --add-opens java.base/java.lang=ALL-UNNAMED, - --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED, --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED, - - --add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED, - --add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED, - --add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED, - --add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED, - --add-opens=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED, - --add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED, - --add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED, - --add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED, - --add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED, - --add-opens=jdk.compiler/com.sun.tools.javac.jvm=ALL-UNNAMED + --add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED, + org.apache.maven.plugins maven-dependency-plugin ${maven.dependency.plugin.version} @@ -522,6 +515,7 @@ org.apache.maven.plugins maven-compiler-plugin + ${maven.compiler.plugin.version} @@ -541,6 +535,14 @@ org.projectlombok lombok-maven-plugin ${lombok-maven-plugin.version} + + + org.projectlombok + lombok + ${lombok.version} + + + generate-sources @@ -557,6 +559,7 @@ org.apache.maven.plugins maven-javadoc-plugin + ${maven-javadoc-plugin.version} **/*Dagger* @@ -574,7 +577,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.8 + ${nexus-staging-maven-plugin.version} true sonatype @@ -585,7 +588,7 @@ org.apache.maven.plugins maven-source-plugin - 3.0.1 + ${maven-source-plugin.version} attach-sources @@ -598,7 +601,7 @@ org.apache.maven.plugins maven-gpg-plugin - 1.6 + ${maven-gpg-plugin.version} @@ -614,8 +617,9 @@ org.apache.maven.plugins maven-javadoc-plugin + ${maven-javadoc-plugin.version} - -Xdoclint:none + -Xdoclint:none @@ -633,11 +637,11 @@ org.apache.maven.plugins maven-project-info-reports-plugin - 2.7 + ${maven-project-info-reports-plugin.version} maven-release-plugin - 2.5.2 + ${maven-release-plugin.version} release true @@ -647,7 +651,7 @@ org.apache.maven.plugins maven-jar-plugin - 2.6 + ${maven-jar-plugin.version} @@ -670,7 +674,7 @@ org.codehaus.mojo buildnumber-maven-plugin - 1.3 + ${buildnumber-maven-plugin.version} validate @@ -687,7 +691,7 @@ org.apache.maven.plugins maven-deploy-plugin - 2.8.1 + ${maven-deploy-plugin.version} ${deploy.disabled} diff --git a/scripts/mvn_deploy.sh b/scripts/mvn_deploy.sh new file mode 100755 index 000000000..dbf870f1f --- /dev/null +++ b/scripts/mvn_deploy.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e + +echo "$GPG_SECRET_KEY" | base64 --decode | $GPG_EXECUTABLE --import --no-tty --batch --yes || true +echo "$GPG_OWNERTRUST" | base64 --decode | $GPG_EXECUTABLE --import-ownertrust --no-tty --batch --yes || true + +mvn clean deploy -ntp --settings scripts/settings.xml gpg:sign -Prelease -DskipTests -U || exit 1 \ No newline at end of file diff --git a/.travis/settings.xml b/scripts/settings.xml similarity index 100% rename from .travis/settings.xml rename to scripts/settings.xml