From 296cb625b6e884164ea0b7740d5f5b82c27a22fd Mon Sep 17 00:00:00 2001 From: AssahBismarkabah Date: Wed, 17 Jul 2024 13:41:01 +0100 Subject: [PATCH] update datasafe-storage-api Readme update: datasafe-storage-api and improve test-coverage --- .../datasafe-storage-api/README.md | 125 +++++++++++++++- .../api/UriBasedAuthStorageService.java | 6 +- .../api/RegexDelegatingStorageTest.java | 62 ++++++++ .../api/UriBasedAuthStorageServiceTest.java | 45 +++++- .../api/UserBasedDelegatingStorageTest.java | 134 ++++++++++++++++++ docs/diagrams/uri_routing_flowchart.puml | 18 +++ 6 files changed, 383 insertions(+), 7 deletions(-) create mode 100644 datasafe-storage/datasafe-storage-api/src/test/java/de/adorsys/datasafe/storage/api/RegexDelegatingStorageTest.java create mode 100644 datasafe-storage/datasafe-storage-api/src/test/java/de/adorsys/datasafe/storage/api/UserBasedDelegatingStorageTest.java create mode 100644 docs/diagrams/uri_routing_flowchart.puml diff --git a/datasafe-storage/datasafe-storage-api/README.md b/datasafe-storage/datasafe-storage-api/README.md index 1e74cc197..0c6f37eba 100644 --- a/datasafe-storage/datasafe-storage-api/README.md +++ b/datasafe-storage/datasafe-storage-api/README.md @@ -1,5 +1,124 @@ # Storage API -This module exposes storage API used by other modules. Use -[StorageService](src/main/java/de/adorsys/datasafe/storage/api/StorageService.java) interface, provided by this module, -if you want to write your own adapter. \ No newline at end of file +This module provides an abstraction layer for various storage operations, enabling interaction with different types +of storage backends through a unified interface. It allows for the reading, writing, listing, and removal of data stored in different locations such as S3 buckets, local file systems, based on URI schemes or patterns. +It exposes storage API used by other modules. + +- Use [StorageService](src/main/java/de/adorsys/datasafe/storage/api/StorageService.java) interface, provided by this module, +if you want to write your own adapter. + +## Key Types and Interfaces + +#### StorageCheckService +- **Purpose:** To check if a specified resource exists at a given location. +- **Key Method:** +```java +boolean objectExists(AbsoluteLocation location); +``` +#### StorageListService +- **Purpose:** To list resources at a given location. +- **Key Method:** +```java +Stream> list(AbsoluteLocation location); +``` +#### StorageReadService +- **Purpose:** To read data from a specified resource location. +- **Key Method:** +```java +InputStream read(AbsoluteLocation location); +``` + +#### StorageRemoveService +- **Purpose:** To remove a specified resource location. +- **Key Method:** +```java +void remove(AbsoluteLocation location); +``` +#### StorageWriteService +- **Purpose:** To write data to a specified resource location +- **Key Method:** +```java +OutputStream write(WithCallback locationWithCallback); +``` +- **Additional Method:** +```java +default Optional flushChunkSize(AbsoluteLocation location) { ... } +``` +#### StorageService +- **Purpose:** Combines all storage operations into a single interface. +- **Implements:** +* StorageCheckService +* StorageListService +* StorageReadService +* StorageRemoveService +* StorageWriteService + +## Key Classes + +#### BaseDelegatingStorage +- **Purpose:** Abstract base class that delegates storage operations to actual storage implementations. +- **Method:** Implements methods from StorageService and delegates them to an abstract service method. +```java +protected abstract StorageService service(AbsoluteLocation location); +``` +#### RegexDelegatingStorage +- **Purpose:** Delegates storage operations based on regex matching of URIs. + +- **Key Fields:** +```java +private final Map storageByPattern; +``` +- **Implementation of service method** +```java +protected StorageService service(AbsoluteLocation location) { ... } +``` +#### SchemeDelegatingStorage +- **Purpose:** Delegates storage operations based on URI schemes. + +- **Key Fields:** +```java +private final Map storageByScheme; +``` +- **Implementation of service method** +```java +protected StorageService service(AbsoluteLocation location) { ... } +``` +#### UriBasedAuthStorageService +- **Purpose:** Manages storage connections based on URIs containing credentials, such as S3 URIs. + +- **Key Fields:** +```java +private final Map clientByItsAccessKey = new ConcurrentHashMap<>(); +private final Function bucketExtractor; +private final Function regionExtractor; +private final Function endpointExtractor; +private final Function storageServiceBuilder; +``` +- **Key Inner Class: AccessId** +```java +private final String accessKey; +private final String secretKey; +private final String region; +private final String bucketName; +private final String endpoint; +private final URI withoutCreds; +private final URI onlyHostPart; +``` + +#### UserBasedDelegatingStorage +- **Purpose:** Delegates storage operations based on user-specific bucket mappings. + +- **Key Fields:** +```java +private final Map clientByBucket = new ConcurrentHashMap<>(); +private final List amazonBuckets; +private final Function storageServiceBuilder; +``` +### URI Routing Flowchart: +This flowchart depicts how storage operations are routed based on URIs and patterns: +![URI Routing flowchart](http://www.plantuml.com/plantuml/dpng/dL1DImCn4BtlhnZtj0N1GtiKQTdMAXHR3D9Z6PFPDfWcaang_VTck-lgKWNn56RcVUIzSM3q7FSckz1McgW8hilHLJdQbCuo7VacorRaWxD53EGl8NzAJzwy0NX7C4L6WHL1OETnIp1PtUU3JBm7fdsXqZMaQtjCn0ullk7JVkNTGQiaYX2jhZGfq9R9LoW9AkUR2ILhkuKtpJiueDSkXixu6UKBMHKwzytio4KO9l79Me0OrZQbSL5rb1Jce2Nr6PKs54vZmY-SH0EtQGKD9E-MhKYVx58d_YljiXwxgAArIuSvMV9Q_l2Jx95Cs_PvUtNjDJsL1YKQKuUjyMV8K-mf6TeYDvJFJonVoIDhPt_bzWhufqQ_Xp-ePE9kkTuiPlFPmxGOP6EoAkxD1m00) +## Conclusion +This module provides a robust and flexible framework for managing data storage in a system with multiple storage backends. It offers several key benefits: +- **Pluggability:** Easily add new storage implementations without modifying existing code. +- **Testability:** Simplify testing with a clear API and mocking capabilities. +- **Maintainability:** Centralize storage logic and reduce code duplication. \ No newline at end of file diff --git a/datasafe-storage/datasafe-storage-api/src/main/java/de/adorsys/datasafe/storage/api/UriBasedAuthStorageService.java b/datasafe-storage/datasafe-storage-api/src/main/java/de/adorsys/datasafe/storage/api/UriBasedAuthStorageService.java index f3796b52b..d97f8c958 100644 --- a/datasafe-storage/datasafe-storage-api/src/main/java/de/adorsys/datasafe/storage/api/UriBasedAuthStorageService.java +++ b/datasafe-storage/datasafe-storage-api/src/main/java/de/adorsys/datasafe/storage/api/UriBasedAuthStorageService.java @@ -19,9 +19,9 @@ public class UriBasedAuthStorageService extends BaseDelegatingStorage { private final Map clientByItsAccessKey = new ConcurrentHashMap<>(); - private final Function bucketExtractor; - private final Function regionExtractor; - private final Function endpointExtractor; + final Function bucketExtractor; + final Function regionExtractor; + final Function endpointExtractor; // Builder to create S3 or other kind of Storage service private final Function storageServiceBuilder; diff --git a/datasafe-storage/datasafe-storage-api/src/test/java/de/adorsys/datasafe/storage/api/RegexDelegatingStorageTest.java b/datasafe-storage/datasafe-storage-api/src/test/java/de/adorsys/datasafe/storage/api/RegexDelegatingStorageTest.java new file mode 100644 index 000000000..be5cd3e4d --- /dev/null +++ b/datasafe-storage/datasafe-storage-api/src/test/java/de/adorsys/datasafe/storage/api/RegexDelegatingStorageTest.java @@ -0,0 +1,62 @@ +package de.adorsys.datasafe.storage.api; +import de.adorsys.datasafe.types.api.resource.AbsoluteLocation; +import de.adorsys.datasafe.types.api.resource.BasePrivateResource; +import de.adorsys.datasafe.types.api.resource.WithCallback; +import de.adorsys.datasafe.types.api.shared.BaseMockitoTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; + +import java.util.Collections; +import java.util.Map; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; + +public class RegexDelegatingStorageTest extends BaseMockitoTest{ + + @Mock + private StorageService service; + private RegexDelegatingStorage tested; + private AbsoluteLocation location; + + @BeforeEach + void setUp() { + Map storageByPattern = Collections.singletonMap(Pattern.compile("s3://.*"), service); + tested = new RegexDelegatingStorage(storageByPattern); + location = new AbsoluteLocation<>(BasePrivateResource.forPrivate("s3://bucket")); + } + @Test + void objectExists() { + tested.objectExists(location); + verify(service).objectExists(location); + } + @Test + void list() { + tested.list(location); + verify(service).list(location); + } + @Test + void read() { + tested.read(location); + verify(service).read(location); + } + @Test + void remove() { + tested.remove(location); + verify(service).remove(location); + } + @Test + void write() { + tested.write(WithCallback.noCallback(location)); + verify(service).write(any(WithCallback.class)); + } + @Test + void objectExistsWithNoMatch() { + AbsoluteLocation badlocation = new AbsoluteLocation<>(BasePrivateResource.forPrivate("file://bucket")); + assertThrows(IllegalArgumentException.class, () -> tested.objectExists(badlocation)); + } + +} diff --git a/datasafe-storage/datasafe-storage-api/src/test/java/de/adorsys/datasafe/storage/api/UriBasedAuthStorageServiceTest.java b/datasafe-storage/datasafe-storage-api/src/test/java/de/adorsys/datasafe/storage/api/UriBasedAuthStorageServiceTest.java index 03e6958c3..1dade06bd 100644 --- a/datasafe-storage/datasafe-storage-api/src/test/java/de/adorsys/datasafe/storage/api/UriBasedAuthStorageServiceTest.java +++ b/datasafe-storage/datasafe-storage-api/src/test/java/de/adorsys/datasafe/storage/api/UriBasedAuthStorageServiceTest.java @@ -2,12 +2,14 @@ import de.adorsys.datasafe.types.api.resource.AbsoluteLocation; import de.adorsys.datasafe.types.api.resource.BasePrivateResource; +import de.adorsys.datasafe.types.api.resource.PrivateResource; import de.adorsys.datasafe.types.api.resource.WithCallback; import de.adorsys.datasafe.types.api.shared.BaseMockitoTest; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; @@ -42,8 +44,49 @@ void init() { when(getService.apply(argumentCaptor.capture())).thenReturn(storage); tested = new UriBasedAuthStorageService(getService); } + @Test + void testDefaultConstructor() { + UriBasedAuthStorageService service = new UriBasedAuthStorageService(getService); + assertThat(service).isNotNull(); + } - @MethodSource("fixture") + @Test + void testCustomConstructor() { + Function segmentator = uri -> new String[] {"region", "bucket"}; + UriBasedAuthStorageService service = new UriBasedAuthStorageService(getService, segmentator); + assertThat(service).isNotNull(); + } + @Test + void testRegionExtractor() { + Function segmentator = uri -> new String[] {"region", "bucket"}; + UriBasedAuthStorageService service = new UriBasedAuthStorageService(getService, segmentator); + + URI uri = URI.create("http://host.com/region/bucket"); + String region = service.regionExtractor.apply(uri); + assertThat(region).isEqualTo("region"); + } + + @Test + void testBucketExtractor() { + Function segmentator = uri -> new String[] {"region", "bucket"}; + UriBasedAuthStorageService service = new UriBasedAuthStorageService(getService, segmentator); + + URI uri = URI.create("http://host.com/region/bucket"); + String bucket = service.bucketExtractor.apply(uri); + assertThat(bucket).isEqualTo("bucket"); + } + + @Test + void testEndpointExtractor() { + Function segmentator = uri -> new String[] {"region", "bucket"}; + UriBasedAuthStorageService service = new UriBasedAuthStorageService(getService, segmentator); + + URI uri = URI.create("http://host.com:8080/region/bucket"); + String endpoint = service.endpointExtractor.apply(uri); + assertThat(endpoint).isEqualTo("http://host.com:8080/"); + } + + @MethodSource("fixture") @ParameterizedTest void objectExists(MappedItem item) { tested.objectExists(item.getUri()); diff --git a/datasafe-storage/datasafe-storage-api/src/test/java/de/adorsys/datasafe/storage/api/UserBasedDelegatingStorageTest.java b/datasafe-storage/datasafe-storage-api/src/test/java/de/adorsys/datasafe/storage/api/UserBasedDelegatingStorageTest.java new file mode 100644 index 000000000..cabb327eb --- /dev/null +++ b/datasafe-storage/datasafe-storage-api/src/test/java/de/adorsys/datasafe/storage/api/UserBasedDelegatingStorageTest.java @@ -0,0 +1,134 @@ +package de.adorsys.datasafe.storage.api; + +import de.adorsys.datasafe.storage.api.actions.StorageWriteService; +import de.adorsys.datasafe.types.api.callback.ResourceWriteCallback; +import de.adorsys.datasafe.types.api.resource.AbsoluteLocation; +import de.adorsys.datasafe.types.api.resource.BasePrivateResource; +import de.adorsys.datasafe.types.api.resource.WithCallback; +import de.adorsys.datasafe.types.api.shared.BaseMockitoTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; + +import java.io.OutputStream; +import java.util.Optional; + +import java.util.List; +import java.util.function.Function; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +public class UserBasedDelegatingStorageTest extends BaseMockitoTest { + @Mock + private StorageService storage; + @Mock + private Function storageServiceBuilder; + private UserBasedDelegatingStorage tested; + private static final List AMAZON_BUCKETS = List.of("bucket1","bucket2" ); + + private AbsoluteLocation locationUser1 = new AbsoluteLocation<>( + BasePrivateResource.forPrivate("s3://datasafe-test1/073047da-dd68-4f70-b9bf-5759d7e30c85/users/user-1/private/files/") + ); + private AbsoluteLocation locationUser3 = new AbsoluteLocation<>( + BasePrivateResource.forPrivate("s3://datasafe-test1/073047da-dd68-4f70-b9bf-5759d7e30c85/users/user-3/private/files/") + ); + private AbsoluteLocation invalidLocation = new AbsoluteLocation<>( + BasePrivateResource.forPrivate("invalid://path") + ); + + @BeforeEach + void init() { + when(storageServiceBuilder.apply(any())).thenReturn(storage); + tested = new UserBasedDelegatingStorage(storageServiceBuilder, AMAZON_BUCKETS); + } + + @Test + void serviceUser1() { + tested.objectExists(locationUser1); + + verify(storageServiceBuilder).apply("bucket2"); + verify(storage).objectExists(locationUser1); + } + + @Test + void serviceUser3() { + tested.objectExists(locationUser3); + + verify(storageServiceBuilder).apply("bucket2"); + verify(storage).objectExists(locationUser3); + } + + @Test + void serviceInvalidLocation() { + assertThrows(IllegalStateException.class, () -> tested.objectExists(invalidLocation)); + + verify(storageServiceBuilder, never()).apply(any()); + verify(storage, never()).objectExists(any()); + } + + @Test + void flushChunkSizeUser1() { + Optional chunkSize = tested.flushChunkSize(locationUser1); + + verify(storageServiceBuilder).apply("bucket2"); + verify(storage).flushChunkSize(locationUser1); + assertThat(chunkSize).isEmpty(); + } + + @Test + void flushChunkSizeUser3() { + Optional chunkSize = tested.flushChunkSize(locationUser3); + + verify(storageServiceBuilder).apply("bucket2"); + verify(storage).flushChunkSize(locationUser3); + assertThat(chunkSize).isEmpty(); + } + + @Test + void listDelegates(){ + tested.list(locationUser1); + + verify(storageServiceBuilder).apply("bucket2"); + verify(storage).list(locationUser1); + } + + @Test + void readDelegates(){ + tested.read(locationUser1); + + verify(storageServiceBuilder).apply("bucket2"); + verify(storage).read(locationUser1); + } + + @Test + void removeDelegates(){ + tested.remove(locationUser1); + + verify(storageServiceBuilder).apply("bucket2"); + verify(storage).remove(locationUser1); + } + + @Test + void writeDelegates() { + tested.write(WithCallback.noCallback(locationUser1)); + + verify(storageServiceBuilder).apply("bucket2"); + verify(storage).write(any(WithCallback.class)); + } + + @Test + void defaultFlushChunkSize() { + + StorageWriteService defaultStorageWriteService = new StorageWriteService() { + @Override + public OutputStream write(WithCallback locationWithCallback) { + return null; + } + }; + Optional chunkSize = defaultStorageWriteService.flushChunkSize(locationUser1); + assertThat(chunkSize).isEmpty(); + } +} diff --git a/docs/diagrams/uri_routing_flowchart.puml b/docs/diagrams/uri_routing_flowchart.puml new file mode 100644 index 000000000..428d2ae44 --- /dev/null +++ b/docs/diagrams/uri_routing_flowchart.puml @@ -0,0 +1,18 @@ +@startuml + +!include + +Person(user, "User", "Uses the DataSafe storage API") +System(datasafestorage, "DataSafe Storage API", "Provides a unified interface for interacting with various storage backends") +System_Ext(s3, "AWS S3", "Cloud storage service") +System_Ext(filesystem, "Local File System", "Local storage") +System_Ext(otherstorage, "Other Storage Service", "Generic storage service") + +Rel(user, datasafestorage, "Uses", "API Calls") +Rel(datasafestorage, s3, "Delegates", "Storage Operations") +Rel(datasafestorage, filesystem, "Delegates", "Storage Operations") +Rel(datasafestorage, otherstorage, "Delegates", "Storage Operations") + +SHOW_LEGEND() + +@enduml \ No newline at end of file