Skip to content

Commit

Permalink
Merge branch 'feature/upgrade-0-7-1' into 'develop'
Browse files Browse the repository at this point in the history
Feature/upgrade 0 7 1

See merge request upm-inesdata/inesdata-connector!26
  • Loading branch information
ralconada-gmv committed Jun 28, 2024
2 parents 8c8cc26 + bd26b6e commit fcdea1f
Show file tree
Hide file tree
Showing 16 changed files with 171 additions and 35 deletions.
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package org.upm.inesdata.countelements;

import jakarta.json.Json;
import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration;
import org.eclipse.edc.runtime.metamodel.annotation.Extension;
import org.eclipse.edc.runtime.metamodel.annotation.Inject;
import org.eclipse.edc.runtime.metamodel.annotation.Provider;
Expand All @@ -12,6 +11,8 @@
import org.eclipse.edc.transaction.spi.TransactionContext;
import org.eclipse.edc.transform.spi.TypeTransformerRegistry;
import org.eclipse.edc.web.spi.WebService;
import org.eclipse.edc.web.spi.configuration.ApiContext;
import org.eclipse.edc.web.spi.configuration.context.ManagementApiUrl;
import org.upm.inesdata.countelements.controller.CountElementsApiController;
import org.upm.inesdata.countelements.service.CountElementsServiceImpl;
import org.upm.inesdata.countelements.transformer.JsonObjectFromCountElementTransformer;
Expand Down Expand Up @@ -39,9 +40,6 @@ public class CountElementsApiExtension implements ServiceExtension {
@Inject(required = false)
private HealthCheckService healthCheckService;

@Inject
private ManagementApiConfiguration config;

@Inject
private TypeManager typeManager;

Expand Down Expand Up @@ -75,6 +73,6 @@ public void initialize(ServiceExtensionContext context) {
managementApiTransformerRegistry.register(new JsonObjectFromCountElementTransformer(factory, jsonLdMapper));

var countElementsApiController = new CountElementsApiController(countElementsService(), managementApiTransformerRegistry);
webService.registerResource(config.getContextAlias(), countElementsApiController);
webService.registerResource(ApiContext.MANAGEMENT, countElementsApiController);
}
}
3 changes: 3 additions & 0 deletions extensions/extended-data-plane-public-api/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Extended data plane public API

Provides an extension of 'DataPlanePublicApiV2Extension' that allows performing HTTP-PULL from AmazonS3 destinations.
9 changes: 9 additions & 0 deletions extensions/extended-data-plane-public-api/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
plugins {
`java-library`
id("com.gmv.inesdata.edc-application")
}

dependencies {
api(libs.edc.spi.core)
implementation(libs.edc.data.plane.public.api)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
/*
* Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0
*
* SPDX-License-Identifier: Apache-2.0
*
* Contributors:
* Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
*
*/

package org.upm.inesdata.extendeddataplanepublic;

import org.eclipse.edc.connector.dataplane.api.controller.DataPlanePublicApiV2Controller;
import org.eclipse.edc.connector.dataplane.spi.Endpoint;
import org.eclipse.edc.connector.dataplane.spi.iam.DataPlaneAuthorizationService;
import org.eclipse.edc.connector.dataplane.spi.iam.PublicEndpointGeneratorService;
import org.eclipse.edc.connector.dataplane.spi.pipeline.PipelineService;
import org.eclipse.edc.runtime.metamodel.annotation.Extension;
import org.eclipse.edc.runtime.metamodel.annotation.Inject;
import org.eclipse.edc.runtime.metamodel.annotation.Setting;
import org.eclipse.edc.runtime.metamodel.annotation.SettingContext;
import org.eclipse.edc.spi.system.ExecutorInstrumentation;
import org.eclipse.edc.spi.system.Hostname;
import org.eclipse.edc.spi.system.ServiceExtension;
import org.eclipse.edc.spi.system.ServiceExtensionContext;
import org.eclipse.edc.web.spi.WebServer;
import org.eclipse.edc.web.spi.WebService;
import org.eclipse.edc.web.spi.configuration.ApiContext;
import org.eclipse.edc.web.spi.configuration.WebServiceConfigurer;
import org.eclipse.edc.web.spi.configuration.WebServiceSettings;

import java.util.concurrent.Executors;

/**
* This extension provides generic endpoints which are open to public participants of the Dataspace to execute
* requests on the actual data source.
*/
@Extension(value = ExtendedDataPlanePublicApiExtension.NAME)
public class ExtendedDataPlanePublicApiExtension implements ServiceExtension {
public static final String NAME = "Data Plane Public API";

private static final int DEFAULT_PUBLIC_PORT = 8185;
private static final String PUBLIC_CONTEXT_PATH = "/api/v2/public";

@SettingContext("Public API context setting key")
private static final String PUBLIC_CONFIG_KEY = "web.http." + ApiContext.PUBLIC;

@Setting(value = "Base url of the public API endpoint without the trailing slash. This should correspond to the values configured " +
"in '" + DEFAULT_PUBLIC_PORT + "' and '" + PUBLIC_CONTEXT_PATH + "'.", defaultValue = "http://<HOST>:" + DEFAULT_PUBLIC_PORT + PUBLIC_CONTEXT_PATH)
private static final String PUBLIC_ENDPOINT = "edc.dataplane.api.public.baseurl";

private static final int DEFAULT_THREAD_POOL = 10;
private static final WebServiceSettings PUBLIC_SETTINGS = WebServiceSettings.Builder.newInstance()
.apiConfigKey(PUBLIC_CONFIG_KEY)
.contextAlias(ApiContext.PUBLIC)
.defaultPath(PUBLIC_CONTEXT_PATH)
.defaultPort(DEFAULT_PUBLIC_PORT)
.name(NAME)
.build();

@Inject
private WebServer webServer;

@Inject
private WebServiceConfigurer webServiceConfigurer;

@Inject
private PipelineService pipelineService;

@Inject
private WebService webService;

@Inject
private ExecutorInstrumentation executorInstrumentation;

@Inject
private DataPlaneAuthorizationService authorizationService;

@Inject
private PublicEndpointGeneratorService generatorService;

@Inject
private Hostname hostname;

@Override
public String name() {
return NAME;
}

@Override
public void initialize(ServiceExtensionContext context) {
var config = context.getConfig(PUBLIC_CONFIG_KEY);
var configuration = webServiceConfigurer.configure(config, webServer, PUBLIC_SETTINGS);
var executorService = executorInstrumentation.instrument(
Executors.newFixedThreadPool(DEFAULT_THREAD_POOL),
"Data plane proxy transfers"
);

var publicEndpoint = context.getSetting(PUBLIC_ENDPOINT, null);
if (publicEndpoint == null) {
publicEndpoint = "http://%s:%d%s".formatted(hostname.get(), configuration.getPort(), configuration.getPath());
context.getMonitor().warning("Config property '%s' was not specified, the default '%s' will be used.".formatted(PUBLIC_ENDPOINT, publicEndpoint));
}
var endpoint = Endpoint.url(publicEndpoint);
generatorService.addGeneratorFunction("HttpData", dataAddress -> endpoint);
generatorService.addGeneratorFunction("AmazonS3", dataAddress -> endpoint);

var publicApiController = new DataPlanePublicApiV2Controller(pipelineService, executorService, authorizationService);
webService.registerResource(ApiContext.PUBLIC, publicApiController);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
org.upm.inesdata.extendeddataplanepublic.ExtendedDataPlanePublicApiExtension
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.upm.inesdata.federated;

import jakarta.json.Json;
import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration;
import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfigurationExtension;
import org.eclipse.edc.connector.controlplane.transform.edc.from.JsonObjectFromAssetTransformer;
import org.eclipse.edc.connector.controlplane.transform.edc.to.JsonObjectToAssetTransformer;
import org.eclipse.edc.jsonld.spi.JsonLd;
Expand All @@ -16,6 +16,7 @@
import org.eclipse.edc.transform.spi.TypeTransformerRegistry;
import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry;
import org.eclipse.edc.web.spi.WebService;
import org.eclipse.edc.web.spi.configuration.ApiContext;
import org.upm.inesdata.federated.controller.FederatedCatalogCacheApiController;
import org.upm.inesdata.federated.service.FederatedCatalogCacheServiceImpl;
import org.upm.inesdata.spi.federated.FederatedCatalogCacheService;
Expand All @@ -34,9 +35,6 @@ public class FederatedCatalogCacheApiExtension implements ServiceExtension {
@Inject
private WebService webService;

@Inject
private ManagementApiConfiguration config;

@Inject
private TypeManager typeManager;

Expand Down Expand Up @@ -85,6 +83,6 @@ public void initialize(ServiceExtensionContext context) {

var federatedCatalogCacheApiController = new FederatedCatalogCacheApiController(this.federatedCatalogCacheService(), managementApiTransformerRegistry,
validator,monitor);
webService.registerResource(config.getContextAlias(), federatedCatalogCacheApiController);
webService.registerResource(ApiContext.MANAGEMENT, federatedCatalogCacheApiController);
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.upm.inesdata.federated.sql.index;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.eclipse.edc.catalog.spi.FederatedCacheStore;
import org.eclipse.edc.catalog.spi.FederatedCatalogCache;
import org.eclipse.edc.runtime.metamodel.annotation.Extension;
import org.eclipse.edc.runtime.metamodel.annotation.Inject;
import org.eclipse.edc.runtime.metamodel.annotation.Provider;
Expand All @@ -20,7 +20,7 @@
/**
* Extension that stores federatedCatalogs in SQL databases
*/
@Provides({ FederatedCacheStore.class, PaginatedFederatedCacheStoreIndex.class })
@Provides({ FederatedCatalogCache.class, PaginatedFederatedCacheStoreIndex.class })
@Extension(value = "SQL federatedCatalog index")
public class SqlFederatedCacheServiceExtension implements ServiceExtension {

Expand Down Expand Up @@ -64,7 +64,7 @@ public void initialize(ServiceExtensionContext context) {
getDialect(), queryExecutor);

context.registerService(PaginatedFederatedCacheStoreIndex.class, sqlFederatedCacheStore);
context.registerService(FederatedCacheStore.class, sqlFederatedCacheStore);
context.registerService(FederatedCatalogCache.class, sqlFederatedCacheStore);
}

private SqlFederatedCatalogStatements getDialect() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,10 @@ public void save(Catalog catalog) {
* @throws EdcPersistenceException if a SQL error occurs during the query.
*/
@Override
public Collection<Catalog> query(List<Criterion> query) {
public Collection<Catalog> query(QuerySpec query) {
return transactionContext.execute(() -> {
try (var connection = getConnection()) {
QuerySpec querySpec = QuerySpec.Builder.newInstance().filter(query).build();
SqlQueryStatement queryStatement = databaseStatements.createQuery(querySpec);
SqlQueryStatement queryStatement = databaseStatements.createQuery(query);
return queryExecutor.query(connection, true, this::mapResultSetToCatalog, queryStatement.getQueryAsString(),
queryStatement.getParameters()).collect(Collectors.toList());
} catch (SQLException e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import jakarta.json.Json;
import org.eclipse.edc.api.validation.DataAddressValidator;
import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration;
import org.eclipse.edc.connector.controlplane.api.management.asset.validation.AssetValidator;
import org.eclipse.edc.connector.controlplane.services.spi.asset.AssetService;
import org.eclipse.edc.connector.controlplane.transform.edc.from.JsonObjectFromAssetTransformer;
Expand All @@ -18,6 +17,7 @@
import org.eclipse.edc.transform.spi.TypeTransformerRegistry;
import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry;
import org.eclipse.edc.web.spi.WebService;
import org.eclipse.edc.web.spi.configuration.ApiContext;
import org.upm.inesdata.storageasset.controller.StorageAssetApiController;
import org.upm.inesdata.storageasset.service.S3Service;
import software.amazon.awssdk.regions.Region;
Expand Down Expand Up @@ -47,9 +47,6 @@ public class StorageAssetApiExtension implements ServiceExtension {
@Inject
private WebService webService;

@Inject
private ManagementApiConfiguration config;

@Inject
private TypeManager typeManager;

Expand Down Expand Up @@ -104,6 +101,6 @@ public void initialize(ServiceExtensionContext context) {
var storageAssetApiController = new StorageAssetApiController(assetService, managementApiTransformerRegistry,
validator,s3Service,
jsonLd, bucketName, regionName);
webService.registerResource(config.getContextAlias(), storageAssetApiController);
webService.registerResource(ApiContext.MANAGEMENT, storageAssetApiController);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ private long getFileSize(InputStream inputStream) throws IOException {
private void setStorageProperties(Asset asset, String fileName) {
asset.getPrivateProperties().put("storageAssetFile", fileName);
asset.getDataAddress().setKeyName(fileName);
asset.getDataAddress().setType("InesDataStore");
asset.getDataAddress().setType("AmazonS3");
asset.getDataAddress().getProperties().put(CoreConstants.EDC_NAMESPACE+ "bucketName", bucketName);
asset.getDataAddress().getProperties().put(CoreConstants.EDC_NAMESPACE+"region", region);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package org.upm.inesdata.vocabulary;

import jakarta.json.Json;
import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration;
import org.eclipse.edc.runtime.metamodel.annotation.Extension;
import org.eclipse.edc.runtime.metamodel.annotation.Inject;
import org.eclipse.edc.runtime.metamodel.annotation.Provider;
Expand All @@ -14,6 +13,7 @@
import org.eclipse.edc.transform.spi.TypeTransformerRegistry;
import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry;
import org.eclipse.edc.web.spi.WebService;
import org.eclipse.edc.web.spi.configuration.ApiContext;
import org.upm.inesdata.spi.vocabulary.VocabularyIndex;
import org.upm.inesdata.spi.vocabulary.VocabularyService;
import org.upm.inesdata.vocabulary.controller.VocabularyApiController;
Expand Down Expand Up @@ -46,9 +46,6 @@ public class VocabularyApiExtension implements ServiceExtension {
@Inject(required = false)
private HealthCheckService healthCheckService;

@Inject
private ManagementApiConfiguration config;

@Inject
private TypeManager typeManager;

Expand Down Expand Up @@ -90,7 +87,7 @@ public void initialize(ServiceExtensionContext context) {

validator.register(EDC_VOCABULARY_TYPE, VocabularyValidator.instance());
var vocabularyApiController = new VocabularyApiController(this.vocabularyService(), managementApiTransformerRegistry, monitor, validator);
webService.registerResource(config.getContextAlias(), vocabularyApiController);
webService.registerResource(ApiContext.MANAGEMENT, vocabularyApiController);

// contribute to the liveness probe
if (healthCheckService != null) {
Expand Down
16 changes: 10 additions & 6 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,15 @@ format.version = "1.1"

[versions]
assertj = "3.26.0"
aws = "2.25.66"
edc = "0.7.0"
aws = "2.26.7"
edc = "0.7.1"
edc-aws = "0.7.1-SNAPSHOT"
jakarta-json = "2.0.1"
jersey = "3.0.0"
jersey = "3.1.7"
junit-pioneer = "2.2.0"
jupiter = "5.10.2"
mockito = "5.2.0"
parsson = "1.1.4"
parsson = "1.1.6"
postgres = "42.7.3"
rsApi = "4.0.0"
rseeApi = "10.0.0"
Expand All @@ -20,6 +21,7 @@ swagger-annotations-jakarta = "2.2.21"
edc-api-asset = { module = "org.eclipse.edc:asset-api", version.ref = "edc" }
edc-api-core = { module = "org.eclipse.edc:api-core", version.ref = "edc" }
edc-api-management-config = { module = "org.eclipse.edc:management-api-configuration", version.ref = "edc" }
edc-api-control-config = { module = "org.eclipse.edc:control-api-configuration", version.ref = "edc" }
edc-auth-spi = { module = "org.eclipse.edc:auth-spi", version.ref = "edc" }
edc-build-plugin = { module = "org.eclipse.edc.edc-build:org.eclipse.edc.edc-build.gradle.plugin", version.ref = "edc" }
edc-configuration-filesystem = { module = "org.eclipse.edc:configuration-filesystem", version.ref = "edc" }
Expand All @@ -35,6 +37,8 @@ edc-data-plane-core = { module = "org.eclipse.edc:data-plane-core", version.ref
edc-data-plane-http = { module = "org.eclipse.edc:data-plane-http", version.ref = "edc" }
edc-data-plane-selector-api = { module = "org.eclipse.edc:data-plane-selector-api", version.ref = "edc" }
edc-data-plane-selector-core = { module = "org.eclipse.edc:data-plane-selector-core", version.ref = "edc" }
edc-data-plane-selector-control-api = { module = "org.eclipse.edc:data-plane-selector-control-api", version.ref = "edc" }
edc-data-plane-selector-client = { module = "org.eclipse.edc:data-plane-selector-client", version.ref = "edc" }
edc-data-plane-self-registration = { module = "org.eclipse.edc:data-plane-self-registration", version.ref = "edc" }
edc-dsp = { module = "org.eclipse.edc:dsp", version.ref = "edc" }
edc-dsp-api-configuration = { module = "org.eclipse.edc:dsp-http-api-configuration", version.ref = "edc" }
Expand Down Expand Up @@ -76,8 +80,8 @@ edc-sql-transfer-process-store = { module = "org.eclipse.edc:transfer-process-st
edc-sql-data-plane-store = { module = "org.eclipse.edc:data-plane-store-sql", version.ref = "edc" }

# EDC aws s3 stuff
edc-aws-s3-core = { module = "org.eclipse.edc:aws-s3-core", version.ref = "edc" }
edc-data-plane-aws-s3 = { module = "org.eclipse.edc:data-plane-aws-s3", version.ref = "edc" }
edc-aws-s3-core = { module = "org.eclipse.edc:aws-s3-core", version.ref = "edc-aws" }
edc-data-plane-aws-s3 = { module = "org.eclipse.edc:data-plane-aws-s3", version.ref = "edc-aws" }

# EDC Observability
edc-observability-api = { module = "org.eclipse.edc:api-observability", version.ref = "edc" }
Expand Down
4 changes: 4 additions & 0 deletions launchers/connector/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ dependencies {
// Common libs
implementation(libs.edc.dsp)
implementation(libs.edc.management.api)
implementation(libs.edc.api.control.config)

// Temporary libs
implementation(libs.edc.configuration.filesystem)
Expand Down Expand Up @@ -89,6 +90,9 @@ dependencies {
// Count elements
implementation(project(":extensions:count-elements-api"))

//Data plane public api
implementation(project(":extensions:extended-data-plane-public-api"))

runtimeOnly(libs.edc.transaction.local)
runtimeOnly(libs.postgres)
}
Expand Down
1 change: 1 addition & 0 deletions settings.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ include(":extensions:federated-catalog-cache-sql")
include(":extensions:federated-catalog-cache-api")
include(":extensions:count-elements-api")
include(":extensions:count-elements-sql")
include(":extensions:extended-data-plane-public-api")

// Connector
include(":launchers:connector")
Loading

0 comments on commit fcdea1f

Please sign in to comment.