Skip to content

Commit

Permalink
External Postgres support (#79)
Browse files Browse the repository at this point in the history
* #73 Update Yaci Store: PostgreSQL support and schema management

* Fix artifact save path and clarify comment formatting

* Update external DB management message in YaciStoreService
  • Loading branch information
satran004 authored Nov 12, 2024
1 parent 6596e92 commit 84d8276
Show file tree
Hide file tree
Showing 9 changed files with 182 additions and 16 deletions.
3 changes: 3 additions & 0 deletions applications/cli/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ dependencies {
implementation 'org.apache.commons:commons-text:1.9'

implementation 'org.codehaus.janino:janino:3.1.8'

runtimeOnly 'org.postgresql:postgresql:42.7.1'

compileOnly 'org.projectlombok:lombok'

developmentOnly 'org.springframework.boot:spring-boot-devtools'
Expand Down
10 changes: 10 additions & 0 deletions applications/cli/config/application.properties
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,13 @@ bp.create.enabled=true
#socat.port=3333
#prometheus.port=12798


######################################################
#To configure an external database for Yaci Store (Indexer),
# uncomment the following properties and provide the required values
#Only PostgreSQL is supported for now for external database
######################################################

#yaci.store.db.url=jdbc:postgresql://localhost:5433/yaci_indexer?currentSchema=dev
#yaci.store.db.username=user
#yaci.store.db.password=
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.bloxbean.cardano.yacicli.commands.common.Groups;
import com.bloxbean.cardano.yacicli.common.CommandContext;
import com.bloxbean.cardano.yacicli.localcluster.ClusterConfig;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.shell.Availability;
Expand All @@ -15,6 +16,7 @@
@Slf4j
public class YaciStoreCommands {
private final YaciStoreService yaciStoreService;
private final YaciStoreCustomDbHelper yaciStoreCustomDbHelper;

@ShellMethod(value = "Show recent Yaci Store logs", key = "yaci-store-logs")
@ShellMethodAvailability("yaciStoreEnableAvailability")
Expand Down Expand Up @@ -42,6 +44,12 @@ public void checkYaciStoreStatus() {
writeLn(info("Yaci Store Status: disable"));
}

@ShellMethod(value = "Drop yaci store db or schema (for external Postgres)", key = "yaci-store-drop-db")
public void dropYaciStoreDb() {
String clusterName = CommandContext.INSTANCE.getProperty(ClusterConfig.CLUSTER_NAME);
yaciStoreCustomDbHelper.dropDatabase(clusterName);
}

public Availability yaciStoreEnableAvailability() {
return CommandContext.INSTANCE.getCurrentMode() == CommandContext.Mode.LOCAL_CLUSTER
? Availability.available()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package com.bloxbean.cardano.yacicli.localcluster.yacistore;

import com.bloxbean.cardano.yacicli.localcluster.ClusterService;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.nio.file.Path;
import java.sql.*;

import static com.bloxbean.cardano.yacicli.util.ConsoleWriter.*;
import static com.bloxbean.cardano.yacicli.util.ConsoleWriter.success;

@Component
@RequiredArgsConstructor
@Getter
public class YaciStoreCustomDbHelper {
private final ClusterService clusterService;

//External postgres db configuration
@Value("${yaci.store.db.url:#{null}}")
private String storeDbUrl;
@Value("${yaci.store.db.username:#{null}}")
private String storeDbUsername;
@Value("${yaci.store.db.password:#{null}}")
private String storeDbPassword;

public void dropDatabase(String clusterName) {
if (storeDbUrl == null || storeDbUrl.isEmpty() || !storeDbUrl.contains("postgresql")) {
//Looks like default db.
Path clusterFolder = clusterService.getClusterFolder(clusterName);
Path nodeFolder = clusterFolder.resolve("node");
String dbDir = "yaci_store";
Path dbPath = nodeFolder.resolve(dbDir);

writeLn(info("Deleting Yaci Store db folder : " + dbPath.toFile().getAbsolutePath()));
if (dbPath.toFile().exists()) {
try {
FileUtils.deleteDirectory(dbPath.toFile());
writeLn(success("Yaci Store db folder deleted successfully"));
} catch (IOException e) {
writeLn(error("Yaci store db could not be deleted : " + dbPath.toAbsolutePath()));
}
}

return;
}

if(storeDbUrl != null && storeDbUrl.contains("postgres")) {
dropPostgresSqlSchema(storeDbUrl, storeDbUsername, storeDbPassword);
}
}

private void dropPostgresSqlSchema(String url, String user, String password) {
Connection conn = null;
try {
Class.forName("org.postgresql.Driver");

// Establish the connection
conn = DriverManager.getConnection(url, user, password);

String schema = conn.getSchema();

writeLn("Got schema: " + schema);
// Try to drop the schema
String dropSchemaSQL = "DROP SCHEMA " + schema + " CASCADE";
try (Statement stmt = conn.createStatement()) {
stmt.executeUpdate(dropSchemaSQL);
writeLn("Schema '" + schema + "' dropped successfully.");
} catch (SQLException e) {
// Handle specific SQLState codes
String sqlState = e.getSQLState();
if ("42501".equals(sqlState)) {
writeLn(error("Insufficient privileges to drop the schema '" + schema + "'."));
} else if ("3F000".equals(sqlState)) {
writeLn(error("Schema '" + schema + "' does not exist."));
} else {
writeLn(error("An error occurred while trying to drop the schema:"));
e.printStackTrace();
}
}

} catch (ClassNotFoundException e) {
writeLn(error("PostgreSQL JDBC Driver not found."));
e.printStackTrace();
} catch (SQLException e) {
writeLn(error("Database connection error occurred."));
e.printStackTrace();
} finally {
// Close the connection
if (conn != null) {
try { conn.close(); } catch (SQLException e) { e.printStackTrace(); }
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public class YaciStoreService {
private final ClusterConfig clusterConfig;
private final JreResolver jreResolver;
private final YaciStoreConfigBuilder yaciStoreConfigBuilder;
private final YaciStoreCustomDbHelper customDBHelper;

private List<Process> processes = new ArrayList<>();

Expand Down Expand Up @@ -138,17 +139,39 @@ private Process startStoreApp(ClusterInfo clusterInfo, Era era) throws IOExcepti
writeLn(info("Java Path: " + javaExecPath));
}

//Set custom db info if provided through env file (docker) or application.properties
if (customDBHelper.getStoreDbUrl() != null) {
builder.environment().put("SPRING_DATASOURCE_URL", customDBHelper.getStoreDbUrl());

writeLn(info("Yaci Store DB Url: " + customDBHelper.getStoreDbUrl()));
writeLn(info("Yaci Store DB User: " + customDBHelper.getStoreDbUsername()));
}
if (customDBHelper.getStoreDbUsername() != null)
builder.environment().put("SPRING_DATASOURCE_USERNAME", customDBHelper.getStoreDbUsername());
if (customDBHelper.getStoreDbPassword() != null)
builder.environment().put("SPRING_DATASOURCE_PASSWORD", customDBHelper.getStoreDbPassword());

Process process = builder.start();

writeLn(success("Yaci store starting ..."));
AtomicBoolean started = new AtomicBoolean(false);
AtomicBoolean intersectNotFoundAlreadyShown = new AtomicBoolean(false);
ProcessStream processStream =
new ProcessStream(process.getInputStream(), line -> {
logs.add(line);
if (line != null && line.contains("Started YaciStoreApplication")) {
writeLn(infoLabel("OK", "Yaci Store Started"));
started.set(true);
}

if (line != null && customDBHelper.getStoreDbUrl() != null && !customDBHelper.getStoreDbUrl().isEmpty()) {
if (!intersectNotFoundAlreadyShown.get() && line.contains("IntersactNotFound")) {
writeLn(warn("Looks like some issue while starting yaci store."));
writeLn(warn("Please check the logs for more details. Command: yaci-store-logs"));
writeLn(warn("Please verify if you are using an empty schema while creating a new devnet."));
intersectNotFoundAlreadyShown.set(true);
}
}
});
Future<?> future = Executors.newSingleThreadExecutor().submit(processStream);

Expand All @@ -168,6 +191,17 @@ private Process startStoreApp(ClusterInfo clusterInfo, Era era) throws IOExcepti
"If so, please check the process and kill it manually. e.g. kill -9 <pid>"));
}

if (customDBHelper.getStoreDbUrl() != null && !customDBHelper.getStoreDbUrl().isEmpty()) {
writeLn("");
writeLn(info("######################### Important ########################################################################################"));
writeLn("!!!! Yaci Store is connecting to an external database: " + customDBHelper.getStoreDbUrl());
writeLn("Automatic management of an external database may not be possible during 'reset' or when creating a new devnet with 'create-node'.");
writeLn("You can use the 'yaci-store-drop-db' command to drop the schema. If that doesn’t work, " +
"please drop the schema manually before performing a reset or creating a new devnet.");
writeLn("Use the 'yaci-store-logs' command to verify if Yaci Store has started successfully.");
writeLn("###########################################################################################################################");
}

return process;
}

Expand Down Expand Up @@ -249,20 +283,7 @@ public void handleClusterStopped(ClusterStopped clusterStopped) {

@EventListener
public void handleClusterDeleted(ClusterDeleted clusterDeleted) {
Path clusterFolder = clusterService.getClusterFolder(clusterDeleted.getClusterName());
Path nodeFolder = clusterFolder.resolve("node");
String dbDir = "yaci_store";
Path dbPath = nodeFolder.resolve(dbDir);

writeLn(info("Deleting Yaci Store db folder : " + dbPath.toFile().getAbsolutePath()));
if (dbPath.toFile().exists()) {
try {
FileUtils.deleteDirectory(dbPath.toFile());
writeLn(success("Yaci Store db folder deleted successfully"));
} catch (IOException e) {
writeLn(error("Yaci store db could not be deleted : " + dbPath.toAbsolutePath()));
}
}
customDBHelper.dropDatabase(clusterDeleted.getClusterName());
}

private void killForcibly(Process process) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,21 @@
"allPublicConstructors": true,
"allPublicFields": true,
"allDeclaredMethods": true
},
{
"name":"org.postgresql.Driver",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
"name":"org.postgresql.core.QueryExecutorCloseAction",
"fields":[{"name":"pgStream"}]
},
{
"name":"org.postgresql.jdbc.PgStatement",
"fields":[{"name":"cancelTimerTask"}, {"name":"isClosed"}, {"name":"statementState"}]
},
{
"name":"org.postgresql.util.PGobject"
}
]

2 changes: 1 addition & 1 deletion applications/store-build/Earthfile
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,5 @@ store-native:
RUN --mount=type=cache,target=/root/.gradle ./gradlew --no-daemon -i -Pversion=${APP_VERSION} clean build nativeCompile

# Save artifacts
SAVE ARTIFACT applications/all/build/native/nativeCompile/yaci-store-all yaci-store
SAVE ARTIFACT applications/all/build/native/nativeCompile/yaci-store yaci-store
SAVE ARTIFACT store.git.properties yaci-store.git.properties
10 changes: 10 additions & 0 deletions config/env
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,13 @@ IS_DOCKER=true

#topup_addresses=addr_test1qzlwg5c3mpr0cz5td0rvr5rvcgf02al05cqgd2wzv7pud6chpzk4elx4jh2f7xtftjrdxddr88wg6sfszu8r3gktpjtqrr00q9:2000,addr_test1qqwpl7h3g84mhr36wpetk904p7fchx2vst0z696lxk8ujsjyruqwmlsm344gfux3nsj6njyzj3ppvrqtt36cp9xyydzqzumz82:1000


############################################################
#To configure an external database for Yaci Store (Indexer),
#uncomment the following properties and provide the required values
#Only PostgreSQL is supported for now for external database
############################################################

#yaci_store_db_url=jdbc:postgresql://192.168.0.68:5433/yaci_indexer?currentSchema=dev
#yaci_store_db_username=user
#yaci_store_db_password=
2 changes: 1 addition & 1 deletion config/version
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
tag=0.10.0-preview1
tag=0.10.0-preview2
revision=

0 comments on commit 84d8276

Please sign in to comment.