From 1584064cbcbe3e16d262098a1301bd90da7d85ae Mon Sep 17 00:00:00 2001 From: Peter Kraft Date: Fri, 11 Oct 2024 17:57:27 -0700 Subject: [PATCH] Use Cloud Database Locally (#645) Add support for using a DBOS Cloud Postgres server for local development. --- dbos-config.schema.json | 4 + .../applications/deploy-app-code.ts | 70 +------------- packages/dbos-cloud/applications/types.ts | 8 ++ packages/dbos-cloud/cli.ts | 28 ++++-- packages/dbos-cloud/cloudutils.ts | 1 + packages/dbos-cloud/configutils.ts | 1 + packages/dbos-cloud/databases/databases.ts | 96 +++++++++++++++++-- src/dbos-runtime/config.ts | 8 +- src/dbos-runtime/migrate.ts | 9 +- tests/dbos-runtime/config.test.ts | 26 +++++ 10 files changed, 158 insertions(+), 93 deletions(-) diff --git a/dbos-config.schema.json b/dbos-config.schema.json index 6d2c34763..3556d6788 100644 --- a/dbos-config.schema.json +++ b/dbos-config.schema.json @@ -59,6 +59,10 @@ "type": "string", "description": "If using SSL/TLS to securely connect to a database, path to an SSL root certificate file" }, + "local_suffix": { + "type": "boolean", + "description": "Whether to suffix app_db_name with '_local'. Set to true when doing local development using a DBOS Cloud database." + }, "app_db_client": { "type": "string", "description": "Specify the database client to use to connect to the application database", diff --git a/packages/dbos-cloud/applications/deploy-app-code.ts b/packages/dbos-cloud/applications/deploy-app-code.ts index 4771f5946..1678a791f 100644 --- a/packages/dbos-cloud/applications/deploy-app-code.ts +++ b/packages/dbos-cloud/applications/deploy-app-code.ts @@ -21,11 +21,10 @@ import { Application } from "./types.js"; import JSZip from "jszip"; import fg from "fast-glob"; import chalk from "chalk"; -import { createUserDb, UserDBInstance } from "../databases/databases.js"; import { registerApp } from "./register-app.js"; -import { input, select } from "@inquirer/prompts"; import { Logger } from "winston"; import { loadConfigFile } from "../configutils.js"; +import { chooseAppDBServer } from "../databases/databases.js"; type DeployOutput = { ApplicationName: string; @@ -292,70 +291,3 @@ async function isAppRegistered(logger: Logger, host: string, appName: string, us } return app; } - -async function chooseAppDBServer(logger: Logger, host: string, userCredentials: DBOSCloudCredentials, userDBName: string = ""): Promise { - // List existing database instances. - let userDBs: UserDBInstance[] = []; - const bearerToken = "Bearer " + userCredentials.token; - try { - const res = await axios.get(`https://${host}/v1alpha1/${userCredentials.organization}/databases`, { - headers: { - "Content-Type": "application/json", - Authorization: bearerToken, - }, - }); - userDBs = res.data as UserDBInstance[]; - } catch (e) { - const errorLabel = `Failed to list databases`; - const axiosError = e as AxiosError; - if (isCloudAPIErrorResponse(axiosError.response?.data)) { - handleAPIErrors(errorLabel, axiosError); - } else { - logger.error(`${errorLabel}: ${(e as Error).message}`); - } - return ""; - } - - if (userDBName) { - // Check if the database instance exists or not. - const dbExists = userDBs.some((db) => db.PostgresInstanceName === userDBName); - if (dbExists) { - return userDBName; - } - } - - if (userDBName || userDBs.length === 0) { - // If not, prompt the user to provision one. - if (!userDBName) { - logger.info("No database found, provisioning a database instance (server)..."); - userDBName = await input({ - message: "Database instance name?", - default: `${userCredentials.userName}-db-server`, - }); - } else { - logger.info(`Database instance ${userDBName} not found, provisioning a new one...`); - } - - // Use a default user name and auto generated password. - const appDBUserName = "dbos_user"; - const appDBPassword = Buffer.from(Math.random().toString()).toString("base64"); - const res = await createUserDb(host, userDBName, appDBUserName, appDBPassword, true); - if (res !== 0) { - return ""; - } - } else if (userDBs.length > 1) { - // If there is more than one database instances, prompt the user to select one. - userDBName = await select({ - message: "Choose a database instance for this app:", - choices: userDBs.map((db) => ({ - name: db.PostgresInstanceName, - value: db.PostgresInstanceName, - })), - }); - } else { - // Use the only available database server. - userDBName = userDBs[0].PostgresInstanceName; - logger.info(`Using database instance: ${userDBName}`); - } - return userDBName; -} diff --git a/packages/dbos-cloud/applications/types.ts b/packages/dbos-cloud/applications/types.ts index 0d6ceb8d9..183b68a4e 100644 --- a/packages/dbos-cloud/applications/types.ts +++ b/packages/dbos-cloud/applications/types.ts @@ -29,3 +29,11 @@ export function prettyPrintApplicationVersion(version: ApplicationVersion) { console.log(`Version: ${version.Version}`); console.log(`Creation Timestamp: ${version.CreationTime}`); } + +export interface UserDBInstance { + readonly PostgresInstanceName: string; + readonly Status: string; + readonly HostName: string; + readonly Port: number; + readonly DatabaseUsername: string; +} diff --git a/packages/dbos-cloud/cli.ts b/packages/dbos-cloud/cli.ts index 4c4214418..84c107cab 100755 --- a/packages/dbos-cloud/cli.ts +++ b/packages/dbos-cloud/cli.ts @@ -246,11 +246,11 @@ databaseCommands databaseCommands .command("reset-password") .description("Reset password for a Postgres database instance") - .argument("", "database instance name") + .argument("[name]", "database instance name") .option("-W, --password ", "Specify the database user password") - .action(async (dbName: string, options: { password: string }) => { + .action(async (dbName: string | undefined, options: { password: string }) => { if (!options.password) { - options.password = prompt("Database Password: ", { echo: "*" }); + options.password = prompt("Database Password (must contain at least 8 characters): ", { echo: "*" }); } const exitCode = await resetDBCredentials(DBOSCloudHost, dbName, options.password); process.exit(exitCode); @@ -305,16 +305,30 @@ databaseCommands databaseCommands .command("connect") .description(`Load cloud database connection information into ${dbosConfigFilePath}`) - .argument("", "database instance name") + .argument("[name]", "database instance name") .option("-W, --password ", "Specify the database user password") - .action(async (dbname: string, options: { password: string | undefined; }) => { + .action(async (dbname: string | undefined, options: { password: string | undefined; }) => { if (!options.password) { - options.password = prompt("Database Password: ", { echo: "*" }); + options.password = prompt("Database Password (must contain at least 8 characters): ", { echo: "*" }); } - const exitCode = await connect(DBOSCloudHost, dbname, options.password); + const exitCode = await connect(DBOSCloudHost, dbname, options.password, false); process.exit(exitCode); }); + databaseCommands + .command("local") + .description(`Configure ${dbosConfigFilePath} to use a DBOS Cloud database for local development`) + .argument("[name]", "database instance name") + .option("-W, --password ", "Specify the database user password") + .action(async (dbname: string | undefined, options: { password: string | undefined; }) => { + if (!options.password) { + options.password = prompt("Database Password (must contain at least 8 characters): ", { echo: "*" }); + } + const exitCode = await connect(DBOSCloudHost, dbname, options.password, true); + process.exit(exitCode); + }); + + ///////////////////// /* USER DASHBOARDS */ ///////////////////// diff --git a/packages/dbos-cloud/cloudutils.ts b/packages/dbos-cloud/cloudutils.ts index ed023519e..7a769cbc4 100644 --- a/packages/dbos-cloud/cloudutils.ts +++ b/packages/dbos-cloud/cloudutils.ts @@ -367,3 +367,4 @@ async function registerUser(host: string, credentials: DBOSCloudCredentials, log } return; } + diff --git a/packages/dbos-cloud/configutils.ts b/packages/dbos-cloud/configutils.ts index 7b62b0f18..79d1f816a 100644 --- a/packages/dbos-cloud/configutils.ts +++ b/packages/dbos-cloud/configutils.ts @@ -11,6 +11,7 @@ export interface ConfigFile { username: string; password?: string; app_db_name: string; + local_suffix: boolean; }; } diff --git a/packages/dbos-cloud/databases/databases.ts b/packages/dbos-cloud/databases/databases.ts index b735d1329..efdbd34c9 100644 --- a/packages/dbos-cloud/databases/databases.ts +++ b/packages/dbos-cloud/databases/databases.ts @@ -3,14 +3,9 @@ import { isCloudAPIErrorResponse, handleAPIErrors, getCloudCredentials, getLogge import { Logger } from "winston"; import { ConfigFile, loadConfigFile, writeConfigFile } from "../configutils.js"; import { copyFileSync, existsSync } from "fs"; +import { UserDBInstance } from "../applications/types.js"; +import { input, select } from "@inquirer/prompts"; -export interface UserDBInstance { - readonly PostgresInstanceName: string; - readonly Status: string; - readonly HostName: string; - readonly Port: number; - readonly DatabaseUsername: string; -} function isValidPassword(logger: Logger, password: string): boolean { if (password.length < 8 || password.length > 128) { @@ -257,9 +252,15 @@ async function getUserDBInfo(host: string, dbName: string, userCredentials?: DBO return res.data as UserDBInstance; } -export async function resetDBCredentials(host: string, dbName: string, appDBPassword: string) { +export async function resetDBCredentials(host: string, dbName: string | undefined, appDBPassword: string) { const logger = getLogger(); const userCredentials = await getCloudCredentials(host, logger); + + dbName = await chooseAppDBServer(logger, host, userCredentials, dbName) + if (dbName === "") { + return 1 + } + const bearerToken = "Bearer " + userCredentials.token; if (!isValidPassword(logger, appDBPassword)) { @@ -336,9 +337,16 @@ export async function restoreUserDB(host: string, dbName: string, targetName: st } } -export async function connect(host: string, dbName: string, password: string) { +export async function connect(host: string, dbName: string | undefined, password: string, local_suffix: boolean) { const logger = getLogger(); + const userCredentials = await getCloudCredentials(host, logger); + + dbName = await chooseAppDBServer(logger, host, userCredentials, dbName) + if (dbName === "") { + return 1 + } + try { if(!existsSync(dbosConfigFilePath)) { logger.error(`Error: ${dbosConfigFilePath} not found`); @@ -350,7 +358,7 @@ export async function connect(host: string, dbName: string, password: string) { copyFileSync(dbosConfigFilePath, backupConfigFilePath); logger.info("Retrieving cloud database info..."); - const userDBInfo = await getUserDBInfo(host, dbName); + const userDBInfo = await getUserDBInfo(host, dbName, userCredentials); console.log(`Postgres Instance Name: ${userDBInfo.PostgresInstanceName}`); console.log(`Host Name: ${userDBInfo.HostName}`); console.log(`Port: ${userDBInfo.Port}`); @@ -363,6 +371,7 @@ export async function connect(host: string, dbName: string, password: string) { config.database.port = userDBInfo.Port; config.database.username = userDBInfo.DatabaseUsername; config.database.password = password; + config.database.local_suffix = local_suffix; writeConfigFile(config, dbosConfigFilePath); logger.info(`Cloud database connection information loaded into ${dbosConfigFilePath}`) return 0; @@ -377,3 +386,70 @@ export async function connect(host: string, dbName: string, password: string) { return 1; } } + +export async function chooseAppDBServer(logger: Logger, host: string, userCredentials: DBOSCloudCredentials, userDBName: string = ""): Promise { + // List existing database instances. + let userDBs: UserDBInstance[] = []; + const bearerToken = "Bearer " + userCredentials.token; + try { + const res = await axios.get(`https://${host}/v1alpha1/${userCredentials.organization}/databases`, { + headers: { + "Content-Type": "application/json", + Authorization: bearerToken, + }, + }); + userDBs = res.data as UserDBInstance[]; + } catch (e) { + const errorLabel = `Failed to list databases`; + const axiosError = e as AxiosError; + if (isCloudAPIErrorResponse(axiosError.response?.data)) { + handleAPIErrors(errorLabel, axiosError); + } else { + logger.error(`${errorLabel}: ${(e as Error).message}`); + } + return ""; + } + + if (userDBName) { + // Check if the database instance exists or not. + const dbExists = userDBs.some((db) => db.PostgresInstanceName === userDBName); + if (dbExists) { + return userDBName; + } + } + + if (userDBName || userDBs.length === 0) { + // If not, prompt the user to provision one. + if (!userDBName) { + logger.info("No database found, provisioning a database instance (server)..."); + userDBName = await input({ + message: "Database instance name?", + default: `${userCredentials.userName}-db-server`, + }); + } else { + logger.info(`Database instance ${userDBName} not found, provisioning a new one...`); + } + + // Use a default user name and auto generated password. + const appDBUserName = "dbos_user"; + const appDBPassword = Buffer.from(Math.random().toString()).toString("base64"); + const res = await createUserDb(host, userDBName, appDBUserName, appDBPassword, true); + if (res !== 0) { + return ""; + } + } else if (userDBs.length > 1) { + // If there is more than one database instances, prompt the user to select one. + userDBName = await select({ + message: "Choose a database instance for this app:", + choices: userDBs.map((db) => ({ + name: db.PostgresInstanceName, + value: db.PostgresInstanceName, + })), + }); + } else { + // Use the only available database server. + userDBName = userDBs[0].PostgresInstanceName; + logger.info(`Using database instance: ${userDBName}`); + } + return userDBName; +} diff --git a/src/dbos-runtime/config.ts b/src/dbos-runtime/config.ts index fea9e1dbc..e232dcca2 100644 --- a/src/dbos-runtime/config.ts +++ b/src/dbos-runtime/config.ts @@ -32,6 +32,7 @@ export interface ConfigFile { app_db_client?: UserDatabaseName; migrate?: string[]; rollback?: string[]; + local_suffix: boolean; }; http?: { cors_middleware?: boolean; @@ -59,7 +60,7 @@ export function substituteEnvVars(content: string): string { /** * Loads config file as a ConfigFile. * @param {string} configFilePath - The path to the config file to be loaded. - * @returns + * @returns */ export function loadConfigFile(configFilePath: string): ConfigFile { try { @@ -78,7 +79,7 @@ export function loadConfigFile(configFilePath: string): ConfigFile { /** * Writes a YAML.Document object to configFilePath. - * @param {YAML.Document} configFile - The config file to be written. + * @param {YAML.Document} configFile - The config file to be written. * @param {string} configFilePath - The path to the config file to be written to. */ export function writeConfigFile(configFile: YAML.Document, configFilePath: string) { @@ -95,13 +96,14 @@ export function writeConfigFile(configFile: YAML.Document, configFilePath: strin } export function constructPoolConfig(configFile: ConfigFile) { + const databaseName = configFile.database.local_suffix === true ? `${configFile.database.app_db_name}_local` : configFile.database.app_db_name const poolConfig: PoolConfig = { host: configFile.database.hostname, port: configFile.database.port, user: configFile.database.username, password: configFile.database.password, connectionTimeoutMillis: configFile.database.connectionTimeoutMillis || 3000, - database: configFile.database.app_db_name, + database: databaseName }; if (!poolConfig.database) { diff --git a/src/dbos-runtime/migrate.ts b/src/dbos-runtime/migrate.ts index f0068660c..eb74bab3f 100644 --- a/src/dbos-runtime/migrate.ts +++ b/src/dbos-runtime/migrate.ts @@ -12,19 +12,20 @@ export async function migrate(configFile: ConfigFile, logger: GlobalLogger) { if (!(await checkDatabaseExists(configFile, logger))) { const postgresConfig: PoolConfig = constructPoolConfig(configFile) + const app_database = postgresConfig.database postgresConfig.database = "postgres" const postgresClient = new Client(postgresConfig); let connection_failed = true; try { await postgresClient.connect() connection_failed = false; - await postgresClient.query(`CREATE DATABASE ${configFile.database.app_db_name}`); + await postgresClient.query(`CREATE DATABASE ${app_database}`); } catch (e) { if (e instanceof Error) { if (connection_failed) { logger.error(`Error connecting to database ${postgresConfig.host}:${postgresConfig.port} with user ${postgresConfig.user}: ${e.message}`); } else { - logger.error(`Error creating database ${configFile.database.app_db_name}: ${e.message}`); + logger.error(`Error creating database ${app_database}: ${e.message}`); } } else { logger.error(e); @@ -71,10 +72,10 @@ export async function checkDatabaseExists(configFile: ConfigFile, logger: Global try { await pgUserClient.connect(); // Try to establish a connection await pgUserClient.end(); - logger.info(`Database ${configFile.database.app_db_name} exists!`) + logger.info(`Database ${pgUserConfig.database} exists!`) return true; // If successful, return true } catch (error) { - logger.info(`Database ${configFile.database.app_db_name} does not exist, creating...`) + logger.info(`Database ${pgUserConfig.database} does not exist, creating...`) return false; // If connection fails, return false } } diff --git a/tests/dbos-runtime/config.test.ts b/tests/dbos-runtime/config.test.ts index a0b60b92c..812dd6f86 100644 --- a/tests/dbos-runtime/config.test.ts +++ b/tests/dbos-runtime/config.test.ts @@ -248,6 +248,32 @@ describe("dbos-config", () => { expect(dbosConfig.poolConfig.ssl).toEqual({ rejectUnauthorized: false }); }); + + test("local_suffix works", async () => { + const localMockDBOSConfigYamlString = ` + database: + hostname: 'localhost' + port: 1234 + username: 'some user' + password: \${PGPASSWORD} + connectionTimeoutMillis: 3000 + app_db_name: 'some_db' + local_suffix: true + `; + jest.restoreAllMocks(); + jest.spyOn(utils, "readFileSync").mockReturnValue(localMockDBOSConfigYamlString); + const [dbosConfig, _dbosRuntimeConfig]: [DBOSConfig, DBOSRuntimeConfig] = parseConfigFile(mockCLIOptions); + const poolConfig = dbosConfig.poolConfig; + expect(poolConfig.host).toBe("localhost"); + expect(poolConfig.port).toBe(1234); + expect(poolConfig.user).toBe("some user"); + expect(poolConfig.password).toBe(process.env.PGPASSWORD); + expect(poolConfig.connectionTimeoutMillis).toBe(3000); + expect(poolConfig.database).toBe("some_db_local"); + expect(dbosConfig.poolConfig.ssl).toBe(false); + }); + + test("ssl defaults off for localhost", async () => { const localMockDBOSConfigYamlString = ` database: