Use Cloud Database Locally (#645)

Add support for using a DBOS Cloud Postgres server for local
development.

dbos-config.schema.json

@@ -59,6 +59,10 @@
           "type": "string",
           "description": "If using SSL/TLS to securely connect to a database, path to an SSL root certificate file"
         },
+        "local_suffix": {
+          "type": "boolean",
+          "description": "Whether to suffix app_db_name with '_local'. Set to true when doing local development using a DBOS Cloud database."
+        },
         "app_db_client": {
           "type": "string",
           "description": "Specify the database client to use to connect to the application database",

packages/dbos-cloud/applications/deploy-app-code.ts

@@ -21,11 +21,10 @@ import { Application } from "./types.js";
 import JSZip from "jszip";
 import fg from "fast-glob";
 import chalk from "chalk";
-import { createUserDb, UserDBInstance } from "../databases/databases.js";
 import { registerApp } from "./register-app.js";
-import { input, select } from "@inquirer/prompts";
 import { Logger } from "winston";
 import { loadConfigFile } from "../configutils.js";
+import { chooseAppDBServer } from "../databases/databases.js";
 
 type DeployOutput = {
   ApplicationName: string;
@@ -292,70 +291,3 @@ async function isAppRegistered(logger: Logger, host: string, appName: string, us
   }
   return app;
 }
-
-async function chooseAppDBServer(logger: Logger, host: string, userCredentials: DBOSCloudCredentials, userDBName: string = ""): Promise<string> {
-  // List existing database instances.
-  let userDBs: UserDBInstance[] = [];
-  const bearerToken = "Bearer " + userCredentials.token;
-  try {
-    const res = await axios.get(`https://${host}/v1alpha1/${userCredentials.organization}/databases`, {
-      headers: {
-        "Content-Type": "application/json",
-        Authorization: bearerToken,
-      },
-    });
-    userDBs = res.data as UserDBInstance[];
-  } catch (e) {
-    const errorLabel = `Failed to list databases`;
-    const axiosError = e as AxiosError;
-    if (isCloudAPIErrorResponse(axiosError.response?.data)) {
-      handleAPIErrors(errorLabel, axiosError);
-    } else {
-      logger.error(`${errorLabel}: ${(e as Error).message}`);
-    }
-    return "";
-  }
-
-  if (userDBName) {
-    // Check if the database instance exists or not.
-    const dbExists = userDBs.some((db) => db.PostgresInstanceName === userDBName);
-    if (dbExists) {
-      return userDBName;
-    }
-  }
-
-  if (userDBName || userDBs.length === 0) {
-    // If not, prompt the user to provision one.
-    if (!userDBName) {
-      logger.info("No database found, provisioning a database instance (server)...");
-      userDBName = await input({
-        message: "Database instance name?",
-        default: `${userCredentials.userName}-db-server`,
-      });
-    } else {
-      logger.info(`Database instance ${userDBName} not found, provisioning a new one...`);
-    }
-
-    // Use a default user name and auto generated password.
-    const appDBUserName = "dbos_user";
-    const appDBPassword = Buffer.from(Math.random().toString()).toString("base64");
-    const res = await createUserDb(host, userDBName, appDBUserName, appDBPassword, true);
-    if (res !== 0) {
-      return "";
-    }
-  } else if (userDBs.length > 1) {
-    // If there is more than one database instances, prompt the user to select one.
-    userDBName = await select({
-      message: "Choose a database instance for this app:",
-      choices: userDBs.map((db) => ({
-        name: db.PostgresInstanceName,
-        value: db.PostgresInstanceName,
-      })),
-    });
-  } else {
-    // Use the only available database server.
-    userDBName = userDBs[0].PostgresInstanceName;
-    logger.info(`Using database instance: ${userDBName}`);
-  }
-  return userDBName;
-}

packages/dbos-cloud/applications/types.ts

@@ -29,3 +29,11 @@ export function prettyPrintApplicationVersion(version: ApplicationVersion) {
   console.log(`Version: ${version.Version}`);
   console.log(`Creation Timestamp: ${version.CreationTime}`);
 }
+
+export interface UserDBInstance {
+  readonly PostgresInstanceName: string;
+  readonly Status: string;
+  readonly HostName: string;
+  readonly Port: number;
+  readonly DatabaseUsername: string;
+}

packages/dbos-cloud/cli.ts

@@ -246,11 +246,11 @@ databaseCommands
 databaseCommands
   .command("reset-password")
   .description("Reset password for a Postgres database instance")
-  .argument("<name>", "database instance name")
+  .argument("[name]", "database instance name")
   .option("-W, --password <string>", "Specify the database user password")
-  .action(async (dbName: string, options: { password: string }) => {
+  .action(async (dbName: string | undefined, options: { password: string }) => {
     if (!options.password) {
-      options.password = prompt("Database Password: ", { echo: "*" });
+      options.password = prompt("Database Password (must contain at least 8 characters): ", { echo: "*" });
     }
     const exitCode = await resetDBCredentials(DBOSCloudHost, dbName, options.password);
     process.exit(exitCode);
@@ -305,16 +305,30 @@ databaseCommands
 databaseCommands
   .command("connect")
   .description(`Load cloud database connection information into ${dbosConfigFilePath}`)
-  .argument("<name>", "database instance name")
+  .argument("[name]", "database instance name")
   .option("-W, --password <string>", "Specify the database user password")
-  .action(async (dbname: string, options: { password: string | undefined; }) => {
+  .action(async (dbname: string | undefined, options: { password: string | undefined; }) => {
     if (!options.password) {
-      options.password = prompt("Database Password: ", { echo: "*" });
+      options.password = prompt("Database Password (must contain at least 8 characters): ", { echo: "*" });
     }
-    const exitCode = await connect(DBOSCloudHost, dbname, options.password);
+    const exitCode = await connect(DBOSCloudHost, dbname, options.password, false);
     process.exit(exitCode);
   });
 
+  databaseCommands
+  .command("local")
+  .description(`Configure ${dbosConfigFilePath} to use a DBOS Cloud database for local development`)
+  .argument("[name]", "database instance name")
+  .option("-W, --password <string>", "Specify the database user password")
+  .action(async (dbname: string | undefined, options: { password: string | undefined; }) => {
+    if (!options.password) {
+      options.password = prompt("Database Password (must contain at least 8 characters): ", { echo: "*" });
+    }
+    const exitCode = await connect(DBOSCloudHost, dbname, options.password, true);
+    process.exit(exitCode);
+  });
+
+
 /////////////////////
 /* USER DASHBOARDS */
 /////////////////////

packages/dbos-cloud/cloudutils.ts

@@ -367,3 +367,4 @@ async function registerUser(host: string, credentials: DBOSCloudCredentials, log
   }
   return;
 }
+

packages/dbos-cloud/configutils.ts

@@ -11,6 +11,7 @@ export interface ConfigFile {
     username: string;
     password?: string;
     app_db_name: string;
+    local_suffix: boolean;
   };
 }
 

packages/dbos-cloud/databases/databases.ts

@@ -3,14 +3,9 @@ import { isCloudAPIErrorResponse, handleAPIErrors, getCloudCredentials, getLogge
 import { Logger } from "winston";
 import { ConfigFile, loadConfigFile, writeConfigFile } from "../configutils.js";
 import { copyFileSync, existsSync } from "fs";
+import { UserDBInstance } from "../applications/types.js";
+import { input, select } from "@inquirer/prompts";
 
-export interface UserDBInstance {
-  readonly PostgresInstanceName: string;
-  readonly Status: string;
-  readonly HostName: string;
-  readonly Port: number;
-  readonly DatabaseUsername: string;
-}
 
 function isValidPassword(logger: Logger, password: string): boolean {
   if (password.length < 8 || password.length > 128) {
@@ -257,9 +252,15 @@ async function getUserDBInfo(host: string, dbName: string, userCredentials?: DBO
   return res.data as UserDBInstance;
 }
 
-export async function resetDBCredentials(host: string, dbName: string, appDBPassword: string) {
+export async function resetDBCredentials(host: string, dbName: string | undefined, appDBPassword: string) {
   const logger = getLogger();
   const userCredentials = await getCloudCredentials(host, logger);
+
+  dbName = await chooseAppDBServer(logger, host, userCredentials, dbName)
+  if (dbName === "") {
+    return 1
+  }
+
   const bearerToken = "Bearer " + userCredentials.token;
 
   if (!isValidPassword(logger, appDBPassword)) {
@@ -336,9 +337,16 @@ export async function restoreUserDB(host: string, dbName: string, targetName: st
   }
 }
 
-export async function connect(host: string, dbName: string, password: string) {
+export async function connect(host: string, dbName: string | undefined, password: string, local_suffix: boolean) {
   const logger = getLogger();
 
+  const userCredentials = await getCloudCredentials(host, logger);
+
+  dbName = await chooseAppDBServer(logger, host, userCredentials, dbName)
+  if (dbName === "") {
+    return 1
+  }
+
   try {
     if(!existsSync(dbosConfigFilePath)) {
       logger.error(`Error: ${dbosConfigFilePath} not found`);
@@ -350,7 +358,7 @@ export async function connect(host: string, dbName: string, password: string) {
     copyFileSync(dbosConfigFilePath, backupConfigFilePath);
 
     logger.info("Retrieving cloud database info...");
-    const userDBInfo = await getUserDBInfo(host, dbName);
+    const userDBInfo = await getUserDBInfo(host, dbName, userCredentials);
     console.log(`Postgres Instance Name: ${userDBInfo.PostgresInstanceName}`);
     console.log(`Host Name: ${userDBInfo.HostName}`);
     console.log(`Port: ${userDBInfo.Port}`);
@@ -363,6 +371,7 @@ export async function connect(host: string, dbName: string, password: string) {
     config.database.port = userDBInfo.Port;
     config.database.username = userDBInfo.DatabaseUsername;
     config.database.password = password;
+    config.database.local_suffix = local_suffix;
     writeConfigFile(config, dbosConfigFilePath);
     logger.info(`Cloud database connection information loaded into ${dbosConfigFilePath}`)
     return 0;
@@ -377,3 +386,70 @@ export async function connect(host: string, dbName: string, password: string) {
     return 1;
   }
 }
+
+export async function chooseAppDBServer(logger: Logger, host: string, userCredentials: DBOSCloudCredentials, userDBName: string = ""): Promise<string> {
+  // List existing database instances.
+  let userDBs: UserDBInstance[] = [];
+  const bearerToken = "Bearer " + userCredentials.token;
+  try {
+    const res = await axios.get(`https://${host}/v1alpha1/${userCredentials.organization}/databases`, {
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: bearerToken,
+      },
+    });
+    userDBs = res.data as UserDBInstance[];
+  } catch (e) {
+    const errorLabel = `Failed to list databases`;
+    const axiosError = e as AxiosError;
+    if (isCloudAPIErrorResponse(axiosError.response?.data)) {
+      handleAPIErrors(errorLabel, axiosError);
+    } else {
+      logger.error(`${errorLabel}: ${(e as Error).message}`);
+    }
+    return "";
+  }
+
+  if (userDBName) {
+    // Check if the database instance exists or not.
+    const dbExists = userDBs.some((db) => db.PostgresInstanceName === userDBName);
+    if (dbExists) {
+      return userDBName;
+    }
+  }
+
+  if (userDBName || userDBs.length === 0) {
+    // If not, prompt the user to provision one.
+    if (!userDBName) {
+      logger.info("No database found, provisioning a database instance (server)...");
+      userDBName = await input({
+        message: "Database instance name?",
+        default: `${userCredentials.userName}-db-server`,
+      });
+    } else {
+      logger.info(`Database instance ${userDBName} not found, provisioning a new one...`);
+    }
+
+    // Use a default user name and auto generated password.
+    const appDBUserName = "dbos_user";
+    const appDBPassword = Buffer.from(Math.random().toString()).toString("base64");
+    const res = await createUserDb(host, userDBName, appDBUserName, appDBPassword, true);
+    if (res !== 0) {
+      return "";
+    }
+  } else if (userDBs.length > 1) {
+    // If there is more than one database instances, prompt the user to select one.
+    userDBName = await select({
+      message: "Choose a database instance for this app:",
+      choices: userDBs.map((db) => ({
+        name: db.PostgresInstanceName,
+        value: db.PostgresInstanceName,
+      })),
+    });
+  } else {
+    // Use the only available database server.
+    userDBName = userDBs[0].PostgresInstanceName;
+    logger.info(`Using database instance: ${userDBName}`);
+  }
+  return userDBName;
+}

src/dbos-runtime/config.ts

@@ -32,6 +32,7 @@ export interface ConfigFile {
     app_db_client?: UserDatabaseName;
     migrate?: string[];
     rollback?: string[];
+    local_suffix: boolean;
   };
   http?: {
     cors_middleware?: boolean;
@@ -59,7 +60,7 @@ export function substituteEnvVars(content: string): string {
 /**
  * Loads config file as a ConfigFile.
  * @param {string} configFilePath - The path to the config file to be loaded.
- * @returns 
+ * @returns
  */
 export function loadConfigFile(configFilePath: string): ConfigFile {
   try {
@@ -78,7 +79,7 @@ export function loadConfigFile(configFilePath: string): ConfigFile {
 
 /**
  * Writes a YAML.Document object to configFilePath.
- * @param {YAML.Document} configFile - The config file to be written. 
+ * @param {YAML.Document} configFile - The config file to be written.
  * @param {string} configFilePath - The path to the config file to be written to.
  */
 export function writeConfigFile(configFile: YAML.Document, configFilePath: string) {
@@ -95,13 +96,14 @@ export function writeConfigFile(configFile: YAML.Document, configFilePath: strin
 }
 
 export function constructPoolConfig(configFile: ConfigFile) {
+  const databaseName = configFile.database.local_suffix === true ? `${configFile.database.app_db_name}_local` : configFile.database.app_db_name
    const poolConfig: PoolConfig = {
     host: configFile.database.hostname,
     port: configFile.database.port,
     user: configFile.database.username,
     password: configFile.database.password,
     connectionTimeoutMillis: configFile.database.connectionTimeoutMillis || 3000,
-    database: configFile.database.app_db_name,
+    database: databaseName
   };
 
   if (!poolConfig.database) {

src/dbos-runtime/migrate.ts

@@ -12,19 +12,20 @@ export async function migrate(configFile: ConfigFile, logger: GlobalLogger) {
 
   if (!(await checkDatabaseExists(configFile, logger))) {
     const postgresConfig: PoolConfig = constructPoolConfig(configFile)
+    const app_database = postgresConfig.database
     postgresConfig.database = "postgres"
     const postgresClient = new Client(postgresConfig);
     let connection_failed = true;
     try {
       await postgresClient.connect()
       connection_failed = false;
-      await postgresClient.query(`CREATE DATABASE ${configFile.database.app_db_name}`);
+      await postgresClient.query(`CREATE DATABASE ${app_database}`);
     } catch (e) {
       if (e instanceof Error) {
         if (connection_failed) {
           logger.error(`Error connecting to database ${postgresConfig.host}:${postgresConfig.port} with user ${postgresConfig.user}: ${e.message}`);
         } else {
-          logger.error(`Error creating database ${configFile.database.app_db_name}: ${e.message}`);
+          logger.error(`Error creating database ${app_database}: ${e.message}`);
         }
       } else {
         logger.error(e);
@@ -71,10 +72,10 @@ export async function checkDatabaseExists(configFile: ConfigFile, logger: Global
   try {
     await pgUserClient.connect(); // Try to establish a connection
     await pgUserClient.end();
-    logger.info(`Database ${configFile.database.app_db_name} exists!`)
+    logger.info(`Database ${pgUserConfig.database} exists!`)
     return true; // If successful, return true
   } catch (error) {
-    logger.info(`Database ${configFile.database.app_db_name} does not exist, creating...`)
+    logger.info(`Database ${pgUserConfig.database} does not exist, creating...`)
     return false; // If connection fails, return false
   }
 }