diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..6ac2397 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,23 @@ +name: test + +on: + push: + branches: + - master + - main + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: erlef/setup-beam@v1 + with: + otp-version: "26.0.2" + gleam-version: "1.2.1" + rebar3-version: "3" + # elixir-version: "1.15.4" + - run: gleam deps download + - run: gleam test + - run: gleam format --check src test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..599be4e --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.beam +*.ez +/build +erl_crash.dump diff --git a/README.md b/README.md new file mode 100644 index 0000000..8242fed --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# feather + +[![Package Version](https://img.shields.io/hexpm/v/feather)](https://hex.pm/packages/feather) +[![Hex Docs](https://img.shields.io/badge/hex-docs-ffaff3)](https://hexdocs.pm/feather/) + +```sh +gleam add feather +``` +Add the following fields to your gleam.toml file: + +```toml +# this can of course be anything you like +migrations_dir = "./priv/migrations" +schemafile = "./schema.sql" +``` + +Then run the command `gleam run -m feather -- new "Initial schema migration"` and make any changes you like. + +Running the command `gleam run -m feather -- schema` will create the file ./schema.sql, (or whatever you set in your gleam.toml) with the schema of your database after all migrations have been applied. + +```gleam +import feather +import gleam/result +import gleam/erlang +import sqlight + +pub fn main() { + let assert Ok(priv_dir) = erlang.priv_directory("my_module_name") + use migrations <- result.try(feather.get_migrations(priv_dir <> "/migrations")) + use connection <- feather.connect(feather.Config(..feather.default_config(), file: "./database.db")) + feather.migrate(migrations, on: connection) +} +``` + +Further documentation can be found at . + +## Development + +```sh +gleam run # Run the project +gleam test # Run the tests +gleam shell # Run an Erlang shell +``` diff --git a/gleam.toml b/gleam.toml new file mode 100644 index 0000000..74c3e0a --- /dev/null +++ b/gleam.toml @@ -0,0 +1,28 @@ +name = "feather" +version = "1.0.0" + +# Fill out these fields if you intend to generate HTML documentation or publish +# your project to the Hex package manager. +# +description = "A companion library to sqlight" +licences = ["Apache-2.0"] +repository = { type = "github", user = "VioletBuse", repo = "feather" } +# links = [{ title = "Website", href = "https://gleam.run" }] +# +# For a full reference of all the available options, you can have a look at +# https://gleam.run/writing-gleam/gleam-toml/. + +[dependencies] +gleam_stdlib = ">= 0.34.0 and < 2.0.0" +sqlight = ">= 0.9.0 and < 1.0.0" +simplifile = ">= 2.0.0 and < 3.0.0" +filepath = ">= 1.0.0 and < 2.0.0" +justin = ">= 1.0.1 and < 2.0.0" +gloml = ">= 0.1.3 and < 1.0.0" +argv = ">= 1.0.2 and < 2.0.0" +gleam_erlang = ">= 0.25.0 and < 1.0.0" +puddle = ">= 0.5.0 and < 1.0.0" +gleam_otp = ">= 0.10.0 and < 1.0.0" + +[dev-dependencies] +gleeunit = ">= 1.0.0 and < 2.0.0" diff --git a/manifest.toml b/manifest.toml new file mode 100644 index 0000000..c283d6a --- /dev/null +++ b/manifest.toml @@ -0,0 +1,31 @@ +# This file was generated by Gleam +# You typically do not need to edit this file + +packages = [ + { name = "argv", version = "1.0.2", build_tools = ["gleam"], requirements = [], otp_app = "argv", source = "hex", outer_checksum = "BA1FF0929525DEBA1CE67256E5ADF77A7CDDFE729E3E3F57A5BDCAA031DED09D" }, + { name = "esqlite", version = "0.8.8", build_tools = ["rebar3"], requirements = [], otp_app = "esqlite", source = "hex", outer_checksum = "374902457C7D94DC9409C98D3BDD1CA0D50A60DC9F3BDF1FD8EB74C0DCDF02D6" }, + { name = "filepath", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "EFB6FF65C98B2A16378ABC3EE2B14124168C0CE5201553DE652E2644DCFDB594" }, + { name = "gleam_erlang", version = "0.25.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_erlang", source = "hex", outer_checksum = "054D571A7092D2A9727B3E5D183B7507DAB0DA41556EC9133606F09C15497373" }, + { name = "gleam_otp", version = "0.10.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_stdlib"], otp_app = "gleam_otp", source = "hex", outer_checksum = "0B04FE915ACECE539B317F9652CAADBBC0F000184D586AAAF2D94C100945D72B" }, + { name = "gleam_stdlib", version = "0.38.0", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "663CF11861179AF415A625307447775C09404E752FF99A24E2057C835319F1BE" }, + { name = "gleeunit", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "F7A7228925D3EE7D0813C922E062BFD6D7E9310F0BEE585D3A42F3307E3CFD13" }, + { name = "gloml", version = "0.1.3", build_tools = ["gleam"], requirements = ["gleam_stdlib", "toml"], otp_app = "gloml", source = "hex", outer_checksum = "D70229ACD487010B2D1CB57FFCCB0D2BB38CEC885DC9688D51D1020A579AC057" }, + { name = "justin", version = "1.0.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "justin", source = "hex", outer_checksum = "7FA0C6DB78640C6DC5FBFD59BF3456009F3F8B485BF6825E97E1EB44E9A1E2CD" }, + { name = "puddle", version = "0.5.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_otp", "gleam_stdlib"], otp_app = "puddle", source = "hex", outer_checksum = "1D199F61CAB692DA84CE8153C9351DC21C68C62BCE75782AFAAD5EE780144806" }, + { name = "simplifile", version = "2.0.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "95219227A43FCFE62C6E494F413A1D56FF953B68FE420698612E3D89A1EFE029" }, + { name = "sqlight", version = "0.9.0", build_tools = ["gleam"], requirements = ["esqlite", "gleam_stdlib"], otp_app = "sqlight", source = "hex", outer_checksum = "2D9C9BA420A5E7DCE7DB2DAAE4CAB0BE6218BEB48FD1531C583550B3D1316E94" }, + { name = "toml", version = "0.7.0", build_tools = ["mix"], requirements = [], otp_app = "toml", source = "hex", outer_checksum = "0690246A2478C1DEFD100B0C9B89B4EA280A22BE9A7B313A8A058A2408A2FA70" }, +] + +[requirements] +argv = { version = ">= 1.0.2 and < 2.0.0" } +filepath = { version = ">= 1.0.0 and < 2.0.0" } +gleam_erlang = { version = ">= 0.25.0 and < 1.0.0" } +gleam_otp = { version = ">= 0.10.0 and < 1.0.0"} +gleam_stdlib = { version = ">= 0.34.0 and < 2.0.0" } +gleeunit = { version = ">= 1.0.0 and < 2.0.0" } +gloml = { version = ">= 0.1.3 and < 1.0.0" } +justin = { version = ">= 1.0.1 and < 2.0.0" } +puddle = { version = ">= 0.5.0 and < 1.0.0" } +simplifile = { version = ">= 2.0.0 and < 3.0.0" } +sqlight = { version = ">= 0.9.0 and < 1.0.0" } diff --git a/src/feather.gleam b/src/feather.gleam new file mode 100644 index 0000000..8de0be2 --- /dev/null +++ b/src/feather.gleam @@ -0,0 +1,124 @@ +import feather/migrate +import gleam/int +import gleam/option.{type Option, None} +import gleam/result +import sqlight.{type Connection, type Error} + +/// Runs the feather cli to generate new migrations and dump the schema +/// you probably don't wanna run this yourself... +/// run `gleam run -m feather` to find out more +pub fn main() { + migrate.main() +} + +pub type JournalMode { + JournalDelete + JournalTruncate + JournalPersist + JournalMemory + JournalWal + JournalOff +} + +pub type SyncMode { + SyncExtra + SyncFull + SyncNormal + SyncOff +} + +pub type TempStore { + TempStoreDefault + TempStoreFile + TempStoreMemory +} + +pub type Config { + Config( + file: String, + journal_mode: JournalMode, + synchronous: SyncMode, + temp_store: TempStore, + mmap_size: Option(Int), + page_size: Option(Int), + ) +} + +pub fn default_config() -> Config { + Config( + "./sqlite.db", + journal_mode: JournalWal, + synchronous: SyncNormal, + temp_store: TempStoreMemory, + mmap_size: None, + page_size: None, + ) +} + +pub fn connect(config: Config) -> Result(Connection, Error) { + use connection <- result.try(sqlight.open(config.file)) + + let journal_mode = case config.journal_mode { + JournalOff -> "OFF" + JournalWal -> "WAL" + JournalDelete -> "DELETE" + JournalMemory -> "MEMORY" + JournalTruncate -> "TRUNCATE" + JournalPersist -> "PERSIST" + } + + let sync = case config.synchronous { + SyncOff -> "OFF" + SyncFull -> "FULL" + SyncExtra -> "EXTRA" + SyncNormal -> "NORMAL" + } + + let temp_store = case config.temp_store { + TempStoreFile -> "FILE" + TempStoreMemory -> "MEMORY" + TempStoreDefault -> "DEFAULT" + } + + use _ <- result.try(sqlight.exec( + "PRAGMA journal_mode = " <> journal_mode <> ";", + connection, + )) + use _ <- result.try(sqlight.exec( + "PRAGMA synchronous = " <> sync <> ";", + connection, + )) + use _ <- result.try(sqlight.exec( + "PRAGMA temp_store = " <> temp_store <> ";", + connection, + )) + use _ <- result.try( + option.map(config.mmap_size, int.to_string) + |> option.map(fn(size) { + sqlight.exec("PRAGMA mmap_size = " <> size <> ";", connection) + }) + |> option.unwrap(Ok(Nil)), + ) + + use _ <- result.try( + option.map(config.page_size, int.to_string) + |> option.map(fn(size) { + sqlight.exec("PRAGMA page_size = " <> size <> ";", connection) + }) + |> option.unwrap(Ok(Nil)), + ) + + Ok(connection) +} + +/// runs "PRAGMA optimize;" before closing the connection. +/// If the connections are long-lived, then consider running +/// this periodically anyways. +pub fn disconnect(connection: Connection) { + let _ = sqlight.exec("PRAGMA optimize;", connection) + sqlight.close(connection) +} + +pub fn optimize(connection: Connection) { + sqlight.exec("PRAGMA optimize;", connection) +} diff --git a/src/feather/migrate.gleam b/src/feather/migrate.gleam new file mode 100644 index 0000000..5fd5aa5 --- /dev/null +++ b/src/feather/migrate.gleam @@ -0,0 +1,358 @@ +import argv +import filepath +import gleam/bool +import gleam/dynamic +import gleam/erlang +import gleam/int +import gleam/io +import gleam/list +import gleam/option.{type Option, None, Some} +import gleam/regex +import gleam/result +import gleam/string +import gloml +import justin +import simplifile +import sqlight.{type Connection} + +const helptext = " + gleam run -m feather/migrate -- [options] + + commands: + new + Generate a new migration script. A timestamp will + be prepended as the migration id, to ensure ordering + of migration scripts. + + schema + Dump the schema of the sqlite database into a schema.sql + file. Pass in the path (absolute or relative) to the + migrations directory. +" + +fn get_migrations_dir() -> String { + simplifile.read("gleam.toml") + |> result.nil_error + |> result.map(gloml.decode(_, dynamic.field("migrations_dir", dynamic.string))) + |> result.map(result.map_error(_, fn(_) { Nil })) + |> result.flatten + |> result.unwrap("./migrations") +} + +fn get_schema_file() -> String { + simplifile.read("gleam.toml") + |> result.nil_error + |> result.map(gloml.decode(_, dynamic.field("schemafile", dynamic.string))) + |> result.map(result.map_error(_, fn(_) { Nil })) + |> result.flatten + |> result.unwrap("./schema.sql") +} + +/// Runs the feather cli to generate new migrations and dump the schema +/// you probably don't wanna run this yourself... +/// run `gleam run -m feather` to find out more +pub fn main() { + let help_flag = + list.any(argv.load().arguments, fn(flag) { + case flag { + "--help" | "-h" | "help" -> True + _ -> False + } + }) + + case argv.load().arguments { + ["new", ..rest] -> handle_new_cmd(rest, help: help_flag) + ["schema", ..rest] -> handle_schema_dump(rest, help: help_flag) + _ -> io.println(helptext) + } +} + +const new_cmd_helptext = " + gleam run -m feather/migrate -- new + options: + --dir, --migrations-dir, -d Migrations directory location, default: ./migrations + --help, -h Show this help message +" + +fn handle_new_cmd(args: List(String), help help_flag: Bool) { + let timestamp = erlang.system_time(erlang.Second) + + let #(_, dir) = + list.find(list.window_by_2(args), fn(tuple) { + case tuple { + #("--dir", _) | #("--migrations-dir", _) | #("-d", _) -> True + _ -> False + } + }) + |> result.unwrap(#("default", get_migrations_dir())) + + case help_flag, args { + True, _ -> io.println(new_cmd_helptext) + _, [] -> io.println("Please provide a migration name") + _, [name, ..] -> { + let filename = + int.to_string(timestamp) <> "_" <> justin.snake_case(name) <> ".sql" + let path = filepath.join(dir, filename) + + let _ = simplifile.create_file(path) + let _ = simplifile.write(path, "-- " <> name) + + Nil + } + } +} + +const schema_dump_helptext = " + gleam run -m feather/migrate -- schema + options: + --migrations-dir, -d Migrations directory location, default: ./migrations + --file-name, -f Name of the resulting file, default: ./schema.sql +" + +fn handle_schema_dump(args: List(String), help help_flag: Bool) { + let #(_, migrations_dir) = + list.find(list.window_by_2(args), fn(window) { + case window { + #("--migrations-dir", _) | #("-d", _) -> True + _ -> False + } + }) + |> result.unwrap(#("", get_migrations_dir())) + + let #(_, outfile) = + list.find(list.window_by_2(args), fn(window) { + case window { + #("--file-name", _) | #("-f", _) -> True + _ -> False + } + }) + |> result.unwrap(#("", get_schema_file())) + + case help_flag { + True -> io.println(schema_dump_helptext) + False -> { + use connection <- sqlight.with_connection(":memory:") + + let schema_result = { + use migrations <- result.try(get_migrations(migrations_dir)) + use _ <- result.try(migrate(migrations, connection)) + use sql_dumps <- result.try( + sqlight.query( + "SELECT * FROM sqlite_schema", + connection, + [], + dynamic.tuple5( + dynamic.string, + dynamic.string, + dynamic.string, + dynamic.int, + dynamic.optional(dynamic.string), + ), + ) + |> result.map_error(TransactionError), + ) + + Ok( + list.filter(sql_dumps, fn(row) { + case row.4 { + Some(_) -> True + None -> False + } + }) + |> list.map(fn(row) { option.unwrap(row.4, "") }), + ) + } + + case schema_result { + Error(err) -> { + io.debug(err) + + Nil + } + Ok(sql_list) -> { + let _ = + list.map(sql_list, fn(str) { str <> ";\n\n" }) + |> string.concat + |> simplifile.write(outfile, _) + + Nil + } + } + } + } +} + +/// Migrations with an id and a sql script +/// +pub type Migration { + Migration(id: Int, up: String) +} + +/// Migration error type +pub type MigrationError { + /// Folder that you gave does not exist + DirectoryNotExist(String) + /// The migration script file name is not valid + InvalidMigrationName(String) + /// The migration script file has a non-integer id + InvalidMigrationId(String) + /// Error starting or comitting the migration transaction + TransactionError(sqlight.Error) + /// Error reading/writing/creating the transactions table + MigrationsTableError(sqlight.Error) + /// Error applying the migrations script + MigrationScriptError(Int, sqlight.Error) +} + +fn migration_error(error: MigrationError) -> fn(a) -> MigrationError { + fn(_) { error } +} + +/// Pass in a list of migrations and a sqlight connection +/// +pub fn migrate( + migrations: List(Migration), + on connection: Connection, +) -> Result(Nil, MigrationError) { + let transaction = { + use _ <- result.try( + sqlight.exec("begin transaction;", connection) + |> result.map_error(TransactionError), + ) + use _ <- result.try( + sqlight.exec( + "create table if not exists storch_migrations (id integer, applied integer);", + connection, + ) + |> result.map_error(MigrationsTableError), + ) + + let migrations_decoder = dynamic.tuple2(dynamic.int, sqlight.decode_bool) + + let applications = + list.try_each(migrations, fn(migration) { + use migrated <- result.try( + sqlight.query( + "select id, applied from storch_migrations where id = ?;", + on: connection, + with: [sqlight.int(migration.id)], + expecting: migrations_decoder, + ) + |> result.map_error(MigrationsTableError), + ) + + let already_applied = case migrated { + [] -> False + [#(_, applied)] -> applied + _ -> + panic as "Multiple migrations with the same id in the storch migrations table" + } + + use <- bool.guard(when: already_applied, return: Ok(Nil)) + + use _ <- result.try( + sqlight.exec(migration.up, connection) + |> result.map_error(MigrationScriptError(migration.id, _)), + ) + use _ <- result.try( + sqlight.query( + "insert into storch_migrations (id, applied) values (?,?) returning *;", + on: connection, + with: [sqlight.int(migration.id), sqlight.bool(True)], + expecting: migrations_decoder, + ) + |> result.map_error(MigrationsTableError), + ) + + Ok(Nil) + }) + + use _ <- result.try(applications) + + use _ <- result.try( + sqlight.exec("commit;", connection) |> result.map_error(TransactionError), + ) + Ok(Nil) + } + + case transaction { + Ok(_) -> { + Ok(Nil) + } + Error(err) -> { + io.println("error running migration") + io.debug(err) + io.println("rolling back") + let _ = sqlight.exec("rollback;", connection) + Error(err) + } + } +} + +/// Get a list of migrations from a folder in the filesystem +/// migration files *must* end in .sql and start with an integer id followed by an underscore +/// example: 0000001_init.sql +/// +/// you could store these in the priv directory if you like, that's probably the best way +pub fn get_migrations( + in directory: String, +) -> Result(List(Migration), MigrationError) { + use filenames <- result.try(get_migration_filenames(directory)) + use raw_migrations <- result.try(read_migrations(filenames)) + + list.map(raw_migrations, fn(raw) { Migration(raw.0, raw.1) }) + |> list.sort(fn(a, b) { int.compare(a.id, b.id) }) + |> Ok +} + +fn read_migrations( + scripts paths: List(String), +) -> Result(List(#(Int, String)), MigrationError) { + list.try_map(paths, fn(path) { + let filename = filepath.base_name(path) + + use #(id, _) <- result.try( + string.split_once(filename, "_") + |> result.map_error(migration_error(InvalidMigrationName(filename))), + ) + use id <- result.try( + int.parse(id) |> result.map_error(migration_error(InvalidMigrationId(id))), + ) + + let assert Ok(contents) = simplifile.read(path) + + Ok(#(id, contents)) + }) +} + +fn get_migration_filenames( + in directory: String, +) -> Result(List(String), MigrationError) { + use is_dir <- result.try( + simplifile.is_directory(directory) + |> result.map_error(migration_error(DirectoryNotExist(directory))), + ) + use <- bool.guard(when: !is_dir, return: Error(DirectoryNotExist(directory))) + + use filenames_raw <- result.try( + simplifile.get_files(directory) + |> result.map_error(migration_error(DirectoryNotExist(directory))), + ) + + list.map(filenames_raw, fn(path) { + use extension <- result.try(filepath.extension(path)) + let base_path = filepath.directory_name(path) + let filename = filepath.base_name(path) |> filepath.strip_extension + + use <- bool.guard(when: extension != "sql", return: Error(Nil)) + use <- bool.guard(when: base_path != directory, return: Error(Nil)) + + use #(numbers, _) <- result.try(string.split_once(filename, "_")) + + use regex <- result.try(regex.from_string("^[0-9]+$") |> result.nil_error) + use <- bool.guard(when: !regex.check(regex, numbers), return: Error(Nil)) + Ok(path) + }) + |> result.values + |> Ok +} diff --git a/src/feather/pool.gleam b/src/feather/pool.gleam new file mode 100644 index 0000000..ee6aaa2 --- /dev/null +++ b/src/feather/pool.gleam @@ -0,0 +1,47 @@ +import feather +import gleam/erlang/process.{type Subject} +import gleam/function +import gleam/otp/actor +import gleam/result +import puddle +import sqlight.{type Connection} + +pub type Pool(a) = + Subject(puddle.ManagerMessage(Connection, a)) + +pub fn start( + config: feather.Config, + count: Int, +) -> Result(Pool(a), actor.StartError) { + puddle.start(count, fn() { feather.connect(config) |> result.nil_error }) +} + +pub fn with_connection(pool: Pool(a), timeout: Int, fxn: fn(Connection) -> a) { + puddle.apply(pool, fxn, timeout, function.identity) +} + +/// This will panic if you end the transaction yourself! +pub fn with_transaction( + pool: Pool(Result(a, Nil)), + timeout: Int, + fxn: fn(Connection) -> Result(a, Nil), +) { + let result = + with_connection(pool, timeout, fn(connection) { + use _ <- result.try( + sqlight.exec("BEGIN TRANSACTION;", connection) |> result.nil_error, + ) + case fxn(connection) { + Ok(val) -> { + let assert Ok(_) = sqlight.exec("COMMIT TRANSACTION;", connection) + Ok(val) + } + Error(Nil) -> { + let _ = sqlight.exec("ROLLBACK TRANSACTION;", connection) + Error(Nil) + } + } + }) + + result.flatten(result) +} diff --git a/test/feather_test.gleam b/test/feather_test.gleam new file mode 100644 index 0000000..3831e7a --- /dev/null +++ b/test/feather_test.gleam @@ -0,0 +1,12 @@ +import gleeunit +import gleeunit/should + +pub fn main() { + gleeunit.main() +} + +// gleeunit test functions end in `_test` +pub fn hello_world_test() { + 1 + |> should.equal(1) +}