Merge pull request #144 from dukefirehawk/bugfix/mysql

Bugfix/mysql

doc/deployment/docker/README.md

@@ -1,6 +1,6 @@
-# Runing Ancillary Docker Services
+# Docker Services
 
-The required ancillary services required by the framework can be run using the compose files provided in this folder.
+The required applications by the framework can be run using the docker compose files provided in this folder.
 
 ## PostreSQL
 
@@ -30,12 +30,12 @@ The required ancillary services required by the framework can be run using the c
     psql --username postgres
     ```
 
-### Create database, user and access
+### Create PostgreSQL database, user and grant access
 
-    ```psql
-    postgres=# create database orm_test;
-    postgres=# create user test with encrypted password 'test123';
-    postgres=# grant all privileges on database orm_test to test;
+    ```sql
+    create database orm_test;
+    create user test with encrypted password 'test123';
+    grant all privileges on database orm_test to test;
     ```
 
 ## MariaDB
@@ -59,6 +59,20 @@ The required ancillary services required by the framework can be run using the c
     docker logs maria-mariadb-1 -f
     ```
 
+### Create MariaDB database, user and grant access
+
+    ```sql
+    create database orm_test;
+
+    -- Granting localhost access only
+    create user 'test'@'localhost' identified by 'test123';
+    grant all privileges on orm_test.* to 'test'@'localhost';
+
+    -- Granting localhost and remote access
+    create user 'test'@'%' identified by 'test123';
+    grant all privileges on orm_test.* to 'test'@'%';
+    ```
+
 ## MySQL
 
 ### Starting the MySQL container
@@ -80,6 +94,20 @@ The required ancillary services required by the framework can be run using the c
     docker logs mysql-mysql-1 -f
     ```
 
+### Create MySQL database, user and grant access
+
+    ```sql
+    create database orm_test;
+
+    -- Granting localhost access only
+    create user 'test'@'localhost' identified by 'test123';
+    grant all privileges on orm_test.* to 'test'@'localhost';
+
+    -- Granting localhost and remote access
+    create user 'test'@'%' identified by 'test123';
+    grant all privileges on orm_test.* to 'test'@'%';
+    ```
+
 ## MongoDB
 
 ### Starting the MongoDB container

doc/deployment/docker/docker-compose-pg.yml

@@ -10,7 +10,7 @@ services:
     volumes:
       - "db:/var/lib/postgresql/data"
     networks:
-      - webnet
+      - appnet
 
   pgadmin4:
     image: dpage/pgadmin4:latest

packages/orm/angel_migration/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Change Log
 
+## 8.2.1
+
+* Updated README
+
 ## 8.2.0
 
 * Require Dart >= 3.3

packages/orm/angel_migration/README.md

@@ -5,20 +5,14 @@
 [![Discord](https://img.shields.io/discord/1060322353214660698)](https://discord.gg/3X6bxTUdCM)
 [![License](https://img.shields.io/github/license/dart-backend/angel)](https://github.com/dart-backend/angel/tree/master/packages/orm/angel_migration/LICENSE)
 
-A basic database migration framework built for Angel3 ORM.
+This package contains the abstract classes for implementing database migration in Angel3 framework. It is designed to work with Angel3 ORM. Please refer to the implementation in the [ORM Migration Runner](<https://pub.dev/packages/angel3_migration_runner>) package for more details.
 
-## Supported database
-
-* PostgreSQL version 10 or later
-* MariaDB 10.2.x or later
-* MySQL 8.x or later
-
-## Features
+## Supported Features
 
 * Create tables based on ORM models
 * Drop tables based on ORM models
-* Add new tables based ORM models
+* Add new tables based on ORM models
 
 ## Limitation
 
-* Alter table/fields based on updated ORM models not supported
+* Alter table/fields based on updated ORM models is not supported

packages/orm/angel_migration/pubspec.yaml

@@ -1,6 +1,6 @@
 name: angel3_migration
-version: 8.2.0
-description: Database migration runtime for Angel3 ORM. Use this package to define schemas.
+version: 8.2.1
+description: The abstract classes for implementing database migration in Angel3 framework. Designed to work with Angel3 ORM.
 homepage: https://angel3-framework.web.app/
 repository: https://github.com/dart-backend/angel/tree/master/packages/orm/angel_migration
 environment:

packages/orm/angel_migration_runner/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Change Log
 
+## 8.2.1
+
+* Updated README
+* Updated examples
+* Updated `PostgresMigrationRunner` error handling
+* Fixed `MySqlMigrationRunner` migration issues
+* Added test cases for `PostgreSQL`, `MySQL` and `MariaDB`
+* Added auto increment integer primary key suppport to MySQL and MariaDB
+
 ## 8.2.0
 
 * Require Dart >= 3.3

packages/orm/angel_migration_runner/README.md

@@ -5,9 +5,7 @@
 [![Discord](https://img.shields.io/discord/1060322353214660698)](https://discord.gg/3X6bxTUdCM)
 [![License](https://img.shields.io/github/license/dart-backend/angel)](https://github.com/dart-backend/angel/tree/master/packages/orm/angel_migration_runner/LICENSE)
 
-Database migration runner for Angel3 ORM.
-
-Supported database:
+This package contains the implementation of the database migration for the following databases. It is designed to work with Angel3 ORM.
 
 * PostgreSQL 10.x or greater
 * MariaDB 10.2.x or greater

packages/orm/angel_migration_runner/example/main.dart

@@ -2,28 +2,40 @@ import 'dart:io';
 
 import 'package:angel3_migration/angel3_migration.dart';
 import 'package:angel3_migration_runner/angel3_migration_runner.dart';
+import 'package:angel3_migration_runner/mysql.dart';
 import 'package:angel3_migration_runner/postgres.dart';
 import 'package:angel3_orm/angel3_orm.dart';
+import 'package:mysql_client/mysql_client.dart';
 import 'package:postgres/postgres.dart';
 
 import 'todo.dart';
 
 void main(List<String> args) async {
+  // Run migration on PostgreSQL database
+  postgresqlMigration(args);
+
+  // Run migration on MySQL database
+  mysqlMigration(args);
+}
+
+void postgresqlMigration(List<String> args) async {
   var host = Platform.environment['DB_HOST'] ?? 'localhost';
   var database = Platform.environment['DB_NAME'] ?? 'demo';
   var username = Platform.environment['DB_USERNAME'] ?? 'demouser';
   var password = Platform.environment['DB_PASSWORD'] ?? 'demo123';
 
   print("$host $database $username $password");
 
-  Connection conn = await Connection.open(Endpoint(
-      host: host,
-      port: 5432,
-      database: database,
-      username: username,
-      password: password));
+  Connection conn = await Connection.open(
+      Endpoint(
+          host: host,
+          port: 5432,
+          database: database,
+          username: username,
+          password: password),
+      settings: ConnectionSettings(sslMode: SslMode.disable));
 
-  var postgresqlMigrationRunner = PostgresMigrationRunner(
+  var runner = PostgresMigrationRunner(
     conn,
     migrations: [
       UserMigration(),
@@ -32,27 +44,34 @@ void main(List<String> args) async {
     ],
   );
 
-  /*
+  runMigrations(runner, args);
+}
+
+void mysqlMigration(List<String> args) async {
+  var host = Platform.environment['MYSQL_HOST'] ?? 'localhost';
+  var database = Platform.environment['MYSQL_DB'] ?? 'orm_test';
+  var username = Platform.environment['MYSQL_USERNAME'] ?? 'test';
+  var password = Platform.environment['MYSQL_PASSWORD'] ?? 'test123';
+
   var mySQLConn = await MySQLConnection.createConnection(
       host: host,
       port: 3306,
       databaseName: database,
       userName: username,
       password: password,
-      secure: false);
+      secure: true);
 
   // ignore: unused_local_variable
-  var mysqlMigrationRunner = MySqlMigrationRunner(
+  var runner = MySqlMigrationRunner(
     mySQLConn,
     migrations: [
       UserMigration(),
       TodoMigration(),
       FooMigration(),
     ],
   );
-  */
 
-  runMigrations(postgresqlMigrationRunner, args);
+  runMigrations(runner, args);
 }
 
 class FooMigration extends Migration {

packages/orm/angel_migration_runner/lib/src/cli.dart

@@ -50,6 +50,7 @@ class _RefreshCommand extends Command {
 
   @override
   String get name => 'refresh';
+
   @override
   String get description =>
       'Resets the database, and then re-runs all migrations.';
@@ -67,8 +68,9 @@ class _RollbackCommand extends Command {
 
   @override
   String get name => 'rollback';
+
   @override
-  String get description => 'Undoes the last batch of migrations.';
+  String get description => 'Undo the last batch of migrations.';
 
   final MigrationRunner migrationRunner;
 

packages/orm/angel_migration_runner/lib/src/mariadb/runner.dart

@@ -7,6 +7,7 @@ import '../runner.dart';
 import '../util.dart';
 import 'schema.dart';
 
+/// A MariaDB database migration runner.
 class MariaDbMigrationRunner implements MigrationRunner {
   final _log = Logger('MariaDbMigrationRunner');
 
@@ -17,8 +18,8 @@ class MariaDbMigrationRunner implements MigrationRunner {
 
   MariaDbMigrationRunner(this.connection,
       {Iterable<Migration> migrations = const [], bool connected = false}) {
-    if (migrations.isNotEmpty == true) migrations.forEach(addMigration);
-    _connected = connected == true;
+    if (migrations.isNotEmpty) migrations.forEach(addMigration);
+    _connected = connected;
   }
 
   @override
@@ -39,7 +40,7 @@ class MariaDbMigrationRunner implements MigrationRunner {
 
     await connection.query('''
     CREATE TABLE IF NOT EXISTS migrations (
-      id serial,
+      id integer NOT NULL AUTO_INCREMENT,
       batch integer,
       path varchar(255),
       PRIMARY KEY(id)

packages/orm/angel_migration_runner/lib/src/mariadb/schema.dart

@@ -5,6 +5,7 @@ import 'package:mysql1/mysql1.dart';
 
 import 'table.dart';
 
+/// A MariaDB database schema generator
 class MariaDbSchema extends Schema {
   final _log = Logger('MariaDbSchema');
 

packages/orm/angel_migration_runner/lib/src/mariadb/table.dart

@@ -48,6 +48,12 @@ abstract class MariaDbGenerator {
       buf.write(' UNIQUE');
     } else if (column.indexType == IndexType.primaryKey) {
       buf.write(' PRIMARY KEY');
+
+      // For int based primary key, apply NOT NULL
+      // and AUTO_INCREMENT
+      if (column.type == ColumnType.int) {
+        buf.write(' NOT NULL AUTO_INCREMENT');
+      }
     }
 
     for (var ref in column.externalReferences) {
@@ -105,7 +111,7 @@ class MariaDbTable extends Table {
 
     if (indexBuf.isNotEmpty) {
       for (var i = 0; i < indexBuf.length; i++) {
-        buf.write(',\n${indexBuf[$1]}');
+        buf.write(',\n${indexBuf[i]}');
       }
     }
   }

packages/orm/angel_migration_runner/lib/src/mysql/runner.dart

@@ -7,6 +7,7 @@ import '../runner.dart';
 import '../util.dart';
 import 'schema.dart';
 
+/// A MySQL database migration runner.
 class MySqlMigrationRunner implements MigrationRunner {
   final _log = Logger('MysqlMigrationRunner');
 
@@ -17,8 +18,10 @@ class MySqlMigrationRunner implements MigrationRunner {
 
   MySqlMigrationRunner(this.connection,
       {Iterable<Migration> migrations = const [], bool connected = false}) {
-    if (migrations.isNotEmpty == true) migrations.forEach(addMigration);
-    _connected = connected == true;
+    if (migrations.isNotEmpty) {
+      migrations.forEach(addMigration);
+    }
+    _connected = connected;
   }
 
   @override
@@ -39,15 +42,18 @@ class MySqlMigrationRunner implements MigrationRunner {
 
     await connection.execute('''
     CREATE TABLE IF NOT EXISTS migrations (
-      id serial,
+      id integer NOT NULL AUTO_INCREMENT,
       batch integer,
-      path varchar(255),
+      path varchar(500),
       PRIMARY KEY(id)
     );
     ''').then((result) {
+      //print(result);
       _log.info('Check and create "migrations" table');
     }).catchError((e) {
+      //print(e);
       _log.severe('Failed to create "migrations" table.');
+      throw e;
     });
   }
 
@@ -56,8 +62,9 @@ class MySqlMigrationRunner implements MigrationRunner {
     await _init();
     var result = await connection.execute('SELECT path from migrations;');
     var existing = <String>[];
-    if (result.rows.isNotEmpty) {
-      existing = result.rows.first.assoc().values.cast<String>().toList();
+    for (var item in result.rows) {
+      var rec = item.assoc().values.first ?? "";
+      existing.add(rec.replaceAll("\\", "\\\\"));
     }
     var toRun = <String>[];
 
@@ -110,8 +117,9 @@ class MySqlMigrationRunner implements MigrationRunner {
     result = await connection
         .execute('SELECT path from migrations WHERE batch = $curBatch;');
     var existing = <String>[];
-    if (result.rows.isNotEmpty) {
-      existing = result.rows.first.assoc().values.cast<String>().toList();
+    for (var item in result.rows) {
+      var rec = item.assoc().values.first ?? "";
+      existing.add(rec.replaceAll("\\", "\\\\"));
     }
     var toRun = <String>[];
 
@@ -140,11 +148,15 @@ class MySqlMigrationRunner implements MigrationRunner {
     await _init();
     var result = await connection
         .execute('SELECT path from migrations ORDER BY batch DESC;');
+
+    // "mysql_client" driver will auto convert path containing "\\" to "\".
+    // So need to revert "\" back to "\\" for the migration logic to work
     var existing = <String>[];
-    if (result.rows.isNotEmpty) {
-      var firstRow = result.rows.first;
-      existing = firstRow.assoc().values.cast<String>().toList();
+    for (var item in result.rows) {
+      var rec = item.assoc().values.first ?? "";
+      existing.add(rec.replaceAll("\\", "\\\\"));
     }
+
     var toRun = existing.where(migrations.containsKey).toList();
 
     if (toRun.isNotEmpty) {