Merge pull request #23 from keystone-scim/mysql-store

Add MySQL store

.github/workflows/unit_tests.yaml

@@ -24,6 +24,13 @@ jobs:
           --health-retries 5
         ports:
           - 5432:5432
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: supersecret
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=10s --health-retries=10
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2

README.md

@@ -50,7 +50,7 @@ its different backends.
 **Keystone** implements the SCIM 2.0 REST API.  If you run your identity management
 operations with an identity manager that supports user provisioning (e.g., Azure AD, Okta, etc.),
 you can use **Keystone** to persist directory changes. Keystone v0.1.0 supports two
-persistence layers: PostgreSQL and Azure Cosmos DB.
+persistence layers: PostgreSQL, MongoDB, Azure Cosmos DB, and MySQL.
 
 <div align="center">
     <img src="./logo/how-it-works.png" alt="logo" />
@@ -66,8 +66,9 @@ Key features:
   * [Azure Cosmos DB](https://docs.microsoft.com/en-us/azure/cosmos-db/introduction)
   * [PostgreSQL](https://www.postgresql.org) (version 10 or higher)
   * [MongoDB](https://www.mongodb.com/docs/) (version 3.6 or higher)
+  * [MySQL](https://www.mongodb.com/docs/) (version 5.7.8 or higher)
 * Azure Key Vault bearer token retrieval.
-* Extensible store: Can't use MongoDB, Cosmos DB, or PostgreSQL?  Open an issue and/or consider
+* Extensible store: Can't use MongoDB, Cosmos DB, PostgreSQL, or MySQL?  Open an issue and/or consider
   [becoming a contributor](./CONTRIBUTING.md) by implementing your own data store.
 
 ## Configure the API

keystone_scim/__init__.py

@@ -5,7 +5,7 @@
 _red = "\033[0;31m"
 _nc = "\033[0m"
 
-VERSION = "0.1.0"
+VERSION = "0.2.0-rc.0"
 LOGO = """
                             ..............                            
                            .--------------. :                         

keystone_scim/cmd.py

@@ -4,11 +4,12 @@
 
 from keystone_scim import VERSION, LOGO, InterceptHandler
 from keystone_scim.store.mongodb_store import set_up
-from keystone_scim.store.postgresql_store import set_up_schema
+from keystone_scim.store import postgresql_store
+from keystone_scim.store import mysql_store
 from keystone_scim.util.logger import get_log_handler
 
-# Initialize logger prior to loading any other modules:
 
+# Initialize logger prior to loading any other 3rd party modules:
 logger = logging.getLogger()
 logger.propagate = True
 if int(os.environ.get("JSON_LOGS", "0")) == 1:
@@ -46,7 +47,9 @@ async def print_logo(_logger):
 
 async def serve(port: int = 5001):
     if CONFIG.get("store.pg.host") is not None:
-        set_up_schema()
+        postgresql_store.set_up_schema()
+    elif CONFIG.get("store.mysql.host") is not None:
+        mysql_store.set_up_schema()
     elif CONFIG.get("store.mongo.host") or CONFIG.get("store.mongo.dsn"):
         await set_up()
 

keystone_scim/rest/__init__.py

@@ -1,6 +1,7 @@
 from aiohttp_catcher import Catcher, canned, catch
 from azure.cosmos.exceptions import CosmosResourceNotFoundError
 from psycopg2.errors import UniqueViolation
+from pymysql.err import IntegrityError
 from pymongo.errors import DuplicateKeyError
 
 from keystone_scim.models import DEFAULT_ERROR_SCHEMA
@@ -18,7 +19,7 @@ async def get_error_handling_mw():
         catch(CosmosResourceNotFoundError).with_status_code(404).and_return(
             "Resource not found").with_additional_fields(err_schemas),
 
-        catch(UniqueViolation, DuplicateKeyError, ResourceAlreadyExists).with_status_code(409).and_return(
+        catch(IntegrityError, UniqueViolation, DuplicateKeyError, ResourceAlreadyExists).with_status_code(409).and_return(
             "Resource already exists").with_additional_fields(err_schemas),
 
         catch(UnauthorizedRequest).with_status_code(401).and_return("Unauthorized request").with_additional_fields(

keystone_scim/store/mysql_models.py

@@ -0,0 +1,45 @@
+import sqlalchemy as sa
+from sqlalchemy.dialects.mysql import JSON
+
+from keystone_scim.util.config import Config
+
+metadata = sa.MetaData()
+
+CONFIG = Config()
+_schema = CONFIG.get("store.mysql.schema", "public")
+
+
+users = sa.Table(
+    "users", metadata,
+    sa.Column("id", sa.VARCHAR, primary_key=True),
+    sa.Column("externalId", sa.VARCHAR),
+    sa.Column("locale", sa.VARCHAR),
+    sa.Column("name", JSON, nullable=False),
+    sa.Column("schemas", JSON, nullable=False),
+    sa.Column("userName", sa.VARCHAR, nullable=False),
+    sa.Column("displayName", sa.VARCHAR, nullable=False),
+    sa.Column("active", sa.Boolean, default=True),
+    sa.Column("customAttributes", JSON)
+)
+
+groups = sa.Table(
+    "groups", metadata,
+    sa.Column("id", sa.VARCHAR, primary_key=True),
+    sa.Column("displayName", sa.VARCHAR, nullable=False),
+    sa.Column("schemas", JSON, nullable=False)
+)
+
+users_groups = sa.Table(
+    "users_groups", metadata,
+    sa.Column("userId", sa.VARCHAR, sa.ForeignKey("users.id")),
+    sa.Column("groupId", sa.VARCHAR, sa.ForeignKey("groups.id"))
+)
+
+user_emails = sa.Table(
+    "user_emails", metadata,
+    sa.Column("id", sa.VARCHAR, primary_key=True),
+    sa.Column("userId", sa.VARCHAR, sa.ForeignKey("users.id")),
+    sa.Column("primary", sa.Boolean, default=True),
+    sa.Column("value", sa.VARCHAR),
+    sa.Column("type", sa.VARCHAR)
+)

keystone_scim/store/mysql_queries.py

@@ -0,0 +1,46 @@
+# scim_schema = "CREATE SCHEMA IF NOT EXISTS `{}`;"
+
+users_tbl = """
+    CREATE TABLE IF NOT EXISTS `users` (
+        `id` VARCHAR(256) PRIMARY KEY,
+        `externalId` VARCHAR(1024),
+        `locale` VARCHAR(64),
+        `name` JSON NOT NULL,
+        `schemas` JSON NOT NULL,
+        `userName` VARCHAR(512) UNIQUE NOT NULL,
+        `displayName` VARCHAR(1024) NOT NULL,
+        `customAttributes` JSON,
+        `active` BOOLEAN,
+        INDEX USING BTREE (`userName`)
+    );
+"""
+
+groups_tbl = """
+    CREATE TABLE IF NOT EXISTS `groups` (
+        `id` VARCHAR(256) PRIMARY KEY,
+        `displayName` VARCHAR(512) UNIQUE NOT NULL,
+        `schemas` JSON NOT NULL,
+        INDEX USING BTREE (`displayName`)
+    );
+"""
+
+users_groups_tbl = """
+    CREATE TABLE IF NOT EXISTS `users_groups` (
+        `userId` VARCHAR(256) NOT NULL,
+        `groupId` VARCHAR(256) NOT NULL,
+        PRIMARY KEY(`userId`, `groupId`)
+    );
+"""
+
+user_emails_tbl = """
+    CREATE TABLE IF NOT EXISTS `user_emails` (
+        `id` VARCHAR(256) PRIMARY KEY,
+        `userId` VARCHAR(256) NOT NULL,
+        `value` VARCHAR(512) NOT NULL,
+        `primary` BOOLEAN DEFAULT TRUE,
+        `type` VARCHAR(128) DEFAULT 'work',
+        INDEX USING BTREE (`value`)
+    );
+"""
+
+ddl_queries = [users_tbl, groups_tbl, users_groups_tbl, user_emails_tbl]