From a9b8f3cb4ef079b0c0402a20b1d6962eacafa113 Mon Sep 17 00:00:00 2001 From: Yash Mittal Date: Fri, 19 Apr 2024 22:58:17 +0530 Subject: [PATCH] feat: fix some bugs and add github actions --- .github/workflows/tests.yml | 26 +++++++ docker-compose.yml | 27 ++++++- package.json | 5 +- src/checks.ts | 5 +- src/commands.ts | 22 +++++- src/schemaGenerator.ts | 77 ++++++++++--------- src/utils/utils.ts | 2 +- test/cli.ts | 3 +- test/schema-generation.spec.ts | 130 +++++++++++++++++++++++++++++++++ test/schemas/enum.json | 2 +- yarn.lock | 4 +- 11 files changed, 251 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/tests.yml create mode 100644 test/schema-generation.spec.ts diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..a578e80 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,26 @@ +name: Node.js CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [18.x] + + steps: + - uses: actions/checkout@v2 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Install dependencies + run: npm install + - name: Run tests + run: npm test \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index aeef4a7..7308096 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,31 @@ services: POSTGRES_PASSWORD: ${DATABASE_PASSWORD} POSTGRES_DB: ${DATABASE_NAME} + minio: + image: minio/minio + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio-data:/data + environment: + MINIO_ROOT_USER: ${MINIO_USERNAME} + MINIO_ROOT_PASSWORD: ${MINIO_PASSWORD} + command: server --console-address ":9001" /data + + createbuckets: + image: minio/mc + depends_on: + - minio + entrypoint: > + /bin/sh -c " + /usr/bin/mc config host add myminio http://minio:9000 ${MINIO_USERNAME} ${MINIO_PASSWORD}; + /usr/bin/mc rm -r --force myminio/${MINIO_BUCKETNAME}; + /usr/bin/mc mb myminio/${MINIO_BUCKETNAME}; + /usr/bin/mc anonymous set public myminio/${MINIO_BUCKETNAME}; + exit 0; + " + volumes: dataset-db: - \ No newline at end of file + minio-data: \ No newline at end of file diff --git a/package.json b/package.json index afface8..f0583aa 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,10 @@ { "name": "@techsavvyash/dynamo-prisma", - "version": "0.0.1", + "version": "0.0.2", "description": "Create Prisma Models Dynamically", "main": "src/index.ts", "scripts": { + "test": "jest", "cleanup": "rm -r ./prisma" }, "author": "Himanshu, Sooraj, Yash", @@ -17,7 +18,7 @@ }, "devDependencies": { "@types/jest": "^29.5.12", - "jest": "^29.5.0", + "jest": "^29.7.0", "prisma": "5.12.1", "supertest": "^6.3.3", "ts-jest": "^29.1.0", diff --git a/src/checks.ts b/src/checks.ts index ad73033..3a2a276 100644 --- a/src/checks.ts +++ b/src/checks.ts @@ -26,11 +26,12 @@ export function checkJSON( const newModelObjects = []; const models = jsonData.schema.map((model) => { if (!existingData.models.includes(model.schemaName)) { + newModelObjects.push(model); + return model.schemaName; + } else { console.warn( `Model ${model.schemaName} is already defined in the schema, please use a different name, skipping this one.` ); - newModelObjects.push(model); - return model.schemaName; } }); const newEnums = []; diff --git a/src/commands.ts b/src/commands.ts index 5009ac3..0a4da70 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -27,9 +27,21 @@ export function runDBPull() { exec("npx prisma db pull", (error, stdout, stderr) => { if (error) { console.error('Error executing "npx prisma db pull"', error); - return; + throw new Error( + JSON.stringify({ + error: true, + message: `Error while performing npx prisma db pull ${error}`, + }) + ); } - console.log("Prisma migrations fetched from the database."); + console.log( + "🚀 Prisma schema has been populated with old tables fetched from the database" + ); + return { + status: true, + message: + "Prisma schema has been populated with old tables fetched from the database.", + }; }); } @@ -57,6 +69,12 @@ export function validateAndMigrate(migrateModels: string[]) { }) .catch((error) => { console.error("An error occurred:", error); + throw new Error( + JSON.stringify({ + error: true, + message: `Error validating and migrating Prisma schema: ${error}`, + }) + ); }); return { diff --git a/src/schemaGenerator.ts b/src/schemaGenerator.ts index 87f5261..523424c 100644 --- a/src/schemaGenerator.ts +++ b/src/schemaGenerator.ts @@ -6,10 +6,13 @@ import { createModels } from "./dsl-helper"; import { checkJSON } from "./checks"; import { createSchema, print } from "prisma-schema-dsl"; import { Schema } from "./types/dynamoPrisma.types"; -import { parseExistingEnums, parseExistingModels } from "./utils/utils"; +import { parseExistingEnums, parsePrismaSchemaModels } from "./utils/utils"; import { validateAndMigrate } from "./commands"; -export async function generateIfNoSchema(jsonData: Schema): Promise { +export async function generateIfNoSchema( + jsonData: Schema, + prismaFilePath: string +): Promise { if (!jsonData.dataSource || !jsonData.generator) { throw new Error( JSON.stringify({ @@ -38,22 +41,22 @@ export async function generateIfNoSchema(jsonData: Schema): Promise { const schemaString = await print(schema); result = Generator + "\n" + DataSource + "\n" + schemaString; - console.warn("schema generated"); const migrateModels: string[] = []; - fs.mkdirSync("./prisma", { recursive: true }); - fs.writeFile("./prisma/schema.prisma", result, (err) => { - if (err) { - return { - status: false, + try { + fs.mkdirSync(prismaFilePath.split("/schema.prisma")[0], { + recursive: true, + }); + fs.writeFileSync(prismaFilePath, result); + console.log("🚀 Prisma schema generated successfully!"); + migrateModels.push(...jsonData.schema.map((model) => model.schemaName)); + } catch (err) { + throw new Error( + JSON.stringify({ + error: true, message: "Error writing Prisma schema", - error: err, - }; - } else { - console.log("Prisma schema generated successfully!"); - migrateModels.push(...jsonData.schema.map((model) => model.schemaName)); - // validateAndMigrate(migrateModels); - } - }); + }) + ); + } return migrateModels; } @@ -68,28 +71,22 @@ export async function generateSchemaWhenFilePresent( const Enum = jsonData.enum ? jsonData.enum! : []; const schema = createSchema(models, Enum, undefined, undefined); const schemaString = await print(schema); - fs.appendFileSync(prismaFilePath, "\n\n" + schemaString, "utf8"); const migrateModels: string[] = []; - fs.mkdirSync("./prisma", { recursive: true }); - fs.writeFile( - "./prisma/schema.prisma", - fs.readFileSync(prismaFilePath, "utf8"), - (err) => { - if (err) { - throw new Error( - JSON.stringify({ - status: false, - message: "Error writing Prisma schema", - error: err, - }) - ); - } else { - console.log("🚀 Prisma schema generated successfully!"); - migrateModels.push(...jsonData.schema.map((model) => model.schemaName)); - // validateAndMigrate(migrateModels); - } - } - ); + + try { + fs.appendFileSync(prismaFilePath, "\n\n" + schemaString, "utf8"); + console.log("🚀 Prisma schema generated successfully!"); + migrateModels.push(...jsonData.schema.map((model) => model.schemaName)); + } catch (err) { + throw new Error( + JSON.stringify({ + status: false, + message: "Error writing Prisma schema", + error: err, + }) + ); + } + return migrateModels; } @@ -103,11 +100,11 @@ export async function generatePrismaSchemaFile( jsonData: Schema, prismaFilePath: string = "./prisma/schema.prisma", failOnWarn: boolean = false -) { +): Promise { const prismaFileExists = fs.existsSync(prismaFilePath); const models = prismaFileExists - ? parseExistingModels(fs.readFileSync(prismaFilePath, "utf8")) + ? parsePrismaSchemaModels(fs.readFileSync(prismaFilePath, "utf8")) : []; const enums = prismaFileExists ? parseExistingEnums(fs.readFileSync(prismaFilePath, "utf8")) @@ -119,6 +116,6 @@ export async function generatePrismaSchemaFile( console.log("📝 Prisma Schema file exists."); return await generateSchemaWhenFilePresent(jsonData, prismaFilePath); } else { - return await generateIfNoSchema(jsonData); + return await generateIfNoSchema(jsonData, prismaFilePath); } } diff --git a/src/utils/utils.ts b/src/utils/utils.ts index de96c42..c483ca5 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -95,7 +95,7 @@ export function checkIllegalCombinationOfFieldAttributes( } // TODO: Change this to use the `pp package`(https://github.com/techsavvyash/pp) package which uses internal prisma DMMF -export function parseExistingModels(fileContent: string) { +export function parsePrismaSchemaModels(fileContent: string) { const modelRegex = /model\s+(\w+)\s+{/g; const models: string[] = []; let match; diff --git a/test/cli.ts b/test/cli.ts index 2e6854a..f4355c5 100644 --- a/test/cli.ts +++ b/test/cli.ts @@ -1,6 +1,6 @@ /** * @description This is just for manual testing - * run: npx ts-node src/cli.ts ./test/schemas/no_unique.json + * run: npx ts-node test/cli.ts ./test/schemas/no_unique.json */ import { generatePrismaSchemaFile } from "../src/schemaGenerator"; import { readJsonFile } from "../src/utils/utils"; @@ -16,6 +16,7 @@ export async function main(argv: string[]) { const filePath = argv[2]; const data = readJsonFile(filePath); const migrateModels: any = await generatePrismaSchemaFile(data); + console.log("migration models: ", migrateModels); validateAndMigrate(migrateModels); return filePath; diff --git a/test/schema-generation.spec.ts b/test/schema-generation.spec.ts new file mode 100644 index 0000000..37fcad8 --- /dev/null +++ b/test/schema-generation.spec.ts @@ -0,0 +1,130 @@ +import * as fs from "fs"; + +import { generatePrismaSchemaFile } from "../src"; +import { + parseExistingEnums, + parsePrismaSchemaModels, + readJsonFile, +} from "../src/utils/utils"; + +describe("tests for schema generation", () => { + beforeEach(() => { + try { + fs.rmdirSync("./prisma", { recursive: true }); + } catch (err) { + console.error("Error deleting prisma folder"); + } + }); + + it("should generate a schema.prisma for a basic file with no problems", async () => { + const fileContent = readJsonFile("./test/schemas/demo.json"); + await generatePrismaSchemaFile(fileContent).then((migrationModels) => { + expect(migrationModels).toBeDefined(); + expect(fs.existsSync("./prisma/schema.prisma")).toBe(true); + const schemaPrismaContent = fs.readFileSync( + "./prisma/schema.prisma", + "utf8" + ); + expect(schemaPrismaContent).toBeDefined(); + console.log("schemaPrismaContent: ", schemaPrismaContent); + const models = parsePrismaSchemaModels(schemaPrismaContent); + console.log("models: ", models); + expect(models).toContain("User"); + expect(models).toContain("Post"); + expect(models).toContain("Comment"); + }); + }); + + it("should generate a schema.prisma for a file with dashes in schema and field name", async () => { + const fileContent = readJsonFile("./test/schemas/dash_name.json"); + const migrationModels = await generatePrismaSchemaFile(fileContent); + expect(migrationModels).toBeDefined(); + expect(fs.existsSync("./prisma/schema.prisma")).toBe(true); + const schemaPrismaContent = fs.readFileSync( + "./prisma/schema.prisma", + "utf8" + ); + expect(schemaPrismaContent).toBeDefined(); + console.log("schemaPrismaContent: ", schemaPrismaContent); + const models = parsePrismaSchemaModels(schemaPrismaContent); + console.log("models: ", models); + expect(models).toContain("dash_name"); + }); + + it("should generate a schema.prisma for a file with no unique or id field", async () => { + const fileContent = readJsonFile("./test/schemas/no_unique.json"); + await generatePrismaSchemaFile(fileContent).then((migrationModels) => { + expect(migrationModels).toBeDefined(); + expect(fs.existsSync("./prisma/schema.prisma")).toBe(true); + const schemaPrismaContent = fs.readFileSync( + "./prisma/schema.prisma", + "utf8" + ); + expect(schemaPrismaContent).toBeDefined(); + console.log("schemaPrismaContent: ", schemaPrismaContent); + const models = parsePrismaSchemaModels(schemaPrismaContent); + console.log("models: ", models); + expect(models).toContain("no_unique"); + }); + }); + + it("should generate a schema.prisma for a file with whitespaces in schema and field name", async () => { + const fileContent = readJsonFile("./test/schemas/whitespace_name.json"); + await generatePrismaSchemaFile(fileContent).then((migrationModels) => { + expect(migrationModels).toBeDefined(); + expect(fs.existsSync("./prisma/schema.prisma")).toBe(true); + const schemaPrismaContent = fs.readFileSync( + "./prisma/schema.prisma", + "utf8" + ); + expect(schemaPrismaContent).toBeDefined(); + console.log("schemaPrismaContent: ", schemaPrismaContent); + const models = parsePrismaSchemaModels(schemaPrismaContent); + console.log("models: ", models); + expect(models).toContain("white_space_name"); + }); + }); + + it("should generate a schema.prisma for a file with vector embeddings in data fields", async () => { + const fileContent = readJsonFile("./test/schemas/vector_embeddings.json"); + await generatePrismaSchemaFile(fileContent).then((migrationModels) => { + expect(migrationModels).toBeDefined(); + expect(fs.existsSync("./prisma/schema.prisma")).toBe(true); + const schemaPrismaContent = fs.readFileSync( + "./prisma/schema.prisma", + "utf8" + ); + expect(schemaPrismaContent).toBeDefined(); + console.log("schemaPrismaContent: ", schemaPrismaContent); + const models = parsePrismaSchemaModels(schemaPrismaContent); + console.log("models: ", models); + expect(models).toContain("vector_embedding"); + const pattern = /@default\("text-embedding-ada-002"\)/; + expect(pattern.test(schemaPrismaContent)).toBe(true); + }); + }); + + it("should generate a schema.prisma for a file with enums", async () => { + const fileContent = readJsonFile("./test/schemas/enum.json"); + await generatePrismaSchemaFile(fileContent).then((migrationModels) => { + expect(migrationModels).toBeDefined(); + expect(fs.existsSync("./prisma/schema.prisma")).toBe(true); + const schemaPrismaContent = fs.readFileSync( + "./prisma/schema.prisma", + "utf8" + ); + expect(schemaPrismaContent).toBeDefined(); + console.log("schemaPrismaContent: ", schemaPrismaContent); + const models = parsePrismaSchemaModels(schemaPrismaContent); + console.log("models: ", models); + expect(models).toContain("User"); + const enums = parseExistingEnums(schemaPrismaContent); + console.log("enums: ", enums); + expect(enums).toBeDefined(); + expect(enums.length).toBeGreaterThan(0); + expect(enums).toContain("UserType"); + const pattern = /@default\("text-embedding-3-large"\)/; + expect(pattern.test(schemaPrismaContent)).toBe(true); + }); + }); +}); diff --git a/test/schemas/enum.json b/test/schemas/enum.json index 1f6265c..6a2b385 100644 --- a/test/schemas/enum.json +++ b/test/schemas/enum.json @@ -155,7 +155,7 @@ "unique": false, "isForeignKey": true, "vectorEmbed": true, - "embeddingAlgo": "whatever" + "embeddingAlgo": "text-embedding-3-large" } ], "description": "Comment model" diff --git a/yarn.lock b/yarn.lock index 3b2e315..8a3ea99 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2418,9 +2418,9 @@ jest-worker@^29.7.0: merge-stream "^2.0.0" supports-color "^8.0.0" -jest@^29.5.0: +jest@^29.7.0: version "29.7.0" - resolved "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz" + resolved "https://registry.yarnpkg.com/jest/-/jest-29.7.0.tgz#994676fc24177f088f1c5e3737f5697204ff2613" integrity sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw== dependencies: "@jest/core" "^29.7.0"