From f5cf650bdd09957431c1bd3c9c8d4a9cca0ee15a Mon Sep 17 00:00:00 2001 From: Igor Katsuba Date: Tue, 7 May 2024 11:55:09 +0300 Subject: [PATCH] fix(api): fix performance issue for /runs/end Close #507 --- apps/api/project.json | 10 +++++ .../db/entities/src/lib/task/task.entity.ts | 2 + .../db/entities/src/lib/task/task.service.ts | 24 +++-------- libs/api/http/runs/src/lib/runs.controller.ts | 40 +++++++++---------- migrations/.snapshot-postgres.json | 7 ++++ migrations/Migration20240507084336.ts | 13 ++++++ package.json | 1 + scripts/create-migrations.ts | 28 +++++++++++++ 8 files changed, 86 insertions(+), 39 deletions(-) create mode 100644 migrations/Migration20240507084336.ts create mode 100644 scripts/create-migrations.ts diff --git a/apps/api/project.json b/apps/api/project.json index 881a99e..30b21d0 100644 --- a/apps/api/project.json +++ b/apps/api/project.json @@ -55,6 +55,16 @@ ] } }, + "create-migrations": { + "executor": "nx:run-commands", + "options": { + "commands": [ + { + "command": "npm run migrations:create" + } + ] + } + }, "lint": { "executor": "@nx/eslint:lint", "outputs": ["{options.outputFile}"] diff --git a/libs/api/db/entities/src/lib/task/task.entity.ts b/libs/api/db/entities/src/lib/task/task.entity.ts index 58a5687..953d96c 100644 --- a/libs/api/db/entities/src/lib/task/task.entity.ts +++ b/libs/api/db/entities/src/lib/task/task.entity.ts @@ -1,5 +1,6 @@ import { Entity, + Index, LoadStrategy, ManyToOne, PrimaryKey, @@ -29,6 +30,7 @@ export class TaskEntity implements Task { @Property({ nullable: true }) configuration: string; + @Index() @Property() hash: string; diff --git a/libs/api/db/entities/src/lib/task/task.service.ts b/libs/api/db/entities/src/lib/task/task.service.ts index 0109341..99bcbc9 100644 --- a/libs/api/db/entities/src/lib/task/task.service.ts +++ b/libs/api/db/entities/src/lib/task/task.service.ts @@ -27,24 +27,10 @@ export class TaskService { }); } - async findTaskWithoutCache(workspaceId: string, hash: string) { - return this.taskRepository.findOne( - { - execution: { - runGroup: { - workspace: { - id: workspaceId, - }, - }, - }, - hash, - cacheStatus: 'cache-miss', - }, - { - orderBy: { - startTime: 'desc', - }, - } - ); + async findTaskWithoutCache(hash: string) { + return this.taskRepository.findOne({ + hash, + cacheStatus: 'cache-miss', + }); } } diff --git a/libs/api/http/runs/src/lib/runs.controller.ts b/libs/api/http/runs/src/lib/runs.controller.ts index 263040f..3348214 100644 --- a/libs/api/http/runs/src/lib/runs.controller.ts +++ b/libs/api/http/runs/src/lib/runs.controller.ts @@ -78,30 +78,30 @@ export class RunsController { const workspace = await this.workspaceService.getWorkspace(workspaceId); - for (const task of data.tasks) { - const prevTask = - task.cacheStatus === 'cache-miss' - ? task - : (await this.taskService.findTaskWithoutCache( - workspaceId, - task.hash - )) ?? task; + Promise.all( + data.tasks.map(async (task) => { + const prevTask = + task.cacheStatus === 'cache-miss' + ? task + : (await this.taskService.findTaskWithoutCache(task.hash)) ?? + task; - const executionTime = - Date.parse(task.endTime) - Date.parse(task.startTime); + const executionTime = + Date.parse(task.endTime) - Date.parse(task.startTime); - const prevExecutionTime = - Date.parse(prevTask.endTime) - Date.parse(prevTask.startTime); + const prevExecutionTime = + Date.parse(prevTask.endTime) - Date.parse(prevTask.startTime); - const diff = prevExecutionTime - executionTime; + const diff = prevExecutionTime - executionTime; - await this.stats?.trackTaskExecutionTime?.( - workspace, - task, - executionTime, - diff - ); - } + await this.stats?.trackTaskExecutionTime?.( + workspace, + task, + executionTime, + diff + ); + }) + ).catch(console.error); const runGroup = (await this.runGroupService.findOne(data.run.runGroup)) ?? diff --git a/migrations/.snapshot-postgres.json b/migrations/.snapshot-postgres.json index 9b045af..5814799 100644 --- a/migrations/.snapshot-postgres.json +++ b/migrations/.snapshot-postgres.json @@ -327,6 +327,13 @@ "name": "task_entity", "schema": "public", "indexes": [ + { + "columnNames": ["hash"], + "composite": false, + "keyName": "task_entity_hash_index", + "primary": false, + "unique": false + }, { "keyName": "task_entity_pkey", "columnNames": ["id"], diff --git a/migrations/Migration20240507084336.ts b/migrations/Migration20240507084336.ts new file mode 100644 index 0000000..4f41fd9 --- /dev/null +++ b/migrations/Migration20240507084336.ts @@ -0,0 +1,13 @@ +import { Migration } from '@mikro-orm/migrations'; + +export class Migration20240507084336 extends Migration { + async up(): Promise { + this.addSql( + 'create index "task_entity_hash_index" on "task_entity" ("hash");' + ); + } + + async down(): Promise { + this.addSql('drop index "task_entity_hash_index";'); + } +} diff --git a/package.json b/package.json index 89d5ec2..a64c9b4 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "0.0.0-development", "license": "MIT", "scripts": { + "migrations:create": "ts-node -r tsconfig-paths/register -P scripts/tsconfig.json scripts/create-migrations.ts", "migrations:run": "ts-node -r tsconfig-paths/register -P scripts/tsconfig.json scripts/migrations.ts", "start:prod": "npm run migrations:run && node dist/apps/api/main.js" }, diff --git a/scripts/create-migrations.ts b/scripts/create-migrations.ts new file mode 100644 index 0000000..36bc8f7 --- /dev/null +++ b/scripts/create-migrations.ts @@ -0,0 +1,28 @@ +import { MikroORM } from '@mikro-orm/core'; +import { NestFactory } from '@nestjs/core'; +import { AppModule } from '../apps/api/src/app/app.module'; +import { Logger } from 'nestjs-pino'; + +(async () => { + const app = await NestFactory.create(AppModule, { + autoFlushLogs: true, + bufferLogs: true, + }); + + const logger = app.get(Logger); + + app.useLogger(logger); + + logger.log('Starting migrations...'); + + const orm = app.get(MikroORM); + + const migrator = orm.getMigrator(); + await migrator.createMigration(); + + logger.log('Migrations created.'); + + await orm.close(true); +})().catch((e) => { + console.error(e); +});