diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..30fea21b --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +.env +.env.local +package-lock.json +package.json diff --git a/database/base-provider.ts b/database/base-provider.ts new file mode 100644 index 00000000..f3bb64c4 --- /dev/null +++ b/database/base-provider.ts @@ -0,0 +1,302 @@ +import { EntityObserver } from './utils/observer'; +import { Kysely } from 'kysely'; +import { BaseDatabase, BaseTable, ObserverData, Event } from './types'; + +export const TRUE = 1; +export const FALSE = 0; + +export abstract class SQLiteDataProvider { + protected static _instance: any; + protected static _db: any; + + /** + * Get the singleton instance of the provider + */ + public static get shared(): SQLiteDataProvider { + if (!this._instance) { + this._instance = new (this as any)(); + } + return this._instance; + } + + /** + * Get the database instance + */ + public static get db(): Kysely { + if (!this._db) { + throw new Error('Database not initialized'); + } + return this._db; + } + + /** + * Initialize the database instance + */ + public static initialize(db: Kysely) { + this._db = db; + } + + /** + * Reset the singleton instance and database connection + */ + public static reset() { + this._instance = undefined; + this._db = undefined; + } + + protected constructor() { + // Protected constructor to enforce singleton pattern + if (SQLiteDataProvider._instance) { + throw new Error('Use shared to get the singleton instance'); + } + } + + /** + * Get the database instance for this provider + */ + protected get db(): Kysely { + return (this.constructor as typeof SQLiteDataProvider).db; + } +} + +export class BaseSQLiteProvider { + static entity: string; + static observer: EntityObserver = new EntityObserver(); + static db: Kysely; + + static dbSelect() { + return this.db.selectFrom(this.entity).selectAll(); + } + + static dbDelete() { + return this.db.deleteFrom(this.entity); + } + + static dbInsert() { + return this.db.insertInto(this.entity); + } + + static dbUpdate() { + return this.db.updateTable(this.entity); + } + + static async selectOne(ulid: string) { + return await this.dbSelect().where('ulid', '=', ulid).executeTakeFirst(); + } + + static async selectByParent(parentUlid: string) { + return await this.dbSelect().where('parent_ulid', '=', parentUlid).execute(); + } + + static async selectByUlids(ulids: string[]) { + return await this.dbSelect().where('ulid', 'in', ulids).execute(); + } + + static async selectAll() { + return await this.dbSelect().execute(); + } + + static async countByParent(parentUlid: string) { + return await this.db + .selectFrom(this.entity) + .select([(b) => b.fn.count('ulid').as('count')]) + .where('parent_ulid', '=', parentUlid) + .executeTakeFirst(); + } + + static async selectDirty() { + return await this.dbSelect() + .where('is_dirty', '=', TRUE) + .where('is_deleted', '=', FALSE) + .where('last_synced_at', 'is not', null) + .execute(); + } + + static async selectCreated() { + return await this.dbSelect() + .where('is_dirty', '=', TRUE) + .where('is_deleted', '=', FALSE) + .where('last_synced_at', 'is', null) + .execute(); + } + + static async selectDeleted() { + return await this.dbSelect().where('is_deleted', '=', TRUE).execute(); + } + + static async upsertSynced(ulid: string, data: any, parent_ulid: string = '') { + const values = { + parent_ulid, + data: typeof data === 'string' ? data : JSON.stringify(data), + timestamp: Date.now(), + is_deleted: FALSE, + is_dirty: FALSE, + last_synced_at: Date.now(), + }; + + return await this.dbInsert() + .values({ ulid, ...values }) + .onConflict((oc) => oc.column('ulid').doUpdateSet(values)) + .executeTakeFirst(); + } + + static async upsertDirty(ulid: string, data: any, parent_ulid: string = '') { + const values = { + parent_ulid, + data: typeof data === 'string' ? data : JSON.stringify(data), + timestamp: Date.now(), + is_deleted: FALSE, + is_dirty: TRUE, + last_synced_at: Date.now(), + }; + + return await this.dbInsert() + .values({ ulid, ...values }) + .onConflict((oc) => oc.column('ulid').doUpdateSet(values)) + .executeTakeFirst(); + } + + static async insertDirty(ulid: string, data: any, parent_ulid: string = '') { + return await this.dbInsert() + .values({ + ulid, + parent_ulid, + data: typeof data === 'string' ? data : JSON.stringify(data), + timestamp: Date.now(), + is_dirty: TRUE, + last_synced_at: null, + is_deleted: FALSE, + }) + .executeTakeFirst(); + } + + static async insertDirtyMultiple( + rows: { ulid: string; data: any; parentUlid?: string }[] + ) { + const values = rows.map(({ ulid, data, parentUlid: parent_ulid = '' }) => ({ + ulid, + parent_ulid, + data: typeof data === 'string' ? data : JSON.stringify(data), + timestamp: Date.now(), + is_dirty: TRUE, + last_synced_at: null, + is_deleted: FALSE, + })); + + return await this.dbInsert().values(values).executeTakeFirst(); + } + + static async insertSynced(ulid: string, data: any, parent_ulid: string = '') { + return await this.dbInsert() + .values({ + ulid, + parent_ulid, + data: typeof data === 'string' ? data : JSON.stringify(data), + timestamp: Date.now(), + is_dirty: FALSE, + last_synced_at: Date.now(), + is_deleted: FALSE, + }) + .executeTakeFirst(); + } + + static async insertSyncedMultiple( + rows: { ulid: string; data: any; parentUlid?: string }[] + ) { + const values = rows.map(({ ulid, data, parentUlid: parent_ulid = '' }) => ({ + ulid, + parent_ulid, + data: typeof data === 'string' ? data : JSON.stringify(data), + timestamp: Date.now(), + is_dirty: FALSE, + last_synced_at: Date.now(), + is_deleted: FALSE, + })); + return await this.dbInsert().values(values).executeTakeFirst(); + } + + static async updateDirty(ulid: string, data: any) { + return await this.dbUpdate() + .set({ + data: typeof data === 'string' ? data : JSON.stringify(data), + is_dirty: TRUE, + timestamp: Date.now(), + }) + .where('ulid', '=', ulid) + .executeTakeFirst(); + } + + static async updateSynced(ulid: string, data?: any) { + return await this.dbUpdate() + .set({ + ...(data + ? { data: typeof data === 'string' ? data : JSON.stringify(data) } + : {}), + is_dirty: FALSE, + last_synced_at: Date.now(), + timestamp: Date.now(), + }) + .where('ulid', '=', ulid) + .executeTakeFirst(); + } + + static async markDeleted(ulid: string) { + return await this.dbUpdate() + .set({ + is_dirty: TRUE, + is_deleted: TRUE, + timestamp: Date.now(), + }) + .where('ulid', '=', ulid) + .executeTakeFirst(); + } + + static async markDeletedByParent(parentUlid: string) { + return await this.dbUpdate() + .set({ + is_dirty: TRUE, + is_deleted: TRUE, + timestamp: Date.now(), + }) + .where('parent_ulid', '=', parentUlid) + .execute(); + } + + static async purgeDeleted() { + return await this.dbDelete().where('is_deleted', '=', TRUE).execute(); + } + + static async purgeAll() { + await this.dbDelete().execute(); + } + + static async purge(ulid: string) { + return await this.dbDelete().where('ulid', '=', ulid).execute(); + } + + static async purgeByParent(parentUlid: string) { + return await this.dbDelete().where('parent_ulid', '=', parentUlid).execute(); + } + + static async purgeUlids(ulids: string[]) { + return await this.dbDelete().where('ulid', 'in', ulids).execute(); + } + + static subscribe( + event: Event, + observer: ObserverCallback + ): () => void { + return this.observer.subscribe(String(this.entity), event, observer); + } + + static unsubscribe( + event: Event, + observer: ObserverCallback + ): void { + this.observer.unsubscribe(String(this.entity), event, observer); + } + + protected static notify(event: Event, data: ObserverData) { + this.observer.notify(String(this.entity), event, data); + } +} diff --git a/database/config.ts b/database/config.ts new file mode 100644 index 00000000..c6e82bc3 --- /dev/null +++ b/database/config.ts @@ -0,0 +1,52 @@ +import { Kysely, sql } from 'kysely'; +import { ExpoDialect } from 'kysely-expo'; +import { BaseTable } from './types'; + +export function createDatabase>(config: { + name: string; + debug?: boolean; + onError?: (error: any) => void; +}) { + const dialect = new ExpoDialect({ + database: config.name, + debug: config.debug ?? false, + onError: config.onError, + }); + + return new Kysely({ + dialect, + }); +} + +export async function initializeTable( + db: Kysely, + tableName: string +) { + await db.schema + .createTable(tableName) + .ifNotExists() + .addColumn('parent_ulid', 'text') + .addColumn('ulid', 'text', (col) => col.primaryKey()) + .addColumn('data', 'text') + .addColumn('timestamp', 'integer', (col) => + col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull() + ) + .addColumn('last_synced_at', 'integer') + .addColumn('is_dirty', 'integer', (col) => col.notNull()) + .addColumn('is_deleted', 'integer', (col) => col.defaultTo(false).notNull()) + .execute(); + + await db.schema + .createIndex(`${tableName}_timestamp`) + .ifNotExists() + .on(tableName) + .column('timestamp') + .execute(); + + await db.schema + .createIndex(`${tableName}_parent_ulid`) + .ifNotExists() + .on(tableName) + .column('parent_ulid') + .execute(); +} diff --git a/database/index.ts b/database/index.ts new file mode 100644 index 00000000..6c1fc84c --- /dev/null +++ b/database/index.ts @@ -0,0 +1,4 @@ +export { SQLiteDataProvider } from './provider'; +export * from './types'; +export * from './config'; +export * from './base-provider'; diff --git a/database/provider-v2.ts b/database/provider-v2.ts new file mode 100644 index 00000000..f5c07a1d --- /dev/null +++ b/database/provider-v2.ts @@ -0,0 +1,18 @@ +import { BaseEndpoint, BaseRecord, BaseTable } from './types'; +import { SQLiteDataProvider } from './provider'; + +// Generic database type that extends BaseTable +export interface GenericDatabase { + [key: string]: BaseTable; +} + +export class SQLiteDataProviderV2 extends SQLiteDataProvider { + constructor( + entity: keyof DB, + endpoint: BaseEndpoint, + queryKey: string, + db: any // The actual database instance will be passed in + ) { + super(db, entity, endpoint, queryKey); + } +} diff --git a/database/provider.ts b/database/provider.ts new file mode 100644 index 00000000..6501ae57 --- /dev/null +++ b/database/provider.ts @@ -0,0 +1,303 @@ +import { Kysely } from 'kysely'; +import { QueryClient } from '@tanstack/react-query'; +import { BaseEndpoint, BaseRecord, BaseTable, CreateParams, DataWithSyncStatus, Event, ObserverData, UpdateParams } from './types'; +import { EntityObserver } from '@htk/utils/observer'; +import { RecordID } from './types'; + +// Create a shared queryClient instance +export const queryClient = new QueryClient(); + +// Constants with proper type assertions +export const TRUE = 1 as const; +export const FALSE = 0 as const; + +export class SQLiteDataProvider> { + protected db: Kysely; + public entity: keyof DB; + protected endpoint: BaseEndpoint; + protected queryKey: string; + static observer: EntityObserver = new EntityObserver(); + + constructor( + db: Kysely, + entity: keyof DB, + endpoint: BaseEndpoint, + queryKey: string + ) { + this.db = db; + this.entity = entity; + this.endpoint = endpoint; + this.queryKey = queryKey; + } + + async list(): Promise[]> { + const records = await this.db + .selectFrom(this.entity as string) + .selectAll() + .where('is_deleted', '=', FALSE) + .orderBy('timestamp', 'desc') + .execute() as unknown as BaseTable[]; + + return records.map((record) => ({ + ...(JSON.parse(record.data) as T), + isDirty: record.is_dirty === TRUE, + isCreatedOnServer: record.last_synced_at !== null, + })); + } + + async listAll(): Promise[]> { + const records = await this.db + .selectFrom(this.entity as string) + .selectAll() + .orderBy('timestamp', 'desc') + .execute() as unknown as BaseTable[]; + + return records.map((record) => ({ + ...(JSON.parse(record.data) as T), + isDirty: record.is_dirty === TRUE, + isCreatedOnServer: record.last_synced_at !== null, + })); + } + + async getOne(ulid: RecordID): Promise> { + const record = await this.db + .selectFrom(this.entity as string) + .selectAll() + .where('ulid', '=', ulid as any) + .where('is_deleted', '=', FALSE as any) + .executeTakeFirst() as unknown as BaseTable; + + if (!record) { + throw new Error(`Record with ulid ${ulid} not found in ${String(this.entity)}`); + } + + return { + ...(JSON.parse(record.data) as T), + isDirty: record.is_dirty === TRUE, + isCreatedOnServer: record.last_synced_at !== null, + }; + } + + async create(params: CreateParams): Promise> { + const { data, isDirty = true, parentUlid = '' } = params; + const now = Date.now(); + + const result = await this.db + .insertInto(this.entity as string) + .values({ + ulid: data.ulid, + data: JSON.stringify(data), + parent_ulid: parentUlid, + timestamp: now, + is_dirty: isDirty ? TRUE : FALSE, + is_deleted: FALSE, + last_synced_at: isDirty ? null : now, + } as any) + .executeTakeFirst(); + + if (!result || result.numInsertedOrUpdatedRows === 0n) { + throw new Error(`Failed to insert record into ${String(this.entity)}`); + } + + return { + ...data, + isDirty: true, + isCreatedOnServer: false, + }; + } + + async update(params: UpdateParams): Promise> { + const { ulid, data, isDirty = true } = params; + const now = Date.now(); + const preparedData = { + ...data, + updatedAt: now, + } as T; + + const result = await this.db + .updateTable(this.entity as string) + .set({ + data: JSON.stringify(preparedData), + timestamp: now, + is_dirty: isDirty ? TRUE : FALSE, + } as any) + .where('ulid', '=', ulid as any) + .executeTakeFirst(); + + if (!result || result.numUpdatedRows === 0n) { + throw new Error(`Failed to update record in ${String(this.entity)}`); + } + + return { + ...preparedData, + isDirty: true, + isCreatedOnServer: false, + }; + } + + async upsert( + data: T, + dirty: boolean = true, + updateIfDirty: boolean = true + ): Promise> { + const baseValues = { + parent_ulid: '', + data: JSON.stringify(data), + timestamp: Date.now(), + is_deleted: FALSE, + is_dirty: dirty ? TRUE : FALSE, + }; + + const createValues = { + ...baseValues, + ulid: data.ulid, + last_synced_at: dirty ? null : Date.now(), + }; + + const updateValues = { + ...baseValues, + ...(!dirty ? { last_synced_at: Date.now() } : {}), + }; + + await this.db + .insertInto(this.entity as string) + .values(createValues as any) + .onConflict((oc) => { + const set = oc.column('ulid').doUpdateSet(updateValues as any); + if (!dirty && !updateIfDirty) { + return set.where('is_dirty' as any, '=', FALSE); + } + return set; + }) + .executeTakeFirstOrThrow(); + + return { + ...data, + isDirty: dirty, + isCreatedOnServer: false, + }; + } + + async delete(ulid: RecordID): Promise { + const result = await this.db + .updateTable(this.entity as string) + .set({ + is_deleted: TRUE, + } as any) + .where('ulid', '=', ulid as any) + .executeTakeFirst(); + + if (!result || result.numUpdatedRows === 0n) { + throw new Error(`Failed to delete record in ${String(this.entity)}`); + } + + return ulid; + } + + async purge(ulid: RecordID) { + await this.db + .deleteFrom(this.entity as string) + .where('ulid', '=', ulid as any ) + .execute(); + } + + async purgeByUlids(ulids: RecordID[]) { + await this.db + .deleteFrom(this.entity as string) + .where('ulid', 'in', ulids as any) + .execute(); + } + + async purgeOtherThanUlids(ulids: RecordID[]) { + await this.db + .deleteFrom(this.entity as string) + .where('ulid', 'not in', ulids as any) + .execute(); + } + + async purgeAll() { + await this.db.deleteFrom(this.entity as string).execute(); + } + + protected async listDirty(): Promise { + const records = await this.db + .selectFrom(this.entity as string) + .selectAll() + .where('is_dirty', '=', TRUE as any) + .execute() as unknown as BaseTable[]; + + return records.map((record) => JSON.parse(record.data) as T); + } + + protected async listDeleted(): Promise { + const records = await this.db + .selectFrom(this.entity as string) + .selectAll() + .where('is_deleted', '=', TRUE as any) + .execute() as unknown as BaseTable[]; + + return records.map((record) => JSON.parse(record.data) as T); + } + + async sync(all: boolean = false): Promise { + const count = (await this.list()).length; + let isSynced = false; + + const isDirtySynced = await this.syncDirty(true); + + if (count === 0 || all) { + await this._fetchAll(); + isSynced = true; + } + + isSynced = isSynced || isDirtySynced; + return isSynced; + } + + async syncDirty(invalidateCache: boolean = false) { + const dirtyEntries = await this.listDirty(); + const deletedEntries = await this.listDeleted(); + + if (dirtyEntries.length === 0 && deletedEntries.length === 0) { + return false; + } + + const response = await this.endpoint.sync({ + entries: dirtyEntries.map((x) => this.prepareDataForSync(x)), + removed: deletedEntries.map((x) => x.ulid), + }); + + await this.purgeByUlids(response.data.removed); + + for (const entry of response.data.entries) { + await this.upsert(entry, false); + } + + if (invalidateCache) { + this.invalidateCache(response.data.entries); + } + + return true; + } + + invalidateCache(entries: T[]) { + queryClient.invalidateQueries({ queryKey: [this.queryKey] }); + + for (const entry of entries) { + queryClient.invalidateQueries({ queryKey: [this.queryKey, entry.ulid] }); + } + } + + protected async _fetchAll() { + const response = await this.endpoint.list(); + for (const entry of response.data.list) { + await this.upsert(entry, false, false); + } + + await this.purgeOtherThanUlids(response.data.list.map((x) => x.ulid)); + } + + prepareDataForSync(data: T) { + return data; + } +} diff --git a/database/types.ts b/database/types.ts new file mode 100644 index 00000000..c5263069 --- /dev/null +++ b/database/types.ts @@ -0,0 +1,69 @@ +import { Selectable } from 'kysely'; + +export type RecordID = string; + +export interface BaseTable { + parent_ulid: string; + ulid: RecordID; + data: string; + timestamp: number; + last_synced_at: number | null; + is_dirty: number; + is_deleted: number; +} + +export interface BaseRecord { + ulid: string; + parent_ulid?: string; + data: string; + timestamp: number; + is_dirty: number; + is_deleted: number; + last_synced_at: number | null; +} + +export interface BaseEndpoint { + list(): Promise<{ data: { list: T[] } }>; + fetch(id: string): Promise<{ data: { entry: T } }>; + sync(data: { entries: any[]; removed: string[] }): Promise<{ + data: { + entries: T[]; + removed: string[]; + }; + }>; +} + +export interface CreateParams { + data: T; + isDirty?: boolean; + parentUlid?: string; +} + +export interface UpdateParams { + ulid: string; + data: T; + isDirty?: boolean; +} + +export type DataWithSyncStatus = T & { + isDirty: boolean; + isCreatedOnServer: boolean; +}; + +export type ObserverData = { + ulids: string[]; +}; + +export type Event = + | 'all' + | 'createLocal' + | 'updateLocal' + | 'deleteLocal' + | 'synced' + | 'deletedSynced'; + +export type BaseTableRecord = Selectable; + +export interface BaseDatabase { + [key: string]: BaseTable; +} diff --git a/database/utils/observer.ts b/database/utils/observer.ts new file mode 100644 index 00000000..d2fa4c00 --- /dev/null +++ b/database/utils/observer.ts @@ -0,0 +1,28 @@ +export type ObserverCallback = (data: T, event: E) => void; + +export class EntityObserver { + private observers: Map>>> = new Map(); + + subscribe(entity: string, event: E, callback: ObserverCallback): () => void { + if (!this.observers.has(entity)) { + this.observers.set(entity, new Map()); + } + const entityObservers = this.observers.get(entity)!; + + if (!entityObservers.has(event)) { + entityObservers.set(event, new Set()); + } + const eventObservers = entityObservers.get(event)!; + + eventObservers.add(callback); + return () => this.unsubscribe(entity, event, callback); + } + + unsubscribe(entity: string, event: E, callback: ObserverCallback): void { + this.observers.get(entity)?.get(event)?.delete(callback); + } + + notify(entity: string, event: E, data: T): void { + this.observers.get(entity)?.get(event)?.forEach(callback => callback(data, event)); + } +}