Merge pull request #25 from tiktok/chao/incremental-copy

feat: incremental copy

packages/pnpm-sync-lib/etc/pnpm-sync-lib.api.md

@@ -48,7 +48,7 @@ export interface ILogMessageCallbackOptions {
 
 // @beta (undocumented)
 export interface IPnpmSyncCopyOptions {
-    ensureFolder: (folderPath: string) => Promise<void>;
+    ensureFolderAsync: (folderPath: string) => Promise<void>;
     forEachAsyncWithConcurrency: <TItem>(iterable: Iterable<TItem>, callback: (item: TItem) => Promise<void>, options: {
         concurrency: number;
     }) => Promise<void>;
@@ -60,7 +60,7 @@ export interface IPnpmSyncCopyOptions {
 // @beta (undocumented)
 export interface IPnpmSyncPrepareOptions {
     dotPnpmFolder: string;
-    ensureFolder: (folderPath: string) => Promise<void>;
+    ensureFolderAsync: (folderPath: string) => Promise<void>;
     lockfilePath: string;
     logMessageCallback: (options: ILogMessageCallbackOptions) => void;
     readPnpmLockfile: (lockfilePath: string, options: {

packages/pnpm-sync-lib/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pnpm-sync-lib",
-  "version": "0.2.2",
+  "version": "0.2.3",
   "description": "API library for integrating \"pnpm-sync\" with your toolchain",
   "repository": {
     "type": "git",

packages/pnpm-sync-lib/src/interfaces.ts

@@ -16,6 +16,12 @@ export interface ITargetFolder {
   folderPath: string;
 }
 
+export interface ISyncItem {
+  absolutePath: string;
+  isDirectory: boolean;
+  isFile: boolean;
+}
+
 /**
  * @beta
  */

packages/pnpm-sync-lib/src/pnpmSyncCopy.ts

@@ -4,10 +4,11 @@ import process from 'node:process';
 import {
   ILogMessageCallbackOptions,
   IPnpmSyncJson,
+  ISyncItem,
   LogMessageIdentifier,
   LogMessageKind
 } from './interfaces';
-import { pnpmSyncGetJsonVersion } from './utilities';
+import { getFilesInDirectory, pnpmSyncGetJsonVersion } from './utilities';
 
 /**
  * @beta
@@ -40,7 +41,7 @@ export interface IPnpmSyncCopyOptions {
    * Environment-provided API to avoid an NPM dependency.
    * The "pnpm-sync" NPM package provides a reference implementation.
    */
-  ensureFolder: (folderPath: string) => Promise<void>;
+  ensureFolderAsync: (folderPath: string) => Promise<void>;
   /**
    * A callback for reporting events during the operation.
    *
@@ -63,7 +64,8 @@ export interface IPnpmSyncCopyOptions {
  * @beta
  */
 export async function pnpmSyncCopyAsync(options: IPnpmSyncCopyOptions): Promise<void> {
-  const { getPackageIncludedFiles, forEachAsyncWithConcurrency, ensureFolder, logMessageCallback } = options;
+  const { getPackageIncludedFiles, forEachAsyncWithConcurrency, ensureFolderAsync, logMessageCallback } =
+    options;
   let pnpmSyncJsonPath = options.pnpmSyncJsonPath;
 
   pnpmSyncJsonPath = path.resolve(process.cwd(), pnpmSyncJsonPath);
@@ -128,11 +130,19 @@ export async function pnpmSyncCopyAsync(options: IPnpmSyncCopyOptions): Promise<
 
   const startTime = process.hrtime.bigint();
 
-  // clear the destination folder first
+  // init the map to track files that already processed
+  const targetFolderFileToIsProcessed: Map<string, ISyncItem> = new Map();
+
   for (const targetFolder of targetFolders) {
     const destinationPath = path.resolve(pnpmSyncJsonFolder, targetFolder.folderPath);
-    // TODO: optimize this
-    await fs.promises.rm(destinationPath, { recursive: true, force: true });
+    if (!fs.existsSync(destinationPath)) {
+      continue;
+    }
+    const existFileStatInoInTargetFolder: Array<ISyncItem> = await getFilesInDirectory(destinationPath);
+    // all files are not processed in the beginning
+    for (const item of existFileStatInoInTargetFolder) {
+      targetFolderFileToIsProcessed.set(item.absolutePath, item);
+    }
   }
 
   await forEachAsyncWithConcurrency(
@@ -144,17 +154,48 @@ export async function pnpmSyncCopyAsync(options: IPnpmSyncCopyOptions): Promise<
         const copySourcePath: string = path.join(sourcePath, npmPackFile);
         const copyDestinationPath: string = path.join(destinationPath, npmPackFile);
 
-        await ensureFolder(path.dirname(copyDestinationPath));
-
-        // create a hard link to the destination path
-        await fs.promises.link(copySourcePath, copyDestinationPath);
+        if (!targetFolderFileToIsProcessed.has(copyDestinationPath)) {
+          // if not exist in target folder, we just copy it
+          await ensureFolderAsync(path.dirname(copyDestinationPath));
+          await fs.promises.link(copySourcePath, copyDestinationPath);
+        } else {
+          // if exist in target folder, check if it still point to the source Inode number
+          // in our copy implementation, we use hard link to copy files
+          // so that, we can utilize the file inode info to determine the equality of two files
+          const sourceFileIno = (await fs.promises.stat(copySourcePath)).ino;
+          const destinationFileIno = (await fs.promises.stat(copyDestinationPath)).ino;
+          if (sourceFileIno !== destinationFileIno) {
+            await fs.promises.unlink(copyDestinationPath);
+            await fs.promises.link(copySourcePath, copyDestinationPath);
+          }
+
+          // to keep track which file already processed
+          targetFolderFileToIsProcessed.delete(copyDestinationPath);
+        }
       }
     },
     {
       concurrency: 10
     }
   );
 
+  // delete unprocessed files in target folders
+  const unprocessedDirectories: string[] = [];
+  for (const [absolutePath, item] of targetFolderFileToIsProcessed) {
+    if (item.isFile) {
+      await fs.promises.unlink(absolutePath);
+    } else {
+      unprocessedDirectories.push(absolutePath);
+    }
+  }
+
+  // delete empty folders in target folders as well
+  for (const directory of unprocessedDirectories) {
+    if ((await fs.promises.readdir(directory)).length === 0) {
+      await fs.promises.rmdir(directory);
+    }
+  }
+
   const endTime = process.hrtime.bigint();
   const executionTimeInMs: number = Number(endTime - startTime) / 1e6;
 

packages/pnpm-sync-lib/src/pnpmSyncPrepare.ts

@@ -33,7 +33,7 @@ export interface IPnpmSyncPrepareOptions {
    * Environment-provided API to avoid an NPM dependency.
    * The "pnpm-sync" NPM package provides a reference implementation.
    */
-  ensureFolder: (folderPath: string) => Promise<void>;
+  ensureFolderAsync: (folderPath: string) => Promise<void>;
 
   /**
    * Environment-provided API to avoid an NPM dependency.
@@ -63,7 +63,7 @@ export interface IPnpmSyncPrepareOptions {
  * @beta
  */
 export async function pnpmSyncPrepareAsync(options: IPnpmSyncPrepareOptions): Promise<void> {
-  const { ensureFolder, readPnpmLockfile, logMessageCallback } = options;
+  const { ensureFolderAsync, readPnpmLockfile, logMessageCallback } = options;
   let { lockfilePath, dotPnpmFolder } = options;
 
   // get the pnpm-lock.yaml path
@@ -186,7 +186,7 @@ export async function pnpmSyncPrepareAsync(options: IPnpmSyncPrepareOptions): Pr
     // it is possible that node_modules folder for a package is not exist yet
     // but we need to generate .pnpm-sync.json for that package
     if (!fs.existsSync(pnpmSyncJsonFolder)) {
-      await ensureFolder(pnpmSyncJsonFolder);
+      await ensureFolderAsync(pnpmSyncJsonFolder);
     }
 
     const expectedPnpmSyncJsonVersion: string = pnpmSyncGetJsonVersion();

packages/pnpm-sync-lib/src/utilities.ts

@@ -1,3 +1,7 @@
+import fs, { type Dirent } from 'fs';
+import path from 'path';
+import { ISyncItem } from './interfaces';
+
 /**
  * Get .pnpm-sync.json version
  *
@@ -6,3 +10,27 @@
 export function pnpmSyncGetJsonVersion(): string {
   return require('../package.json').version;
 }
+
+export async function getFilesInDirectory(directory: string): Promise<ISyncItem[]> {
+  const returnFileList: ISyncItem[] = [];
+  await getFilesInDirectoryHelper(directory, returnFileList);
+  return returnFileList;
+}
+
+async function getFilesInDirectoryHelper(directory: string, returnFileList: ISyncItem[]): Promise<void> {
+  const itemList: Array<Dirent> = await fs.promises.readdir(directory, { withFileTypes: true });
+
+  for (const item of itemList) {
+    const absolutePath: string = path.join(directory, item.name);
+    if (item.isDirectory()) {
+      await getFilesInDirectoryHelper(absolutePath, returnFileList);
+    }
+
+    // the list should include both files and directories
+    returnFileList.push({
+      absolutePath,
+      isDirectory: item.isDirectory(),
+      isFile: item.isFile()
+    });
+  }
+}

packages/pnpm-sync/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pnpm-sync",
-  "version": "0.2.2",
+  "version": "0.2.3",
   "description": "Recopy injected dependencies whenever a project is rebuilt in your PNPM workspace",
   "keywords": [
     "rush",

packages/pnpm-sync/src/start.ts

@@ -51,7 +51,7 @@ program
         pnpmSyncJsonPath: process.cwd() + '/node_modules/.pnpm-sync.json',
         getPackageIncludedFiles: PackageExtractor.getPackageIncludedFilesAsync,
         forEachAsyncWithConcurrency: Async.forEachAsync,
-        ensureFolder: FileSystem.ensureFolderAsync,
+        ensureFolderAsync: FileSystem.ensureFolderAsync,
         logMessageCallback: logMessage
       });
     } catch (error) {
@@ -74,7 +74,7 @@ program
       await pnpmSyncPrepareAsync({
         lockfilePath: lockfile,
         dotPnpmFolder: store,
-        ensureFolder: FileSystem.ensureFolderAsync,
+        ensureFolderAsync: FileSystem.ensureFolderAsync,
         readPnpmLockfile: async (
           lockfilePath: string,
           options: {