Improve large watcher events payload experience (#9583)

packages/core/core/src/Parcel.js

@@ -426,10 +426,9 @@ export default class Parcel {
         }
 
         let isInvalid = this.#requestTracker.respondToFSEvents(
-          events.map(e => ({
-            type: e.type,
-            path: toProjectPath(resolvedOptions.projectRoot, e.path),
-          })),
+          events,
+          resolvedOptions.projectRoot,
+          Number.POSITIVE_INFINITY,
         );
         if (isInvalid && this.#watchQueue.getNumWaiting() === 0) {
           if (this.#watchAbortController) {

packages/core/core/src/RequestTracker.js

@@ -2,7 +2,7 @@
 
 import type {AbortSignal} from 'abortcontroller-polyfill/dist/cjs-ponyfill';
 import type {Async, EnvMap} from '@parcel/types';
-import type {EventType, Options as WatcherOptions} from '@parcel/watcher';
+import type {Options as WatcherOptions, Event} from '@parcel/watcher';
 import type WorkerFarm from '@parcel/workers';
 import type {
   ContentGraphOpts,
@@ -750,23 +750,51 @@ export class RequestGraph extends ContentGraph<
   }
 
   respondToFSEvents(
-    events: Array<{|path: ProjectPath, type: EventType|}>,
+    events: Array<Event>,
+    projectRoot: string,
+    threshold: number,
   ): boolean {
     let didInvalidate = false;
-    for (let {path: _filePath, type} of events) {
+    let count = 0;
+    let predictedTime = 0;
+    let startTime = Date.now();
+
+    for (let {path: _path, type} of events) {
+      if (++count === 256) {
+        let duration = Date.now() - startTime;
+        predictedTime = duration * (events.length >> 8);
+        if (predictedTime > threshold) {
+          logger.warn({
+            origin: '@parcel/core',
+            message:
+              'Building with clean cache. Cache invalidation took too long.',
+            meta: {
+              trackableEvent: 'cache_invalidation_timeout',
+              watcherEventCount: events.length,
+              predictedTime,
+            },
+          });
+          throw new Error(
+            'Responding to file system events exceeded threshold, start with empty cache.',
+          );
+        }
+      }
+
+      let _filePath = toProjectPath(projectRoot, _path);
       let filePath = fromProjectPathRelative(_filePath);
       let hasFileRequest = this.hasContentKey(filePath);
 
       // If we see a 'create' event for the project root itself,
       // this means the project root was moved and we need to
       // re-run all requests.
       if (type === 'create' && filePath === '') {
-        // $FlowFixMe(incompatible-call) `trackableEvent` isn't part of the Diagnostic interface
         logger.verbose({
           origin: '@parcel/core',
           message:
             'Watcher reported project root create event. Invalidate all nodes.',
-          trackableEvent: 'project_root_create',
+          meta: {
+            trackableEvent: 'project_root_create',
+          },
         });
         for (let [id, node] of this.nodes.entries()) {
           if (node?.type === REQUEST) {
@@ -860,6 +888,17 @@ export class RequestGraph extends ContentGraph<
       }
     }
 
+    let duration = Date.now() - startTime;
+    logger.verbose({
+      origin: '@parcel/core',
+      message: `RequestGraph.respondToFSEvents duration: ${duration}`,
+      meta: {
+        trackableEvent: 'fsevent_response_time',
+        duration,
+        predictedTime,
+      },
+    });
+
     return didInvalidate && this.invalidNodeIds.size > 0;
   }
 
@@ -994,9 +1033,11 @@ export default class RequestTracker {
   }
 
   respondToFSEvents(
-    events: Array<{|path: ProjectPath, type: EventType|}>,
+    events: Array<Event>,
+    projectRoot: string,
+    threshold: number,
   ): boolean {
-    return this.graph.respondToFSEvents(events);
+    return this.graph.respondToFSEvents(events, projectRoot, threshold);
   }
 
   hasInvalidRequests(): boolean {
@@ -1363,24 +1404,48 @@ async function loadRequestGraph(options): Async<RequestGraph> {
     let opts = getWatcherOptions(options);
     let snapshotKey = `snapshot-${cacheKey}`;
     let snapshotPath = path.join(options.cacheDir, snapshotKey + '.txt');
+
+    let timeout = setTimeout(() => {
+      logger.warn({
+        origin: '@parcel/core',
+        message: `Retrieving file system events since last build...\nThis can take upto a minute after branch changes or npm/yarn installs.`,
+      });
+    }, 5000);
+    let startTime = Date.now();
     let events = await options.inputFS.getEventsSince(
       options.watchDir,
       snapshotPath,
       opts,
     );
+    clearTimeout(timeout);
+
+    logger.verbose({
+      origin: '@parcel/core',
+      message: `File system event count: ${events.length}`,
+      meta: {
+        trackableEvent: 'watcher_events_count',
+        watcherEventCount: events.length,
+        duration: Date.now() - startTime,
+      },
+    });
 
     requestGraph.invalidateUnpredictableNodes();
     requestGraph.invalidateOnBuildNodes();
     requestGraph.invalidateEnvNodes(options.env);
     requestGraph.invalidateOptionNodes(options);
-    requestGraph.respondToFSEvents(
-      (options.unstableFileInvalidations || events).map(e => ({
-        type: e.type,
-        path: toProjectPath(options.projectRoot, e.path),
-      })),
-    );
 
-    return requestGraph;
+    try {
+      requestGraph.respondToFSEvents(
+        options.unstableFileInvalidations || events,
+        options.projectRoot,
+        10000,
+      );
+      return requestGraph;
+    } catch (e) {
+      // This error means respondToFSEvents timed out handling the invalidation events
+      // In this case we'll return a fresh RequestGraph
+      return new RequestGraph();
+    }
   }
 
   return new RequestGraph();

packages/core/core/src/types.js

@@ -31,7 +31,7 @@ import type {FileSystem} from '@parcel/fs';
 import type {Cache} from '@parcel/cache';
 import type {PackageManager} from '@parcel/package-manager';
 import type {ProjectPath} from './projectPath';
-import type {EventType} from '@parcel/watcher';
+import type {Event} from '@parcel/watcher';
 import type {FeatureFlags} from '@parcel/feature-flags';
 import type {BackendType} from '@parcel/watcher';
 
@@ -295,10 +295,7 @@ export type ParcelOptions = {|
   shouldTrace: boolean,
   shouldPatchConsole: boolean,
   detailedReport?: ?DetailedReportOptions,
-  unstableFileInvalidations?: Array<{|
-    path: FilePath,
-    type: EventType,
-  |}>,
+  unstableFileInvalidations?: Array<Event>,
 
   inputFS: FileSystem,
   outputFS: FileSystem,

packages/core/diagnostic/src/diagnostic.js

@@ -46,6 +46,18 @@ export type DiagnosticCodeFrame = {|
   codeHighlights: Array<DiagnosticCodeHighlight>,
 |};
 
+type JSONValue =
+  | null
+  | void // ? Is this okay?
+  | boolean
+  | number
+  | string
+  | Array<JSONValue>
+  | JSONObject;
+
+/** A JSON object (as in "map") */
+type JSONObject = {[key: string]: JSONValue, ...};
+
 /**
  * A style agnostic way of emitting errors, warnings and info.
  * Reporters are responsible for rendering the message, codeframes, hints, ...
@@ -72,6 +84,9 @@ export type Diagnostic = {|
 
   /** A URL to documentation to learn more about the diagnostic. */
   documentationURL?: string,
+
+  /** Diagnostic specific metadata (optional) */
+  meta?: JSONObject,
 |};
 
 // This type should represent all error formats Parcel can encounter...

packages/core/types/index.js

@@ -15,7 +15,7 @@ import type {Cache} from '@parcel/cache';
 import type {AST as _AST, ConfigResult as _ConfigResult} from './unsafe';
 import type {TraceMeasurement} from '@parcel/profiler';
 import type {FeatureFlags} from '@parcel/feature-flags';
-import type {EventType, BackendType} from '@parcel/watcher';
+import type {Event, BackendType} from '@parcel/watcher';
 
 /** Plugin-specific AST, <code>any</code> */
 export type AST = _AST;
@@ -313,7 +313,7 @@ export type InitialParcelOptions = {|
   +lazyIncludes?: string[],
   +lazyExcludes?: string[],
   +shouldBundleIncrementally?: boolean,
-  +unstableFileInvalidations?: Array<{|path: FilePath, type: EventType|}>,
+  +unstableFileInvalidations?: Array<Event>,
 
   +inputFS?: FileSystem,
   +outputFS?: FileSystem,