feat(gatsby-dev-cli): add verdaccio support (gatsbyjs#11525)

* feat(gatsby-dev-cli): add verdaccio support [wip]

* yuck - don't look

* just to verify verdaccio

* fix gitignore and add htpasswd with dummy gatsby-dev user

* some logging to understand why it tries to handle all the packages

* just temporarily so it hopefully at least works

* moar data

* don't copy all the packages right now

* don't rely on package name starting with "gatsby" heuristic

* tmp commit

* tmp

* fs change

* tmp

* a

* updates

* remove random files that shouldn't be here

* tmp

* revert dep changes

* ignore changes to package.json created during publishing

* testing of dep change in gatsby-cli is picked up by gatsby-dev-cli

* let's see if gatsby-dev will handle case of no node_modules

* sanity check

* adjust console message

* add missing devDeps

* more missing devDeps

* grab package.json from unpkg if package is not yet installed locally

* correctly ignore package.json changes during publishing

* cleaning things up

* htpasswd loc

* restore gatsby package.json

* adjust ignore files

* gatsby-dev-cli: ignore tests

* remove extra ln

* less console output

* turns out you don't need htpasswd after all

* use exact when installing local packages  - timestamp is used in version

* why it failed?

* use yarn for publishing - see verdaccio/verdaccio#997

* v0.1.1

* add temporary .npmrc file to allow anonymous publishes

* add some jsdocs

* update yarn.lock

* oops

* debug why still can't publish anon

* adjust npmrc content

* doh

* Revert "debug why still can't publish anon"

This reverts commit fc1045a.

* handle package installation of deps didn't change

* skip version changes for packages in gatsby mono-repo, queue copying instead of doing it immediately

* fix tests

* packages stored in verdaccio storage seems to be picked up by jest and causing trouble, so let's ignore verdaccio storage directory

* bye "wat", you will be missed

* bail early to avoid conditional bracket hell

* more bailing early

* more concise

* new line

* use exaca to simplify running yarn

* clean up www/.gitignore

* handle not existing packages

* remove unused (was needed for basic auth)

* cleanup temp file changes when process exit in middle of publishing

* ops

* Apply suggestions from code review

Co-Authored-By: pieh <[email protected]>

* cleanup unpkg response handler

* move stuff around

* remove tmp comment

* document traversePackageDocs fn

* restrcuture files

* add basic tests for traversingPackageDeps

* add basic tests for generating packages to publish (based on dep graph)

* fix tests on windows

* handle circular dependencies when constructing list of packages to publish

* remove cat ;(

* restore original e2e-test script

* update gitignore

Author: pieh

jest.config.js

@@ -11,7 +11,12 @@ const builtTestsDirs = pkgs
   .filter(p => fs.existsSync(path.join(p, `src`)))
   .map(p => path.join(p, `__tests__`))
 const distDirs = pkgs.map(p => path.join(p, `dist`))
-const ignoreDirs = [].concat(gatsbyBuildDirs, builtTestsDirs, distDirs)
+const ignoreDirs = [`<rootDir>/packages/gatsby-dev-cli/verdaccio`].concat(
+  gatsbyBuildDirs,
+  builtTestsDirs,
+  distDirs
+)
+
 const coverageDirs = pkgs.map(p => path.join(p, `src/**/*.js`))
 const useCoverage = !!process.env.GENERATE_JEST_REPORT
 

packages/babel-plugin-remove-graphql-queries/package.json

@@ -5,7 +5,8 @@
   "devDependencies": {
     "@babel/cli": "^7.0.0",
     "@babel/core": "^7.0.0",
-    "babel-preset-gatsby-package": "^0.1.4"
+    "babel-preset-gatsby-package": "^0.1.4",
+    "cross-env": "^5.1.4"
   },
   "license": "MIT",
   "main": "index.js",

packages/babel-preset-gatsby/package.json

@@ -21,6 +21,8 @@
     "watch": "babel -w src --out-dir . --ignore **/__tests__"
   },
   "devDependencies": {
-    "babel-preset-gatsby-package": "^0.1.4"
+    "@babel/cli": "^7.0.0",
+    "babel-preset-gatsby-package": "^0.1.4",
+    "cross-env": "^5.1.4"
   }
 }

packages/gatsby-dev-cli/.gitignore

@@ -28,3 +28,6 @@ node_modules
 
 decls
 dist
+
+# verdaccio local storage
+verdaccio

packages/gatsby-dev-cli/.npmignore

@@ -1,2 +1,3 @@
 src
 flow-typed
+verdaccio

packages/gatsby-dev-cli/package.json

@@ -13,9 +13,13 @@
     "@babel/runtime": "^7.0.0",
     "chokidar": "^1.7.0",
     "configstore": "^3.1.0",
+    "execa": "^1.0.0",
     "fs-extra": "^4.0.1",
     "is-absolute": "^0.2.6",
     "lodash": "^4.17.10",
+    "request": "2.88.0",
+    "signal-exit": "^3.0.2",
+    "verdaccio": "^3.11.1",
     "yargs": "^8.0.2"
   },
   "devDependencies": {
@@ -32,9 +36,9 @@
   "main": "index.js",
   "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-dev-cli",
   "scripts": {
-    "build": "babel src --out-dir dist",
+    "build": "babel src --out-dir dist --ignore **/__tests__",
     "prepare": "cross-env NODE_ENV=production npm run build",
     "test": "echo \"Error: no test specified\" && exit 1",
-    "watch": "babel -w src --out-dir dist"
+    "watch": "babel -w src --out-dir dist --ignore **/__tests__"
   }
 }

packages/gatsby-dev-cli/src/__tests__/watch.js

@@ -59,10 +59,21 @@ describe(`watching`, () => {
       expect(fs.copy).not.toHaveBeenCalled()
     })
 
-    it(`copies files on watch event`, () => {
+    it(`it doesn't copy files before ready event`, async () => {
       const filePath = path.join(process.cwd(), `packages/gatsby/dist/index.js`)
       watch(...args)
-      callEventCallback(`add`, filePath)
+      await callEventCallback(`add`, filePath)
+
+      expect(fs.copy).toHaveBeenCalledTimes(0)
+    })
+
+    it(`copies files after ready event`, async () => {
+      const filePath = path.join(process.cwd(), `packages/gatsby/dist/index.js`)
+      watch(...args)
+      await callEventCallback(`add`, filePath)
+      await callReadyCallback()
+
+      // console.log(`checking`)
 
       expect(fs.copy).toHaveBeenCalledTimes(1)
       expect(fs.copy).toHaveBeenCalledWith(
@@ -72,14 +83,15 @@ describe(`watching`, () => {
       )
     })
 
-    it(`copies cache-dir files`, () => {
+    it(`copies cache-dir files`, async () => {
       watch(...args)
 
       const filePath = path.join(
         process.cwd(),
         `packages/gatsby/cache-dir/register-service-worker.js`
       )
-      callEventCallback(`add`, filePath)
+      await callEventCallback(`add`, filePath)
+      await callReadyCallback()
 
       expect(fs.copy).toHaveBeenCalledTimes(2)
       expect(fs.copy).toHaveBeenLastCalledWith(
@@ -122,9 +134,9 @@ describe(`watching`, () => {
       global.process = realProcess
     })
 
-    it(`does not exit if scanOnce is not defined`, () => {
+    it(`does not exit if scanOnce is not defined`, async () => {
       watch(...args)
-      callReadyCallback()
+      await callReadyCallback()
 
       expect(process.exit).not.toHaveBeenCalled()
     })

packages/gatsby-dev-cli/src/index.js

@@ -73,15 +73,14 @@ let packages = Object.keys(
   _.merge({}, localPkg.dependencies, localPkg.devDependencies)
 )
 
+// get list of packages from monorepo
+const monoRepoPackages = fs.readdirSync(path.join(gatsbyLocation, `packages`))
+
 if (argv.copyAll) {
-  packages = fs.readdirSync(path.join(gatsbyLocation, `packages`))
+  packages = monoRepoPackages
 } else {
-  const { dependencies } = JSON.parse(
-    fs.readFileSync(path.join(gatsbyLocation, `packages/gatsby/package.json`))
-  )
-  packages = packages
-    .concat(Object.keys(dependencies))
-    .filter(p => p.startsWith(`gatsby`))
+  // intersect dependencies with monoRepoPackags to get list of packages to watch
+  packages = _.intersection(monoRepoPackages, packages)
 }
 
 if (!argv.packages && _.isEmpty(packages)) {
@@ -104,4 +103,5 @@ gatsby-dev will pick them up.
 watch(gatsbyLocation, argv.packages || packages, {
   quiet: argv.quiet,
   scanOnce: argv.scanOnce,
+  monoRepoPackages,
 })

packages/gatsby-dev-cli/src/local-npm-registry/cleanup-tasks.js

@@ -0,0 +1,19 @@
+const signalExit = require(`signal-exit`)
+
+const cleanupTasks = new Set()
+
+exports.registerCleanupTask = taskFn => {
+  cleanupTasks.add(taskFn)
+  return () => {
+    const result = taskFn()
+    cleanupTasks.delete(taskFn)
+    return result
+  }
+}
+
+signalExit(() => {
+  if (cleanupTasks.size) {
+    console.log(`Process exitted in middle of publishing - cleaning up`)
+    cleanupTasks.forEach(taskFn => taskFn())
+  }
+})