fix: get npm v6 content from tarball

.github/workflows/ci-cli.yml

@@ -100,5 +100,3 @@ jobs:
         run: echo "::add-matcher::.github/matchers/tap.json"
       - name: Test
         run: npm test --ignore-scripts -w cli
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CONTRIBUTING.md

@@ -179,11 +179,6 @@ adding a new major version to the site.
      fetch the latest version of the documentation from GitHub.
    * `spec`: The registry spec for the version. This will be used
      to fetch the latest version in that range from the registry.
-   * `useBranch`: A boolean that controls whether the content for this
-     version will be fetched from GitHub. The default is false, which
-     means the content will be fetched directly from the registry tarball.
-     It is preferred to use the registry but for some legacy versions,
-     the content was only updated on GitHub and never published.
    * `resolved`: This should not be edited manually. This is a reference
      to the last fetched version of the content for this release. If
      a future fetch is done and this field matches what is returned

cli/lib/extract.js

@@ -43,40 +43,6 @@ const unpackTarball = async ({ release, cwd, dir }) => {
   return result
 }
 
-const unpackTree = async ({ release, cwd, dir }) => {
-  const dirParts = dir.split(sep)
-  const child = dirParts.pop()
-  const parent = join(...dirParts)
-
-  // to get the sha of the dir, we have to get the parent
-  // and find the child as an entry and get its sha
-  const sha = await gh.getDirectory(release.branch, parent)
-    .then(paths => paths.find((p) => p.name === child).sha)
-
-  const files = await gh.getAllFiles(sha)
-
-  // tar makes the directories for us when unpacking but we
-  // need to to that manually here
-  const dirs = [...new Set(files.map((f) => join(cwd, dirname(f.path))))]
-  await Promise.all(dirs.map((d) => fs.mkdir(d, { recursive: true })))
-
-  await Promise.all(
-    files.map(async (file) => {
-      const buffer = await gh.getFile({ sha: file.sha })
-      return fs.writeFile(
-        join(cwd, file.path),
-        Transform.sync(buffer, {
-          path: file.path,
-          release,
-        }),
-        'utf-8'
-      )
-    })
-  )
-
-  return files.map((f) => f.path)
-}
-
 const getNav = async ({ path, release }) => {
   const nav = await gh.getFile({ ref: release.branch, path })
 
@@ -157,11 +123,7 @@ const unpackRelease = async (
   // the tree of the doc directory's sha which has all the docs
   // we need in it. Note that this requires the docs to all be
   // built in source, which is true for v6 but not for v9 and later.
-  const files = release.useBranch ? await unpackTree({
-    release,
-    cwd,
-    dir: release.src,
-  }) : await unpackTarball({
+  const files = await unpackTarball({
     release,
     cwd,
     dir: builtPath,

cli/lib/gh.js

@@ -20,34 +20,6 @@ const getFile = async ({ sha, ref, path }) => {
   return Buffer.from(data.content, data.encoding)
 }
 
-const getAllFiles = async (sha) => {
-  const {
-    data: { tree },
-  } = await octokit.git.getTree({
-    ...opts,
-    tree_sha: sha,
-    recursive: true,
-  })
-
-  return tree
-    .filter((f) => f.type === 'blob')
-    .map((f) => ({
-      ...f,
-      // return file paths that can be used on the
-      // system to write files
-      path: f.path.split(posix.sep).join(sep),
-    }))
-}
-
-const getDirectory = async (ref, dir) => {
-  const { data } = await octokit.repos.getContent({
-    ...opts,
-    ref,
-    path: dir.split(sep).join(posix.sep),
-  })
-  return data
-}
-
 const pathExists = async (ref, path) => {
   try {
     await octokit.repos.getContent({
@@ -66,12 +38,7 @@ const pathExists = async (ref, path) => {
 }
 
 module.exports = {
-  octokit,
   getFile,
-  getAllFiles,
-  getDirectory,
   pathExists,
-  owner,
-  repo,
   nwo: `${owner}/${repo}`,
 }

cli/releases.json

@@ -1,8 +1,7 @@
 [
   {
     "id": "v6",
-    "branch": "release/v6",
-    "useBranch": true
+    "branch": "release/v6"
   },
   {
     "id": "v7",

cli/test/index.js

@@ -1,7 +1,9 @@
 const t = require('tap')
-const { resolve, join } = require('path')
+const { resolve, join, posix } = require('path')
 const fs = require('fs/promises')
 const pacote = require('pacote')
+const yaml = require('yaml')
+const semver = require('semver')
 
 const navPath = resolve(
   __dirname,
@@ -16,7 +18,6 @@ const getReleases = () => [
   {
     id: 'v6',
     branch: 'release/v6',
-    useBranch: true,
   },
   {
     id: 'v7',
@@ -33,9 +34,12 @@ const getReleases = () => [
 ]
 
 const mockBuild = async ({ releases, packument = {}, testdir: testdirOpts }) => {
+  const rawNav = await fs.readFile(navPath, 'utf-8')
+  const nav = yaml.parse(rawNav)
+
   const testdir = t.testdir({
     'releases.json': JSON.stringify(releases),
-    'nav.yml': await fs.readFile(navPath, 'utf-8'),
+    'nav.yml': rawNav,
     content: {},
     ...testdirOpts,
   })
@@ -46,7 +50,7 @@ const mockBuild = async ({ releases, packument = {}, testdir: testdirOpts }) =>
       // so by default they all need to exist
       switch (r.id.slice(1)) {
         case '6':
-          return '6.14.17'
+          return '6.14.18'
         case '7':
           return '7.24.2'
         case '8':
@@ -61,6 +65,13 @@ const mockBuild = async ({ releases, packument = {}, testdir: testdirOpts }) =>
     packument.latest = packument.versions[packument.versions.length - 1]
   }
 
+  const navSection = (ref) => {
+    const id = ref === 'latest' ? `v${semver.major(packument.latest)}` : posix.basename(ref)
+    const { variants } = nav.find(c => c.url === '/cli')
+    const { children } = variants.find(v => posix.basename(v.url) === id)
+    return yaml.stringify(children).replace(new RegExp(`/cli/${id}/`, 'g'), '/')
+  }
+
   const build = t.mock('../lib/build', {
     pacote: {
       ...pacote,
@@ -76,6 +87,16 @@ const mockBuild = async ({ releases, packument = {}, testdir: testdirOpts }) =>
         }
       },
     },
+    '../lib/gh.js': {
+      getFile: async ({ ref }) => navSection(ref),
+      pathExists: async (ref, p) => {
+        if (ref.includes('v6') && p.includes('docs/lib/content')) {
+          return null
+        }
+        return p
+      },
+      nwo: `npm/cli`,
+    },
   })
 
   return {
@@ -116,7 +137,7 @@ t.test('prereleases', async (t) => {
   const releases = getReleases()
   const { build, testdir } = await mockBuild({
     releases,
-    packument: { versions: ['6.14.17', '7.24.2', '8.19.3', '9.0.0-pre.2'], latest: '8.19.3' },
+    packument: { versions: ['6.14.18', '7.24.2', '8.19.3', '9.0.0-pre.2'], latest: '8.19.3' },
   })
 
   await build({ prerelease: false })

cli/test/transform.js

@@ -1,8 +1,10 @@
 const t = require('tap')
 const fm = require('front-matter')
-const Transform = require('../lib/transform')
 
 const transform = ({ id, path }) => {
+  const Transform = t.mock('../lib/transform', {
+    '../lib/gh.js': { nwo: 'npm/cli' },
+  })
   const transformed = Transform.sync('---\n---\n', {
     release: {
       id: id,