From 614612d1a8c8e07107f6079f22310c8bfe165b87 Mon Sep 17 00:00:00 2001 From: Stefan Buck Date: Thu, 22 Sep 2022 23:44:33 +0200 Subject: [PATCH] feat: Add glob support (#2) --- README.md | 46 ++++++++++++++-- helper.js | 63 +++++++++++++++++++++ package-lock.json | 138 ++++++++++++++++++++++++++++++---------------- package.json | 4 +- script.js | 53 +++++++----------- 5 files changed, 214 insertions(+), 90 deletions(-) create mode 100644 helper.js diff --git a/README.md b/README.md index ede03f3..0f92405 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # octoherd-script-get-files -> An Octoherd script to download files from repositories +> The easiest way to download files from GitHub. [![@latest](https://img.shields.io/npm/v/octoherd-script-get-files.svg)](https://www.npmjs.com/package/octoherd-script-get-files) [![Build Status](https://github.com/stefanbuck/octoherd-script-get-files/workflows/Test/badge.svg)](https://github.com/stefanbuck/octoherd-script-get-files/actions?query=workflow%3ATest+branch%3Amain) @@ -12,7 +12,7 @@ Minimal usage ```js npx octoherd-script-get-files \ --source README.md \ - --target ./out + --output ./out ``` Pass all options as CLI flags to avoid user prompts @@ -20,22 +20,56 @@ Pass all options as CLI flags to avoid user prompts ```js npx octoherd-script-get-files \ -T ghp_0123456789abcdefghjklmnopqrstuvwxyzA \ - -R "stefanbuck/*" \ + -R "octolinker/*" \ --source README.md \ - --target ./out + --output ./out ``` ## Options | option | type | description | | --------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--source` | string | **Required.** Path to the destination directory | -| `--target` | string | **Required.** File path to download. Note: Directories are not supported yet | +| `--source` | string | **Required.** File to download. This can also be a Glob see [example](#examples). | +| `--output` | string | **Required.** Specify a path to place the downloaded file or directory (instead of using the current working directory). Directories specified in the path will be created by this command. | | `--ignore-archived` or `--no-ignore-archived` | boolean | Default `true`. Ignores archive repositories | | `--octoherd-token`, `-T` | string | A personal access token ([create](https://github.com/settings/tokens/new?scopes=repo)). Script will create one if option is not set | | `--octoherd-repos`, `-R` | array of strings | One or multiple space-separated repositories in the form of `repo-owner/repo-name`. `repo-owner/*` will find all repositories for one owner. `*` will find all repositories the user has access to. Will prompt for repositories if not set | | `--octoherd-bypass-confirms` | boolean | Bypass prompts to confirm mutating requests | +## Examples + +Download a single file + +```js +npx octoherd-script-get-files -R octolinker/octolinker --source=README.md --output=./out +``` + +Download a single file by full path + +```js +npx octoherd-script-get-files -R octolinker/octolinker --source=.github/PULL_REQUEST_TEMPLATE.md --output=./out +``` + +Download recursively all files with a certain file extension + +```js +npx octoherd-script-get-files -R octolinker/octolinker --source='**/*.html' --output=./out +``` + +Download recursively all files from a specific folder + +```js +npx octoherd-script-get-files -R octolinker/octolinker --source='.github/**/*' --output=./out +``` + +Download everything + +```js +npx octoherd-script-get-files -R octolinker/octolinker --source='**/*' --output=./out +``` + +Don't know how to write Glob? Check out DigitalOcean's amazing [Glob testing tool](https://www.digitalocean.com/community/tools/glob). + ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md) diff --git a/helper.js b/helper.js new file mode 100644 index 0000000..b48c7a8 --- /dev/null +++ b/helper.js @@ -0,0 +1,63 @@ +import os from 'os' +import fs from 'fs' +import path from 'path' +import minimatch from "minimatch" + +export async function downloadFile(octokit, repository, target, source) { + const filepath = [repository.owner.login, repository.name, source].join('/'); + + return octokit.request( + `GET /repos/{owner}/{repo}/contents/${source}`, + { + owner: repository.owner.login, + repo: repository.name, + headers: { + Accept: "application/vnd.github.v3.raw" + } + } + ).then((res) => { + octokit.log.info(`Download ${filepath}`); + + // Expand the ~ character to a users home directory + const newTarget = target.replace("~", os.homedir) + + const targetFile = path.join(newTarget, repository.owner.login, repository.name, source); + const targetPath = path.join(newTarget, repository.owner.login, repository.name, path.dirname(source)); + + if (!fs.existsSync(targetPath)) { + fs.mkdirSync(targetPath, { recursive: true }); + } + + fs.writeFileSync(targetFile, res.data) + }).catch(error => { + if (error.status === 404) { + octokit.log.warn(`File ${filepath} not found`); + return false; + } + + throw error; + }) +} + +export async function getListOfFilesToDownload(octokit, repository, source) { + const res = await octokit.request( + `GET /repos/{owner}/{repo}/git/trees/HEAD?recursive=true`, + { + owner: repository.owner.login, + repo: repository.name, + headers: { + Accept: "application/vnd.github.v3.raw" + } + }).catch(error => { + if (error.status === 409) { + octokit.log.warn(`Git Repository is empty`); + return { data: { tree: [] } }; + } + + throw error; + }) + + const tree = res?.data?.tree?.filter(item => item.type === 'blob').map(item => item.path); + + return tree.filter(item => minimatch(item, source)) +} diff --git a/package-lock.json b/package-lock.json index 16e6fa4..7741c9c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,12 +9,13 @@ "version": "0.0.0-development", "license": "ISC", "dependencies": { - "@octoherd/cli": "^3.4.11" + "@octoherd/cli": "^3.4.11", + "is-glob": "^4.0.3", + "minimatch": "^5.1.0" }, "bin": { "octoherd-script-get-files": "cli.js" - }, - "devDependencies": {} + } }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", @@ -352,12 +353,11 @@ "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "balanced-match": "^1.0.0" } }, "node_modules/braces": { @@ -501,6 +501,25 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/del/node_modules/globby": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "dependencies": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/deprecation": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", @@ -651,23 +670,24 @@ "node": ">= 6" } }, - "node_modules/globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "node_modules/glob/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "dependencies": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "^1.1.7" }, "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": "*" } }, "node_modules/graceful-fs": { @@ -858,14 +878,14 @@ } }, "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.0.tgz", + "integrity": "sha512-9TPBGGak4nHfGZsPBohm9AWg6NoT7QTCehS3BIJABslyZbzxfV78QM2Y6+i741OPZIafFAaiiEMh5OyIrJPgtg==", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^2.0.1" }, "engines": { - "node": "*" + "node": ">=10" } }, "node_modules/mkdirp": { @@ -1548,12 +1568,11 @@ "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" }, "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "balanced-match": "^1.0.0" } }, "braces": { @@ -1655,6 +1674,21 @@ "p-map": "^4.0.0", "rimraf": "^3.0.2", "slash": "^3.0.0" + }, + "dependencies": { + "globby": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "requires": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + } + } } }, "deprecation": { @@ -1758,6 +1792,25 @@ "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" + }, + "dependencies": { + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "requires": { + "brace-expansion": "^1.1.7" + } + } } }, "glob-parent": { @@ -1768,19 +1821,6 @@ "is-glob": "^4.0.1" } }, - "globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "requires": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - } - }, "graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -1910,11 +1950,11 @@ "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==" }, "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.0.tgz", + "integrity": "sha512-9TPBGGak4nHfGZsPBohm9AWg6NoT7QTCehS3BIJABslyZbzxfV78QM2Y6+i741OPZIafFAaiiEMh5OyIrJPgtg==", "requires": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^2.0.1" } }, "mkdirp": { diff --git a/package.json b/package.json index 989289d..64fcb0a 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,9 @@ "author": "Stefan Buck (stefanbuck.com)", "license": "ISC", "dependencies": { - "@octoherd/cli": "^3.4.11" + "@octoherd/cli": "^3.4.11", + "is-glob": "^4.0.3", + "minimatch": "^5.1.0" }, "release": { "branches": [ diff --git a/script.js b/script.js index 11d82fe..4d51397 100644 --- a/script.js +++ b/script.js @@ -1,8 +1,7 @@ // @ts-check -import os from 'os' -import fs from 'fs' -import path from 'path' +import isGlob from 'is-glob'; +import { getListOfFilesToDownload, downloadFile } from './helper.js'; /** @type boolean */ let hasRepoScope; @@ -15,9 +14,9 @@ let hasRepoScope; * @param {object} options * @param {boolean} [options.ignoreArchived] Ignores archive repositories * @param {string} options.source Path to the destination directory - * @param {string} options.target File path to download. Note: Directories are not supported yet + * @param {string} options.output File path to download. Note: Directories are not supported yet */ -export async function script(octokit, repository, { source, target = process.cwd(), ignoreArchived = true }) { +export async function script(octokit, repository, { source, output = process.cwd(), ignoreArchived = true }) { if (!hasRepoScope) { const { headers } = await octokit.request("HEAD /"); const scopes = new Set(headers["x-oauth-scopes"].split(", ")); @@ -40,40 +39,26 @@ export async function script(octokit, repository, { source, target = process.cwd process.exit(1); } - if (!target) { - octokit.log.error('Please specify a source file to download with --source=README.md --target=./out') + if (!output) { + octokit.log.error('Please specify a source file to download with --source=README.md --output=./out') process.exit(1); } - await octokit.request( - `GET /repos/{owner}/{repo}/contents/${source}`, - { - owner: repository.owner.login, - repo: repository.name, - headers: { - Accept: "application/vnd.github.v3.raw" - } - } - ).then((res) => { - octokit.log.info(`Download ${source}`); - - // Expand the ~ character to a users home directory - const newTarget = target.replace("~", os.homedir) + let filesToDownload = [source] - const targetFile = path.join(newTarget, repository.owner.login, repository.name, source); - const targetPath = path.join(newTarget, repository.owner.login, repository.name, path.dirname(source)); + if (isGlob(source)) { + filesToDownload = await getListOfFilesToDownload(octokit, repository, source); + } - if (!fs.existsSync(targetPath)) { - fs.mkdirSync(targetPath, { recursive: true }); - } + if (filesToDownload.length > 1) { + octokit.log.debug(`Start downloading ${filesToDownload.length} files`); + } - fs.writeFileSync(targetFile, res.data) - }).catch(error => { - if (error.status === 404) { - octokit.log.warn(`File ${source} not found`); - return false; - } + if (filesToDownload.length === 0) { + octokit.log.warn(`No matches found for ${source}`); + } - throw error; - }) + for (const item of filesToDownload) { + await downloadFile(octokit, repository, output, item) + } }