From 7818c30d6ec5fdb8a0d0133148d07144ed7b4ef6 Mon Sep 17 00:00:00 2001 From: Avi Press Date: Wed, 17 Feb 2021 12:13:50 -0800 Subject: [PATCH] Backport all fixes in main branch to 0.1.x (#33) * version bump 0.2.0 * Version bump 1.0.0 * Add timeout to exec(npm ls) call (#15) * Update package.json * version bump 1.0.2 * Readme update * Request timeout (#16) * Request timeout * version bump 1.0.3 * Intermediate package opt out (#19) * version bump 1.0.4 * Relicense as Apache-2.0 (#20) * Update License file to full Apache-2.0 text * version bump 1.0.5 * Update link * Disable scarf-js analytics when yarn is the installing package manager (#21) * Hash package names and versions of grandparent and root (#23) * version bump 1.0.6 * Update README download badge link * Readme updates * Set maxBuffer for Dependency tree output (#25) Tried to use scarf for my project botium-bindings. I am receiving this error message on _npm install_: SyntaxError: Unexpected end of JSON input at JSON.parse () at /home/ftreml/dev/botium-bindings/node_modules/@scarf/scarf/report.js:122:27 at ChildProcess.exithandler (child_process.js:301:5) at ChildProcess.emit (events.js:198:13) at maybeClose (internal/child_process.js:982:16) at Process.ChildProcess._handle.onexit (internal/child_process.js:259:5) Seems that the stdout stream for _npm -ls_ (dependency tree) is truncated. Possible fix: add _maxBuffer_ to child process options. * update to link to new blogpost * Bump lodash from 4.17.15 to 4.17.19 (#28) Bumps [lodash](https://github.com/lodash/lodash) from 4.17.15 to 4.17.19. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.15...4.17.19) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Enable analytics when target package has no parent. (#27) * version bump 1.1.0 * Added nix files * Update README.md * add nix to linguist ignore * Used like for missing verb. (#30) * License copyright notice * delete trailing whitespace * bump version of 0.1.x branch to 0.1.8 * fixup, duplicate const Co-authored-by: Botium Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Tim Dysinger Co-authored-by: Tim Dysinger Co-authored-by: Phil de Joux --- .gitattributes | 1 + .gitignore | 5 +- LICENSE | 222 +- README.md | 71 +- WHY.org | 9 +- nix/default.nix | 17 + nix/generate.sh | 7 + nix/node-env.nix | 542 +++ nix/node-packages.nix | 6265 +++++++++++++++++++++++++ package-lock.json | 30 +- package.json | 6 +- report.js | 205 +- test/example-ls-output.json | 108 + test/report.test.js | 115 +- test/top-level-package-ls-output.json | 115 + 15 files changed, 7591 insertions(+), 127 deletions(-) create mode 100644 .gitattributes create mode 100644 nix/default.nix create mode 100755 nix/generate.sh create mode 100644 nix/node-env.nix create mode 100644 nix/node-packages.nix create mode 100644 test/example-ls-output.json create mode 100644 test/top-level-package-ls-output.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..789b161 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.nix linguist-detectable=false diff --git a/.gitignore b/.gitignore index b512c09..116ad5d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ -node_modules \ No newline at end of file +.envrc +/*.nix +/result* +node_modules diff --git a/LICENSE b/LICENSE index 2471d97..584253b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,201 @@ -MIT License - -Copyright (c) 2019 Scarf Systems, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Scarf Systems, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index e57d478..e364b3c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ ![](https://github.com/scarf-sh/scarf-js/workflows/CI/badge.svg) [![npm version](https://badge.fury.io/js/%40scarf%2Fscarf.svg)](https://badge.fury.io/js/%40scarf%2Fscarf) -![](https://img.shields.io/npm/dw/@scarf/scarf) +![](https://img.shields.io/npm/dw/@scarf/scarf) + Scarf is like Google Analytics for your npm packages. By sending some basic details after installation, this package can help you can gain insights into how @@ -36,7 +37,7 @@ Head to your package's dashboard on Scarf to see your reports when available. #### Configuration Users of your package will be opted in by default and can opt out by setting the -`SCARF_ANALYTICS=false` environment variable. If you'd Scarf analytics to +`SCARF_ANALYTICS=false` environment variable. If you'd like Scarf analytics to instead be opt-in, you can set this by adding an entry to your `package.json` @@ -58,32 +59,53 @@ to opt in. Regardless of the default state, Scarf will log what it is doing to users who haven't explictly opted in or out. -### What information does Scarf provide me as a package author? +By default, scarf-js will only trigger analytics when your package is installed as a dependency of another package, or is being installed globally. This ensures that scarf-js analytics will not be triggered on `npm install` being run _within your project_. To change this, you can add: + +```json5 +// your-package/package.json + +{ + // ... + "scarfSettings": { + "allowTopLevel": true + } + // ... +} +``` + +### FAQ + +#### What information does scarf-js provide me as a package author? - Understanding your user-base - Which companies are using your package? - Is your project growing or shrinking? Where? On which platforms? - Which versions of your package are being used? -### As a user of a package using Scarf, what information does Scarf send about me? +#### As a user of a package using scarf-js, what information does scarf-js send about me? + +*Scarf does not store personally identifying information.* Scarf aims to collect information that is helpful for: +- Package maintainence +- Identifying which companies are using a particular package, in order to set up support agreements between developers and companies. + +Specifically, scarf-js sends: - The operating system you are using -- Your IP address will be used to look up any available company information. The - IP address itself will be subsequently deleted. -- Limited dependency tree information. Scarf sends the package name and version for - certain packages (provided they are not scoped packages, `@org/package-name`, - which are assumed to be private): - - Packages in the dependency tree that directly depend on - Scarf. - - Packages that depend on a package that depends on Scarf. +- Your IP address will be used to look up any available company information. _Scarf does not store the actual IP address_ +- Limited dependency tree information. Scarf sends the name and version of the package(s) that directly depend on scarf-js. Additionally, scarf-js will send SHA256-hashed name and version for the following packages in the dependency tree: + - Packages that depend on a package that depends on scarf-js. - The root package of the dependency tree. - -### As a user of a package using Scarf, how can I opt out of analytics? +This allows Scarf to provide maintainers information about which public packages are using their own, without exposing identifying details of non-public packages. + +You can have scarf-js print the exact JSON payload it sends by settings `SCARF_VERBOSE=true` in your environment. + +#### As a user of a package using scarf-js, how can I opt out of analytics? Scarf's analytics help support developers of the open source packages you are using, so enabling analytics is appreciated. However, if you'd like to opt out, you can add your preference to your project's `package.json`: + ```json5 // your-package/package.json @@ -104,10 +126,29 @@ export SCARF_ANALYTICS=false Either route will disable Scarf for all packages. +#### I distribute a package on npm, and scarf-js is in our dependency tree. Can I disable the analytics for my downstream dependents? + +Yes. By opting out of analytics via `package.json`, any package upstream will have analytics disbabled. + +```json5 +// your-package/package.json + +{ + // ... + "scarfSettings": { + "enabled": false + } + // ... +} +``` + +Installers of your packages will have scarf-js disabled for all dependencies upstream from yours. + + ### Developing Setting the environment variable `SCARF_LOCAL_PORT=8080` will configure Scarf to -use http://localhost:${SCARF_LOCAL_PORT} as the analytics endpoint host. +use http://localhost:8080 as the analytics endpoint host. ### Future work diff --git a/WHY.org b/WHY.org index 8150c35..fdbaa4f 100644 --- a/WHY.org +++ b/WHY.org @@ -21,7 +21,7 @@ What makes things even harder for these developers is, as Andrew Nesbitt puts it in this excellent [[https://www.youtube.com/watch?v=hW4wUpoBHr8][talk]]: #+BEGIN_QUOTE -Maintainers are working mostly in the dark +Maintainers are working mostly in the dark #+END_QUOTE This is an important point. Package maintainers have little insight into how @@ -43,7 +43,7 @@ statistics, as do npm and others. Unfortunately, the data they collect is still limited, and whether the data is even exposed to package maintainers directly varies between package managers. -So what can we, the software development community do about this? +So what can we, the software development community do about this? One thing we could all do would be to convince our employers to support the awesome people that build and maintain the OSS they rely on. But you knew that @@ -106,6 +106,9 @@ is, there's something you can do to make things better for everyone! If you're interested in adding Scarf to your npm package or learning more: -- Scarf homepage: [[https://scarf.sh/library]] +- Scarf homepage: [[https://scarf.sh]] - GitHub: [[https://github.com/scarf-sh/scarf-js][https://github.com/scarf-sh/scarf-js]] - npm: [[https://www.npmjs.com/package/@scarf/scarf][https://www.npmjs.com/package/@scarf/scarf]] + +UPDATE: a more recent post which expands on some of these ideas: [[https://about.scarf.sh/post/analytics-and-open-source-sustainability]] + diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 0000000..c970861 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,17 @@ +# This file has been generated by node2nix 1.8.0. Do not edit! + +{pkgs ? import { + inherit system; + }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-12_x"}: + +let + nodeEnv = import ./node-env.nix { + inherit (pkgs) stdenv python2 utillinux runCommand writeTextFile; + inherit nodejs; + libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null; + }; +in +import ./node-packages.nix { + inherit (pkgs) fetchurl fetchgit; + inherit nodeEnv; +} \ No newline at end of file diff --git a/nix/generate.sh b/nix/generate.sh new file mode 100755 index 0000000..b422a53 --- /dev/null +++ b/nix/generate.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env nix-shell +#!nix-shell "" -i bash -p nodePackages.node2nix + +exec node2nix \ + --nodejs-12 \ + --input ../package.json \ + --development diff --git a/nix/node-env.nix b/nix/node-env.nix new file mode 100644 index 0000000..e1abf53 --- /dev/null +++ b/nix/node-env.nix @@ -0,0 +1,542 @@ +# This file originates from node2nix + +{stdenv, nodejs, python2, utillinux, libtool, runCommand, writeTextFile}: + +let + python = if nodejs ? python then nodejs.python else python2; + + # Create a tar wrapper that filters all the 'Ignoring unknown extended header keyword' noise + tarWrapper = runCommand "tarWrapper" {} '' + mkdir -p $out/bin + + cat > $out/bin/tar <> $out/nix-support/hydra-build-products + ''; + }; + + includeDependencies = {dependencies}: + stdenv.lib.optionalString (dependencies != []) + (stdenv.lib.concatMapStrings (dependency: + '' + # Bundle the dependencies of the package + mkdir -p node_modules + cd node_modules + + # Only include dependencies if they don't exist. They may also be bundled in the package. + if [ ! -e "${dependency.name}" ] + then + ${composePackage dependency} + fi + + cd .. + '' + ) dependencies); + + # Recursively composes the dependencies of a package + composePackage = { name, packageName, src, dependencies ? [], ... }@args: + builtins.addErrorContext "while evaluating node package '${packageName}'" '' + DIR=$(pwd) + cd $TMPDIR + + unpackFile ${src} + + # Make the base dir in which the target dependency resides first + mkdir -p "$(dirname "$DIR/${packageName}")" + + if [ -f "${src}" ] + then + # Figure out what directory has been unpacked + packageDir="$(find . -maxdepth 1 -type d | tail -1)" + + # Restore write permissions to make building work + find "$packageDir" -type d -exec chmod u+x {} \; + chmod -R u+w "$packageDir" + + # Move the extracted tarball into the output folder + mv "$packageDir" "$DIR/${packageName}" + elif [ -d "${src}" ] + then + # Get a stripped name (without hash) of the source directory. + # On old nixpkgs it's already set internally. + if [ -z "$strippedName" ] + then + strippedName="$(stripHash ${src})" + fi + + # Restore write permissions to make building work + chmod -R u+w "$strippedName" + + # Move the extracted directory into the output folder + mv "$strippedName" "$DIR/${packageName}" + fi + + # Unset the stripped name to not confuse the next unpack step + unset strippedName + + # Include the dependencies of the package + cd "$DIR/${packageName}" + ${includeDependencies { inherit dependencies; }} + cd .. + ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + ''; + + pinpointDependencies = {dependencies, production}: + let + pinpointDependenciesFromPackageJSON = writeTextFile { + name = "pinpointDependencies.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + function resolveDependencyVersion(location, name) { + if(location == process.env['NIX_STORE']) { + return null; + } else { + var dependencyPackageJSON = path.join(location, "node_modules", name, "package.json"); + + if(fs.existsSync(dependencyPackageJSON)) { + var dependencyPackageObj = JSON.parse(fs.readFileSync(dependencyPackageJSON)); + + if(dependencyPackageObj.name == name) { + return dependencyPackageObj.version; + } + } else { + return resolveDependencyVersion(path.resolve(location, ".."), name); + } + } + } + + function replaceDependencies(dependencies) { + if(typeof dependencies == "object" && dependencies !== null) { + for(var dependency in dependencies) { + var resolvedVersion = resolveDependencyVersion(process.cwd(), dependency); + + if(resolvedVersion === null) { + process.stderr.write("WARNING: cannot pinpoint dependency: "+dependency+", context: "+process.cwd()+"\n"); + } else { + dependencies[dependency] = resolvedVersion; + } + } + } + } + + /* Read the package.json configuration */ + var packageObj = JSON.parse(fs.readFileSync('./package.json')); + + /* Pinpoint all dependencies */ + replaceDependencies(packageObj.dependencies); + if(process.argv[2] == "development") { + replaceDependencies(packageObj.devDependencies); + } + replaceDependencies(packageObj.optionalDependencies); + + /* Write the fixed package.json file */ + fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2)); + ''; + }; + in + '' + node ${pinpointDependenciesFromPackageJSON} ${if production then "production" else "development"} + + ${stdenv.lib.optionalString (dependencies != []) + '' + if [ -d node_modules ] + then + cd node_modules + ${stdenv.lib.concatMapStrings (dependency: pinpointDependenciesOfPackage dependency) dependencies} + cd .. + fi + ''} + ''; + + # Recursively traverses all dependencies of a package and pinpoints all + # dependencies in the package.json file to the versions that are actually + # being used. + + pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }@args: + '' + if [ -d "${packageName}" ] + then + cd "${packageName}" + ${pinpointDependencies { inherit dependencies production; }} + cd .. + ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + fi + ''; + + # Extract the Node.js source code which is used to compile packages with + # native bindings + nodeSources = runCommand "node-sources" {} '' + tar --no-same-owner --no-same-permissions -xf ${nodejs.src} + mv node-* $out + ''; + + # Script that adds _integrity fields to all package.json files to prevent NPM from consulting the cache (that is empty) + addIntegrityFieldsScript = writeTextFile { + name = "addintegrityfields.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + function augmentDependencies(baseDir, dependencies) { + for(var dependencyName in dependencies) { + var dependency = dependencies[dependencyName]; + + // Open package.json and augment metadata fields + var packageJSONDir = path.join(baseDir, "node_modules", dependencyName); + var packageJSONPath = path.join(packageJSONDir, "package.json"); + + if(fs.existsSync(packageJSONPath)) { // Only augment packages that exist. Sometimes we may have production installs in which development dependencies can be ignored + console.log("Adding metadata fields to: "+packageJSONPath); + var packageObj = JSON.parse(fs.readFileSync(packageJSONPath)); + + if(dependency.integrity) { + packageObj["_integrity"] = dependency.integrity; + } else { + packageObj["_integrity"] = "sha1-000000000000000000000000000="; // When no _integrity string has been provided (e.g. by Git dependencies), add a dummy one. It does not seem to harm and it bypasses downloads. + } + + if(dependency.resolved) { + packageObj["_resolved"] = dependency.resolved; // Adopt the resolved property if one has been provided + } else { + packageObj["_resolved"] = dependency.version; // Set the resolved version to the version identifier. This prevents NPM from cloning Git repositories. + } + + if(dependency.from !== undefined) { // Adopt from property if one has been provided + packageObj["_from"] = dependency.from; + } + + fs.writeFileSync(packageJSONPath, JSON.stringify(packageObj, null, 2)); + } + + // Augment transitive dependencies + if(dependency.dependencies !== undefined) { + augmentDependencies(packageJSONDir, dependency.dependencies); + } + } + } + + if(fs.existsSync("./package-lock.json")) { + var packageLock = JSON.parse(fs.readFileSync("./package-lock.json")); + + if(packageLock.lockfileVersion !== 1) { + process.stderr.write("Sorry, I only understand lock file version 1!\n"); + process.exit(1); + } + + if(packageLock.dependencies !== undefined) { + augmentDependencies(".", packageLock.dependencies); + } + } + ''; + }; + + # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes + reconstructPackageLock = writeTextFile { + name = "addintegrityfields.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + var packageObj = JSON.parse(fs.readFileSync("package.json")); + + var lockObj = { + name: packageObj.name, + version: packageObj.version, + lockfileVersion: 1, + requires: true, + dependencies: {} + }; + + function augmentPackageJSON(filePath, dependencies) { + var packageJSON = path.join(filePath, "package.json"); + if(fs.existsSync(packageJSON)) { + var packageObj = JSON.parse(fs.readFileSync(packageJSON)); + dependencies[packageObj.name] = { + version: packageObj.version, + integrity: "sha1-000000000000000000000000000=", + dependencies: {} + }; + processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies); + } + } + + function processDependencies(dir, dependencies) { + if(fs.existsSync(dir)) { + var files = fs.readdirSync(dir); + + files.forEach(function(entry) { + var filePath = path.join(dir, entry); + var stats = fs.statSync(filePath); + + if(stats.isDirectory()) { + if(entry.substr(0, 1) == "@") { + // When we encounter a namespace folder, augment all packages belonging to the scope + var pkgFiles = fs.readdirSync(filePath); + + pkgFiles.forEach(function(entry) { + if(stats.isDirectory()) { + var pkgFilePath = path.join(filePath, entry); + augmentPackageJSON(pkgFilePath, dependencies); + } + }); + } else { + augmentPackageJSON(filePath, dependencies); + } + } + }); + } + } + + processDependencies("node_modules", lockObj.dependencies); + + fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2)); + ''; + }; + + prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}: + let + forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com"; + in + '' + # Pinpoint the versions of all dependencies to the ones that are actually being used + echo "pinpointing versions of dependencies..." + source $pinpointDependenciesScriptPath + + # Patch the shebangs of the bundled modules to prevent them from + # calling executables outside the Nix store as much as possible + patchShebangs . + + # Deploy the Node.js package by running npm install. Since the + # dependencies have been provided already by ourselves, it should not + # attempt to install them again, which is good, because we want to make + # it Nix's responsibility. If it needs to install any dependencies + # anyway (e.g. because the dependency parameters are + # incomplete/incorrect), it fails. + # + # The other responsibilities of NPM are kept -- version checks, build + # steps, postprocessing etc. + + export HOME=$TMPDIR + cd "${packageName}" + runHook preRebuild + + ${stdenv.lib.optionalString bypassCache '' + ${stdenv.lib.optionalString reconstructLock '' + if [ -f package-lock.json ] + then + echo "WARNING: Reconstruct lock option enabled, but a lock file already exists!" + echo "This will most likely result in version mismatches! We will remove the lock file and regenerate it!" + rm package-lock.json + else + echo "No package-lock.json file found, reconstructing..." + fi + + node ${reconstructPackageLock} + ''} + + node ${addIntegrityFieldsScript} + ''} + + npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} rebuild + + if [ "''${dontNpmInstall-}" != "1" ] + then + # NPM tries to download packages even when they already exist if npm-shrinkwrap is used. + rm -f npm-shrinkwrap.json + + npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} install + fi + ''; + + # Builds and composes an NPM package including all its dependencies + buildNodePackage = + { name + , packageName + , version + , dependencies ? [] + , buildInputs ? [] + , production ? true + , npmFlags ? "" + , dontNpmInstall ? false + , bypassCache ? false + , reconstructLock ? false + , preRebuild ? "" + , dontStrip ? true + , unpackPhase ? "true" + , buildPhase ? "true" + , ... }@args: + + let + extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "preRebuild" "unpackPhase" "buildPhase" ]; + in + stdenv.mkDerivation ({ + name = "node_${name}-${version}"; + buildInputs = [ tarWrapper python nodejs ] + ++ stdenv.lib.optional (stdenv.isLinux) utillinux + ++ stdenv.lib.optional (stdenv.isDarwin) libtool + ++ buildInputs; + + inherit nodejs; + + inherit dontStrip; # Stripping may fail a build for some package deployments + inherit dontNpmInstall preRebuild unpackPhase buildPhase; + + compositionScript = composePackage args; + pinpointDependenciesScript = pinpointDependenciesOfPackage args; + + passAsFile = [ "compositionScript" "pinpointDependenciesScript" ]; + + installPhase = '' + # Create and enter a root node_modules/ folder + mkdir -p $out/lib/node_modules + cd $out/lib/node_modules + + # Compose the package and all its dependencies + source $compositionScriptPath + + ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} + + # Create symlink to the deployed executable folder, if applicable + if [ -d "$out/lib/node_modules/.bin" ] + then + ln -s $out/lib/node_modules/.bin $out/bin + fi + + # Create symlinks to the deployed manual page folders, if applicable + if [ -d "$out/lib/node_modules/${packageName}/man" ] + then + mkdir -p $out/share + for dir in "$out/lib/node_modules/${packageName}/man/"* + do + mkdir -p $out/share/man/$(basename "$dir") + for page in "$dir"/* + do + ln -s $page $out/share/man/$(basename "$dir") + done + done + fi + + # Run post install hook, if provided + runHook postInstall + ''; + } // extraArgs); + + # Builds a development shell + buildNodeShell = + { name + , packageName + , version + , src + , dependencies ? [] + , buildInputs ? [] + , production ? true + , npmFlags ? "" + , dontNpmInstall ? false + , bypassCache ? false + , reconstructLock ? false + , dontStrip ? true + , unpackPhase ? "true" + , buildPhase ? "true" + , ... }@args: + + let + extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" ]; + + nodeDependencies = stdenv.mkDerivation ({ + name = "node-dependencies-${name}-${version}"; + + buildInputs = [ tarWrapper python nodejs ] + ++ stdenv.lib.optional (stdenv.isLinux) utillinux + ++ stdenv.lib.optional (stdenv.isDarwin) libtool + ++ buildInputs; + + inherit dontStrip; # Stripping may fail a build for some package deployments + inherit dontNpmInstall unpackPhase buildPhase; + + includeScript = includeDependencies { inherit dependencies; }; + pinpointDependenciesScript = pinpointDependenciesOfPackage args; + + passAsFile = [ "includeScript" "pinpointDependenciesScript" ]; + + installPhase = '' + mkdir -p $out/${packageName} + cd $out/${packageName} + + source $includeScriptPath + + # Create fake package.json to make the npm commands work properly + cp ${src}/package.json . + chmod 644 package.json + ${stdenv.lib.optionalString bypassCache '' + if [ -f ${src}/package-lock.json ] + then + cp ${src}/package-lock.json . + fi + ''} + + # Go to the parent folder to make sure that all packages are pinpointed + cd .. + ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + + ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} + + # Expose the executables that were installed + cd .. + ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + + mv ${packageName} lib + ln -s $out/lib/node_modules/.bin $out/bin + ''; + } // extraArgs); + in + stdenv.mkDerivation { + name = "node-shell-${name}-${version}"; + + buildInputs = [ python nodejs ] ++ stdenv.lib.optional (stdenv.isLinux) utillinux ++ buildInputs; + buildCommand = '' + mkdir -p $out/bin + cat > $out/bin/shell < { return (rootPackage && rootPackage.scarfSettings && rootPackage.scarfSettings.enabled) || process.env.SCARF_ANALYTICS === 'true' } -// We don't send any paths, we don't send any scoped package names or versions +// Packages that depend on Scarf can configure whether to should fire when +// `npm install` is being run directly from within the package, rather than from a +// dependent package +function allowTopLevel (rootPackage) { + return rootPackage && rootPackage.scarfSettings && rootPackage.scarfSettings.allowTopLevel +} + +function parentIsRoot (dependencyToReport) { + return dependencyToReport.parent.name === dependencyToReport.rootPackage.name && + dependencyToReport.parent.version === dependencyToReport.rootPackage.version +} + +function isTopLevel (dependencyToReport) { + return parentIsRoot(dependencyToReport) && !process.env.npm_config_global +} + +function isGlobal (dependencyToReport) { + return parentIsRoot(dependencyToReport) && !!process.env.npm_config_global +} + +function hashWithDefault (toHash, defaultReturn) { + let crypto + try { + crypto = require('crypto') + } catch (err) { + logIfVerbose('node crypto module unavailable') + } + + if (crypto && toHash) { + return crypto.createHash('sha256').update(toHash, 'utf-8').digest('hex') + } else { + return defaultReturn + } +} + +// We don't send any paths, hash package names and versions function redactSensitivePackageInfo (dependencyInfo) { - const scopedRegex = /@\S+\// - const privatePackageRewrite = '@private/private' - const privateVersionRewrite = '0' - if (dependencyInfo.grandparent && dependencyInfo.grandparent.name.match(scopedRegex)) { - dependencyInfo.grandparent.name = privatePackageRewrite - dependencyInfo.grandparent.version = privateVersionRewrite + if (dependencyInfo.grandparent && dependencyInfo.grandparent.name) { + dependencyInfo.grandparent.nameHash = hashWithDefault(dependencyInfo.grandparent.name, privatePackageRewrite) + dependencyInfo.grandparent.versionHash = hashWithDefault(dependencyInfo.grandparent.version, privateVersionRewrite) } - if (dependencyInfo.rootPackage && dependencyInfo.rootPackage.name.match(scopedRegex)) { - dependencyInfo.rootPackage.name = privateVersionRewrite - dependencyInfo.rootPackage.version = privateVersionRewrite + + if (dependencyInfo.rootPackage && dependencyInfo.rootPackage.name) { + dependencyInfo.rootPackage.nameHash = hashWithDefault(dependencyInfo.rootPackage.name, privatePackageRewrite) + dependencyInfo.rootPackage.versionHash = hashWithDefault(dependencyInfo.rootPackage.version, privateVersionRewrite) } + delete (dependencyInfo.rootPackage.packageJsonPath) delete (dependencyInfo.rootPackage.path) + delete (dependencyInfo.rootPackage.name) + delete (dependencyInfo.rootPackage.version) delete (dependencyInfo.parent.path) delete (dependencyInfo.scarf.path) if (dependencyInfo.grandparent) { delete (dependencyInfo.grandparent.path) + delete (dependencyInfo.grandparent.name) + delete (dependencyInfo.grandparent.version) } return dependencyInfo } -async function getDependencyInfo () { - return new Promise((resolve, reject) => { - exec(`cd ${rootPath} && npm ls @scarf/scarf --json --long`, { timeout: execTimeout }, function (error, stdout, stderr) { - if (error) { - return reject(new Error(`Scarf received an error from npm -ls: ${error}`)) - } +/* + Scarf-js is automatically disabled when being run inside of a yarn install. + The `npm_execpath` environment variable tells us which package manager is + running our install + */ +function isYarn () { + const execPath = module.exports.npmExecPath() || '' + return ['yarn', 'yarn.js', 'yarnpkg', 'yarn.cmd', 'yarnpkg.cmd'] + .some(packageManBinName => execPath.endsWith(packageManBinName)) +} - try { - const output = JSON.parse(stdout) +function processDependencyTreeOutput (resolve, reject) { + return function (error, stdout, stderr) { + if (error && !stdout) { + return reject(new Error(`Scarf received an error from npm -ls: ${error} | ${stderr}`)) + } - let depsToScarf = findScarfInFullDependencyTree(output) - depsToScarf = depsToScarf.filter(depChain => depChain.length > 2) - if (depsToScarf.length === 0) { - return reject(new Error('No Scarf parent package found')) - } + try { + const output = JSON.parse(stdout) - const rootPackageDetails = rootPackageDepInfo(output) + const depsToScarf = findScarfInFullDependencyTree(output).filter(depChain => depChain.length >= 2) + if (!depsToScarf.length) { + return reject(new Error('No Scarf parent package found')) + } + const rootPackageDetails = rootPackageDepInfo(output) + + const dependencyInfo = depsToScarf.map(depChain => { + return { + scarf: depChain[depChain.length - 1], + parent: depChain[depChain.length - 2], + grandparent: depChain[depChain.length - 3], + rootPackage: rootPackageDetails, + anyInChainDisabled: depChain.some(dep => { + return (dep.scarfSettings || {}).enabled === false + }) + } + }) - const dependencyInfo = depsToScarf.map(depChain => { - return { - scarf: depChain[depChain.length - 1], - parent: depChain[depChain.length - 2], - grandparent: depChain[depChain.length - 3], // might be undefined - rootPackage: rootPackageDetails - } - }) + dependencyInfo.forEach(d => { + d.parent.scarfSettings = Object.assign(makeDefaultSettings(), d.parent.scarfSettings || {}) + }) - dependencyInfo.forEach(d => { - d.parent.scarfSettings = Object.assign(makeDefaultSettings(), d.parent.scarfSettings || {}) - }) + // Here, we find the dependency chain that corresponds to the scarf package we're currently in + const dependencyToReport = dependencyInfo.find(dep => (dep.scarf.path === module.exports.dirName())) || dependencyInfo[0] + if (!dependencyToReport) { + return reject(new Error(`Couldn't find dependency info for path ${module.exports.dirName()}`)) + } - // Here, we find the dependency chain that corresponds to the scarf package we're currently in - const dependencyToReport = dependencyInfo.find(dep => (dep.scarf.path === __dirname)) - if (!dependencyToReport) { - return reject(new Error(`Couldn't find dependency info for path ${__dirname}`)) - } + // If any intermediate dependency in the chain of deps that leads to scarf + // has disabled Scarf, we must respect that setting unless the user overrides it. + if (dependencyToReport.anyInChainDisabled && !userHasOptedIn(dependencyToReport.rootPackage)) { + return reject(new Error('Scarf has been disabled via a package.json in the dependency chain.')) + } - return resolve(dependencyToReport) - } catch (err) { - logIfVerbose(err, console.error) - return reject(err) + if (isTopLevel(dependencyToReport) && !isGlobal(dependencyToReport) && !allowTopLevel(rootPackageDetails)) { + return reject(new Error('The package depending on Scarf is the root package being installed, but Scarf is not configured to run in this case. To enable it, set `scarfSettings.allowTopLevel = true` in your package.json')) } - }) + + return resolve(dependencyToReport) + } catch (err) { + logIfVerbose(err, console.error) + return reject(err) + } + } +} + +async function getDependencyInfo () { + return new Promise((resolve, reject) => { + exec(`cd ${rootPath} && npm ls @scarf/scarf --json --long`, { timeout: execTimeout, maxBuffer: 1024 * 1024 * 1024 }, processDependencyTreeOutput(resolve, reject)) }) } async function reportPostInstall () { const scarfApiToken = process.env.SCARF_API_TOKEN - const dependencyInfo = await getDependencyInfo() + + const dependencyInfo = await module.exports.getDependencyInfo() if (!dependencyInfo.parent || !dependencyInfo.parent.name) { - return Promise.reject(new Error('No parent, nothing to report')) + return Promise.reject(new Error('No parent found, nothing to report')) } const rootPackage = dependencyInfo.rootPackage + if (!userHasOptedIn(rootPackage) && isYarn()) { + return Promise.reject(new Error('Package manager is yarn. scarf-js is unable to inform user of analytics. Aborting.')) + } + await new Promise((resolve, reject) => { if (dependencyInfo.parent.scarfSettings.defaultOptIn) { if (userHasOptedOut(rootPackage)) { @@ -135,7 +216,7 @@ async function reportPostInstall () { if (!userHasOptedIn(rootPackage)) { rateLimitedUserLog(optedInLogRateLimitKey, ` The dependency '${dependencyInfo.parent.name}' is tracking installation - statistics using Scarf (https://scarf.sh), which helps open-source developers + statistics using scarf-js (https://scarf.sh), which helps open-source developers fund and maintain their projects. Scarf securely logs basic installation details when this package is installed. The Scarf npm library is open source and permissively licensed at https://github.com/scarf-sh/scarf-js. For more @@ -154,7 +235,7 @@ async function reportPostInstall () { } rateLimitedUserLog(optedOutLogRateLimitKey, ` The dependency '${dependencyInfo.parent.name}' would like to track - installation statistics using Scarf (https://scarf.sh), which helps + installation statistics using scarf-js (https://scarf.sh), which helps open-source developers fund and maintain their projects. Reporting is disabled by default for this package. When enabled, Scarf securely logs basic installation details when this package is installed. The Scarf npm library is @@ -330,6 +411,9 @@ function packageDetailsFromDepInfo (tree) { } function rootPackageDepInfo (packageInfo) { + if (process.env.npm_config_global) { + packageInfo = Object.values(packageInfo.dependencies)[0] + } const info = packageDetailsFromDepInfo(packageInfo) info.packageJsonPath = `${packageInfo.path}/package.json` return info @@ -386,6 +470,21 @@ function writeCurrentTimeToLogHistory (rateLimitKey, history) { fs.writeFileSync(module.exports.tmpFileName(), JSON.stringify(history)) } +module.exports = { + redactSensitivePackageInfo, + hasHitRateLimit, + getRateLimitedLogHistory, + rateLimitedUserLog, + tmpFileName, + dirName, + processDependencyTreeOutput, + npmExecPath, + getDependencyInfo, + reportPostInstall, + hashWithDefault, + findScarfInFullDependencyTree +} + if (require.main === module) { try { reportPostInstall().catch(e => { @@ -400,11 +499,3 @@ if (require.main === module) { process.exit(0) } } - -module.exports = { - redactSensitivePackageInfo, - hasHitRateLimit, - getRateLimitedLogHistory, - rateLimitedUserLog, - tmpFileName -} diff --git a/test/example-ls-output.json b/test/example-ls-output.json new file mode 100644 index 0000000..90494be --- /dev/null +++ b/test/example-ls-output.json @@ -0,0 +1,108 @@ +{ + "name": "scarfed-lib-consumer-consumer", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "dependencies": { + "scarfed-lib-consumer": { + "_from": "file:/tmp/scarfed-lib-consumer-9.9.9.tgz", + "_id": "scarfed-lib-consumer@9.9.9", + "_location": "/scarfed-lib-consumer", + "_requiredBy": [ + "/" + ], + "_resolved": "/tmp/scarfed-lib-consumer-9.9.9.tgz", + "_spec": "file:../../../../tmp/scarfed-lib-consumer-9.9.9.tgz", + "_where": "/path/scarfed-lib-consumer-consumer", + "author": "", + "bundleDependencies": false, + "dependencies": { + "scarfed-library": { + "_from": "file:/tmp/scarfed-library-1.0.0.tgz", + "_id": "scarfed-library@1.0.0", + "_location": "/scarfed-library", + "_requiredBy": [ + "/scarfed-lib-consumer" + ], + "_resolved": "/tmp/scarfed-library-1.0.0.tgz", + "_spec": "file:../../../../tmp/scarfed-library-1.0.0.tgz", + "_where": "/path/scarfed-lib-consumer-consumer", + "author": "", + "bundleDependencies": false, + "dependencies": { + "@scarf/scarf": { + "_from": "file:/tmp/scarf-scarf-1.0.3.tgz", + "_id": "@scarf/scarf@1.0.3", + "_location": "/@scarf/scarf", + "_phantomChildren": {}, + "_requiredBy": [ + "/scarfed-library" + ], + "_resolved": "/tmp/scarf-scarf-1.0.3.tgz", + "_where": "/path/scarfed-lib-consumer-consumer", + "files": [ + "report.js", + "test/*.js" + ], + "version": "1.0.3", + "dependencies": {}, + "optionalDependencies": {}, + "_dependencies": {}, + "path": "/path/scarfed-lib-consumer-consumer/node_modules/@scarf/scarf", + "error": null, + "extraneous": false, + "_parent": "[Circular]", + "_found": "explicit" + } + }, + "deprecated": false, + "main": "index.js", + "name": "scarfed-library", + "scarfSettings": { + "defaultOptIn": true + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "version": "1.0.0", + "readme": "ERROR: No README data found!", + "_args": [ + [ + "scarfed-library@file:../../../../tmp/scarfed-library-1.0.0.tgz", + "/path/scarfed-lib-consumer-consumer" + ] + ], + "devDependencies": {}, + "optionalDependencies": {}, + "_dependencies": { + "@scarf/scarf": "file:///tmp/scarf-scarf-1.0.3.tgz" + }, + "path": "/path/scarfed-lib-consumer-consumer/node_modules/scarfed-library", + "error": "[Circular]", + "extraneous": false, + "_parent": "[Circular]", + "_found": "implicit" + } + }, + "deprecated": false, + "main": "index.js", + "name": "scarfed-lib-consumer", + "scarfSettings": { + "enabled": false + }, + "version": "9.9.9", + "_dependencies": { + "scarfed-library": "file:///tmp/scarfed-library-1.0.0.tgz" + }, + "path": "/path/scarfed-lib-consumer-consumer/node_modules/scarfed-lib-consumer" + } + }, + "_id": "scarfed-lib-consumer-consumer@1.0.0", + "_dependencies": { + "scarfed-lib-consumer": "file:///tmp/scarfed-lib-consumer-9.9.9.tgz" + }, + "path": "/path/scarfed-lib-consumer-consumer" +} diff --git a/test/report.test.js b/test/report.test.js index 58c0336..575d416 100644 --- a/test/report.test.js +++ b/test/report.test.js @@ -3,6 +3,7 @@ const fs = require('fs') const rateLimitKey = 'testKey' const tmpFileReturnVal = './scarf-js-test-history.log' +const scarfExecPath = '/path/scarfed-lib-consumer-consumer/node_modules/@scarf/scarf' function wipeLogHistoryIfPresent () { try { @@ -15,6 +16,9 @@ describe('Reporting tests', () => { report.tmpFileName = jest.fn(() => { return tmpFileReturnVal }) + report.dirName = jest.fn(() => { + return scarfExecPath + }) wipeLogHistoryIfPresent() }) @@ -31,13 +35,15 @@ describe('Reporting tests', () => { }) test('Redact sensitive data', () => { - const rootPackageName = '@org/scarfed-lib-consumer' + const rootPackageName = '@org/scarfed-lib-consumer-consumer' const rootPackageVersion = '1.0.0' + const grandparentName = 'scarfed-lib-consumer' + const grandparentVersion = '1.0.1' const depInfo = { scarf: { name: '@scarf/scarf', version: '0.0.1', path: '/local/directory/deeper/deeper' }, parent: { name: 'scarfed-library', version: '1.0.0', scarfSettings: { defaultOptIn: true }, path: '/local/directory/deeper/' }, - grandparent: { name: rootPackageName, version: rootPackageVersion, path: '/local/directory/' }, + grandparent: { name: grandparentName, version: grandparentVersion, path: '/local/directory/' }, rootPackage: { name: rootPackageName, version: rootPackageVersion, packageJsonPath: '/local/directory', path: '/local/directory' } } @@ -49,13 +55,15 @@ describe('Reporting tests', () => { expect(redacted.rootPackage.path).toBeUndefined() expect(redacted.rootPackage.packageJsonPath).toBeUndefined() - expect(redacted.rootPackage.name).not.toContain('org') - expect(redacted.rootPackage.name).not.toContain('scarfed-lib-consumer') - expect(redacted.grandparent.name).not.toContain('org') - expect(redacted.grandparent.name).not.toContain('scarfed-lib-consumer') + expect(redacted.grandparent.nameHash).toBe(report.hashWithDefault(grandparentName, 'Fail: used hash fallback for name')) + expect(redacted.grandparent.versionHash).toBe(report.hashWithDefault(grandparentVersion, 'Fail: used hash fallback for version')) + expect(redacted.rootPackage.nameHash).toBe(report.hashWithDefault(rootPackageName, 'Fail: used hash fallback for name')) + expect(redacted.rootPackage.versionHash).toBe(report.hashWithDefault(rootPackageVersion, 'Fail: used hash fallback for version')) - expect(redacted.rootPackage.version).toBe('0') - expect(redacted.grandparent.version).toBe('0') + expect(redacted.grandparent.name).toBeUndefined() + expect(redacted.grandparent.version).toBeUndefined() + expect(redacted.rootPackage.name).toBeUndefined() + expect(redacted.rootPackage.version).toBeUndefined() expect(redacted.scarf.name).toBe('@scarf/scarf') expect(redacted.scarf.version).toBe('0.0.1') @@ -63,4 +71,95 @@ describe('Reporting tests', () => { expect(redacted.parent.name).toBe('scarfed-library') expect(redacted.parent.version).toBe('1.0.0') }) + + test('Intermediate packages can disable Scarf for their dependents', async () => { + const exampleLsOutput = fs.readFileSync('./test/example-ls-output.json') + + await expect(new Promise((resolve, reject) => { + return report.processDependencyTreeOutput(resolve, reject)(null, exampleLsOutput, null) + })).rejects.toEqual(new Error('Scarf has been disabled via a package.json in the dependency chain.')) + + const parsedLsOutput = JSON.parse(exampleLsOutput) + delete (parsedLsOutput.dependencies['scarfed-lib-consumer'].scarfSettings) + + await new Promise((resolve, reject) => { + return report.processDependencyTreeOutput(resolve, reject)(null, JSON.stringify(parsedLsOutput), null) + }).then(output => { + expect(output).toBeTruthy() + expect(output.anyInChainDisabled).toBe(false) + }) + }) + + test('Disable when package manager is yarn', async () => { + const parsedLsOutput = dependencyTreeScarfEnabled() + + await new Promise((resolve, reject) => { + return report.processDependencyTreeOutput(resolve, reject)(null, JSON.stringify(parsedLsOutput), null) + }).then(output => { + expect(output).toBeTruthy() + expect(output.anyInChainDisabled).toBe(false) + }) + + // Simulate a yarn install by mocking the env variable npm_execpath + // leading to a yarn executable + report.npmExecPath = jest.fn(() => { + return '/usr/local/lib/node_modules/yarn/bin/yarn.js' + }) + + report.getDependencyInfo = jest.fn(() => { + return Promise.resolve({ + scarf: { name: '@scarf/scarf', version: '0.0.1' }, + parent: { name: 'scarfed-library', version: '1.0.0', scarfSettings: { defaultOptIn: true } }, + grandparent: { name: 'scarfed-lib-consumer', version: '1.0.0' } + }) + }) + + try { + await report.reportPostInstall() + throw new Error("report.reportPostInstall() didn't throw an error") + } catch (err) { + expect(err.message).toContain('yarn') + } + }) + + test('Configurable for top level installs, and enabled for global installs', async () => { + const parsedLsOutput = dependencyTreeTopLevelInstall() + + await expect(new Promise((resolve, reject) => { + return report.processDependencyTreeOutput(resolve, reject)(null, JSON.stringify(parsedLsOutput), null) + })).rejects.toEqual(new Error('The package depending on Scarf is the root package being installed, but Scarf is not configured to run in this case. To enable it, set `scarfSettings.allowTopLevel = true` in your package.json')) + + process.env.npm_config_global = true + + await expect(new Promise((resolve, reject) => { + return report.processDependencyTreeOutput(resolve, reject)(null, JSON.stringify(parsedLsOutput), null) + })).toBeTruthy() + + process.env.npm_config_global = '' + parsedLsOutput.scarfSettings = { allowTopLevel: true } + + await expect(new Promise((resolve, reject) => { + return report.processDependencyTreeOutput(resolve, reject)(null, JSON.stringify(parsedLsOutput), null) + })).toBeTruthy() + + parsedLsOutput.scarfSettings = { allowTopLevel: false } + await expect(new Promise((resolve, reject) => { + return report.processDependencyTreeOutput(resolve, reject)(null, JSON.stringify(parsedLsOutput), null) + })).rejects.toEqual(new Error('The package depending on Scarf is the root package being installed, but Scarf is not configured to run in this case. To enable it, set `scarfSettings.allowTopLevel = true` in your package.json')) + }) }) + +function dependencyTreeScarfEnabled () { + const exampleLsOutput = fs.readFileSync('./test/example-ls-output.json') + + const parsedLsOutput = JSON.parse(exampleLsOutput) + delete (parsedLsOutput.dependencies['scarfed-lib-consumer'].scarfSettings) + + return parsedLsOutput +} + +function dependencyTreeTopLevelInstall () { + const exampleLsOutput = fs.readFileSync('./test/top-level-package-ls-output.json') + const parsedLsOutput = JSON.parse(exampleLsOutput) + return parsedLsOutput +} diff --git a/test/top-level-package-ls-output.json b/test/top-level-package-ls-output.json new file mode 100644 index 0000000..f8c2e25 --- /dev/null +++ b/test/top-level-package-ls-output.json @@ -0,0 +1,115 @@ +{ + "name": "scarfed-lib-consumer", + "version": "9.9.9", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "@scarf/scarf": { + "_from": "file:/tmp/scarf-scarf-1.0.6.tgz", + "_id": "@scarf/scarf@1.0.6", + "_integrity": "sha512-YQ7vXVI6G2qLm14rKRWvnvAtcW2dTwCw5ywQWgaNKfqLjteyQqM3pAlIOAQ9C5ShiKwsFavHpwCUkOG7Sfa67A==", + "_location": "/@scarf/scarf", + "_phantomChildren": {}, + "_requested": { + "type": "file", + "where": "/Users/avi/dev/scarfed-lib-consumer", + "raw": "@scarf/scarf@file:../../../../tmp/scarf-scarf-1.0.6.tgz", + "name": "@scarf/scarf", + "escapedName": "@scarf%2fscarf", + "scope": "@scarf", + "rawSpec": "file:../../../../tmp/scarf-scarf-1.0.6.tgz", + "saveSpec": "file:../../../../tmp/scarf-scarf-1.0.6.tgz", + "fetchSpec": "/tmp/scarf-scarf-1.0.6.tgz" + }, + "_requiredBy": [ + "/" + ], + "_resolved": "/tmp/scarf-scarf-1.0.6.tgz", + "_shasum": "57bb2ce26f6aafa6a1ea33f6978d044240412773", + "_spec": "file:../../../../tmp/scarf-scarf-1.0.6.tgz", + "_where": "/Users/avi/dev/scarfed-lib-consumer", + "author": { + "name": "Scarf Systems" + }, + "bugs": { + "url": "https://github.com/scarf-sh/scarf-js/issues" + }, + "bundleDependencies": false, + "deprecated": false, + "description": "", + "devDependencies": { + "jest": "^25.3.0", + "minimist": "^1.2.2", + "standard": "^14.3.1" + }, + "files": [ + "report.js", + "test/*" + ], + "homepage": "https://github.com/scarf-sh/scarf-js", + "license": "Apache-2.0", + "main": "report.js", + "name": "@scarf/scarf", + "repository": { + "type": "git", + "url": "git+https://github.com/scarf-sh/scarf-js.git" + }, + "scripts": { + "postinstall": "node ./report.js", + "test": "jest" + }, + "standard": { + "globals": [ + "expect", + "test", + "jest", + "beforeAll", + "afterAll", + "describe" + ] + }, + "version": "1.0.6", + "readme": "", + "readmeFilename": "README.md", + "_args": [ + [ + "@scarf/scarf@file:../../../../tmp/scarf-scarf-1.0.6.tgz", + "/Users/avi/dev/scarfed-lib-consumer" + ] + ], + "dependencies": {}, + "optionalDependencies": {}, + "_dependencies": {}, + "path": "/Users/avi/dev/scarfed-lib-consumer/node_modules/@scarf/scarf", + "error": null, + "extraneous": false + } + }, + "readme": "ERROR: No README data found!", + "_id": "scarfed-lib-consumer@9.9.9", + "_shrinkwrap": { + "name": "scarfed-lib-consumer", + "version": "9.9.9", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@scarf/scarf": { + "version": "file:../../../../tmp/scarf-scarf-1.0.6.tgz", + "integrity": "sha512-YQ7vXVI6G2qLm14rKRWvnvAtcW2dTwCw5ywQWgaNKfqLjteyQqM3pAlIOAQ9C5ShiKwsFavHpwCUkOG7Sfa67A==" + } + } + }, + "devDependencies": {}, + "optionalDependencies": {}, + "_dependencies": { + "@scarf/scarf": "file:///tmp/scarf-scarf-1.0.6.tgz" + }, + "path": "/Users/avi/dev/scarfed-lib-consumer", + "error": "[Circular]", + "extraneous": false +}