Skip to content

Commit

Permalink
fix: handle parallel installs (#84)
Browse files Browse the repository at this point in the history
* fix: handle parallel installs

* build: skip unnecessary shims
  • Loading branch information
merceyz authored Feb 2, 2022
1 parent f17384e commit 5cfc6c9
Show file tree
Hide file tree
Showing 15 changed files with 184 additions and 45 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ jobs:
env:
TARGET_BRANCH: ${{github.event.pull_request.base.ref}}

- name: 'Check for type errors'
run: yarn typecheck

build:
strategy:
fail-fast: false
Expand Down
45 changes: 40 additions & 5 deletions .pnp.cjs

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
21 changes: 15 additions & 6 deletions mkshims.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,14 @@ import path from 'path';
import {Engine} from './sources/Engine';
import {SupportedPackageManagerSet} from './sources/types';

const EXCLUDE_SHIMS = new Set([
`vcc.js`,
]);
function shouldGenerateShim(name: string) {
if (name === 'vcc.js') {
return false;
} else if (name.startsWith('vendors')) {
return false;
}
return true;
}

const engine = new Engine();

Expand Down Expand Up @@ -38,7 +43,7 @@ async function main() {
}

for (const binaryName of fs.readdirSync(distDir)) {
if (EXCLUDE_SHIMS.has(binaryName))
if (shouldGenerateShim(binaryName) === false)
continue;

await cmdShim(path.join(distDir, binaryName), path.join(shimsDir, path.basename(binaryName, `.js`)), {createCmdFile: true});
Expand All @@ -55,12 +60,16 @@ async function main() {
const remapPath = (p: string) => path.resolve(__dirname, path.relative(virtualNodewinDir, p));

const easyStatFs = Object.assign(Object.create(fs), {
readFile: (p: string, encoding: string, cb: (err: any, str: string) => void) => fs.readFile(remapPath(p), encoding, cb),
readFile: (p: string, encoding: BufferEncoding, cb: (err: any, str: string) => void) => fs.readFile(remapPath(p), encoding, cb),
stat: (p: string, cb: () => void) => fs.stat(remapPath(p), cb),
});

for (const binaryName of fs.readdirSync(distDir))
for (const binaryName of fs.readdirSync(distDir)) {
if (shouldGenerateShim(binaryName) === false)
continue;

await cmdShim(path.join(virtualNodewinDir, `dist/${binaryName}`), path.join(physicalNodewinDir, path.basename(binaryName, `.js`)), {createCmdFile: true, fs: easyStatFs});
}

console.log(`All shims have been generated.`);
}
Expand Down
7 changes: 5 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@
"@babel/preset-typescript": "^7.13.0",
"@types/debug": "^4.1.5",
"@types/jest": "^26.0.23",
"@types/node": "^13.9.2",
"@types/node": "^17.0.10",
"@types/rimraf": "^3.0.2",
"@types/semver": "^7.1.0",
"@types/tar": "^4.0.3",
"@types/which": "^1.3.2",
Expand All @@ -43,6 +44,7 @@
"eslint-plugin-arca": "^0.9.5",
"jest": "^26.0.0",
"nock": "^13.0.4",
"rimraf": "^3.0.2",
"semver": "^7.1.3",
"supports-color": "^7.1.0",
"tar": "^6.0.1",
Expand All @@ -56,10 +58,11 @@
"which": "^2.0.2"
},
"scripts": {
"build": "rm -rf dist && webpack && ts-node ./mkshims.ts",
"build": "rm -rf dist shims && webpack && ts-node ./mkshims.ts",
"corepack": "ts-node ./sources/main.ts",
"prepack": "node ./.yarn/releases/*.*js build",
"postpack": "rm -rf dist shims",
"typecheck": "tsc --noEmit",
"test": "yarn jest"
},
"files": [
Expand Down
59 changes: 35 additions & 24 deletions sources/corepackUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,39 +87,50 @@ export async function installVersion(installTarget: string, locator: Locator, {s
const url = spec.url.replace(`{}`, locator.reference);
debugUtils.log(`Installing ${locator.name}@${locator.reference} from ${url}`);

return await fsUtils.mutex(installFolder, async () => {
// Creating a temporary folder inside the install folder means that we
// are sure it'll be in the same drive as the destination, so we can
// just move it there atomically once we are done
// Creating a temporary folder inside the install folder means that we
// are sure it'll be in the same drive as the destination, so we can
// just move it there atomically once we are done

const tmpFolder = folderUtils.getTemporaryFolder(installTarget);
const stream = await httpUtils.fetchUrlStream(url);
const tmpFolder = folderUtils.getTemporaryFolder(installTarget);
const stream = await httpUtils.fetchUrlStream(url);

const parsedUrl = new URL(url);
const ext = path.posix.extname(parsedUrl.pathname);
const parsedUrl = new URL(url);
const ext = path.posix.extname(parsedUrl.pathname);

let outputFile: string | null = null;
let outputFile: string | null = null;

let sendTo: any;
if (ext === `.tgz`) {
sendTo = tar.x({strip: 1, cwd: tmpFolder});
} else if (ext === `.js`) {
outputFile = path.join(tmpFolder, path.posix.basename(parsedUrl.pathname));
sendTo = fs.createWriteStream(outputFile);
}
let sendTo: any;
if (ext === `.tgz`) {
sendTo = tar.x({strip: 1, cwd: tmpFolder});
} else if (ext === `.js`) {
outputFile = path.join(tmpFolder, path.posix.basename(parsedUrl.pathname));
sendTo = fs.createWriteStream(outputFile);
}

stream.pipe(sendTo);
stream.pipe(sendTo);

await new Promise(resolve => {
sendTo.on(`finish`, resolve);
});
await new Promise(resolve => {
sendTo.on(`finish`, resolve);
});

await fs.promises.mkdir(path.dirname(installFolder), {recursive: true});
await fs.promises.mkdir(path.dirname(installFolder), {recursive: true});
try {
await fs.promises.rename(tmpFolder, installFolder);
} catch (err) {
if (
err.code === `ENOTEMPTY` ||
// On Windows the error code is EPERM so we check if it is a directory
(err.code === `EPERM` && (await fs.promises.stat(installFolder)).isDirectory())
) {
debugUtils.log(`Another instance of corepack installed ${locator.name}@${locator.reference}`);
await fsUtils.rimraf(tmpFolder);
} else {
throw err;
}
}

debugUtils.log(`Install finished`);
return installFolder;
});
debugUtils.log(`Install finished`);
return installFolder;
}

export async function runVersion(installSpec: { location: string, spec: PackageManagerSpec }, locator: Locator, binName: string, args: Array<string>, context: Context) {
Expand Down
25 changes: 23 additions & 2 deletions sources/fsUtils.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,24 @@
export async function mutex<T>(p: string, cb: () => Promise<T>) {
return await cb();
import fs from 'fs';

export async function rimraf(path: string) {
const [major, minor] = process.versions.node.split(`.`).map(section => Number(section));

if (major > 14 || (major === 14 && minor >= 14)) {
// rm was added in v14.14.0
return fs.promises.rm(path, {recursive: true});
} else if (major > 12 || (major === 12 && minor >= 10)) {
// rmdir got support for recursive in v12.10.0 and was deprecated in v14.14.0
return fs.promises.rmdir(path, {recursive: true});
} else {
const rimraf = await import(`rimraf`);
return new Promise<void>((resolve, reject) => {
rimraf.default(path, err => {
if (err) {
reject(err);
} else {
resolve();
}
});
});
}
}
27 changes: 27 additions & 0 deletions tests/main.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -328,3 +328,30 @@ it(`should support running package managers with bin array`, async () => {
});
});
});

it(`should handle parallel installs`, async () => {
await xfs.mktempPromise(async cwd => {
await xfs.writeJsonPromise(ppath.join(cwd, `package.json` as Filename), {
packageManager: `[email protected]`,
});

await expect(Promise.all([
runCli(cwd, [`yarn`, `--version`]),
runCli(cwd, [`yarn`, `--version`]),
runCli(cwd, [`yarn`, `--version`]),
])).resolves.toMatchObject([
{
stdout: `2.2.2\n`,
exitCode: 0,
},
{
stdout: `2.2.2\n`,
exitCode: 0,
},
{
stdout: `2.2.2\n`,
exitCode: 0,
},
]);
});
});
Binary file added tests/nock/5ed9c2e2a56a83b54950e0282b2c406c.dat
Binary file not shown.
1 change: 1 addition & 0 deletions tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"lib": ["dom", "es2017", "esnext.asynciterable"],
"module": "commonjs",
"resolveJsonModule": true,
"skipLibCheck": true,
"target": "es2017"
}
}
Loading

0 comments on commit 5cfc6c9

Please sign in to comment.