From 41a7353e968409bf5941ddc552cd7a64e74febde Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 09:59:38 -0700 Subject: [PATCH 01/15] Add new python step to pipeline --- azure-pipelines.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d870c41a73..ed71a00609 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -77,6 +77,23 @@ stages: npm install -g tslint --verbose npm run lint displayName: Run Lint + ##### Verify NPM and Yarn are in sync ##### + - job: SyncPackageManagers + displayName: 'Verify NPM & Yarn In-Sync' + pool: + vmImage: 'windows-latest' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.x' + addToPath: true + architecture: 'x64' + - task: PythonScript@0 + inputs: + scriptSource: 'filePath' + scriptPath: 'dependency-verifier.py' + arguments: 'System.PullRequest.TargetBranch' + failOnStderr: true ##### Package and Publish ##### - job: Package displayName: 'Package and Publish' From 883fe50d1851d83749af3f1c2e2c4f63780cf0b3 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 10:23:43 -0700 Subject: [PATCH 02/15] add dependency-verifier --- dependency-verifier.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 dependency-verifier.py diff --git a/dependency-verifier.py b/dependency-verifier.py new file mode 100644 index 0000000000..ba0ae6a514 --- /dev/null +++ b/dependency-verifier.py @@ -0,0 +1,3 @@ +import sys + +sys.exit("The yarn.lock and package-lock appear to be out of sync. Update by doing yarn import or yarn add for the following dependencies.") From 25cbcf795b95d8d1038984ec84f92d71f0053e8a Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 10:41:31 -0700 Subject: [PATCH 03/15] Check if git can be called in pipeline --- dependency-verifier.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dependency-verifier.py b/dependency-verifier.py index ba0ae6a514..3eff5d3738 100644 --- a/dependency-verifier.py +++ b/dependency-verifier.py @@ -1,3 +1,8 @@ import sys +import subprocess -sys.exit("The yarn.lock and package-lock appear to be out of sync. Update by doing yarn import or yarn add for the following dependencies.") +targetBranch = sys.argv[1] + +result = subprocess.getoutput(f"git diff --name-only {targetBranch}..") +sys.exit(result) +sys.exit(f"The yarn.lock and package-lock appear to be out of sync on {targetBranch}. Update by doing yarn import or yarn add for the following dependencies.") From d67969898ff8c78855f0f9066c0966f45b12c64e Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 10:59:20 -0700 Subject: [PATCH 04/15] Use var syntax --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ed71a00609..75992e6e0a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -92,7 +92,7 @@ stages: inputs: scriptSource: 'filePath' scriptPath: 'dependency-verifier.py' - arguments: 'System.PullRequest.TargetBranch' + arguments: '$(System.PullRequest.TargetBranch)' failOnStderr: true ##### Package and Publish ##### - job: Package From 0601849852b6fc6c3fbc412ecc5a90fe5e3e3ee5 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 11:09:48 -0700 Subject: [PATCH 05/15] Fetch branches beforehand --- dependency-verifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependency-verifier.py b/dependency-verifier.py index 3eff5d3738..fca7b9250a 100644 --- a/dependency-verifier.py +++ b/dependency-verifier.py @@ -2,7 +2,7 @@ import subprocess targetBranch = sys.argv[1] - +result = subprocess.getoutput(f"git fetch --all") result = subprocess.getoutput(f"git diff --name-only {targetBranch}..") sys.exit(result) sys.exit(f"The yarn.lock and package-lock appear to be out of sync on {targetBranch}. Update by doing yarn import or yarn add for the following dependencies.") From ce44e694f3293a1554729d5bb3986c94b344398c Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 11:26:57 -0700 Subject: [PATCH 06/15] PushGitBranchesToCheckRemote --- dependency-verifier.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/dependency-verifier.py b/dependency-verifier.py index fca7b9250a..dad3ec9d85 100644 --- a/dependency-verifier.py +++ b/dependency-verifier.py @@ -1,8 +1,33 @@ import sys import subprocess -targetBranch = sys.argv[1] -result = subprocess.getoutput(f"git fetch --all") -result = subprocess.getoutput(f"git diff --name-only {targetBranch}..") -sys.exit(result) -sys.exit(f"The yarn.lock and package-lock appear to be out of sync on {targetBranch}. Update by doing yarn import or yarn add for the following dependencies.") +def main(): + """Check if the dependency updates in package-lock are also updated in yarn.locks""" + targetBranch = sys.argv[1] # Script is called with PR Target Branch Name, Fulfilled by AzDo + subprocess.getoutput(f"git fetch --all") + sys.exit(subprocess.getoutput(f"git branch --all")) + VerifyDependencies(targetBranch) + sys.exit(0) + +def VerifyDependencies(targetBranch): + """Enumerate through all changed files to check diffs.""" + changedFiles = subprocess.getoutput(f"git diff --name-only {targetBranch}..") + npmLockFile = "package-lock.json" + + for file in changedFiles: + fileName = os.path.basename(os.path.realpath(file)) + if fileName == npmLockFile: + NpmChangesMirrorYarnChanges(changedFiles, file, targetBranch) + +def NpmChangesMirrorYarnChanges(changedFiles, packageLockPath, targetBranch): + """Returns successfully if yarn.lock matches packagelock changes, if not, throws exit code""" + yarnLockFile = "yarn.lock" + yarnLockPath = os.path.join(os.path.dirname(packageLockPath), yarnLockFile) + + +main() + + + + +sys.exit(f"The yarn.lock and package-lock appear to be out of sync. Update by doing yarn import or yarn add for the following dependencies.") From a100f12418b2915fa31ac9293229115c62e2bfb4 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 11:30:36 -0700 Subject: [PATCH 07/15] Update checks to branches in azdo --- dependency-verifier.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dependency-verifier.py b/dependency-verifier.py index dad3ec9d85..245108092e 100644 --- a/dependency-verifier.py +++ b/dependency-verifier.py @@ -23,6 +23,16 @@ def NpmChangesMirrorYarnChanges(changedFiles, packageLockPath, targetBranch): """Returns successfully if yarn.lock matches packagelock changes, if not, throws exit code""" yarnLockFile = "yarn.lock" yarnLockPath = os.path.join(os.path.dirname(packageLockPath), yarnLockFile) + outOfDateYarnLocks = [] + + if yarnLockPath in changedFiles: + yarnDiff = subprocess.getoutput(f"git diff {targetBranch}.. {yarnLockPath}") + npmDiff = subprocess.getoutput(f"git diff {targetBranch}.. {packageLockPath}") + + else: + outOfDateYarnLocks += yarnLockPath + sys.exit(f"The yarn.lock and package-lock appear to be out of sync. Update by doing yarn import or yarn add for {outOfDateYarnLocks}.") + main() @@ -30,4 +40,3 @@ def NpmChangesMirrorYarnChanges(changedFiles, packageLockPath, targetBranch): -sys.exit(f"The yarn.lock and package-lock appear to be out of sync. Update by doing yarn import or yarn add for the following dependencies.") From bc5ecd470a02b0b7c5102e28c889c1c1054d025a Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 11:39:55 -0700 Subject: [PATCH 08/15] Yarn lock checker --- dependency-verifier.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/dependency-verifier.py b/dependency-verifier.py index 245108092e..f733da598e 100644 --- a/dependency-verifier.py +++ b/dependency-verifier.py @@ -28,11 +28,35 @@ def NpmChangesMirrorYarnChanges(changedFiles, packageLockPath, targetBranch): if yarnLockPath in changedFiles: yarnDiff = subprocess.getoutput(f"git diff {targetBranch}.. {yarnLockPath}") npmDiff = subprocess.getoutput(f"git diff {targetBranch}.. {packageLockPath}") - + if(DiffsMatch(yarnDiff, npmDiff): + ; + else: + outOfDateYarnLocks += yarnLockPath else: outOfDateYarnLocks += yarnLockPath - sys.exit(f"The yarn.lock and package-lock appear to be out of sync. Update by doing yarn import or yarn add for {outOfDateYarnLocks}.") + if(outOfDateYarnLocks != []): + sys.exit(f"The yarn.lock and package-lock appear to be out of sync. Update by doing yarn import or yarn add for {outOfDateYarnLocks}.") + else: + return 0 # OK, status here is not used + +def GetNpmDependencyUpdates(packageLockDiff): + """Returns a dictionary of [dependency -> version] changes found in diff string of package-lock.json""" + pass + +def GetYarnDependencyUpdates(yarnLockDiff): + """Returns a dictionary of [dependency -> version] changes found in diff string of yarn.lock""" + pass +def DiffsMatch(yarnDiff, npmDiff): + """Returns true if dependency updates are reflected in both diffs.""" + yarnDeps = GetYarnDependencyUpdates(yarnDiff) + npmDeps = GetNpmDependencyUpdates(npmDiff) + for dep in npmDeps: + if dep in yarnDeps and yarnDeps[dep] == npmDeps[dep]: # version changes match + continue + else: + return False + return True main() From 143611c90a001b535fd1b8982f46f8bb5e7f7d01 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 12:06:10 -0700 Subject: [PATCH 09/15] Add code to grab dependencies --- dependency-verifier.py | 60 +++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/dependency-verifier.py b/dependency-verifier.py index f733da598e..721abffc51 100644 --- a/dependency-verifier.py +++ b/dependency-verifier.py @@ -5,7 +5,7 @@ def main(): """Check if the dependency updates in package-lock are also updated in yarn.locks""" targetBranch = sys.argv[1] # Script is called with PR Target Branch Name, Fulfilled by AzDo subprocess.getoutput(f"git fetch --all") - sys.exit(subprocess.getoutput(f"git branch --all")) + sys.exit(subprocess.getoutput(f"git pull origin {targetBranch}")) VerifyDependencies(targetBranch) sys.exit(0) @@ -19,6 +19,38 @@ def VerifyDependencies(targetBranch): if fileName == npmLockFile: NpmChangesMirrorYarnChanges(changedFiles, file, targetBranch) +def GetNpmDependencyUpdates(packageLockDiff): + """Returns a dictionary of [dependency -> [] (can be changed to version in later implementations)] changes found in diff string of package-lock.json""" + # Assumes dependency line starts with "node_modules/DEPENDENCYNAME". Version may or may not come after + dependencies = {} + for line in packageLockDiff.splitlines(): + if line.strip().startswith("node_modules/"): + dependencies[line.strip().split("node_modules/", 1)[1]] = [] + return dependencies + +def GetYarnDependencyUpdates(yarnLockDiff): + """Returns a dictionary of [dependency -> [] (can be changed to version in later implementations)] changes found in diff string of yarn.lock""" + # Assumes dependency line starts with "DEPEDENCY@Version" + dependencies = {} + for line in yarnLockDiff.splitlines(): + if line.startswith('"'): + depAtVers = line.split('"', 1)[1] + dep = depAtVers.rsplit("@", 1)[0] + vers = depAtVers.rsplit("@", 1)[1] + dependencies[dep] = [] # Could add version here later. (TODO) that will probably not happen + return dependencies + +def DiffsMatch(yarnDiff, npmDiff): + """Returns true if dependency updates are reflected in both diffs.""" + yarnDeps = GetYarnDependencyUpdates(yarnDiff) + npmDeps = GetNpmDependencyUpdates(npmDiff) + for dep in npmDeps: + if dep in yarnDeps and yarnDeps[dep] == npmDeps[dep]: # version changes match + continue + else: + return False + return True + def NpmChangesMirrorYarnChanges(changedFiles, packageLockPath, targetBranch): """Returns successfully if yarn.lock matches packagelock changes, if not, throws exit code""" yarnLockFile = "yarn.lock" @@ -28,37 +60,17 @@ def NpmChangesMirrorYarnChanges(changedFiles, packageLockPath, targetBranch): if yarnLockPath in changedFiles: yarnDiff = subprocess.getoutput(f"git diff {targetBranch}.. {yarnLockPath}") npmDiff = subprocess.getoutput(f"git diff {targetBranch}.. {packageLockPath}") - if(DiffsMatch(yarnDiff, npmDiff): - ; + if DiffsMatch(yarnDiff, npmDiff): + pass else: outOfDateYarnLocks += yarnLockPath else: outOfDateYarnLocks += yarnLockPath if(outOfDateYarnLocks != []): - sys.exit(f"The yarn.lock and package-lock appear to be out of sync. Update by doing yarn import or yarn add for {outOfDateYarnLocks}.") + sys.exit(f"The yarn.lock and package-lock appear to be out of sync with the changes made after {targetBranch}. Update by doing yarn import or yarn add for {outOfDateYarnLocks}.") else: return 0 # OK, status here is not used -def GetNpmDependencyUpdates(packageLockDiff): - """Returns a dictionary of [dependency -> version] changes found in diff string of package-lock.json""" - pass - -def GetYarnDependencyUpdates(yarnLockDiff): - """Returns a dictionary of [dependency -> version] changes found in diff string of yarn.lock""" - pass - -def DiffsMatch(yarnDiff, npmDiff): - """Returns true if dependency updates are reflected in both diffs.""" - yarnDeps = GetYarnDependencyUpdates(yarnDiff) - npmDeps = GetNpmDependencyUpdates(npmDiff) - for dep in npmDeps: - if dep in yarnDeps and yarnDeps[dep] == npmDeps[dep]: # version changes match - continue - else: - return False - return True - - main() From 7fdd33df6a150e9ba48c41ba16605d2bd9696f5a Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 12:16:15 -0700 Subject: [PATCH 10/15] Improve code quality --- dependency-verifier.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/dependency-verifier.py b/dependency-verifier.py index 721abffc51..7d6d20f205 100644 --- a/dependency-verifier.py +++ b/dependency-verifier.py @@ -1,11 +1,12 @@ import sys import subprocess +import os def main(): """Check if the dependency updates in package-lock are also updated in yarn.locks""" targetBranch = sys.argv[1] # Script is called with PR Target Branch Name, Fulfilled by AzDo subprocess.getoutput(f"git fetch --all") - sys.exit(subprocess.getoutput(f"git pull origin {targetBranch}")) + subprocess.getoutput(f"git pull origin {targetBranch}") VerifyDependencies(targetBranch) sys.exit(0) @@ -30,7 +31,7 @@ def GetNpmDependencyUpdates(packageLockDiff): def GetYarnDependencyUpdates(yarnLockDiff): """Returns a dictionary of [dependency -> [] (can be changed to version in later implementations)] changes found in diff string of yarn.lock""" - # Assumes dependency line starts with "DEPEDENCY@Version" + # Assumes dependency line starts with "DEPEDENCY@Version dependencies = {} for line in yarnLockDiff.splitlines(): if line.startswith('"'): @@ -67,11 +68,12 @@ def NpmChangesMirrorYarnChanges(changedFiles, packageLockPath, targetBranch): else: outOfDateYarnLocks += yarnLockPath if(outOfDateYarnLocks != []): - sys.exit(f"The yarn.lock and package-lock appear to be out of sync with the changes made after {targetBranch}. Update by doing yarn import or yarn add for {outOfDateYarnLocks}.") + sys.exit(f"The yarn.lock and package-lock appear to be out of sync with the changes made after {targetBranch}. Update by doing yarn import or yarn add dep@package-lock-version for {outOfDateYarnLocks}.") else: return 0 # OK, status here is not used - -main() + +if __name__ == "__main__": + main() From cab88daac65ab18fdfe3fb4cb5852b59676475fa Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 12:37:29 -0700 Subject: [PATCH 11/15] Bug fixes --- dependency-verifier.py | 5 +- .../package-lock.json | 81 +++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/dependency-verifier.py b/dependency-verifier.py index 7d6d20f205..fcdf2a026e 100644 --- a/dependency-verifier.py +++ b/dependency-verifier.py @@ -4,7 +4,7 @@ def main(): """Check if the dependency updates in package-lock are also updated in yarn.locks""" - targetBranch = sys.argv[1] # Script is called with PR Target Branch Name, Fulfilled by AzDo + targetBranch = "nagilson-refresh" # sys.argv[1] # Script is called with PR Target Branch Name, Fulfilled by AzDo subprocess.getoutput(f"git fetch --all") subprocess.getoutput(f"git pull origin {targetBranch}") VerifyDependencies(targetBranch) @@ -12,7 +12,8 @@ def main(): def VerifyDependencies(targetBranch): """Enumerate through all changed files to check diffs.""" - changedFiles = subprocess.getoutput(f"git diff --name-only {targetBranch}..") + # origin/ requires origin/ to be up to date. + changedFiles = subprocess.getoutput(f"git diff --name-only origin/{targetBranch}..").splitlines() npmLockFile = "package-lock.json" for file in changedFiles: diff --git a/vscode-dotnet-runtime-extension/package-lock.json b/vscode-dotnet-runtime-extension/package-lock.json index 509ad7a263..f10ba07dc1 100644 --- a/vscode-dotnet-runtime-extension/package-lock.json +++ b/vscode-dotnet-runtime-extension/package-lock.json @@ -19,6 +19,7 @@ "open": "^8.4.0", "rimraf": "3.0.2", "shelljs": "^0.8.4", + "sudo": "^1.0.3", "ts-loader": "^9.2.6", "tslint": "^5.20.1", "typescript": "4.4.4", @@ -1980,6 +1981,14 @@ "integrity": "sha1-D6LGT5MpF8NDOg3tVTY6rjdBa3w=", "license": "ISC" }, + "node_modules/inpath": { + "version": "1.0.2", + "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/inpath/-/inpath-1.0.2.tgz", + "integrity": "sha1-SsIZcQ7Hpy9GD/lL9CTdPvDlKBc=", + "engines": { + "node": "*" + } + }, "node_modules/interpret": { "version": "1.4.0", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/interpret/-/interpret-1.4.0.tgz", @@ -2537,6 +2546,12 @@ "integrity": "sha1-V0yBOM4dK1hh8LRFedut1gxmFbI=", "license": "MIT" }, + "node_modules/mute-stream": { + "version": "0.0.8", + "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha1-FjDEKyJR/4HiooPelqVJfqkuXg0=", + "license": "ISC" + }, "node_modules/nanoid": { "version": "3.3.1", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/nanoid/-/nanoid-3.3.1.tgz", @@ -2810,6 +2825,14 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pidof": { + "version": "1.0.2", + "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/pidof/-/pidof-1.0.2.tgz", + "integrity": "sha1-+6Dq4cgzWhHrgJn10PPvvEXLTpA=", + "engines": { + "node": "*" + } + }, "node_modules/pkg-dir": { "version": "4.2.0", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -3123,6 +3146,18 @@ "safe-buffer": "^5.1.0" } }, + "node_modules/read": { + "version": "1.0.7", + "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/read/-/read-1.0.7.tgz", + "integrity": "sha1-s9oZvQUkMal2cdRKQmNK33ELQMQ=", + "license": "ISC", + "dependencies": { + "mute-stream": "~0.0.4" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/readable-stream": { "version": "2.3.7", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/readable-stream/-/readable-stream-2.3.7.tgz", @@ -3507,6 +3542,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/sudo": { + "version": "1.0.3", + "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/sudo/-/sudo-1.0.3.tgz", + "integrity": "sha1-zPKGaRIPi3T4K4Rt/38clRIO/yA=", + "dependencies": { + "inpath": "~1.0.2", + "pidof": "~1.0.2", + "read": "~1.0.3" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/supports-color": { "version": "8.1.1", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/supports-color/-/supports-color-8.1.1.tgz", @@ -5561,6 +5609,11 @@ "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/inherits/-/inherits-2.0.4.tgz", "integrity": "sha1-D6LGT5MpF8NDOg3tVTY6rjdBa3w=" }, + "inpath": { + "version": "1.0.2", + "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/inpath/-/inpath-1.0.2.tgz", + "integrity": "sha1-SsIZcQ7Hpy9GD/lL9CTdPvDlKBc=" + }, "interpret": { "version": "1.4.0", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/interpret/-/interpret-1.4.0.tgz", @@ -5925,6 +5978,11 @@ "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/ms/-/ms-2.1.3.tgz", "integrity": "sha1-V0yBOM4dK1hh8LRFedut1gxmFbI=" }, + "mute-stream": { + "version": "0.0.8", + "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha1-FjDEKyJR/4HiooPelqVJfqkuXg0=" + }, "nanoid": { "version": "3.3.1", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/nanoid/-/nanoid-3.3.1.tgz", @@ -6086,6 +6144,11 @@ "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha1-O6ODNzNkbZ0+SZWUbBNlpn+wekI=" }, + "pidof": { + "version": "1.0.2", + "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/pidof/-/pidof-1.0.2.tgz", + "integrity": "sha1-+6Dq4cgzWhHrgJn10PPvvEXLTpA=" + }, "pkg-dir": { "version": "4.2.0", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -6300,6 +6363,14 @@ "safe-buffer": "^5.1.0" } }, + "read": { + "version": "1.0.7", + "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/read/-/read-1.0.7.tgz", + "integrity": "sha1-s9oZvQUkMal2cdRKQmNK33ELQMQ=", + "requires": { + "mute-stream": "~0.0.4" + } + }, "readable-stream": { "version": "2.3.7", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/readable-stream/-/readable-stream-2.3.7.tgz", @@ -6548,6 +6619,16 @@ "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha1-MfEoGzgyYwQ0gxwxDAHMzajL4AY=" }, + "sudo": { + "version": "1.0.3", + "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/sudo/-/sudo-1.0.3.tgz", + "integrity": "sha1-zPKGaRIPi3T4K4Rt/38clRIO/yA=", + "requires": { + "inpath": "~1.0.2", + "pidof": "~1.0.2", + "read": "~1.0.3" + } + }, "supports-color": { "version": "8.1.1", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/supports-color/-/supports-color-8.1.1.tgz", From 6e7e68ed19cf817a603bd497d46785410ccca0a9 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 15:00:26 -0700 Subject: [PATCH 12/15] Several fixes to depedency verifier --- dependency-verifier.py | 66 ++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/dependency-verifier.py b/dependency-verifier.py index fcdf2a026e..3d09b54e92 100644 --- a/dependency-verifier.py +++ b/dependency-verifier.py @@ -1,6 +1,8 @@ import sys import subprocess import os +from pathlib import Path + def main(): """Check if the dependency updates in package-lock are also updated in yarn.locks""" @@ -13,7 +15,7 @@ def main(): def VerifyDependencies(targetBranch): """Enumerate through all changed files to check diffs.""" # origin/ requires origin/ to be up to date. - changedFiles = subprocess.getoutput(f"git diff --name-only origin/{targetBranch}..").splitlines() + changedFiles = [Path(path) for path in subprocess.getoutput(f"git diff --name-only origin/{targetBranch}..").splitlines()] npmLockFile = "package-lock.json" for file in changedFiles: @@ -21,61 +23,63 @@ def VerifyDependencies(targetBranch): if fileName == npmLockFile: NpmChangesMirrorYarnChanges(changedFiles, file, targetBranch) -def GetNpmDependencyUpdates(packageLockDiff): +def GetNpmDependencyUpdates(packageLockDiffLines): """Returns a dictionary of [dependency -> [] (can be changed to version in later implementations)] changes found in diff string of package-lock.json""" # Assumes dependency line starts with "node_modules/DEPENDENCYNAME". Version may or may not come after dependencies = {} - for line in packageLockDiff.splitlines(): - if line.strip().startswith("node_modules/"): - dependencies[line.strip().split("node_modules/", 1)[1]] = [] + for line in packageLockDiffLines: + line = line.strip() + line = line.lstrip("\t") + if line.startswith('"node_modules/'): + dependencies[line.split('"node_modules/', 1)[1].split('"', 1)[0]] = [] # will be "node_modules/dep further" content, need to cull return dependencies -def GetYarnDependencyUpdates(yarnLockDiff): +def GetYarnDependencyUpdates(yarnLockDiffLines): """Returns a dictionary of [dependency -> [] (can be changed to version in later implementations)] changes found in diff string of yarn.lock""" - # Assumes dependency line starts with "DEPEDENCY@Version + # Assumes dependency line starts with DEPEDENCY@Version without whitespace dependencies = {} - for line in yarnLockDiff.splitlines(): - if line.startswith('"'): - depAtVers = line.split('"', 1)[1] - dep = depAtVers.rsplit("@", 1)[0] - vers = depAtVers.rsplit("@", 1)[1] - dependencies[dep] = [] # Could add version here later. (TODO) that will probably not happen + for line in yarnLockDiffLines: + if line == line.lstrip() and "@" in line: + depsAtVers = line.lstrip('"').split(",") # multiple dependencies are possible with diff versions, sep by , + for dependencyAtVers in depsAtVers: + dep = dependencyAtVers.rsplit("@", 1)[0] + vers = dependencyAtVers.rsplit("@", 1)[1] + dependencies[dep] = [] # Could add version here later. (TODO) that will probably not happen return dependencies -def DiffsMatch(yarnDiff, npmDiff): - """Returns true if dependency updates are reflected in both diffs.""" - yarnDeps = GetYarnDependencyUpdates(yarnDiff) - npmDeps = GetNpmDependencyUpdates(npmDiff) +def GetUnmatchedDiffs(yarnDiff, npmDiff): + """Returns [] if dependency updates are reflected in both diffs, elsewise the dependencies out of sync.""" + # v Remove + or - from diff and additional git diff context lines + yarnDeps = GetYarnDependencyUpdates([line[1:] for line in yarnDiff.splitlines() if line.startswith("+") or line.startswith("-")]) + npmDeps = GetNpmDependencyUpdates([line[1:] for line in npmDiff.splitlines() if line.startswith("+") or line.startswith("-")]) + outOfSyncDependencies = [] for dep in npmDeps: if dep in yarnDeps and yarnDeps[dep] == npmDeps[dep]: # version changes match continue else: - return False - return True + outOfSyncDependencies.append(dep) + return outOfSyncDependencies def NpmChangesMirrorYarnChanges(changedFiles, packageLockPath, targetBranch): """Returns successfully if yarn.lock matches packagelock changes, if not, throws exit code""" yarnLockFile = "yarn.lock" - yarnLockPath = os.path.join(os.path.dirname(packageLockPath), yarnLockFile) + yarnLockPath = Path(os.path.join(os.path.dirname(packageLockPath), yarnLockFile)) outOfDateYarnLocks = [] if yarnLockPath in changedFiles: - yarnDiff = subprocess.getoutput(f"git diff {targetBranch}.. {yarnLockPath}") - npmDiff = subprocess.getoutput(f"git diff {targetBranch}.. {packageLockPath}") - if DiffsMatch(yarnDiff, npmDiff): + yarnDiff = subprocess.getoutput(f"git diff origin/{targetBranch}.. -- {str(yarnLockPath)}") + npmDiff = subprocess.getoutput(f"git diff origin/{targetBranch}.. -- {packageLockPath}") + diffSetComplement = GetUnmatchedDiffs(yarnDiff, npmDiff) + if diffSetComplement == []: pass else: - outOfDateYarnLocks += yarnLockPath + outOfDateYarnLocks.append((str(yarnLockPath), diffSetComplement)) else: - outOfDateYarnLocks += yarnLockPath + outOfDateYarnLocks.append(yarnLockPath) if(outOfDateYarnLocks != []): - sys.exit(f"The yarn.lock and package-lock appear to be out of sync with the changes made after {targetBranch}. Update by doing yarn import or yarn add dep@package-lock-version for {outOfDateYarnLocks}.") + sys.exit(f"The yarn.lock and package-lock appear to be out of sync with the changes made after {targetBranch}. Update by doing yarn import or yarn add dep@package-lock-version for {outOfDateYarnLocks}. For sub-dependencies, try adding just the main dependency first.") else: return 0 # OK, status here is not used if __name__ == "__main__": - main() - - - - + main() \ No newline at end of file From d4d543a1017a9bf4d482f906fd5b5184688b6322 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 15:01:25 -0700 Subject: [PATCH 13/15] pull nongeneric branch now that testing is done --- dependency-verifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependency-verifier.py b/dependency-verifier.py index 3d09b54e92..8f058c53c5 100644 --- a/dependency-verifier.py +++ b/dependency-verifier.py @@ -6,7 +6,7 @@ def main(): """Check if the dependency updates in package-lock are also updated in yarn.locks""" - targetBranch = "nagilson-refresh" # sys.argv[1] # Script is called with PR Target Branch Name, Fulfilled by AzDo + targetBranch = sys.argv[1] # Script is called with PR Target Branch Name, Fulfilled by AzDo subprocess.getoutput(f"git fetch --all") subprocess.getoutput(f"git pull origin {targetBranch}") VerifyDependencies(targetBranch) From 4b9820d536d12ea589addc104af4b00eb7d8524b Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 15:07:53 -0700 Subject: [PATCH 14/15] undo sudo testing install --- .../package-lock.json | 81 ------------------- 1 file changed, 81 deletions(-) diff --git a/vscode-dotnet-runtime-extension/package-lock.json b/vscode-dotnet-runtime-extension/package-lock.json index f10ba07dc1..509ad7a263 100644 --- a/vscode-dotnet-runtime-extension/package-lock.json +++ b/vscode-dotnet-runtime-extension/package-lock.json @@ -19,7 +19,6 @@ "open": "^8.4.0", "rimraf": "3.0.2", "shelljs": "^0.8.4", - "sudo": "^1.0.3", "ts-loader": "^9.2.6", "tslint": "^5.20.1", "typescript": "4.4.4", @@ -1981,14 +1980,6 @@ "integrity": "sha1-D6LGT5MpF8NDOg3tVTY6rjdBa3w=", "license": "ISC" }, - "node_modules/inpath": { - "version": "1.0.2", - "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/inpath/-/inpath-1.0.2.tgz", - "integrity": "sha1-SsIZcQ7Hpy9GD/lL9CTdPvDlKBc=", - "engines": { - "node": "*" - } - }, "node_modules/interpret": { "version": "1.4.0", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/interpret/-/interpret-1.4.0.tgz", @@ -2546,12 +2537,6 @@ "integrity": "sha1-V0yBOM4dK1hh8LRFedut1gxmFbI=", "license": "MIT" }, - "node_modules/mute-stream": { - "version": "0.0.8", - "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/mute-stream/-/mute-stream-0.0.8.tgz", - "integrity": "sha1-FjDEKyJR/4HiooPelqVJfqkuXg0=", - "license": "ISC" - }, "node_modules/nanoid": { "version": "3.3.1", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/nanoid/-/nanoid-3.3.1.tgz", @@ -2825,14 +2810,6 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/pidof": { - "version": "1.0.2", - "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/pidof/-/pidof-1.0.2.tgz", - "integrity": "sha1-+6Dq4cgzWhHrgJn10PPvvEXLTpA=", - "engines": { - "node": "*" - } - }, "node_modules/pkg-dir": { "version": "4.2.0", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -3146,18 +3123,6 @@ "safe-buffer": "^5.1.0" } }, - "node_modules/read": { - "version": "1.0.7", - "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/read/-/read-1.0.7.tgz", - "integrity": "sha1-s9oZvQUkMal2cdRKQmNK33ELQMQ=", - "license": "ISC", - "dependencies": { - "mute-stream": "~0.0.4" - }, - "engines": { - "node": ">=0.8" - } - }, "node_modules/readable-stream": { "version": "2.3.7", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/readable-stream/-/readable-stream-2.3.7.tgz", @@ -3542,19 +3507,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/sudo": { - "version": "1.0.3", - "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/sudo/-/sudo-1.0.3.tgz", - "integrity": "sha1-zPKGaRIPi3T4K4Rt/38clRIO/yA=", - "dependencies": { - "inpath": "~1.0.2", - "pidof": "~1.0.2", - "read": "~1.0.3" - }, - "engines": { - "node": ">=0.8" - } - }, "node_modules/supports-color": { "version": "8.1.1", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/supports-color/-/supports-color-8.1.1.tgz", @@ -5609,11 +5561,6 @@ "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/inherits/-/inherits-2.0.4.tgz", "integrity": "sha1-D6LGT5MpF8NDOg3tVTY6rjdBa3w=" }, - "inpath": { - "version": "1.0.2", - "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/inpath/-/inpath-1.0.2.tgz", - "integrity": "sha1-SsIZcQ7Hpy9GD/lL9CTdPvDlKBc=" - }, "interpret": { "version": "1.4.0", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/interpret/-/interpret-1.4.0.tgz", @@ -5978,11 +5925,6 @@ "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/ms/-/ms-2.1.3.tgz", "integrity": "sha1-V0yBOM4dK1hh8LRFedut1gxmFbI=" }, - "mute-stream": { - "version": "0.0.8", - "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/mute-stream/-/mute-stream-0.0.8.tgz", - "integrity": "sha1-FjDEKyJR/4HiooPelqVJfqkuXg0=" - }, "nanoid": { "version": "3.3.1", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/nanoid/-/nanoid-3.3.1.tgz", @@ -6144,11 +6086,6 @@ "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha1-O6ODNzNkbZ0+SZWUbBNlpn+wekI=" }, - "pidof": { - "version": "1.0.2", - "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/pidof/-/pidof-1.0.2.tgz", - "integrity": "sha1-+6Dq4cgzWhHrgJn10PPvvEXLTpA=" - }, "pkg-dir": { "version": "4.2.0", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -6363,14 +6300,6 @@ "safe-buffer": "^5.1.0" } }, - "read": { - "version": "1.0.7", - "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/read/-/read-1.0.7.tgz", - "integrity": "sha1-s9oZvQUkMal2cdRKQmNK33ELQMQ=", - "requires": { - "mute-stream": "~0.0.4" - } - }, "readable-stream": { "version": "2.3.7", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/readable-stream/-/readable-stream-2.3.7.tgz", @@ -6619,16 +6548,6 @@ "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha1-MfEoGzgyYwQ0gxwxDAHMzajL4AY=" }, - "sudo": { - "version": "1.0.3", - "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/sudo/-/sudo-1.0.3.tgz", - "integrity": "sha1-zPKGaRIPi3T4K4Rt/38clRIO/yA=", - "requires": { - "inpath": "~1.0.2", - "pidof": "~1.0.2", - "read": "~1.0.3" - } - }, "supports-color": { "version": "8.1.1", "resolved": "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/supports-color/-/supports-color-8.1.1.tgz", From 0bd81aa1d996aead5eda8361dafa64ad645cc885 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 6 Oct 2022 15:10:31 -0700 Subject: [PATCH 15/15] Add warning that origin/targetbranch must be up to date --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 75992e6e0a..7a55581246 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -79,7 +79,7 @@ stages: displayName: Run Lint ##### Verify NPM and Yarn are in sync ##### - job: SyncPackageManagers - displayName: 'Verify NPM & Yarn In-Sync' + displayName: 'Verify NPM & Yarn In-Sync [Local Copy of Target Branch Must Be Up to Date]' pool: vmImage: 'windows-latest' steps: @@ -250,4 +250,4 @@ stages: npm install -g vsce vsce publish --packagePath vscode-dotnet-sdk-$(version).vsix -p $(VSCODE_MARKETPLACE_TOKEN) displayName: 'Publish to Marketplace' - workingDirectory: '$(System.ArtifactsDirectory)/vscode-dotnet-sdk-extension' \ No newline at end of file + workingDirectory: '$(System.ArtifactsDirectory)/vscode-dotnet-sdk-extension'