From 1d8d861ff468d3885a712416aedb4939d7d30894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20So=C3=B3s?= Date: Wed, 15 Nov 2023 14:29:22 +0100 Subject: [PATCH] Use pana-integrated dartdoc instead of pkg/pub_dartdoc. (#7175) --- CHANGELOG.md | 3 + app/lib/shared/versions.dart | 4 +- app/pubspec.lock | 4 +- app/pubspec.yaml | 2 +- app/test/shared/versions_test.dart | 7 + .../testdata/goldens/packages/oxygen.html | 4 +- .../goldens/packages/oxygen/changelog.html | 4 +- .../goldens/packages/oxygen/example.html | 4 +- .../goldens/packages/oxygen/install.html | 4 +- .../goldens/packages/oxygen/license.html | 4 +- .../goldens/packages/oxygen/score.html | 38 ++-- .../goldens/packages/oxygen/versions.html | 4 +- .../packages/oxygen/versions/1.0.0.html | 4 +- .../oxygen/versions/1.0.0/changelog.html | 4 +- .../oxygen/versions/1.0.0/example.html | 4 +- .../oxygen/versions/1.0.0/install.html | 4 +- .../oxygen/versions/1.0.0/license.html | 4 +- .../packages/oxygen/versions/1.0.0/score.html | 38 ++-- .../packages/oxygen/versions/2.0.0.html | 4 +- pkg/pub_worker/lib/src/analyze.dart | 45 +--- .../lib/src/bin/dartdoc_wrapper.dart | 194 +----------------- pkg/pub_worker/lib/src/bin/pana_wrapper.dart | 41 ++-- pkg/pub_worker/pubspec.lock | 4 +- pkg/pub_worker/pubspec.yaml | 2 +- 24 files changed, 114 insertions(+), 316 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59dbd86816..39ba8af9a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ Important changes to data models, configuration, and migrations between each AppEngine version, listed here to ease deployment and troubleshooting. ## Next Release (replace with git tag when deployed) + * Bumped runtimeVersion to `2023.11.15`. + * Upgraded pana to `0.21.42`. + * Note: Started running `dartdoc` from `pana`. ## `20231115t090700-all` * Hiding invalid popularity scores. diff --git a/app/lib/shared/versions.dart b/app/lib/shared/versions.dart index 9234559ea5..af37a5b989 100644 --- a/app/lib/shared/versions.dart +++ b/app/lib/shared/versions.dart @@ -24,10 +24,10 @@ final RegExp runtimeVersionPattern = RegExp(r'^\d{4}\.\d{2}\.\d{2}$'); /// when the version switch happens. const _acceptedRuntimeVersions = [ // The current [runtimeVersion]. - '2023.11.02', + '2023.11.15', // Fallback runtime versions. + '2023.11.02', '2023.10.18', - '2023.10.10', ]; /// Sets the current runtime versions. diff --git a/app/pubspec.lock b/app/pubspec.lock index 648f6d1213..0eaa851268 100644 --- a/app/pubspec.lock +++ b/app/pubspec.lock @@ -544,10 +544,10 @@ packages: dependency: "direct main" description: name: pana - sha256: "0a2546792c71440b69e4dd73d9cd310dab6404c18ce6c0e8ceeb1453249c5d27" + sha256: d8c24d31fbf66b9c8c6c5e31c39b094c8535ed5bd152dbce9642049e6e76965d url: "https://pub.dev" source: hosted - version: "0.21.40" + version: "0.21.42" path: dependency: "direct main" description: diff --git a/app/pubspec.yaml b/app/pubspec.yaml index 3a77da6eb1..37a0745d86 100644 --- a/app/pubspec.yaml +++ b/app/pubspec.yaml @@ -53,7 +53,7 @@ dependencies: watcher: ^1.0.0 yaml: '^3.0.0' # pana version to be pinned - pana: '0.21.40' + pana: '0.21.42' # 3rd-party packages with pinned versions mailer: '6.0.1' ulid: '2.0.0' diff --git a/app/test/shared/versions_test.dart b/app/test/shared/versions_test.dart index 48c06ad5ef..8684d9e6cc 100644 --- a/app/test/shared/versions_test.dart +++ b/app/test/shared/versions_test.dart @@ -141,6 +141,13 @@ and do not format to also bump the runtimeVersion.''', expect(dependency.version.toString(), dartdocVersion); }); + test('dartdoc version should match pkg/pub_worker', () async { + final content = + await File('../pkg/pub_worker/lib/src/bin/pana_wrapper.dart') + .readAsString(); + expect(content, contains("globalDartdocVersion: '$dartdocVersion'")); + }); + scopedTest('GC is not deleting currently accepted versions', () { for (final version in acceptedRuntimeVersions) { expect(shouldGCVersion(version), isFalse); diff --git a/app/test/task/testdata/goldens/packages/oxygen.html b/app/test/task/testdata/goldens/packages/oxygen.html index 0dbdd49771..d2e582e5ac 100644 --- a/app/test/task/testdata/goldens/packages/oxygen.html +++ b/app/test/task/testdata/goldens/packages/oxygen.html @@ -234,7 +234,7 @@

- 110 + 120
pub points
@@ -320,7 +320,7 @@

- 110 + 120
pub points
diff --git a/app/test/task/testdata/goldens/packages/oxygen/changelog.html b/app/test/task/testdata/goldens/packages/oxygen/changelog.html index 12cfdff345..e16c62b6ed 100644 --- a/app/test/task/testdata/goldens/packages/oxygen/changelog.html +++ b/app/test/task/testdata/goldens/packages/oxygen/changelog.html @@ -240,7 +240,7 @@

- 110 + 120
pub points
@@ -326,7 +326,7 @@

- 110 + 120
pub points
diff --git a/app/test/task/testdata/goldens/packages/oxygen/example.html b/app/test/task/testdata/goldens/packages/oxygen/example.html index 79dad0b042..c584229035 100644 --- a/app/test/task/testdata/goldens/packages/oxygen/example.html +++ b/app/test/task/testdata/goldens/packages/oxygen/example.html @@ -235,7 +235,7 @@

Metadata

- 110 + 120
pub points
@@ -321,7 +321,7 @@

- 110 + 120
pub points
diff --git a/app/test/task/testdata/goldens/packages/oxygen/install.html b/app/test/task/testdata/goldens/packages/oxygen/install.html index 08212c00e7..46a59c43c0 100644 --- a/app/test/task/testdata/goldens/packages/oxygen/install.html +++ b/app/test/task/testdata/goldens/packages/oxygen/install.html @@ -263,7 +263,7 @@

Import it

- 110 + 120
pub points
@@ -349,7 +349,7 @@

- 110 + 120
pub points
diff --git a/app/test/task/testdata/goldens/packages/oxygen/license.html b/app/test/task/testdata/goldens/packages/oxygen/license.html index 6abdce7de3..5ef6fc4684 100644 --- a/app/test/task/testdata/goldens/packages/oxygen/license.html +++ b/app/test/task/testdata/goldens/packages/oxygen/license.html @@ -236,7 +236,7 @@

License

- 110 + 120
pub points
@@ -322,7 +322,7 @@

- 110 + 120
pub points
diff --git a/app/test/task/testdata/goldens/packages/oxygen/score.html b/app/test/task/testdata/goldens/packages/oxygen/score.html index 90aa805622..75ebe316d0 100644 --- a/app/test/task/testdata/goldens/packages/oxygen/score.html +++ b/app/test/task/testdata/goldens/packages/oxygen/score.html @@ -212,7 +212,7 @@

Metadata

- 110 + 120 / 140
pub points
@@ -228,7 +228,7 @@

Metadata

We analyzed this package %%time-ago%% - , and awarded it 110 pub points (of a possible 140): + , and awarded it 120 pub points (of a possible 140):

@@ -362,11 +362,11 @@

- icon indicating section status + icon indicating section status
Provide documentation
-
- 10 +
+ 20 / 20 icon to trigger folding of the section @@ -377,16 +377,26 @@

- 10/10 points: Package has an example and has no issues with screenshots + 10/10 points: 20% or more of the public API has dartdoc comments

+

2 out of 9 API elements (22.2 %) have documentation comments.

+

+ Some symbols that are missing documentation: + oxygen + , + MainClass + , + MainClass + , + text + , + TypeEnum + . +

- - 0/10 points: 20% or more of the public API has dartdoc comments + + 10/10 points: Package has an example and has no issues with screenshots

-
    -
  • 1 out of 11 API elements (9.1 %) have documentation comments.
    • -
-

Providing good documentation for libraries, classes, functions, and other API elements improves code readability and helps developers find and use your API. Document at least 20% of the public API elements.

@@ -533,7 +543,7 @@

- 110 + 120
pub points
@@ -619,7 +629,7 @@

- 110 + 120
pub points
diff --git a/app/test/task/testdata/goldens/packages/oxygen/versions.html b/app/test/task/testdata/goldens/packages/oxygen/versions.html index 3a1c9afd06..9c9ad75159 100644 --- a/app/test/task/testdata/goldens/packages/oxygen/versions.html +++ b/app/test/task/testdata/goldens/packages/oxygen/versions.html @@ -296,7 +296,7 @@

Stable versions of oxygen

- 110 + 120
pub points
@@ -382,7 +382,7 @@

- 110 + 120
pub points
diff --git a/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0.html b/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0.html index 51e4844b65..59910ea792 100644 --- a/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0.html +++ b/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0.html @@ -238,7 +238,7 @@

- 110 + 120
pub points
@@ -324,7 +324,7 @@

- 110 + 120
pub points
diff --git a/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/changelog.html b/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/changelog.html index 13ef4f59d8..ee69dace20 100644 --- a/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/changelog.html +++ b/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/changelog.html @@ -244,7 +244,7 @@

- 110 + 120
pub points
@@ -330,7 +330,7 @@

- 110 + 120
pub points
diff --git a/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/example.html b/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/example.html index c8ec596bae..794b99ff44 100644 --- a/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/example.html +++ b/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/example.html @@ -239,7 +239,7 @@

Metadata

- 110 + 120
pub points
@@ -325,7 +325,7 @@

- 110 + 120
pub points
diff --git a/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/install.html b/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/install.html index 788d24085b..fa4a27e084 100644 --- a/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/install.html +++ b/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/install.html @@ -267,7 +267,7 @@

Import it

- 110 + 120
pub points
@@ -353,7 +353,7 @@

- 110 + 120
pub points
diff --git a/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/license.html b/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/license.html index 4c9efa1fa7..af333cefd1 100644 --- a/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/license.html +++ b/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/license.html @@ -240,7 +240,7 @@

License

- 110 + 120
pub points
@@ -326,7 +326,7 @@

- 110 + 120
pub points
diff --git a/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/score.html b/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/score.html index 273a48bddd..24d4b3b381 100644 --- a/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/score.html +++ b/app/test/task/testdata/goldens/packages/oxygen/versions/1.0.0/score.html @@ -216,7 +216,7 @@

Metadata

- 110 + 120 / 140
pub points
@@ -232,7 +232,7 @@

Metadata

We analyzed this package %%time-ago%% - , and awarded it 110 pub points (of a possible 140): + , and awarded it 120 pub points (of a possible 140):

@@ -366,11 +366,11 @@

- icon indicating section status + icon indicating section status
Provide documentation
-
- 10 +
+ 20 / 20 icon to trigger folding of the section @@ -381,16 +381,26 @@

- 10/10 points: Package has an example and has no issues with screenshots + 10/10 points: 20% or more of the public API has dartdoc comments

+

2 out of 9 API elements (22.2 %) have documentation comments.

+

+ Some symbols that are missing documentation: + oxygen + , + MainClass + , + MainClass + , + text + , + TypeEnum + . +

- - 0/10 points: 20% or more of the public API has dartdoc comments + + 10/10 points: Package has an example and has no issues with screenshots

-
    -
  • 1 out of 11 API elements (9.1 %) have documentation comments.
    • -
-

Providing good documentation for libraries, classes, functions, and other API elements improves code readability and helps developers find and use your API. Document at least 20% of the public API elements.

@@ -537,7 +547,7 @@

- 110 + 120
pub points
@@ -623,7 +633,7 @@

- 110 + 120
pub points
diff --git a/app/test/task/testdata/goldens/packages/oxygen/versions/2.0.0.html b/app/test/task/testdata/goldens/packages/oxygen/versions/2.0.0.html index 0dbdd49771..d2e582e5ac 100644 --- a/app/test/task/testdata/goldens/packages/oxygen/versions/2.0.0.html +++ b/app/test/task/testdata/goldens/packages/oxygen/versions/2.0.0.html @@ -234,7 +234,7 @@

- 110 + 120
pub points
@@ -320,7 +320,7 @@

- 110 + 120
pub points
diff --git a/pkg/pub_worker/lib/src/analyze.dart b/pkg/pub_worker/lib/src/analyze.dart index 8f321020e4..bba97a3154 100644 --- a/pkg/pub_worker/lib/src/analyze.dart +++ b/pkg/pub_worker/lib/src/analyze.dart @@ -26,11 +26,9 @@ final _log = Logger('pub_worker.process_payload'); /// processing takes more than 55 minutes. const _workerTimeout = Duration(minutes: 55); -/// Stop dartdoc if it takes more than 45 minutes. -const _dartdocTimeout = Duration(minutes: 45); - -/// Stop pana if it takes more than 15 minutes. -const _panaTimeout = Duration(minutes: 15); +/// Stop pana if it takes more than 50 minutes. +/// This also includes pana running dartdoc. +const _panaTimeout = Duration(minutes: 50); List encodeJson(Object json) => JsonUtf8Encoder().convert(json); @@ -128,43 +126,6 @@ Future _analyzePackage( log.writeln('STARTED: ${clock.now().toUtc().toIso8601String()}'); log.writeln(''); // empty-line before the next headline - // Run dartdoc - { - log.writeln('### Starting dartdoc'); - final dartdocWrapper = await Isolate.resolvePackageUri(Uri.parse( - 'package:pub_worker/src/bin/dartdoc_wrapper.dart', - )); - final proc = await Process.start( - Platform.resolvedExecutable, - [ - dartdocWrapper!.toFilePath(), - outDir.path, - package, - version, - ], - workingDirectory: outDir.path, - includeParentEnvironment: true, - environment: { - 'CI': 'true', - 'PUB_HOSTED_URL': pubHostedUrl, - 'PUB_CACHE': pubCache, - }, - ); - await proc.stdin.close(); - - await Future.wait([ - proc.stderr.forEach(log.add), - proc.stdout.forEach(log.add), - proc.exitOrTimeout(_dartdocTimeout, () { - log.writeln('TIMEOUT: dartdoc sending SIGTERM/SIGKILL'); - }), - ]).catchError((e) => const [/* ignore */]); - final exitCode = await proc.exitCode; - - log.writeln('### Execution of dartdoc exited $exitCode'); - log.writeln('STOPPED: ${clock.now().toUtc().toIso8601String()}'); - } - // Run the analysis { log.writeln('### Starting pana'); diff --git a/pkg/pub_worker/lib/src/bin/dartdoc_wrapper.dart b/pkg/pub_worker/lib/src/bin/dartdoc_wrapper.dart index 93a6f7ffbe..747d2e0b85 100644 --- a/pkg/pub_worker/lib/src/bin/dartdoc_wrapper.dart +++ b/pkg/pub_worker/lib/src/bin/dartdoc_wrapper.dart @@ -6,13 +6,9 @@ import 'dart:async'; import 'dart:convert' show json, utf8, Utf8Codec; import 'dart:io'; -import 'package:logging/logging.dart' show Logger, Level, LogRecord; -import 'package:pana/pana.dart'; +import 'package:logging/logging.dart' show Logger; import 'package:path/path.dart' as p; -import 'package:pub_dartdoc/pub_dartdoc.dart'; import 'package:pub_dartdoc_data/dartdoc_page.dart'; -import 'package:pub_worker/src/fetch_pubspec.dart'; -import 'package:pub_worker/src/sdks.dart'; import 'package:stream_transform/stream_transform.dart'; import 'package:tar/tar.dart'; @@ -20,155 +16,12 @@ final _log = Logger('dartdoc'); final _utf8 = Utf8Codec(allowMalformed: true); final _jsonUtf8 = json.fuse(utf8); -const _defaultMaxFileCount = 10 * 1000 * 1000; // 10 million files - -// TODO (sigurdm): reduce this back to 2 GiB when -// https://github.com/dart-lang/dartdoc/issues/3311 is resolved. -const _defaultMaxTotalLengthBytes = - 2 * 1024 * 1024 * 1024 + 300 * 1024 * 1024; // 2 GiB + 300 MiB - -/// Program to be used as subprocess for running dartdoc, ensuring that we -/// capture all the output, and only run dartdoc in a subprocess that can -/// timeout and be killed. -Future main(List args) async { - if (args.length != 3) { - print('Usage: dartdoc_wrapper.dart '); - exit(1); - } - final outputFolder = args[0]; - final package = args[1]; - final version = args[2]; - final pubHostedUrl = - Platform.environment['PUB_HOSTED_URL'] ?? 'https://pub.dartlang.org'; - final pubCache = Platform.environment['PUB_CACHE']!; - - // Setup logging - Logger.root.level = Level.INFO; - Logger.root.onRecord.listen((LogRecord rec) { - print('${rec.level.name}: ${rec.time}: ${rec.message}'); - if (rec.error != null) { - print('ERROR: ${rec.error}, ${rec.stackTrace}'); - } - }); - - final workDir = await Directory.systemTemp.createTemp('pkg-'); - try { - await _dartdoc( - outputFolder: outputFolder, - package: package, - version: version, - pubHostedUrl: pubHostedUrl, - pubCache: pubCache, - workDir: workDir.path, - ); - } finally { - await workDir.delete(recursive: true); - } -} - -Future _dartdoc({ +Future postPorcessDartdoc({ required String outputFolder, required String package, required String version, - required String pubHostedUrl, - required String pubCache, - required String workDir, + required String docDir, }) async { - // Fetch the pubspec so we detect which SDK to use for analysis - // TODO: Download the package, extract and load the pubspec.yaml, that way - // we won't have to list versions again later. - final pubspec = await fetchPubspec( - package: package, - version: version, - pubHostedUrl: pubHostedUrl, - ); - - // Discover installed Dart and Flutter SDKs. - // This reads sibling folders to the Dart and Flutter SDK. - // TODO: Install Dart / Flutter SDKs into these folders ondemand in the future! - final dartSdks = await InstalledSdk.fromDirectory( - kind: 'dart', - path: Directory( - Platform.environment['DART_SDK'] ?? - Directory(Platform.resolvedExecutable).parent.parent.path, - ).parent, - ); - final flutterSdks = await InstalledSdk.fromDirectory( - kind: 'flutter', - path: Directory(Platform.environment['FLUTTER_ROOT'] ?? '').parent, - ); - - // Choose Dart and Flutter SDKs for analysis - final dartSdk = InstalledSdk.prioritizedSdk( - dartSdks, - pubspec.dartSdkConstraint, - ); - final flutterSdk = InstalledSdk.prioritizedSdk( - flutterSdks, - pubspec.flutterSdkConstraint, - ); - - final toolEnv = await ToolEnvironment.create( - dartSdkDir: dartSdk?.path, - flutterSdkDir: flutterSdk?.path, - pubCacheDir: pubCache, - environment: {'CI': 'true'}, - useGlobalDartdoc: false, - ); - - final pkgDir = p.join(workDir, 'pkg'); - await Directory(pkgDir).create(recursive: true); - // TODO: Ensure that we set User-Agent correctly here! - await downloadPackage( - package, - version, - destination: pkgDir, - pubHostedUrl: pubHostedUrl, - ); - - _log.info('Running pub upgrade'); - final ret = await toolEnv.runUpgrade(pkgDir, pubspec.usesFlutter); - if (ret.exitCode != 0) { - _log.shout('Failed to run pub upgrade'); - return; - } - - // Create and/or customize dartdoc_options.yaml - final optionsFile = File(p.join(pkgDir, 'dartdoc_options.yaml')); - Map? originalContent; - try { - originalContent = yamlToJson(await optionsFile.readAsString()); - } on IOException { - // pass, ignore missing file - } on FormatException { - // pass, ignore broken file - } - final updatedContent = _customizeDartdocOptions(originalContent); - await optionsFile.writeAsString(json.encode(updatedContent)); - - final docDir = p.join(workDir, 'doc'); - await Directory(docDir).create(recursive: true); - await pubDartDoc([ - '--input', - pkgDir, - '--output', - docDir, - '--no-validate-links', - '--sanitize-html', - '--max-file-count', - '$_defaultMaxFileCount', - '--max-total-size', - '$_defaultMaxTotalLengthBytes', - if (pubspec.usesFlutter) ...[ - '--sdk-dir', - p.join(flutterSdk!.path, 'bin', 'cache', 'dart-sdk') - ] else ...[ - '--sdk-dir', - dartSdk!.path, - ] - ]); - _log.info('Finished running dartdoc'); - _log.info('Running post-processing'); final tmpOutDir = p.join(outputFolder, '_doc'); await Directory(tmpOutDir).create(recursive: true); @@ -229,44 +82,3 @@ Future _dartdoc({ _log.info('Finished .tar.gz archive'); } - -/// Returns a new, pub-specific dartdoc options based on [original]. -/// -/// dartdoc_options.yaml allows to change how doc content is generated. -/// To provide uniform experience across the pub site, and to reduce the -/// potential attack surface (HTML-, and code-injections, code executions), -/// we do not support every option. -/// -/// https://github.com/dart-lang/dartdoc#dartdoc_optionsyaml -/// -/// Discussion on the enabled options: -/// https://github.com/dart-lang/pub-dev/issues/4521#issuecomment-779821098 -Map _customizeDartdocOptions(Map? original) { - final passThroughOptions = {}; - if (original != null && - original.containsKey('dartdoc') && - original['dartdoc'] is Map) { - final dartdoc = original['dartdoc'] as Map; - for (final key in _passThroughKeys) { - if (dartdoc.containsKey(key)) { - passThroughOptions[key] = dartdoc[key]; - } - } - } - return { - 'dartdoc': { - ...passThroughOptions, - 'showUndocumentedCategories': true, - }, - }; -} - -final _passThroughKeys = [ - 'categories', - 'categoryOrder', - // TODO: enable after checking that the relative path doesn't escape the package folder - // 'examplePathPrefix', - 'exclude', - 'include', - 'nodoc', -]; diff --git a/pkg/pub_worker/lib/src/bin/pana_wrapper.dart b/pkg/pub_worker/lib/src/bin/pana_wrapper.dart index f7b88fb40f..a2c8eb2e4f 100644 --- a/pkg/pub_worker/lib/src/bin/pana_wrapper.dart +++ b/pkg/pub_worker/lib/src/bin/pana_wrapper.dart @@ -9,7 +9,7 @@ import 'dart:io' show Directory, File, Platform, exit, gzip; import 'package:logging/logging.dart' show Logger, Level, LogRecord; import 'package:pana/pana.dart'; import 'package:path/path.dart' as p; -import 'package:pub_dartdoc_data/pub_dartdoc_data.dart'; +import 'package:pub_worker/src/bin/dartdoc_wrapper.dart'; import 'package:pub_worker/src/fetch_pubspec.dart'; import 'package:pub_worker/src/sdks.dart'; @@ -19,6 +19,9 @@ final _log = Logger('pana'); /// replaced with a placeholder report. final _reportSizeDropThreshold = 32 * 1024; +/// Stop dartdoc if it takes more than 45 minutes. +const _dartdocTimeout = Duration(minutes: 45); + /// Program to be used as subprocess for running pana, ensuring that we capture /// all the output, and only run analysis in a subprocess that can timeout and /// be killed. @@ -34,6 +37,8 @@ Future main(List args) async { final pubHostedUrl = Platform.environment['PUB_HOSTED_URL'] ?? 'https://pub.dartlang.org'; final pubCache = Platform.environment['PUB_CACHE']!; + final rawDartdocOutputFolder = + await Directory.systemTemp.createTemp('dartdoc-$package'); // Setup logging Logger.root.level = Level.INFO; @@ -96,7 +101,8 @@ Future main(List args) async { pubCacheDir: pubCache, panaCacheDir: Platform.environment['PANA_CACHE'], environment: {'CI': 'true'}, - useGlobalDartdoc: false, + useGlobalDartdoc: true, + globalDartdocVersion: '7.0.0', ); //final dartdocOutputDir = @@ -111,8 +117,9 @@ Future main(List args) async { version: version, options: InspectOptions( pubHostedUrl: Platform.environment['PUB_HOSTED_URL']!, - //TODO: Run dartdoc as part of pana checkRemoteRepository: true, + dartdocTimeout: _dartdocTimeout, + dartdocOutputDir: rawDartdocOutputFolder.path, ), logger: _log, storeResource: (filename, data) async { @@ -122,25 +129,13 @@ Future main(List args) async { }, ); - // Load doc/pub-data.json created by dartdoc - ReportSection docSection; - try { - final docData = PubDartdocData.fromJson(json.decode( - await File(p.join(outputFolder, 'doc', 'pub-data.json')).readAsString(), - ) as Map); - docSection = documentationCoverageSection( - documented: docData.coverage?.documented ?? 0, - total: docData.coverage?.total ?? 0, - ); - } catch (e) { - // ignore the error - // TODO: handle errors more gracefully, or just run dartdoc as part of pana. - // TODO: make a proper link to the task-log, which isn't exposed yet. - docSection = dartdocFailedSection('`dartdoc` failed, see task-log.'); - } - - final updatedReport = summary.report?.joinSection(docSection); - final updatedSummary = summary.change(report: updatedReport); + await postPorcessDartdoc( + outputFolder: outputFolder, + package: package, + version: version, + docDir: rawDartdocOutputFolder.path, + ); + await rawDartdocOutputFolder.delete(recursive: true); // sanity check on pana report size final reportSize = @@ -169,5 +164,5 @@ Future main(List args) async { _log.info('Writing summary.json'); await File( p.join(outputFolder, 'summary.json'), - ).writeAsString(json.encode(updatedSummary)); + ).writeAsString(json.encode(summary)); } diff --git a/pkg/pub_worker/pubspec.lock b/pkg/pub_worker/pubspec.lock index 065233cbef..44c689821d 100644 --- a/pkg/pub_worker/pubspec.lock +++ b/pkg/pub_worker/pubspec.lock @@ -466,10 +466,10 @@ packages: dependency: "direct main" description: name: pana - sha256: "0a2546792c71440b69e4dd73d9cd310dab6404c18ce6c0e8ceeb1453249c5d27" + sha256: d8c24d31fbf66b9c8c6c5e31c39b094c8535ed5bd152dbce9642049e6e76965d url: "https://pub.dev" source: hosted - version: "0.21.40" + version: "0.21.42" path: dependency: transitive description: diff --git a/pkg/pub_worker/pubspec.yaml b/pkg/pub_worker/pubspec.yaml index cd5c917817..03c1e0903b 100644 --- a/pkg/pub_worker/pubspec.yaml +++ b/pkg/pub_worker/pubspec.yaml @@ -9,7 +9,7 @@ dependencies: appengine: ^0.13.6 json_annotation: ^4.3.0 jsontool: ^1.1.0 - pana: ^0.21.40 + pana: ^0.21.42 lints: ^3.0.0 # required for pana meta: ^1.7.0 api_builder: