Schedule compileJava after the internal task if it affects source files (apache#13282)

dweiss · web-flow · commit afe982b3ef55 · 2024-04-09T07:44:07.000+02:00
diff --git a/gradle/generation/antlr.gradle b/gradle/generation/antlr.gradle
@@ -82,7 +82,10 @@ configure(project(":lucene:expressions")) {
   }
 
   def generateAntlr = wrapWithPersistentChecksums(
-      generateAntlrInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+      generateAntlrInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 
   regenerate.dependsOn generateAntlr
 }
diff --git a/gradle/generation/forUtil.gradle b/gradle/generation/forUtil.gradle
@@ -39,7 +39,10 @@ configure(project(":lucene:core")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtilInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtilInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 }
 
 configure(project(":lucene:backward-codecs")) {
@@ -64,7 +67,10 @@ configure(project(":lucene:backward-codecs")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil84Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil84Internal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 
   task generateForUtil90Internal() {
     description "Regenerate gen_ForUtil.py"
@@ -86,6 +92,9 @@ configure(project(":lucene:backward-codecs")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil90Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil90Internal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 }
 
diff --git a/gradle/generation/icu.gradle b/gradle/generation/icu.gradle
@@ -159,8 +159,13 @@ configure(project(":lucene:analysis:icu")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFilesInternal, [ ignoreWithSource: icuCompileTask ])
-  regenerate.dependsOn wrapWithPersistentChecksums(genRbbiInternal)
+  regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFilesInternal, [
+    ignoreWithSource: icuCompileTask,
+    mustRunBefore: [ "compileJava" ]
+  ])
+  regenerate.dependsOn wrapWithPersistentChecksums(genRbbiInternal, [
+    mustRunBefore: [ "compileJava" ]
+  ])
 
   task compileIcuWindows() {
     doFirst {
@@ -281,5 +286,8 @@ configure(project(":lucene:analysis:common")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateUnicodePropsInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  regenerate.dependsOn wrapWithPersistentChecksums(generateUnicodePropsInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 }
diff --git a/gradle/generation/javacc.gradle b/gradle/generation/javacc.gradle
@@ -232,9 +232,14 @@ configure(project(":lucene:queryparser")) {
     description "Regenerate query parsers (javacc syntax definitions)."
     group "generation"
 
-    dependsOn wrapWithPersistentChecksums(javaccParserClassicInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]),
-        wrapWithPersistentChecksums(javaccParserSurroundInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]),
-        wrapWithPersistentChecksums(javaccParserFlexibleInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+    def extraConfig = [
+        andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+        mustRunBefore: [ "compileJava" ]
+    ]
+
+    dependsOn wrapWithPersistentChecksums(javaccParserClassicInternal, extraConfig),
+        wrapWithPersistentChecksums(javaccParserSurroundInternal, extraConfig),
+        wrapWithPersistentChecksums(javaccParserFlexibleInternal, extraConfig)
   }
 
   regenerate.dependsOn javacc
diff --git a/gradle/generation/jflex.gradle b/gradle/generation/jflex.gradle
@@ -51,7 +51,10 @@ configure(project(":lucene:core")) {
     }
   }
 
-  def generateStandardTokenizer = wrapWithPersistentChecksums(generateStandardTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  def generateStandardTokenizer = wrapWithPersistentChecksums(generateStandardTokenizerInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 
   regenerate.dependsOn generateStandardTokenizer
 }
@@ -149,7 +152,11 @@ configure(project(":lucene:analysis:common")) {
     }
   }
 
-  def generateUAX29URLEmailTokenizer = wrapWithPersistentChecksums(generateUAX29URLEmailTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  def generateUAX29URLEmailTokenizer = wrapWithPersistentChecksums(generateUAX29URLEmailTokenizerInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
+
   // UAX29URLEmailTokenizerImpl.jflex includes: ASCIITLD.jflex
   // so we make sure it is up to date. Also, make sure the ordering of internal
   // tasks is enforced (just scheduling top-level wrappers doesn't mean their subtask graphs cannot be reordered).
@@ -197,7 +204,12 @@ configure(project(":lucene:analysis:common")) {
     skeleton = skeletonDefault
   }
 
-  def generateHTMLStripCharFilter = wrapWithPersistentChecksums(generateHTMLStripCharFilterInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  def extraConfig = [
+      andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+      mustRunBefore: [ "compileJava" ]
+  ]
+
+  def generateHTMLStripCharFilter = wrapWithPersistentChecksums(generateHTMLStripCharFilterInternal, extraConfig)
   // HTMLStripCharFilter.jflex includes HTMLCharacterEntities.jflex so we make sure it's up to date.
   configure(generateHTMLStripCharFilter) {
     dependsOn "generateHTMLCharacterEntities"
@@ -206,12 +218,12 @@ configure(project(":lucene:analysis:common")) {
     dependsOn "generateHTMLCharacterEntitiesInternal"
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]),
-      wrapWithPersistentChecksums(generateClassicTokenizerInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ]),
+  regenerate.dependsOn wrapWithPersistentChecksums(generateWikipediaTokenizerInternal, extraConfig),
+      wrapWithPersistentChecksums(generateClassicTokenizerInternal, extraConfig),
       generateUAX29URLEmailTokenizer,
-      wrapWithPersistentChecksums(generateHTMLCharacterEntitiesInternal),
+      wrapWithPersistentChecksums(generateHTMLCharacterEntitiesInternal, extraConfig),
       generateHTMLStripCharFilter,
-      wrapWithPersistentChecksums(generateTldsInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+      wrapWithPersistentChecksums(generateTldsInternal, extraConfig)
 }
 
 class JFlexTask extends DefaultTask {
diff --git a/gradle/generation/moman.gradle b/gradle/generation/moman.gradle
@@ -105,8 +105,13 @@ configure(project(":lucene:core")) {
     description "Regenerate Moman-based sources."
     group "generation"
 
-    dependsOn wrapWithPersistentChecksums(utilGenPackedInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
-    dependsOn wrapWithPersistentChecksums(utilGenLevInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+    def extraConfig = [
+            andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+            mustRunBefore: [ "compileJava" ]
+    ]
+
+    dependsOn wrapWithPersistentChecksums(utilGenPackedInternal, extraConfig)
+    dependsOn wrapWithPersistentChecksums(utilGenLevInternal, extraConfig)
   }
 
   regenerate.dependsOn moman
diff --git a/gradle/generation/regenerate.gradle b/gradle/generation/regenerate.gradle
@@ -105,25 +105,33 @@ configure([
       //
       // The idea is that, given source-generating task "sourceTaskInternal" (note the suffix),
       // we create a bunch of other tasks that perform checksum generation, validation and
-      // source task skipping; example (sourceTask has Internal suffix stripped)
+      // source task skipping. For example, let's say we have a task 'genFooInternal";
+      // the following tasks would be created
       //
-      // ${sourceTask}ChecksumLoad
-      // ${sourceTask}ChecksumSave
-      // ${sourceTask}ChecksumCheck (fails if checksums are inconsistent)
-      // ${sourceTask} dependsOn [checksum-load, ${sourceTask}Internal, checksum-save]
+      // genFooChecksumLoad
+      // genFooChecksumSave
+      // genFooChecksumCheck (fails if checksums are inconsistent)
+      //
+      // and the following set of dependencies would be created (with additional
+      // constraints to run them in this particular order!).
+      //
+      // genFoo.dependsOn [genFooChecksumLoad, genFooInternal, genFooChecksumSave]
       //
       // Checksums are persisted and computed from sourceTask's inputs/outputs. If the
-      // persisted checksums are identical to current checksums, source task
-      // is skipped (via sourceTaskInternal.onlyIf { false }).
+      // persisted checksums are identical to the now-current checksums, the "internal" task
+      // is skipped (using onlyIf { false }).
       //
       // Implementation-wise things get complicated because gradle doesn't have the notion
       // of "ordered" task execution with respect to task AND its dependencies (we can add
       // constraints to each node in the execution graph but not node-and-dependencies).
       //
       // sourceTask - the task to wrap
       // extraConfig - a map with extra (optional) configuration options.
-      //   andThenTasks: other tasks that should be scheduled to run after source task and
-      //     before checksum calculation.
+      //   andThenTasks: any other tasks that should be scheduled to run after the internal task and
+      //     before checksum calculation/ saving. spotless is a good example of this.
+      //   ignoreWithSource: any other tasks that should be ignored if the internal task is ignored
+      //     (checksums are identical)
+      //   mustRunBefore: any tasks which should be scheduled to run after the internal task.
       wrapWithPersistentChecksums = { Task sourceTask, Map<String, Object> extraConfig = [:] ->
         if (!sourceTask.name.endsWith("Internal")) {
           throw new GradleException("Wrapped task must follow the convention name of *Internal: ${sourceTask.name}")
@@ -144,6 +152,13 @@ configure([
         List<Object> andThenTasks = toList(extraConfig.get("andThenTasks"))
         List<Object> ignoreWithSource =  toList(extraConfig.get("ignoreWithSource"))
 
+        // Schedule must-run-afters
+        List<Object> mustRunBefore = toList(extraConfig.get("mustRunBefore"))
+        // TODO: maybe ensure all task refs here are strings?
+        tasks.matching { it.name in mustRunBefore }.configureEach {
+          mustRunAfter sourceTask
+        }
+
         // Create checksum-loader task.
         Task checksumLoadTask = tasks.create("${sourceTaskName}ChecksumLoad", {
           ext {
diff --git a/gradle/generation/snowball.gradle b/gradle/generation/snowball.gradle
@@ -109,5 +109,9 @@ configure(project(":lucene:analysis:common")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(snowballInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"], ignoreWithSource: [downloadSnowballStemmers, downloadSnowballWebsite] ])
+  regenerate.dependsOn wrapWithPersistentChecksums(snowballInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          ignoreWithSource: [downloadSnowballStemmers, downloadSnowballWebsite],
+          mustRunBefore: [ "compileJava" ]
+  ])
 }
diff --git a/gradle/generation/unicode-test-classes.gradle b/gradle/generation/unicode-test-classes.gradle
@@ -43,7 +43,10 @@ configure(project(":lucene:test-framework")) {
       }
     }
   }
-  regenerate.dependsOn wrapWithPersistentChecksums(generateEmojiTokenizationTestInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+  regenerate.dependsOn wrapWithPersistentChecksums(generateEmojiTokenizationTestInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 
   task generateJavaUnicodeWordBreakTestInternal() {
     def unicodeVersion = "12.1"
@@ -71,6 +74,8 @@ configure(project(":lucene:test-framework")) {
     }
   }
 
-  regenerate.dependsOn wrapWithPersistentChecksums(generateJavaUnicodeWordBreakTestInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
-
+  regenerate.dependsOn wrapWithPersistentChecksums(generateJavaUnicodeWordBreakTestInternal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 }
diff --git a/help/regeneration.txt b/help/regeneration.txt
@@ -16,22 +16,6 @@ In many cases hitting (1) means you'll have to do (2) so let's discuss
 these in order.
 
 
-SPECIAL NOTE
-------------
-
-Regeneration tasks currently don't play well with the current gradle version.
-To workaround the issue, compile everything first:
-
-gradlew compileJava
-
-Then run the commands as described in this document, excluding compileJava.
-For example to run "regenerate":
-
-gradlew regenerate -x compileJava
-
-More information: https://github.com/apache/lucene/issues/13240
-
-
 Checksum validation errors
 --------------------------
 

Original file line number	Diff line number	Diff line change
`@@ -82,7 +82,10 @@ configure(project(":lucene:expressions")) {`
`82`	`82`	`}`
`83`	`83`
`84`	`84`	`def generateAntlr = wrapWithPersistentChecksums(`
`85`		`- generateAntlrInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])`
	`85`	`+ generateAntlrInternal, [`
	`86`	`+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],`
	`87`	`+ mustRunBefore: [ "compileJava" ]`
	`88`	`+ ])`
`86`	`89`
`87`	`90`	`regenerate.dependsOn generateAntlr`
`88`	`91`	`}`
Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,10 @@ configure(project(":lucene:core")) {`
`39`	`39`	`}`
`40`	`40`	`}`
`41`	`41`
`42`		`- regenerate.dependsOn wrapWithPersistentChecksums(generateForUtilInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])`
	`42`	`+ regenerate.dependsOn wrapWithPersistentChecksums(generateForUtilInternal, [`
	`43`	`+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],`
	`44`	`+ mustRunBefore: [ "compileJava" ]`
	`45`	`+ ])`
`43`	`46`	`}`
`44`	`47`
`45`	`48`	`configure(project(":lucene:backward-codecs")) {`
`@@ -64,7 +67,10 @@ configure(project(":lucene:backward-codecs")) {`
`64`	`67`	`}`
`65`	`68`	`}`
`66`	`69`
`67`		`- regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil84Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])`
	`70`	`+ regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil84Internal, [`
	`71`	`+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],`
	`72`	`+ mustRunBefore: [ "compileJava" ]`
	`73`	`+ ])`
`68`	`74`
`69`	`75`	`task generateForUtil90Internal() {`
`70`	`76`	`description "Regenerate gen_ForUtil.py"`
`@@ -86,6 +92,9 @@ configure(project(":lucene:backward-codecs")) {`
`86`	`92`	`}`
`87`	`93`	`}`
`88`	`94`
`89`		`- regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil90Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])`
	`95`	`+ regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil90Internal, [`
	`96`	`+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],`
	`97`	`+ mustRunBefore: [ "compileJava" ]`
	`98`	`+ ])`
`90`	`99`	`}`
`91`	`100`
Original file line number	Diff line number	Diff line change
`@@ -159,8 +159,13 @@ configure(project(":lucene:analysis:icu")) {`
`159`	`159`	`}`
`160`	`160`	`}`
`161`	`161`
`162`		`- regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFilesInternal, [ ignoreWithSource: icuCompileTask ])`
`163`		`- regenerate.dependsOn wrapWithPersistentChecksums(genRbbiInternal)`
	`162`	`+ regenerate.dependsOn wrapWithPersistentChecksums(genUtr30DataFilesInternal, [`
	`163`	`+ ignoreWithSource: icuCompileTask,`
	`164`	`+ mustRunBefore: [ "compileJava" ]`
	`165`	`+ ])`
	`166`	`+ regenerate.dependsOn wrapWithPersistentChecksums(genRbbiInternal, [`
	`167`	`+ mustRunBefore: [ "compileJava" ]`
	`168`	`+ ])`
`164`	`169`
`165`	`170`	`task compileIcuWindows() {`
`166`	`171`	`doFirst {`
`@@ -281,5 +286,8 @@ configure(project(":lucene:analysis:common")) {`
`281`	`286`	`}`
`282`	`287`	`}`
`283`	`288`
`284`		`- regenerate.dependsOn wrapWithPersistentChecksums(generateUnicodePropsInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])`
	`289`	`+ regenerate.dependsOn wrapWithPersistentChecksums(generateUnicodePropsInternal, [`
	`290`	`+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],`
	`291`	`+ mustRunBefore: [ "compileJava" ]`
	`292`	`+ ])`
`285`	`293`	`}`
Original file line number	Diff line number	Diff line change
`@@ -109,5 +109,9 @@ configure(project(":lucene:analysis:common")) {`
`109`	`109`	`}`
`110`	`110`	`}`
`111`	`111`
`112`		`- regenerate.dependsOn wrapWithPersistentChecksums(snowballInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"], ignoreWithSource: [downloadSnowballStemmers, downloadSnowballWebsite] ])`
	`112`	`+ regenerate.dependsOn wrapWithPersistentChecksums(snowballInternal, [`
	`113`	`+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],`
	`114`	`+ ignoreWithSource: [downloadSnowballStemmers, downloadSnowballWebsite],`
	`115`	`+ mustRunBefore: [ "compileJava" ]`
	`116`	`+ ])`
`113`	`117`	`}`
Original file line number	Diff line number	Diff line change
`@@ -43,7 +43,10 @@ configure(project(":lucene:test-framework")) {`
`43`	`43`	`}`
`44`	`44`	`}`
`45`	`45`	`}`
`46`		`- regenerate.dependsOn wrapWithPersistentChecksums(generateEmojiTokenizationTestInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])`
	`46`	`+ regenerate.dependsOn wrapWithPersistentChecksums(generateEmojiTokenizationTestInternal, [`
	`47`	`+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],`
	`48`	`+ mustRunBefore: [ "compileJava" ]`
	`49`	`+ ])`
`47`	`50`
`48`	`51`	`task generateJavaUnicodeWordBreakTestInternal() {`
`49`	`52`	`def unicodeVersion = "12.1"`
`@@ -71,6 +74,8 @@ configure(project(":lucene:test-framework")) {`
`71`	`74`	`}`
`72`	`75`	`}`
`73`	`76`
`74`		`- regenerate.dependsOn wrapWithPersistentChecksums(generateJavaUnicodeWordBreakTestInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])`
`75`		`-`
	`77`	`+ regenerate.dependsOn wrapWithPersistentChecksums(generateJavaUnicodeWordBreakTestInternal, [`
	`78`	`+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],`
	`79`	`+ mustRunBefore: [ "compileJava" ]`
	`80`	`+ ])`
`76`	`81`	`}`