From a1f49c0de1763798ed5f1db9646143ee4493c66f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Sep 2025 11:20:16 +0000 Subject: [PATCH 1/2] deps(java): bump org.apache.rat:apache-rat from 0.14 to 0.16.1 Bumps org.apache.rat:apache-rat from 0.14 to 0.16.1. --- updated-dependencies: - dependency-name: org.apache.rat:apache-rat dependency-version: 0.16.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index caa3649800b2..c30cb44d5014 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -51,7 +51,7 @@ procfork = "1.0.6" # unit tests randomizedtesting = "2.8.3" # license checks -rat = "0.15" +rat = "0.16.1" # spatial-extras/ support s2-geometry = "1.0.0" # spatial-extras/ support From 17b8d774d88946a8d011e18bf608ca0b01e41619 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 10 Sep 2025 07:34:22 -0400 Subject: [PATCH 2/2] build: remove special XML processing from ValidateSourcePatternsPlugin This special check is a cosmetic one (placement of the license header in the XML) and only applies to a total of 29 XML files in the source tree. Processing is complex and runs afoul of thread safety issues in the rat API. Remove the check so that we can upgrade rat. XML files are still separately validated to have a correct license via the official usage of this rat plugin. --- .../gradle/plugins/hacks/HacksPlugin.java | 4 +- .../plugins/misc/RootProjectSetupPlugin.java | 2 - .../ValidateSourcePatternsPlugin.java | 236 ------------------ 3 files changed, 1 insertion(+), 241 deletions(-) delete mode 100644 build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/spotless/ValidateSourcePatternsPlugin.java diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/HacksPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/HacksPlugin.java index 89967f68ac8a..b2d374bff5a5 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/HacksPlugin.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/HacksPlugin.java @@ -24,7 +24,6 @@ import org.apache.lucene.gradle.plugins.LuceneGradlePlugin; import org.apache.lucene.gradle.plugins.java.EcjLintPlugin; import org.apache.lucene.gradle.plugins.misc.CheckGradlewScriptsTweakedPlugin; -import org.apache.lucene.gradle.plugins.spotless.ValidateSourcePatternsPlugin; import org.gradle.api.Project; /** This applies various odd hacks that we probably should not need. */ @@ -56,8 +55,7 @@ private void addDummyOutputs(Project project) { task -> { var taskName = task.getName(); return taskName.startsWith(EcjLintPlugin.TASK_PREFIX) - || taskName.equals(CheckGradlewScriptsTweakedPlugin.TASK_NAME) - || taskName.equals(ValidateSourcePatternsPlugin.TASK_NAME); + || taskName.equals(CheckGradlewScriptsTweakedPlugin.TASK_NAME); }) .configureEach( task -> { diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java index 3e44576a7d79..8f37e45e07dc 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java @@ -33,7 +33,6 @@ import org.apache.lucene.gradle.plugins.ide.IdeaSupportPlugin; import org.apache.lucene.gradle.plugins.regenerate.RegenerateTasksSupportPlugin; import org.apache.lucene.gradle.plugins.spotless.GradleGroovyFormatPlugin; -import org.apache.lucene.gradle.plugins.spotless.ValidateSourcePatternsPlugin; import org.gradle.api.Project; import org.gradle.api.file.DuplicatesStrategy; import org.gradle.api.initialization.IncludedBuild; @@ -76,7 +75,6 @@ public void apply(Project rootProject) { plugins.apply(HacksPlugin.class); plugins.apply(WipeGradleTempPlugin.class); plugins.apply(GradleGroovyFormatPlugin.class); - plugins.apply(ValidateSourcePatternsPlugin.class); plugins.apply(ConfigureLockFilePlugin.class); plugins.apply(CheckGradlewScriptsTweakedPlugin.class); diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/spotless/ValidateSourcePatternsPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/spotless/ValidateSourcePatternsPlugin.java deleted file mode 100644 index 1eb57eeb7a77..000000000000 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/spotless/ValidateSourcePatternsPlugin.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.gradle.plugins.spotless; - -import com.google.common.base.Splitter; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.StringWriter; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CodingErrorAction; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.util.Locale; -import java.util.Set; -import java.util.TreeSet; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import javax.inject.Inject; -import org.apache.lucene.gradle.plugins.LuceneGradlePlugin; -import org.apache.rat.Defaults; -import org.apache.rat.analysis.RatHeaderAnalysisException; -import org.apache.rat.document.impl.FileDocument; -import org.gradle.api.DefaultTask; -import org.gradle.api.GradleException; -import org.gradle.api.Project; -import org.gradle.api.file.ConfigurableFileTree; -import org.gradle.api.tasks.CacheableTask; -import org.gradle.api.tasks.IgnoreEmptyDirectories; -import org.gradle.api.tasks.InputFiles; -import org.gradle.api.tasks.PathSensitive; -import org.gradle.api.tasks.PathSensitivity; -import org.gradle.api.tasks.TaskAction; -import org.gradle.api.tasks.compile.JavaCompile; -import org.gradle.internal.logging.progress.ProgressLogger; -import org.gradle.internal.logging.progress.ProgressLoggerFactory; - -/** Checks for invalid usage patterns in source files. */ -public class ValidateSourcePatternsPlugin extends LuceneGradlePlugin { - public static final String TASK_NAME = "validateSourcePatterns"; - - @Override - public void apply(Project project) { - applicableToRootProjectOnly(project); - - project.allprojects(this::configureProject); - } - - private void configureProject(Project project) { - var validateSourcePatternsTask = - project - .getTasks() - .register( - TASK_NAME, - ValidateSourcePatternsTask.class, - task -> { - task.setGroup("Verification"); - task.setDescription("Validate Source Patterns"); - - ConfigurableFileTree sourceFiles = task.getSourceFiles(); - sourceFiles.setDir(project.getLayout().getProjectDirectory()); - - // it seems we only scan XML files - everything else has been moved - // to rat scanning or elsewhere. - sourceFiles.include("**/*.xml"); - - // default excludes. - sourceFiles.exclude("**/build/**"); - sourceFiles.exclude("**/.idea/**"); - sourceFiles.exclude("**/.gradle/**"); - sourceFiles.exclude("**/.git/**"); - - // Don't go into subproject folders (each is scanned individually). - sourceFiles.exclude( - project.getChildProjects().keySet().stream() - .map(name -> name + "/**") - .toList()); - }); - - // Add to all checks. - project.getTasks().named("check").configure(task -> task.dependsOn(validateSourcePatternsTask)); - - // Ensure validation runs prior to any compilation task. - project - .getTasks() - .withType(JavaCompile.class) - .configureEach( - task -> { - task.mustRunAfter(validateSourcePatternsTask); - }); - - // project-specific tuning. - project - .project(":lucene:benchmark") - .getTasks() - .withType(ValidateSourcePatternsTask.class) - .configureEach( - task -> { - task.getSourceFiles().exclude("data/**", "work/**"); - }); - } - - @CacheableTask - public abstract static class ValidateSourcePatternsTask extends DefaultTask { - @InputFiles - @PathSensitive(PathSensitivity.RELATIVE) - @IgnoreEmptyDirectories - public abstract ConfigurableFileTree getSourceFiles(); - - @Inject - protected abstract ProgressLoggerFactory getProgressLoggerFactory(); - - @TaskAction - public void check() { - Set files = getSourceFiles().getFiles(); - getLogger() - .info( - "Input files for scanning:\n{}", - files.stream().map(f -> " - " + f).collect(Collectors.joining("\n"))); - - var xmlCommentPattern = Pattern.compile("(?sm)\\Q\\E"); - var xmlTagPattern = Pattern.compile("(?m)\\s*<[a-zA-Z].*"); - var violations = new TreeSet(); - - ProgressLogger progress = getProgressLoggerFactory().newOperation(this.getClass()); - progress.start(this.getName(), this.getName()); - for (var file : files) { - progress.progress("Scanning " + file.getName()); - - String fileText = readUtf8WithValidation(file); - - if (file.getName().endsWith(".xml")) { - var ratDocument = new FileDocument(file); - checkLicenseHeaderPrecedes( - file, "", xmlTagPattern, xmlCommentPattern, fileText, ratDocument, violations); - } - } - progress.completed(); - - if (!violations.isEmpty()) { - throw new GradleException( - String.format( - Locale.ROOT, - "Found %d source violation(s):\n %s", - violations.size(), - String.join("\n ", violations))); - } - } - - private void checkLicenseHeaderPrecedes( - File file, - String description, - Pattern contentPattern, - Pattern commentPattern, - String fileText, - FileDocument ratDocument, - TreeSet violations) { - Matcher contentMatcher = contentPattern.matcher(fileText); - if (contentMatcher.find()) { - int contentStartPos = contentMatcher.start(); - Matcher commentMatcher = commentPattern.matcher(fileText); - while (commentMatcher.find()) { - if (isLicense(file, commentMatcher.group(1), ratDocument)) { - if (commentMatcher.start() < contentStartPos) { - // This file is all good, so break the loop: - // license header precedes 'description' definition - break; - } else { - reportViolation( - violations, file, description + " declaration precedes license header"); - } - } - } - } - } - - private void reportViolation(TreeSet violations, File file, String name) { - String msg = String.format(Locale.ROOT, "%s: %s", file, name); - getLogger().error(msg); - violations.add(msg); - } - - // See LUCENE-10419 - rat is not thread safe. - private static final Object ratLockBug = new Object(); - private static final Splitter lineSplitter = Splitter.on(Pattern.compile("[\\r\\n]+")); - - private boolean isLicense(File file, String text, FileDocument ratDocument) { - synchronized (ratLockBug) { - var licenseMatcher = Defaults.createDefaultMatcher(); - licenseMatcher.reset(); - return lineSplitter.splitToList(text).stream() - .anyMatch( - it -> { - try { - return licenseMatcher.match(ratDocument, it); - } catch (RatHeaderAnalysisException e) { - throw new GradleException("Could not scan this file with rat: " + file, e); - } - }); - } - } - - private static String readUtf8WithValidation(File file) { - String fileText; - CharsetDecoder validatingDecoder = - StandardCharsets.UTF_8 - .newDecoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT); - try (var is = Files.newInputStream(file.toPath()); - var sw = new StringWriter(); - var reader = new InputStreamReader(is, validatingDecoder)) { - reader.transferTo(sw); - fileText = sw.toString(); - } catch (IOException e) { - throw new GradleException("Could not read: " + file, e); - } - return fileText; - } - } -}