Add explicit "roles.yml" support to testclusters (#82137)

Previously, within tests, the file "roles.yml" (that is used to define
security roles in a cluster) would need to be configured using
`extraConfigFile`. This is effective, but means that there can only be
a single source of security roles for the testcluster.

This change introduces an explicit "securityRoles" setting in
testclusters that will concatenate the provided files into a single
"roles.yml" in the config directory. This makes it possible for
testclusters itself to define standard roles as well as having each
test define additional roles it may need.

Relates: #81400

Author: tvernum

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -390,6 +390,11 @@ public void user(Map<String, String> userSpec) {
         nodes.all(node -> node.user(userSpec));
     }
 
+    @Override
+    public void rolesFile(File rolesYml) {
+        nodes.all(node -> node.rolesFile(rolesYml));
+    }
+
     private void writeUnicastHostsFiles() {
         String unicastUris = nodes.stream().flatMap(node -> node.getAllTransportPortURI().stream()).collect(Collectors.joining("\n"));
         nodes.forEach(node -> {

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -144,6 +144,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final LazyPropertyMap<String, File> extraConfigFiles = new LazyPropertyMap<>("Extra config files", this, FileEntry::new);
     private final LazyPropertyList<FileCollection> extraJarConfigurations = new LazyPropertyList<>("Extra jar files", this);
     private final List<Map<String, String>> credentials = new ArrayList<>();
+    private final List<File> roleFiles = new ArrayList<>();
     final LinkedHashMap<String, String> defaultConfig = new LinkedHashMap<>();
 
     private final Path confPathRepo;
@@ -561,16 +562,7 @@ public synchronized void start() {
             }
         }
 
-        if (credentials.isEmpty() == false) {
-            logToProcessStdout("Setting up " + credentials.size() + " users");
-
-            credentials.forEach(
-                paramMap -> runElasticsearchBinScript(
-                    getVersion().onOrAfter("6.3.0") ? "elasticsearch-users" : "x-pack/users",
-                    paramMap.entrySet().stream().flatMap(entry -> Stream.of(entry.getKey(), entry.getValue())).toArray(String[]::new)
-                )
-            );
-        }
+        configureSecurity();
 
         if (cliSetup.isEmpty() == false) {
             logToProcessStdout("Running " + cliSetup.size() + " setup commands");
@@ -672,6 +664,39 @@ private void copyExtraJars() {
         });
     }
 
+    private void configureSecurity() {
+        if (credentials.isEmpty() == false) {
+            logToProcessStdout("Setting up " + credentials.size() + " users");
+
+            credentials.forEach(
+                paramMap -> runElasticsearchBinScript(
+                    getVersion().onOrAfter("6.3.0") ? "elasticsearch-users" : "x-pack/users",
+                    paramMap.entrySet().stream().flatMap(entry -> Stream.of(entry.getKey(), entry.getValue())).toArray(String[]::new)
+                )
+            );
+        }
+        if (roleFiles.isEmpty() == false) {
+            logToProcessStdout("Setting up roles.yml");
+
+            Path dst = configFile.getParent().resolve("roles.yml");
+            roleFiles.forEach(from -> {
+                if (Files.exists(from.toPath()) == false) {
+                    throw new TestClustersException(
+                        "Can't create roles.yml config file from " + from + " for " + this + " as it does not exist"
+                    );
+                }
+                try {
+                    final Path source = from.toPath();
+                    final String content = Files.readString(source, StandardCharsets.UTF_8);
+                    Files.writeString(dst, content + System.lineSeparator(), StandardCharsets.UTF_8, StandardOpenOption.APPEND);
+                    LOGGER.info("Appended roles file {} to {}", source, dst);
+                } catch (IOException e) {
+                    throw new UncheckedIOException("Can't append roles file " + from + " to " + dst, e);
+                }
+            });
+        }
+    }
+
     private void installModules() {
         logToProcessStdout("Installing " + modules.size() + " modules");
         for (Provider<File> module : modules) {
@@ -730,6 +755,11 @@ public void user(Map<String, String> userSpec) {
         credentials.add(cred);
     }
 
+    @Override
+    public void rolesFile(File rolesYml) {
+        roleFiles.add(rolesYml);
+    }
+
     private void runElasticsearchBinScriptWithInput(String input, String tool, CharSequence... args) {
         if (Files.exists(getDistroDir().resolve("bin").resolve(tool)) == false
             && Files.exists(getDistroDir().resolve("bin").resolve(tool + ".bat")) == false) {
@@ -1373,6 +1403,12 @@ private List<FileTree> getDistributionFiles(Action<PatternFilterable> patternFil
         return files;
     }
 
+    @InputFiles
+    @PathSensitive(PathSensitivity.RELATIVE)
+    public List<File> getRoleFiles() {
+        return roleFiles;
+    }
+
     @Nested
     public List<?> getKeystoreSettings() {
         return keystoreSettings.getNormalizedCollection();

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java

@@ -95,6 +95,8 @@ public interface TestClusterConfiguration {
 
     void user(Map<String, String> userSpec);
 
+    void rolesFile(File rolesYml);
+
     String getHttpSocketURI();
 
     String getTransportPortURI();

x-pack/plugin/security/qa/security-basic/build.gradle

@@ -27,7 +27,7 @@ testClusters.configureEach {
   setting 'xpack.security.authc.token.enabled', 'true'
   setting 'xpack.security.authc.api_key.enabled', 'true'
 
-  extraConfigFile 'roles.yml', file('src/javaRestTest/resources/roles.yml')
+  rolesFile file('src/javaRestTest/resources/roles.yml')
   user username: "admin_user", password: "admin-password"
   user username: "security_test_user", password: "security-test-password", role: "security_test_role"
   user username: "api_key_admin", password: "security-test-password", role: "api_key_admin_role"