Schema Generation for nested yml configurations (#1027)

Author: Sladyn

integrations/pom.xml

@@ -392,6 +392,12 @@
       <version>3.0.0</version>
     </dependency>
 
+    <dependency>
+      <groupId>org.jenkins-ci.plugins</groupId>
+      <artifactId>azure-keyvault</artifactId>
+      <version>1.4</version>
+    </dependency>
+
     <dependency>
       <groupId>org.jenkins-ci.plugins</groupId>
       <artifactId>display-url-api</artifactId>
@@ -429,6 +435,31 @@
 
   <dependencyManagement>
     <dependencies>
+      <dependency>
+        <groupId>com.microsoft.azure</groupId>
+        <artifactId>azure-annotations</artifactId>
+        <version>1.8.0</version>
+      </dependency>
+      <dependency>
+        <groupId>io.reactivex</groupId>
+        <artifactId>rxjava</artifactId>
+        <version>1.3.8</version>
+      </dependency>
+      <dependency>
+        <groupId>com.google.code.gson</groupId>
+        <artifactId>gson</artifactId>
+        <version>2.8.0</version>
+      </dependency>
+      <dependency>
+        <groupId>com.squareup.okhttp3</groupId>
+        <artifactId>logging-interceptor</artifactId>
+        <version>3.12.2</version>
+      </dependency>
+      <dependency>
+        <groupId>com.squareup.okhttp3</groupId>
+        <artifactId>okhttp</artifactId>
+        <version>3.12.2</version>
+      </dependency>
 
       <dependency>
         <groupId>com.github.docker-java</groupId>
@@ -525,7 +556,7 @@
       <dependency>
         <groupId>com.squareup.okio</groupId>
         <artifactId>okio</artifactId>
-        <version>1.13.0</version>
+        <version>1.15.0</version>
       </dependency>
       <dependency>
         <groupId>jaxen</groupId>

integrations/src/test/java/io/jenkins/plugins/casc/AzureKeyVaultTest.java

@@ -0,0 +1,23 @@
+package io.jenkins.plugins.casc;
+
+import io.jenkins.plugins.casc.misc.JenkinsConfiguredWithReadmeRule;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static io.jenkins.plugins.casc.misc.Util.convertYamlFileToJson;
+import static io.jenkins.plugins.casc.misc.Util.validateSchema;
+import static org.hamcrest.Matchers.empty;
+import static org.junit.Assert.assertThat;
+
+public class AzureKeyVaultTest {
+
+    @Rule
+    public JenkinsConfiguredWithReadmeRule j = new JenkinsConfiguredWithReadmeRule();
+
+    @Test
+    public void validJsonSchema() throws Exception {
+        assertThat(
+            validateSchema(convertYamlFileToJson(this, "azureKeyVault.yml")),
+            empty());
+    }
+}

integrations/src/test/java/io/jenkins/plugins/casc/SlackTest.java

@@ -9,7 +9,11 @@
 import org.junit.contrib.java.lang.system.EnvironmentVariables;
 import org.junit.rules.RuleChain;
 
-import static junit.framework.TestCase.assertNotNull;
+import static io.jenkins.plugins.casc.misc.Util.convertYamlFileToJson;
+import static io.jenkins.plugins.casc.misc.Util.validateSchema;
+import static org.hamcrest.Matchers.empty;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
 
 /**
  * @author v1v (Victor Martinez)
@@ -29,4 +33,11 @@ public void configure_slack() throws Exception {
         SlackNotifier.DescriptorImpl slackNotifier = ExtensionList.lookupSingleton(SlackNotifier.DescriptorImpl.class);
         assertNotNull(slackNotifier);
     }
+
+    @Test
+    public void validJsonSchema() throws Exception {
+        assertThat(
+            validateSchema(convertYamlFileToJson(this, "slackSchema.yml")),
+            empty());
+    }
 }

integrations/src/test/java/io/jenkins/plugins/casc/SonarQubeTest.java

@@ -6,10 +6,15 @@
 import io.jenkins.plugins.casc.misc.ConfiguredWithReadme;
 import io.jenkins.plugins.casc.misc.JenkinsConfiguredWithReadmeRule;
 import jenkins.model.GlobalConfiguration;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 
+import static io.jenkins.plugins.casc.misc.Util.convertYamlFileToJson;
+import static io.jenkins.plugins.casc.misc.Util.validateSchema;
+import static org.hamcrest.Matchers.empty;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -38,4 +43,18 @@ public void configure_sonar_globalconfig() throws Exception {
         assertEquals("envVar", triggers.getEnvVar());
     }
 
+    @Test
+    public void validJsonSchema() throws Exception {
+        assertThat(
+            validateSchema(convertYamlFileToJson(this, "sonarSchema.yml")),
+            empty());
+    }
+
+    @Test
+    @Ignore
+    public void validFullJsonSchema() throws Exception {
+        assertThat(
+            validateSchema(convertYamlFileToJson(this, "sonarSchemaFull.yml")),
+            empty());
+    }
 }

integrations/src/test/resources/io/jenkins/plugins/casc/azureKeyVault.yml

@@ -0,0 +1,4 @@
+unclassified:
+  azureKeyVault:
+    keyVaultURL: https://not-a-real-vault.vault.azure.net
+    credentialID: service-principal

integrations/src/test/resources/io/jenkins/plugins/casc/slackSchema.yml

@@ -0,0 +1,4 @@
+unclassified:
+  slackNotifier:
+    teamDomain: workspace
+    tokenCredentialId: slack-token

integrations/src/test/resources/io/jenkins/plugins/casc/sonarSchema.yml

@@ -0,0 +1,8 @@
+unclassified:
+  sonarGlobalConfiguration:
+    buildWrapperEnabled: false
+    installations:
+      - additionalAnalysisProperties: sonar.organization=jenkinsci
+        name: SonarQube
+        serverAuthenticationToken: 'sonarcloud-api-token'
+        serverUrl: 'https://sonarcloud.io'

integrations/src/test/resources/io/jenkins/plugins/casc/sonarSchemaFull.yml

@@ -0,0 +1,15 @@
+unclassified:
+  sonarGlobalConfiguration:                  # mandatory
+    buildWrapperEnabled: true
+    installations:                           # mandatory
+      - name: "TEST"                         # id of the SonarQube configuration - to be used in jobs
+        serverUrl: "http://url:9000"
+        #credentialsId: token-sonarqube       # id of the credentials containing sonar auth token (since 2.9 version)
+        serverAuthenticationToken: "token"   # for retrocompatibility with versions < 2.9
+        mojoVersion: "mojoVersion"
+        additionalProperties: "blah=blah"
+        additionalAnalysisProperties: "additionalAnalysisProperties"
+        triggers:
+          skipScmCause: true
+          skipUpstreamCause: true
+          envVar: "envVar"

plugin/src/main/java/io/jenkins/plugins/casc/Attribute.java

@@ -61,6 +61,7 @@ public class Attribute<Owner, Type> {
     private Setter<Owner, Type> setter;
     private Getter<Owner, Type> getter;
     private boolean secret;
+    private boolean isJsonSchema;
 
     private boolean deprecated;
 
@@ -111,6 +112,14 @@ public Class<? extends AccessRestriction>[] getRestrictions() {
         return restrictions != null ? restrictions : EMPTY;
     }
 
+    /**
+     * Set jsonSchema is used to tell the describe function to call the describe structure
+     * so that it supports and returns a nested structure
+     */
+    public void setJsonSchema(boolean jsonSchema) {
+        isJsonSchema = jsonSchema;
+    }
+
     public boolean isRestricted() {
         return restrictions != null && restrictions.length > 0;
     }
@@ -235,8 +244,12 @@ public CNode describe(Owner instance, ConfigurationContext context) throws Confi
             if (multiple) {
                 Sequence seq = new Sequence();
                 if (o.getClass().isArray()) o = Arrays.asList((Object[]) o);
-                for (Object value : (Iterable) o) {
-                    seq.add(_describe(c, context, value, shouldBeMasked));
+                if (o instanceof Iterable) {
+                    for (Object value : (Iterable) o) {
+                        seq.add(_describe(c, context, value, shouldBeMasked));
+                    }
+                } else {
+                    LOGGER.log(Level.FINE, o.getClass().toString() + " is not iterable");
                 }
                 return seq;
             }
@@ -249,6 +262,45 @@ public CNode describe(Owner instance, ConfigurationContext context) throws Confi
         }
     }
 
+    /**
+     * This function is for the JSONSchemaGeneration
+     * @param instance Owner Instance
+     * @param context Context to be passed
+     * @return CNode object describing the structure of the node
+     */
+    public CNode describeForSchema (Owner instance, ConfigurationContext context) {
+        final Configurator c = context.lookup(type);
+        if (c == null) {
+            return new Scalar("FAILED TO EXPORT\n" + instance.getClass().getName()+"#"+name +
+                ": No configurator found for type " + type);
+        }
+        try {
+            Object o = getType();
+            if (o == null) {
+                return null;
+            }
+
+            // In Export we sensitive only those values which do not get rendered as secrets
+            boolean shouldBeMasked = isSecret(instance);
+            if (multiple) {
+                Sequence seq = new Sequence();
+                if (o.getClass().isArray()) o = Arrays.asList(o);
+                if (o instanceof Iterable) {
+                    for (Object value : (Iterable) o) {
+                        seq.add(_describe(c, context, value, shouldBeMasked));
+                    }
+                }
+                return seq;
+            }
+            return _describe(c, context, o, shouldBeMasked);
+        } catch (Exception e) {
+            // Don't fail the whole export, prefer logging this error
+            LOGGER.log(Level.WARNING, "Failed to export", e);
+            return new Scalar("FAILED TO EXPORT\n" + instance.getClass().getName() + "#" + name + ": "
+                + printThrowable(e));
+        }
+    }
+
     /**
      * Describes a node.
      * @param c Configurator
@@ -261,7 +313,12 @@ public CNode describe(Owner instance, ConfigurationContext context) throws Confi
      */
     private CNode _describe(Configurator c, ConfigurationContext context, Object value, boolean shouldBeMasked)
             throws Exception {
-        CNode node = c.describe(value, context);
+        CNode node;
+        if (isJsonSchema) {
+            node = c.describeStructure(value, context);
+        } else {
+            node = c.describe(value, context);
+        }
         if (shouldBeMasked && node instanceof Scalar) {
             ((Scalar)node).sensitive(true);
         }

plugin/src/main/java/io/jenkins/plugins/casc/ConfigurationContext.java

@@ -30,6 +30,8 @@ public class ConfigurationContext implements ConfiguratorRegistry {
 
     private transient final ConfiguratorRegistry registry;
 
+    private transient String mode;
+
     public ConfigurationContext(ConfiguratorRegistry registry) {
         this.registry = registry;
     }
@@ -62,6 +64,15 @@ public void setUnknown(Unknown unknown) {
         this.unknown = unknown;
     }
 
+    String getMode() {
+        return mode;
+    }
+
+    public void setMode(String mode) {
+        this.mode = mode;
+    }
+
+
 
     // --- delegate methods for ConfigurationContext
 
@@ -123,6 +134,11 @@ public String value() {
         public boolean isAtLeast(Version version) {
             return this.ordinal() >= version.ordinal();
         }
+
+        @Override
+        public String toString() {
+            return value;
+        }
     }
 
     static {

plugin/src/main/java/io/jenkins/plugins/casc/Configurator.java

@@ -34,6 +34,7 @@
 import org.apache.commons.lang.StringUtils;
 import org.jenkinsci.Symbol;
 
+
 /**
  * Define a {@link Configurator} which handles a configuration element, identified by name.
  * @author <a href="mailto:[email protected]">Nicolas De Loof</a>
@@ -167,4 +168,27 @@ default CNode describe(T instance, ConfigurationContext context) throws Exceptio
         return mapping;
     }
 
+    /**
+     * Describe Structure of the attributes, as required by the schema.
+     * @param instance
+     * @param context
+     * @since 1.35
+     * @return CNode describing the attributes.
+     */
+    @CheckForNull
+    default CNode describeStructure(T instance, ConfigurationContext context)
+        throws Exception {
+        Mapping mapping = new Mapping();
+        for (Attribute attribute : getAttributes()) {
+            if (context.getMode().equals("JSONSchema")) {
+                attribute.setJsonSchema(true);
+            }
+            CNode value = attribute.describeForSchema(instance, context);
+            if (value != null) {
+                mapping.put(attribute.getName(), attribute.getType().getSimpleName());
+            }
+        }
+        return mapping;
+    }
+
 }