[INFRA] Support for local paths Nate.infra.relative paths (#532)

* manage identity test docs

* fixing tf module

* code planning

* init relative path support

* updated parsing logic

* additional changes

* updated generated

* add docs for local paths

* updated changes

* dennis updates

* add unit tests

* remove unwanted eslint disable and correct doc

Co-authored-by: Dennis Seah <[email protected]>

Author: NathanielRose

guides/cloud-infra-management.md

@@ -52,3 +52,81 @@ following:
 - **Using arguments** - Pass in your formatted source url for your private AzDO
   repo with the PAT and arbitrary username specified. Example
   `spk infra scaffold --name fabrikam --source https://spk:{$PAT}@dev.azure.com/microsoft/spk/_git/infra_repo --version master --template cluster/environments/azure-single-keyvault`
+
+## Terraform Modules with Local Paths
+
+`spk` now supports Terraform source templates that use a
+[local repository path](https://www.terraform.io/docs/modules/sources.html#local-paths)
+for references to modules. To obtain the modules for further teraform
+deployment, `spk infra generate` will shape a module source value from the
+`source`, `tempate`, and `version` arguments passed.
+
+**Example:**
+
+Template Main.tf
+
+```tf
+"aks-gitops" {
+  source = "../../azure/aks-gitops"
+  acr_enabled              = var.acr_enabled
+  agent_vm_count           = var.agent_vm_count
+  agent_vm_size            = var.agent_vm_size
+  cluster_name             = var.cluster_name
+  dns_prefix               = var.dns_prefix
+  flux_recreate            = var.flux_recreate
+  gc_enabled               = var.gc_enabled
+  gitops_ssh_url           = var.gitops_ssh_url
+  gitops_ssh_key           = var.gitops_ssh_key
+  gitops_path              = var.gitops_path
+  gitops_poll_interval     = var.gitops_poll_interval
+  gitops_label             = var.gitops_label
+  gitops_url_branch        = var.gitops_url_branch
+  ssh_public_key           = var.ssh_public_key
+  resource_group_name      = data.azurerm_resource_group.cluster_rg.name
+  service_principal_id     = var.service_principal_id
+  service_principal_secret = var.service_principal_secret
+  vnet_subnet_id           = tostring(element(module.vnet.vnet_subnet_ids, 0))
+  service_cidr             = var.service_cidr
+  dns_ip                   = var.dns_ip
+  docker_cidr              = var.docker_cidr
+  network_plugin           = var.network_plugin
+  network_policy           = var.network_policy
+  oms_agent_enabled        = var.oms_agent_enabled
+  kubernetes_version       = var.kubernetes_version
+}`;
+
+```
+
+SPK-generated Main.tf
+
+```tf
+"aks-gitops" {
+  source = "github.com/microsoft/bedrock?ref=master//cluster/azure/aks-gitops/"
+  acr_enabled              = var.acr_enabled
+  agent_vm_count           = var.agent_vm_count
+  agent_vm_size            = var.agent_vm_size
+  cluster_name             = var.cluster_name
+  dns_prefix               = var.dns_prefix
+  flux_recreate            = var.flux_recreate
+  gc_enabled               = var.gc_enabled
+  gitops_ssh_url           = var.gitops_ssh_url
+  gitops_ssh_key           = var.gitops_ssh_key
+  gitops_path              = var.gitops_path
+  gitops_poll_interval     = var.gitops_poll_interval
+  gitops_label             = var.gitops_label
+  gitops_url_branch        = var.gitops_url_branch
+  ssh_public_key           = var.ssh_public_key
+  resource_group_name      = data.azurerm_resource_group.cluster_rg.name
+  service_principal_id     = var.service_principal_id
+  service_principal_secret = var.service_principal_secret
+  vnet_subnet_id           = tostring(element(module.vnet.vnet_subnet_ids, 0))
+  service_cidr             = var.service_cidr
+  dns_ip                   = var.dns_ip
+  docker_cidr              = var.docker_cidr
+  network_plugin           = var.network_plugin
+  network_policy           = var.network_policy
+  oms_agent_enabled        = var.oms_agent_enabled
+  kubernetes_version       = var.kubernetes_version
+}`;
+
+```

src/commands/infra/generate.test.ts

@@ -3,11 +3,13 @@ import * as fsExtra from "fs-extra";
 import path from "path";
 import simpleGit from "simple-git/promise";
 import { loadConfigurationFromLocalEnv, readYaml } from "../../config";
+import { getErrorMessage } from "../../lib/errorBuilder";
 import { safeGitUrlForLogging } from "../../lib/gitutils";
-import { removeDir } from "../../lib/ioUtil";
+import { createTempDir, removeDir } from "../../lib/ioUtil";
 import { disableVerboseLogging, enableVerboseLogging } from "../../logger";
 import { InfraConfigYaml } from "../../types";
 import {
+  checkModuleSource,
   checkRemoteGitExist,
   createGenerated,
   DefinitionYAMLExistence,
@@ -19,6 +21,8 @@ import {
   gitCheckout,
   gitClone,
   gitPull,
+  inspectGeneratedSources,
+  moduleSourceModify,
   retryRemoteValidate,
   validateDefinition,
   validateRemoteSource,
@@ -42,6 +46,25 @@ interface GitTestData {
   safeLoggingUrl: string;
 }
 
+const mockTFData = `"aks-gitops" {
+  source = "../../azure/aks-gitops"
+  acr_enabled              = var.acr_enabled
+  agent_vm_count           = var.agent_vm_count
+};`;
+
+const mockSourceInfo = {
+  source: "https://github.com/microsoft/bedrock.git",
+  template: "cluster/environments/azure-single-keyvault",
+  version: "v0.0.1",
+};
+
+const modifedSourceModuleData = `"aks-gitops" {
+  source = "github.com/microsoft/bedrock.git?ref=v0.0.1//cluster/azure/aks-gitops/"
+  acr_enabled              = var.acr_enabled
+  agent_vm_count           = var.agent_vm_count
+};
+`;
+
 beforeAll(() => {
   enableVerboseLogging();
 });
@@ -778,3 +801,79 @@ describe("Validate backend.tfvars file", () => {
     );
   });
 });
+
+describe("test checkModuleSource function", () => {
+  it("positive test", () => {
+    let res = checkModuleSource(`source="../../azure/aks-gitops"`);
+    expect(res).toBeTruthy();
+    res = checkModuleSource(`source='../../azure/aks-gitops'`);
+    expect(res).toBeTruthy();
+    res = checkModuleSource(`source= '../../azure/aks-gitops'`);
+    expect(res).toBeTruthy();
+    res = checkModuleSource(` source = '../../azure/aks-gitops'`);
+    expect(res).toBeTruthy();
+    res = checkModuleSource(` source ='../../azure/aks-gitops'`);
+    expect(res).toBeTruthy();
+  });
+  it("negative test", () => {
+    const res = checkModuleSource(`source="/azure/aks-gitops"`);
+    expect(res).toBeFalsy();
+  });
+});
+
+describe("test moduleSourceModify function", () => {
+  it("positive test", async () => {
+    jest
+      .spyOn(generate, "revparse")
+      .mockResolvedValueOnce("cluster/azure/aks-gitops/");
+
+    const result = await moduleSourceModify(mockSourceInfo, mockTFData);
+    expect(result).toBe(modifedSourceModuleData);
+  });
+  it("negative test", async () => {
+    jest.spyOn(generate, "revparse").mockRejectedValueOnce(Error());
+
+    await expect(
+      moduleSourceModify(mockSourceInfo, mockTFData)
+    ).rejects.toThrow(getErrorMessage("infra-module-source-modify-err"));
+  });
+});
+
+describe("test inspectGeneratedSources function", () => {
+  it("positive test", async () => {
+    const folderName = createTempDir();
+    const fileName = path.join(folderName, "main.tf");
+    fs.writeFileSync(fileName, mockTFData, "utf-8");
+    jest
+      .spyOn(generate, "moduleSourceModify")
+      .mockResolvedValueOnce(modifedSourceModuleData);
+
+    await inspectGeneratedSources(folderName, mockSourceInfo);
+
+    const result = fs.readFileSync(fileName, "utf-8");
+    expect(result).toBe(modifedSourceModuleData);
+  });
+  it("positive test: there are no files", async () => {
+    const folderName = createTempDir();
+    await inspectGeneratedSources(folderName, mockSourceInfo);
+  });
+  it("positive test: file content is not modified if it does not have .tf extension", async () => {
+    const folderName = createTempDir();
+    const fileName = path.join(folderName, "main.txt");
+    fs.writeFileSync(fileName, mockTFData, "utf-8");
+
+    await inspectGeneratedSources(folderName, mockSourceInfo);
+    const result = fs.readFileSync(fileName, "utf-8");
+    expect(result).toBe(mockTFData);
+  });
+  it("negative test", async () => {
+    const folderName = createTempDir();
+    const fileName = path.join(folderName, "main.tf");
+    fs.writeFileSync(fileName, mockTFData, "utf-8");
+    jest.spyOn(generate, "moduleSourceModify").mockRejectedValueOnce(Error());
+
+    await expect(
+      inspectGeneratedSources(folderName, mockSourceInfo)
+    ).rejects.toThrow(getErrorMessage("infra-inspect-generated-sources-err"));
+  });
+});

src/commands/infra/generate.ts

@@ -40,6 +40,8 @@ export enum DefinitionYAMLExistence {
   PARENT_ONLY,
 }
 
+const regexSource = /^\s*source\s*=\s*["'](\.\.?\/[^"']*)["']$/gm;
+
 /**
  * Checks if definition.yaml is present locally to provided project path
  *
@@ -501,6 +503,117 @@ export const singleDefinitionGeneration = async (
   await copyTfTemplate(templatePath, childDirectory, true);
 };
 
+/**
+ * Checks to see if module sources are local
+ *
+ * @param tfFile path to the terraform file in child directory
+ */
+export const checkModuleSource = (tfData: string): boolean => {
+  // Check if the file string matches an instance of a module source value as a local path
+  const matches = tfData.match(regexSource);
+  return matches !== null;
+};
+
+export const revparse = async (sPath: string): Promise<string> => {
+  return await simpleGit(sPath).revparse(["--show-prefix"]);
+};
+
+/**
+ * Checks to see if module sources are local
+ *
+ * @param sourceConfig Array of source configuration
+ */
+export const moduleSourceModify = async (
+  fileSource: SourceInformation,
+  tfData: string
+): Promise<string> => {
+  try {
+    let result = "";
+    const sourceFolder = getSourceFolderNameFromURL(fileSource.source);
+    const sourcePath = path.join(spkTemplatesPath, sourceFolder);
+
+    // Split data by line and iterate
+    for (let line of tfData.split(/\r?\n/)) {
+      // Match line to expected module source format
+      if (line.match(regexSource) !== null) {
+        // Split the line into segments, the third element is the source value
+        const splitLine = line.split(/\s+/);
+        // Filter on module source value
+        const moduleSource = new RegExp(
+          splitLine[3].replace(/['"]+/g, ""),
+          "g"
+        );
+        // Get relative path of terraform module local to the repo
+        const repoModulePath = await revparse(
+          path.join(
+            sourcePath,
+            fileSource.template,
+            splitLine[3].replace(/["']/g, "")
+          )
+        );
+        // Concatenate the Git URL with munged data
+        const gitSource = fileSource.source
+          .replace(/(^\w+:|^)\/\//g, "")
+          .concat("?ref=", fileSource.version, "//", repoModulePath);
+        // Replace the line
+        line = line.replace(moduleSource, gitSource);
+      }
+      result += line + "\n";
+    }
+    return result;
+  } catch (err) {
+    throw buildError(
+      errorStatusCode.EXE_FLOW_ERR,
+      "infra-module-source-modify-err",
+      err
+    );
+  }
+};
+
+/**
+ * Checks to see if module sources are local
+ *
+ * @param sourceConfig Array of source configuration
+ */
+export const inspectGeneratedSources = async (
+  childDirectory: string,
+  sourceConfig: SourceInformation
+): Promise<void> => {
+  try {
+    // Support for local source paths, check template directory .tf files to generate git paths for terraform modules
+    const files = fsExtra.readdirSync(childDirectory, "utf-8");
+    for (const file of files) {
+      if (path.extname(file) === ".tf") {
+        const tfData = fsExtra.readFileSync(
+          path.join(childDirectory, file),
+          "utf8"
+        );
+        const containsLocalSource = checkModuleSource(tfData);
+        if (containsLocalSource) {
+          logger.info(
+            `Local relative paths for module source values detected in terraform file: ${file}`
+          );
+          const mungeData = await moduleSourceModify(sourceConfig, tfData);
+          logger.info(
+            `Terraform File: ${file} local module source values successfully converted to git source paths`
+          );
+          fsExtra.writeFileSync(
+            path.join(childDirectory, file),
+            mungeData,
+            "utf8"
+          );
+        }
+      }
+    }
+  } catch (err) {
+    throw buildError(
+      errorStatusCode.EXE_FLOW_ERR,
+      "infra-inspect-generated-sources-err",
+      err
+    );
+  }
+};
+
 /**
  * Creates "generated" directory if it does not already exists
  *
@@ -543,7 +656,6 @@ export const generateConfig = async (
       createGenerated(parentDirectory);
       createGenerated(childDirectory);
     }
-
     combineVariable(
       // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
       parentInfraConfig.variables!,
@@ -576,6 +688,8 @@ export const generateConfig = async (
       templatePath
     );
   }
+  // Modify generated TF files if it contains local sources
+  await inspectGeneratedSources(childDirectory, sourceConfig);
 };
 
 export const execute = async (

src/lib/i18n.json

@@ -58,6 +58,8 @@
     "infra-scaffold-cmd-values-missing": "Values for name, version and/or 'template were missing. Provide value for values for them.",
 
     "infra-generate-cmd-failed": "Infra generate command was not successfully executed.",
+    "infra-module-source-modify-err": "Could not modify source module.",
+    "infra-inspect-generated-sources-err": "Could not generated sources.",
 
     "infra-defn-yaml-not-found": "{0} was not found in {1}",
     "infra-defn-yaml-invalid": "The {0} file is invalid. There are missing fields. template: {1} source: {2} version: {3}.",