From d01682bc9168856780065eac0d459c3bb6c6843d Mon Sep 17 00:00:00 2001 From: ray chen Date: Thu, 13 Feb 2025 18:45:13 +0000 Subject: [PATCH 1/6] Increase timeout for generation job --- eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml b/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml index 85460cd9dbbc..b6ffaceb81d6 100644 --- a/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml +++ b/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml @@ -27,12 +27,13 @@ parameters: extends: template: /eng/pipelines/templates/stages/1es-redirect.yml parameters: + Use1ESOfficial: false stages: - stage: Build displayName: 'SDK Generation' jobs: - job: - + timeoutInMinutes: 300 variables: - template: /eng/pipelines/templates/variables/image.yml - name: NodeVersion From 886a0f47523c01a70838dccdb85d7013560b0864 Mon Sep 17 00:00:00 2001 From: ray chen Date: Sat, 15 Feb 2025 18:48:28 +0000 Subject: [PATCH 2/6] increase timeout to 60 hours --- eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml b/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml index b6ffaceb81d6..7193a1d24a46 100644 --- a/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml +++ b/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml @@ -33,7 +33,7 @@ extends: displayName: 'SDK Generation' jobs: - job: - timeoutInMinutes: 300 + timeoutInMinutes: 3600 variables: - template: /eng/pipelines/templates/variables/image.yml - name: NodeVersion From ec42f9b3b3bb3f96c75a35922d9f6138c66b1d79 Mon Sep 17 00:00:00 2001 From: ray chen Date: Fri, 21 Feb 2025 04:12:02 +0000 Subject: [PATCH 3/6] Added git op to the wrapper tool --- .../stages/archetype-spec-gen-sdk.yml | 9 +- .../cmd/spec-gen-sdk-runner.js | 0 .../spec-gen-sdk-runner/eslint.config.js | 5 + .../spec-gen-sdk-runner/src/change-files.ts | 96 +++++++ eng/tools/spec-gen-sdk-runner/src/commands.ts | 83 +++++- eng/tools/spec-gen-sdk-runner/src/index.ts | 11 +- eng/tools/spec-gen-sdk-runner/src/utils.ts | 247 +++++++++++++++++- 7 files changed, 431 insertions(+), 20 deletions(-) mode change 100644 => 100755 eng/tools/spec-gen-sdk-runner/cmd/spec-gen-sdk-runner.js create mode 100644 eng/tools/spec-gen-sdk-runner/src/change-files.ts diff --git a/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml b/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml index 7193a1d24a46..2d23e570a9de 100644 --- a/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml +++ b/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml @@ -34,6 +34,7 @@ extends: jobs: - job: timeoutInMinutes: 3600 + variables: - template: /eng/pipelines/templates/variables/image.yml - name: NodeVersion @@ -57,12 +58,12 @@ extends: displayName: Publish SDK artifacts to Pipeline Artifacts condition: and(ne(variables['ValidationResult'], ''), eq(variables['HasSDKArtifact'], 'true')) artifactName: $(sdkArtifactName) - targetPath: "$(System.DefaultWorkingDirectory)/out/generatedSdkArtifacts" + targetPath: "$(System.DefaultWorkingDirectory)/out/stagedArtifacts" - output: pipelineArtifact displayName: Publish API View artifacts to Pipeline Artifacts condition: and(ne(variables['ValidationResult'], ''), eq(variables['HasApiViewArtifact'], 'true')) artifactName: $(ArtifactName) - targetPath: "$(System.DefaultWorkingDirectory)/out/sdkApiViewArtifacts" + targetPath: "$(System.DefaultWorkingDirectory)/out/stagedArtifacts" - output: pipelineArtifact displayName: Publish logs to Pipeline Artifacts condition: ne(variables['ValidationResult'], '') @@ -169,7 +170,7 @@ extends: optional_params="" sdk_gen_info="sdk generation from Config : " - if [ "${{ parameters.ConfigType }}" = "TypeSpec" ]; then + if [ "$(Build.Reason)" != "PullRequest" ] && [ "${{ parameters.ConfigType }}" = "TypeSpec" ]; then optional_params="$optional_params --tsp-config-relative-path ${{ parameters.ConfigPath }}" sdk_gen_info="$sdk_gen_info '${{ parameters.ConfigPath }}'," elif [ "${{ parameters.ConfigType }}" = "OpenAPI" ]; then @@ -178,7 +179,7 @@ extends: fi if [ "$(Build.Reason)" = "PullRequest" ]; then - optional_params="$optional_params --pr-number=$(System.PullRequest.PullRequestNumber)" + optional_params="$optional_params --pr-number $(System.PullRequest.PullRequestNumber)" specPrUrl="${{ parameters.SpecRepoUrl }}/pull/$(System.PullRequest.PullRequestNumber)" sdk_gen_info="$sdk_gen_info spec PR: $specPrUrl" fi diff --git a/eng/tools/spec-gen-sdk-runner/cmd/spec-gen-sdk-runner.js b/eng/tools/spec-gen-sdk-runner/cmd/spec-gen-sdk-runner.js old mode 100644 new mode 100755 diff --git a/eng/tools/spec-gen-sdk-runner/eslint.config.js b/eng/tools/spec-gen-sdk-runner/eslint.config.js index e2399d13ebaf..c1f1492b887b 100644 --- a/eng/tools/spec-gen-sdk-runner/eslint.config.js +++ b/eng/tools/spec-gen-sdk-runner/eslint.config.js @@ -68,11 +68,16 @@ const config = tseslint.config( "@typescript-eslint/restrict-template-expressions": "off", "@typescript-eslint/no-unsafe-assignment": "off", "@typescript-eslint/no-unsafe-argument": "off", + "@typescript-eslint/consistent-indexed-object-style": "off", + "@typescript-eslint/no-unnecessary-condition": "off", + "@typescript-eslint/consistent-type-definitions": "off", + "@typescript-eslint/no-inferrable-types": "off", // We want more flexibility with file names. // https://github.com/sindresorhus/eslint-plugin-unicorn/blob/main/docs/rules/filename-case.md "unicorn/filename-case": "off", "unicorn/prefer-ternary": "off", + "unicorn/no-useless-undefined": "off", // We prefer to have explicitly import at the top of the file, even if the same element is exported again, // which we do in index.ts files. diff --git a/eng/tools/spec-gen-sdk-runner/src/change-files.ts b/eng/tools/spec-gen-sdk-runner/src/change-files.ts new file mode 100644 index 000000000000..cb9551a34c26 --- /dev/null +++ b/eng/tools/spec-gen-sdk-runner/src/change-files.ts @@ -0,0 +1,96 @@ +import path from "node:path"; +import { + execAsync, + getChangedFiles, + searchRelatedParentFolders, + searchRelatedTypeSpecProjectBySharedLibrary, + searchSharedLibrary, +} from "./utils.js"; +import { logMessage } from "./log.js"; +import { SpecGenSdkCmdInput } from "./types.js"; + +const readmeMdRegex = /^readme.md$/; +const typespecProjectRegex = /^tspconfig.yaml$/; +const typespecProjectSharedLibraryRegex = /[^/]+\.Shared/; + +type ChangedSpecs = { + [K in "readmeMd" | "typespecProject"]?: string; +} & { + specs: string[]; +}; + +export async function detectChangedSpecConfigFiles( + commandInput: SpecGenSdkCmdInput, +): Promise { + const prChangedFiles: string[] = getChangedFiles(commandInput.localSpecRepoPath) ?? []; + if (prChangedFiles.length === 0) { + logMessage("No files changed in the PR"); + } + logMessage(`Changed files in the PR: ${prChangedFiles.length}`); + const fileList = prChangedFiles.filter((p) => !p.includes("/scenarios/")); + const { stdout: headCommitRaw } = await execAsync("git rev-parse HEAD"); + const headCommit = headCommitRaw.trim(); // Trim any newline characters + const { stdout: treeIdRaw } = await execAsync(`git rev-parse ${headCommit}^{tree}`); + const treeId = treeIdRaw.trim(); + + logMessage(`Related readme.md and typespec project list:`); + const changedSpecs: ChangedSpecs[] = []; + const readmeMDResult = await searchRelatedParentFolders(fileList, { + searchFileRegex: readmeMdRegex, + specFolder: commandInput.localSpecRepoPath, + treeId, + }); + const typespecProjectResult = await searchRelatedParentFolders(fileList, { + searchFileRegex: typespecProjectRegex, + specFolder: commandInput.localSpecRepoPath, + treeId, + }); + const typespecProjectSharedLibraries = searchSharedLibrary(fileList, { + searchFileRegex: typespecProjectSharedLibraryRegex, + specFolder: commandInput.localSpecRepoPath, + treeId, + }); + const typespecProjectResultSearchedBySharedLibrary = + await searchRelatedTypeSpecProjectBySharedLibrary(typespecProjectSharedLibraries, { + searchFileRegex: typespecProjectRegex, + specFolder: commandInput.localSpecRepoPath, + treeId, + }); + for (const folderPath of Object.keys(typespecProjectResultSearchedBySharedLibrary)) { + if (typespecProjectResult[folderPath]) { + typespecProjectResult[folderPath] = [ + ...typespecProjectResult[folderPath], + ...typespecProjectResultSearchedBySharedLibrary[folderPath], + ]; + } else { + typespecProjectResult[folderPath] = typespecProjectResultSearchedBySharedLibrary[folderPath]; + } + } + const result: { [folderPath: string]: string[] } = {}; + for (const folderPath of Object.keys(readmeMDResult)) { + result[folderPath] = readmeMDResult[folderPath]; + } + + for (const folderPath of Object.keys(typespecProjectResult)) { + result[folderPath] = typespecProjectResult[folderPath]; + } + for (const folderPath of Object.keys(result)) { + const readmeMdPath = path.join(folderPath, "readme.md"); + const cs: ChangedSpecs = { + readmeMd: readmeMdPath, + specs: readmeMDResult[folderPath], + }; + + if (typespecProjectResult[folderPath]) { + delete cs.readmeMd; + cs.specs = typespecProjectResult[folderPath]; + cs.typespecProject = path.join(folderPath, "tspconfig.yaml"); + logMessage(`\t tspconfig.yaml file: ${cs.typespecProject}`); + } else { + logMessage(`\t readme.md file: ${readmeMdPath}`); + } + changedSpecs.push(cs); + } + + return changedSpecs; +} diff --git a/eng/tools/spec-gen-sdk-runner/src/commands.ts b/eng/tools/spec-gen-sdk-runner/src/commands.ts index 0c6a498a0c49..70cb4016076a 100644 --- a/eng/tools/spec-gen-sdk-runner/src/commands.ts +++ b/eng/tools/spec-gen-sdk-runner/src/commands.ts @@ -6,9 +6,11 @@ import { getArgumentValue, runSpecGenSdkCommand, getAllTypeSpecPaths, + resetGitRepo, } from "./utils.js"; import { LogLevel, logMessage, vsoAddAttachment } from "./log.js"; import { SpecGenSdkCmdInput } from "./types.js"; +import { detectChangedSpecConfigFiles } from "./change-files.js"; export async function generateSdkForSingleSpec(): Promise { // Parse the arguments @@ -44,8 +46,61 @@ export async function generateSdkForSingleSpec(): Promise { return statusCode; } +/* Generate SDKs for spec pull request */ +export async function generateSdkForSpecPr(): Promise { + // Parse the arguments + const commandInput: SpecGenSdkCmdInput = parseArguments(); + // Construct the spec-gen-sdk command + const specGenSdkCommand = prepareSpecGenSdkCommand(commandInput); + // Get the spec paths from the changed files + const changedSpecs = await detectChangedSpecConfigFiles(commandInput); + + for (const changedSpec of changedSpecs) { + if (!changedSpec.typespecProject && !changedSpec.readmeMd) { + logMessage("No spec config file found in the changed files", LogLevel.Warn); + continue; + } + if (changedSpec.typespecProject) { + specGenSdkCommand.push("--tsp-config-relative-path", changedSpec.typespecProject); + } + if (changedSpec.readmeMd) { + specGenSdkCommand.push("--readme-relative-path", changedSpec.readmeMd); + } + const changedSpecPath = changedSpec.typespecProject ?? changedSpec.readmeMd; + logMessage(`Generating SDK from ${changedSpecPath}`, LogLevel.Group); + logMessage(`Command:${specGenSdkCommand.join(" ")}`); + try { + await resetGitRepo(commandInput.localSdkRepoPath); + await runSpecGenSdkCommand(specGenSdkCommand); + logMessage("Command executed successfully"); + } catch (error) { + logMessage(`Error executing command:${error}`, LogLevel.Error); + return 1; + } + // Read the execution report to determine if the generation was successful + const executionReportPath = path.join( + commandInput.workingFolder, + `${commandInput.sdkRepoName}_tmp/execution-report.json`, + ); + try { + const executionReport = JSON.parse(fs.readFileSync(executionReportPath, "utf8")); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + const executionResult = executionReport.executionResult; + logMessage(`Execution Result:${executionResult}`); + } catch (error) { + logMessage( + `Error reading execution report at ${executionReportPath}:${error}`, + LogLevel.Error, + ); + return 1; + } + logMessage("ending group logging", LogLevel.EndGroup); + } + return 0; +} + /** - * Generate SDKs for all specs. + * Generate SDKs for batch specs. */ export async function generateSdkForBatchSpecs(runMode: string): Promise { // Parse the arguments @@ -60,9 +115,9 @@ export async function generateSdkForBatchSpecs(runMode: string): Promise let markdownContent = "\n"; let failedContent = `## Spec Failures in the Generation Process\n`; let succeededContent = `## Successful Specs in the Generation Process\n`; - let undefinedContent = `## Disabled Specs in the Generation Process\n`; + let notEnabledContent = `## Specs with SDK Not Enabled\n`; let failedCount = 0; - let undefinedCount = 0; + let notEnabledCount = 0; let succeededCount = 0; // Generate SDKs for each spec @@ -75,6 +130,7 @@ export async function generateSdkForBatchSpecs(runMode: string): Promise } logMessage(`Command:${specGenSdkCommand.join(" ")}`); try { + await resetGitRepo(commandInput.localSdkRepoPath); await runSpecGenSdkCommand(specGenSdkCommand); logMessage("Command executed successfully"); } catch (error) { @@ -96,12 +152,12 @@ export async function generateSdkForBatchSpecs(runMode: string): Promise const executionResult = executionReport.executionResult; logMessage(`Execution Result:${executionResult}`); - if (executionResult === "succeeded") { + if (executionResult === "succeeded" || executionResult === "warning") { succeededContent += `${specConfigPath},`; succeededCount++; - } else if (executionResult === undefined) { - undefinedContent += `${specConfigPath},`; - undefinedCount++; + } else if (executionResult === "notEnabled") { + notEnabledContent += `${specConfigPath},`; + notEnabledCount++; } else { failedContent += `${specConfigPath},`; failedCount++; @@ -118,15 +174,15 @@ export async function generateSdkForBatchSpecs(runMode: string): Promise if (failedCount > 0) { markdownContent += `${failedContent}\n`; } - if (undefinedCount > 0) { - markdownContent += `${undefinedContent}\n`; + if (notEnabledCount > 0) { + markdownContent += `${notEnabledContent}\n`; } if (succeededCount > 0) { markdownContent += `${succeededContent}\n`; } markdownContent += failedCount ? `## Total Failed Specs\n ${failedCount}\n` : ""; - markdownContent += undefinedCount - ? `## Total Disabled Specs in the Configuration\n ${undefinedCount}\n` + markdownContent += notEnabledCount + ? `## Total Specs with SDK not enabled in the Configuration\n ${notEnabledCount}\n` : ""; markdownContent += succeededCount ? `## Total Successful Specs\n ${succeededCount}\n` : ""; markdownContent += `## Total Specs Count\n ${specConfigPaths.length}\n\n`; @@ -251,7 +307,10 @@ function getSpecPaths(runMode: string, specRepoPath: string): string[] { break; } case "sample-typespecs": { - specConfigPaths.push("specification/contosowidgetmanager/Contoso.Management/tspconfig.yaml"); + specConfigPaths.push( + "specification/contosowidgetmanager/Contoso.Management/tspconfig.yaml", + "specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml", + ); } } return specConfigPaths; diff --git a/eng/tools/spec-gen-sdk-runner/src/index.ts b/eng/tools/spec-gen-sdk-runner/src/index.ts index b25a78a36701..9b1914d17d1e 100644 --- a/eng/tools/spec-gen-sdk-runner/src/index.ts +++ b/eng/tools/spec-gen-sdk-runner/src/index.ts @@ -1,14 +1,23 @@ import { exit } from "node:process"; import { getArgumentValue } from "./utils.js"; -import { generateSdkForBatchSpecs, generateSdkForSingleSpec } from "./commands.js"; +import { + generateSdkForBatchSpecs, + generateSdkForSingleSpec, + generateSdkForSpecPr, +} from "./commands.js"; export async function main() { // Get the arguments passed to the script const args: string[] = process.argv.slice(2); + // Log the arguments to the console + console.log("Arguments passed to the script:", args.join(" ")); const runMode: string = getArgumentValue(args, "--rm", ""); + const pullRequestNumber: string = getArgumentValue(args, "--pr-number", ""); let statusCode = 0; if (runMode) { statusCode = await generateSdkForBatchSpecs(runMode); + } else if (pullRequestNumber) { + statusCode = await generateSdkForSpecPr(); } else { statusCode = await generateSdkForSingleSpec(); } diff --git a/eng/tools/spec-gen-sdk-runner/src/utils.ts b/eng/tools/spec-gen-sdk-runner/src/utils.ts index 2041914a9368..1e12e59932e6 100644 --- a/eng/tools/spec-gen-sdk-runner/src/utils.ts +++ b/eng/tools/spec-gen-sdk-runner/src/utils.ts @@ -1,11 +1,35 @@ -import { spawn, spawnSync } from "node:child_process"; +import { spawn, spawnSync, exec } from "node:child_process"; import path from "node:path"; import fs from "node:fs"; import { LogLevel, logMessage } from "./log.js"; +import { promisify } from "node:util"; type Dirent = fs.Dirent; -// Common function to find files recursively with case-insensitive matching +export const execAsync = promisify(exec); + +/** + * Reset unstaged changes in a git repository + * @param repoPath The path to the git repository + * @returns A promise that resolves when the reset is complete + */ +export async function resetGitRepo(repoPath: string): Promise { + try { + const { stderr } = await execAsync("git reset --hard HEAD", { + cwd: repoPath, + }); + if (stderr) { + logMessage(`Warning during git reset: ${stderr}`, LogLevel.Warn); + } + logMessage(`Successfully reset git repo at ${repoPath}`, LogLevel.Info); + } catch (error) { + throw new Error(`Failed to reset git repo at ${repoPath}: ${error}`); + } +} + +/* + * Common function to find files recursively with case-insensitive matching + */ export function findFilesRecursive(directory: string, fileName: string): string[] { let results: string[] = []; const list: Dirent[] = fs.readdirSync(directory, { withFileTypes: true }); @@ -31,6 +55,9 @@ export function getArgumentValue(args: string[], flag: string, defaultValue: str return index !== -1 && args[index + 1] ? args[index + 1] : defaultValue; } +/* + * Get the relative path from the specification folder + */ export function getRelativePathFromSpecification(absolutePath: string): string { const specificationIndex = absolutePath.indexOf("specification/"); if (specificationIndex !== -1) { @@ -39,6 +66,9 @@ export function getRelativePathFromSpecification(absolutePath: string): string { return absolutePath; } +/* + * Run the spec-gen-sdk command + */ export async function runSpecGenSdkCommand(specGenSdkCommand: string[]): Promise { return new Promise((resolve, reject) => { const childProcess = spawn("npx", specGenSdkCommand, { @@ -59,6 +89,9 @@ export async function runSpecGenSdkCommand(specGenSdkCommand: string[]): Promise }); } +/* + * Get the list of all type spec project folder paths + */ export function getAllTypeSpecPaths(specRepoPath: string): string[] { const scriptPath = path.resolve(specRepoPath, "eng/scripts/Get-TypeSpec-Folders.ps1"); const args = [ @@ -85,6 +118,10 @@ export function getAllTypeSpecPaths(specRepoPath: string): string[] { return []; } } + +/* + * Run the PowerShell script + */ export function runPowerShellScript(args: string[]): string | undefined { const result = spawnSync("/usr/bin/pwsh", args, { encoding: "utf8" }); if (result.error) { @@ -94,5 +131,209 @@ export function runPowerShellScript(args: string[]): string | undefined { if (result.stderr) { logMessage(`PowerShell script error output:${result.stderr}`, LogLevel.Error); } - return result.stdout.trim(); + return result.stdout?.trim(); +} + +// Function to call Get-ChangedFiles from PowerShell script +export function getChangedFiles( + specRepoPath: string, + baseCommitish: string = "HEAD^", + targetCommitish: string = "HEAD", + diffFilter: string = "d", +): string[] | undefined { + const scriptPath = path.resolve(specRepoPath, "eng/scripts/ChangedFiles-Functions.ps1"); + const args = [ + "-Command", + `& { . '${scriptPath}'; Get-ChangedFiles '${baseCommitish}' '${targetCommitish}' '${diffFilter}' }`, + ]; + + const output = runPowerShellScript(args); + if (output) { + return output + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + } + return undefined; +} + +/* + * The options for searching related folders + */ +export type FsSearchOptions = { + searchFileRegex: RegExp; + treeId: string; + specFolder: string; +}; + +export type ListTree = { + mode: string; + type: string; + object: string; + file: string; +}[]; + +/** + * Search for the related folder of a file + * @param filePath The file path to search + * @param options The search options + * @returns The related folder of the file + */ +export const searchRelatedFolder = (filePath: string, options: FsSearchOptions) => { + let searchPath = filePath; + + while (searchPath !== "." && searchPath !== path.dirname(searchPath)) { + const fileName = path.basename(searchPath); + if (options.searchFileRegex.test(fileName)) { + return searchPath; + } + searchPath = path.dirname(searchPath); + } + + return undefined; +}; + +/** + * Search for the shared library folder from peer's folder + */ +export const searchSharedLibrary = (fileList: string[], options: FsSearchOptions) => { + const result: { [relatedFolder: string]: string[] } = {}; + fileList.sort(); + let lastFolder: string | undefined = undefined; + + for (const filePath of fileList) { + if (lastFolder !== undefined && filePath.startsWith(lastFolder)) { + result[lastFolder].push(filePath); + } + const relatedFolder = searchRelatedFolder(filePath, options); + if (relatedFolder === undefined) { + continue; + } + if (result[relatedFolder] === undefined) { + result[relatedFolder] = []; + } + result[relatedFolder].push(filePath); + lastFolder = relatedFolder; + } + + return result; +}; + +/* + * Search for the related type spec projects from a shared library + */ +export const searchRelatedTypeSpecProjectBySharedLibrary = async ( + sharedLibraries: { [relatedFolder: string]: string[] }, + options: FsSearchOptions, +) => { + const result: { [relatedFolder: string]: string[] } = {}; + for (const sharedLibrary of Object.keys(sharedLibraries)) { + const parentFolder = path.dirname(sharedLibrary); + const fileNames = await getFilesInFolder(parentFolder, options); + for (const fileName of fileNames) { + const filePath = path.join(parentFolder, fileName); + const subFileNames = await getFilesInFolder(filePath, options); + for (const subFileName of subFileNames) { + if (options.searchFileRegex.test(subFileName)) { + if (!result[filePath]) { + result[filePath] = []; + } + result[filePath] = [...result[filePath], ...sharedLibraries[sharedLibrary]]; + } + } + } + } + return result; +}; + +/** + * Search from the parent folders for a list of files + */ +export const searchRelatedParentFolders = async (fileList: string[], options: FsSearchOptions) => { + const result: { [relatedFolder: string]: string[] } = {}; + fileList.sort(); + + for (const filePath of fileList) { + const relatedParentFolder = await searchRelatedParentFolder(filePath, options); + if (relatedParentFolder === undefined) { + continue; + } + if (result[relatedParentFolder] === undefined) { + result[relatedParentFolder] = []; + } + result[relatedParentFolder].push(filePath); + } + + return result; +}; + +/* + * Search from a nearest parent folder for the specific file pattern + */ +export const searchRelatedParentFolder = async (filePath: string, options: FsSearchOptions) => { + let searchPath = filePath; + + while (searchPath !== ".") { + const fileNames = await getFilesInFolder(searchPath, options); + for (const fileName of fileNames) { + if (options.searchFileRegex.test(fileName)) { + return searchPath; + } + } + searchPath = path.dirname(searchPath); + } + + return undefined; +}; + +/* + * Get the files in a folder + */ +const getFilesInFolder = async ( + searchPath: string, + options: FsSearchOptions, +): Promise => { + const workingFolder = "."; + const workPath = path.resolve(process.cwd(), workingFolder, options.specFolder, searchPath); + // Execute the git command using exec + const { stdout, stderr } = await execAsync(`git ls-tree ${options.treeId} ${workPath}`); + if (stderr) { + throw new Error(`Error executing git ls-tree ${options.treeId} ${workPath}: ${stderr}`); + } + const subTree = gitTreeResultToStringArray(stdout); + if (subTree.length === 0 || (subTree.length > 0 && subTree[0].type !== "tree")) { + return []; + } + const entryPath = path.join(workPath, "/"); + const { stdout: treeEntry, stderr: treeEntryError } = await execAsync( + `git ls-tree ${options.treeId} ${entryPath}`, + ); + if (treeEntryError) { + throw new Error( + `Error executing git ls-tree ${options.treeId} ${entryPath}: ${treeEntryError}`, + ); + } + const subTreeEntry = gitTreeResultToStringArray(treeEntry); + return subTreeEntry.map((item) => item.file.slice(searchPath.length + 1)); +}; + +/* + * Convert the git tree result to a string array + */ +export function gitTreeResultToStringArray(treeResult: string): ListTree { + if (treeResult === "") { + return []; + } + const lines = treeResult.trim().split("\n"); + const resultArray = lines.map((line) => { + const [mode, type, object, file] = line.split(/\s+/); + return { + mode, + type, + object, + file, + }; + }); + + return resultArray; } From c008111f8d82fdfde3d1c6d584ec9ec781d4a6e8 Mon Sep 17 00:00:00 2001 From: ray chen Date: Fri, 21 Feb 2025 04:40:02 +0000 Subject: [PATCH 4/6] clean all files before generation --- eng/tools/spec-gen-sdk-runner/src/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/spec-gen-sdk-runner/src/utils.ts b/eng/tools/spec-gen-sdk-runner/src/utils.ts index 1e12e59932e6..44f1cf7d9957 100644 --- a/eng/tools/spec-gen-sdk-runner/src/utils.ts +++ b/eng/tools/spec-gen-sdk-runner/src/utils.ts @@ -15,7 +15,7 @@ export const execAsync = promisify(exec); */ export async function resetGitRepo(repoPath: string): Promise { try { - const { stderr } = await execAsync("git reset --hard HEAD", { + const { stderr } = await execAsync("git clean -fd && git reset --hard HEAD", { cwd: repoPath, }); if (stderr) { From d2b361a0640ca88c2f7c101a50a50731ce4c1ced Mon Sep 17 00:00:00 2001 From: ray chen Date: Fri, 21 Feb 2025 18:09:46 +0000 Subject: [PATCH 5/6] update specGenCommand for multiple generation --- .../templates/stages/archetype-spec-gen-sdk.yml | 2 +- eng/tools/spec-gen-sdk-runner/src/commands.ts | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml b/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml index 2d23e570a9de..ff68f1737abc 100644 --- a/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml +++ b/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml @@ -33,7 +33,7 @@ extends: displayName: 'SDK Generation' jobs: - job: - timeoutInMinutes: 3600 + timeoutInMinutes: 2400 variables: - template: /eng/pipelines/templates/variables/image.yml diff --git a/eng/tools/spec-gen-sdk-runner/src/commands.ts b/eng/tools/spec-gen-sdk-runner/src/commands.ts index 70cb4016076a..1c5d56187f33 100644 --- a/eng/tools/spec-gen-sdk-runner/src/commands.ts +++ b/eng/tools/spec-gen-sdk-runner/src/commands.ts @@ -55,27 +55,37 @@ export async function generateSdkForSpecPr(): Promise { // Get the spec paths from the changed files const changedSpecs = await detectChangedSpecConfigFiles(commandInput); + let statusCode = 0; + let pushedSpecConfigCount; for (const changedSpec of changedSpecs) { if (!changedSpec.typespecProject && !changedSpec.readmeMd) { logMessage("No spec config file found in the changed files", LogLevel.Warn); continue; } + pushedSpecConfigCount = 0; if (changedSpec.typespecProject) { specGenSdkCommand.push("--tsp-config-relative-path", changedSpec.typespecProject); + pushedSpecConfigCount++; } if (changedSpec.readmeMd) { specGenSdkCommand.push("--readme-relative-path", changedSpec.readmeMd); + pushedSpecConfigCount++; } const changedSpecPath = changedSpec.typespecProject ?? changedSpec.readmeMd; logMessage(`Generating SDK from ${changedSpecPath}`, LogLevel.Group); logMessage(`Command:${specGenSdkCommand.join(" ")}`); + try { await resetGitRepo(commandInput.localSdkRepoPath); await runSpecGenSdkCommand(specGenSdkCommand); logMessage("Command executed successfully"); } catch (error) { logMessage(`Error executing command:${error}`, LogLevel.Error); - return 1; + statusCode = 1; + } + // Pop the spec config path from specGenSdkCommand + for (let index = 0; index < pushedSpecConfigCount * 2; index++) { + specGenSdkCommand.pop(); } // Read the execution report to determine if the generation was successful const executionReportPath = path.join( @@ -92,11 +102,11 @@ export async function generateSdkForSpecPr(): Promise { `Error reading execution report at ${executionReportPath}:${error}`, LogLevel.Error, ); - return 1; + statusCode = 1; } logMessage("ending group logging", LogLevel.EndGroup); } - return 0; + return statusCode; } /** From 8d11f9cf8aa09a063f273cb1f088cd5ca1980e88 Mon Sep 17 00:00:00 2001 From: ray chen Date: Fri, 21 Feb 2025 18:35:29 +0000 Subject: [PATCH 6/6] Log successful info in else --- eng/tools/spec-gen-sdk-runner/src/utils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/tools/spec-gen-sdk-runner/src/utils.ts b/eng/tools/spec-gen-sdk-runner/src/utils.ts index 44f1cf7d9957..9d8e387a7649 100644 --- a/eng/tools/spec-gen-sdk-runner/src/utils.ts +++ b/eng/tools/spec-gen-sdk-runner/src/utils.ts @@ -20,8 +20,9 @@ export async function resetGitRepo(repoPath: string): Promise { }); if (stderr) { logMessage(`Warning during git reset: ${stderr}`, LogLevel.Warn); + } else { + logMessage(`Successfully reset git repo at ${repoPath}`, LogLevel.Info); } - logMessage(`Successfully reset git repo at ${repoPath}`, LogLevel.Info); } catch (error) { throw new Error(`Failed to reset git repo at ${repoPath}: ${error}`); }