Skip to content

Commit 1e7835e

Browse files
google-oss-prow[bot]google-oss-prow[bot]
authored
Merge pull request GoogleCloudPlatform#3771 from cheftako/conductor2
Added conductor runner to generate mock service and resource go files.
2 parents c178e7e + 5c8b7c5 commit 1e7835e

File tree

5 files changed

+213
-31
lines changed

5 files changed

+213
-31
lines changed

‎dev/tools/controllerbuilder/pkg/commands/exportcsv/prompt.go

+14-6
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,10 @@ import (
3535
type PromptOptions struct {
3636
*options.GenerateOptions
3737

38-
ProtoDir string
39-
SrcDir string
40-
Output string
38+
ProtoDir string
39+
SrcDir string
40+
Output string
41+
InputFile string
4142

4243
// StrictInputColumnKeys ensures that all input datapoints have this shape.
4344
// This helps detect typos in the examples.
@@ -56,6 +57,7 @@ func (o *PromptOptions) BindFlags(cmd *cobra.Command) {
5657
cmd.Flags().StringVar(&o.SrcDir, "src-dir", o.SrcDir, "base directory for source code")
5758
cmd.Flags().StringVar(&o.ProtoDir, "proto-dir", o.ProtoDir, "base directory for checkout of proto API definitions")
5859
cmd.Flags().StringVar(&o.Output, "output", o.Output, "the directory to store the prompt outcome")
60+
cmd.Flags().StringVar(&o.InputFile, "input-file", o.InputFile, "the input file to get input from")
5961
cmd.Flags().StringSliceVar(&o.StrictInputColumnKeys, "strict-input-columns", o.StrictInputColumnKeys, "return an error if we see an irregular datapoint for this tool")
6062
}
6163

@@ -129,9 +131,15 @@ func RunPrompt(ctx context.Context, o *PromptOptions) error {
129131
}
130132
}
131133

132-
b, err := io.ReadAll(os.Stdin)
133-
if err != nil {
134-
return fmt.Errorf("reading from stdin: %w", err)
134+
var b []byte
135+
if o.InputFile == "" {
136+
if b, err = io.ReadAll(os.Stdin); err != nil {
137+
return fmt.Errorf("reading from stdin: %w", err)
138+
}
139+
} else {
140+
if b, err = os.ReadFile(o.InputFile); err != nil {
141+
return fmt.Errorf("reading from %s: %w", o.InputFile, err)
142+
}
135143
}
136144

137145
dataPoints, err := x.BuildDataPoints(ctx, "<prompt>", b)

‎experiments/conductor/branches.yaml

+8-2
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ branches:
2424
package: google.cloud.accessapproval.v1
2525
proto: ApprovalRequest
2626
proto-path: google.cloud.accessapproval.v1.ApprovalRequest
27+
proto-service: google.cloud.accessapproval.v1.AccessApproval
28+
host-name: accessapproval.googleapis.com
2729
notes:
2830
- No gcloud command found
2931
- name: ai-customjobs
@@ -37,6 +39,8 @@ branches:
3739
package: ""
3840
proto: ""
3941
proto-path: ""
42+
proto-svc: ""
43+
host_name: ""
4044
notes:
4145
- name: compute-computebackendservice
4246
local: resource-compute-computebackendservice
@@ -47,6 +51,8 @@ branches:
4751
controller: terraform
4852
kind: ""
4953
package: ""
50-
proto: ""
51-
proto-path: ""
54+
proto: BackendService
55+
proto-path: google.cloud.compute.v1.BackendService
56+
proto-service: google.cloud.compute.v1.BackendServices
57+
host-name: compute.googleapis.com
5258
notes:

‎experiments/conductor/cmd/runner/mock_commands.go

+168-16
Original file line numberDiff line numberDiff line change
@@ -190,15 +190,8 @@ func createScriptYaml(opts *RunnerOptions, branch Branch) {
190190

191191
workDir := filepath.Join(opts.branchRepoDir, "mockgcp")
192192

193-
log.Printf("COMMAND: git checkout %s\r\n", branch.Local)
194-
checkout := exec.Command("git", "checkout", branch.Local)
195-
checkout.Dir = workDir
196193
var out strings.Builder
197-
checkout.Stdout = &out
198-
if err := checkout.Run(); err != nil {
199-
log.Fatal(err)
200-
}
201-
log.Printf("BRANCH CHECKOUT: %q\n", out.String())
194+
checkoutBranch(branch, workDir, out)
202195

203196
// Check to see if the script file already exists
204197
scriptFile := fmt.Sprintf("mock%s/testdata/%s/crud/script.yaml", branch.Group, branch.Resource)
@@ -304,15 +297,8 @@ func captureHttpLog(opts *RunnerOptions, branch Branch) {
304297
defer close()
305298
workDir := filepath.Join(opts.branchRepoDir, "mockgcp")
306299

307-
log.Printf("COMMAND: git checkout %s\r\n", branch.Local)
308-
checkout := exec.Command("git", "checkout", branch.Local)
309-
checkout.Dir = workDir
310300
var out strings.Builder
311-
checkout.Stdout = &out
312-
if err := checkout.Run(); err != nil {
313-
log.Fatal(err)
314-
}
315-
log.Printf("BRANCH CHECKOUT: %q\n", out.String())
301+
checkoutBranch(branch, workDir, out)
316302

317303
// Check to see if the script file exists
318304
scriptFile := fmt.Sprintf("mock%s/testdata/%s/crud/script.yaml", branch.Group, branch.Resource)
@@ -376,3 +362,169 @@ func captureHttpLog(opts *RunnerOptions, branch Branch) {
376362
}
377363
log.Printf("BRANCH COMMIT: %q\n", out.String())
378364
}
365+
366+
const MOCK_SERVICE_GO_GEN string = `mock<SERVICE>/service.go
367+
// +tool:mockgcp-service
368+
// http.host: <HTTP_HOST>
369+
// proto.service: <PROTO_SERVICE>`
370+
371+
const MOCK_RESOURCE_GO_GEN string = `mock<SERVICE>/<RESOURCE>.go
372+
// +tool:mockgcp-support
373+
// proto.service: <PROTO_SERVICE>
374+
// proto.message: <PROTO_MESSAGE>`
375+
376+
func generateMockGo(opts *RunnerOptions, branch Branch) {
377+
close := setLoggingWriter(opts, branch)
378+
defer close()
379+
workDir := filepath.Join(opts.branchRepoDir, "mockgcp")
380+
381+
var out strings.Builder
382+
checkoutBranch(branch, workDir, out)
383+
384+
// Check to see if the http log file already exists
385+
logFile := fmt.Sprintf("mock%s/testdata/%s/crud/_http.log", branch.Group, branch.Resource)
386+
logFullPath := filepath.Join(workDir, logFile)
387+
if _, err := os.Stat(logFullPath); errors.Is(err, os.ErrNotExist) {
388+
log.Printf("SKIPPING %s, missing %s\r\n", branch.Name, logFullPath)
389+
return
390+
}
391+
392+
// Check to see if the script file exists
393+
serviceGoFile := fmt.Sprintf("mock%s/service.go", branch.Group)
394+
serviceGoFullPath := filepath.Join(workDir, serviceGoFile)
395+
if _, err := os.Stat(serviceGoFullPath); errors.Is(err, os.ErrNotExist) {
396+
log.Printf("SKIPPING %s, %s already exists\r\n", branch.Name, serviceGoFullPath)
397+
return
398+
}
399+
400+
tmp := strings.ReplaceAll(MOCK_SERVICE_GO_GEN, "<TICK>", "`")
401+
tmp = strings.ReplaceAll(tmp, "<SERVICE>", branch.Group)
402+
tmp = strings.ReplaceAll(tmp, "<HTTP_HOST>", branch.HostName)
403+
service_prompt := strings.ReplaceAll(tmp, "<PROTO_SERVICE>", branch.ProtoSvc)
404+
log.Printf("MOCK SERVICE GEN PROMPT %s\r\n", service_prompt)
405+
406+
tmp = strings.ReplaceAll(MOCK_RESOURCE_GO_GEN, "<TICK>", "`")
407+
tmp = strings.ReplaceAll(tmp, "<SERVICE>", branch.Group)
408+
tmp = strings.ReplaceAll(tmp, "<RESOURCE>", strings.ToLower(branch.Resource))
409+
tmp = strings.ReplaceAll(tmp, "<PROTO_SERVICE>", branch.ProtoSvc)
410+
resource_prompt := strings.ReplaceAll(tmp, "<PROTO_MESSAGE>", branch.ProtoPath)
411+
log.Printf("MOCK RESOURCE GEN PROMPT %s\r\n", resource_prompt)
412+
413+
// Delete then write the service prompt file.
414+
servicePromptPath := filepath.Join(opts.branchRepoDir, "mockgcp", "service_prompt.txt")
415+
if _, err := os.Stat(servicePromptPath); !errors.Is(err, os.ErrNotExist) {
416+
log.Println("COMMAND: cleaning up old service_prompt.txt")
417+
err = os.Remove(servicePromptPath)
418+
if err != nil {
419+
log.Printf("Attempt to clean up service_prompt.txt failed with %v", err)
420+
}
421+
}
422+
log.Println("COMMAND: writing new service_prompt.txt")
423+
if err := os.WriteFile(servicePromptPath, []byte(service_prompt), 0644); err != nil {
424+
log.Fatal(err)
425+
}
426+
427+
// Delete then write the resource prompt file.
428+
resourcePromptPath := filepath.Join(opts.branchRepoDir, "mockgcp", "resource_prompt.txt")
429+
if _, err := os.Stat(resourcePromptPath); !errors.Is(err, os.ErrNotExist) {
430+
log.Println("COMMAND: cleaning up old resource_prompt.txt")
431+
err = os.Remove(resourcePromptPath)
432+
if err != nil {
433+
log.Printf("Attempt to clean up resource_prompt.txt failed with %v", err)
434+
}
435+
}
436+
log.Println("COMMAND: writing new resource_prompt.txt")
437+
if err := os.WriteFile(resourcePromptPath, []byte(resource_prompt), 0644); err != nil {
438+
log.Fatal(err)
439+
}
440+
441+
// Run the controller builder to generate the service go file.
442+
serviceFile := filepath.Join(workDir, fmt.Sprintf("mock%s", branch.Group), "service.go")
443+
if _, err := os.Stat(serviceFile); errors.Is(err, os.ErrNotExist) {
444+
start := time.Now()
445+
log.Printf("COMMAND: controllerbuilder prompt --src-dir %s --proto-dir %s/.build/third_party/googleapis/ --input-file=service_prompt.txt", opts.branchRepoDir, opts.branchRepoDir)
446+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
447+
defer cancel()
448+
service_go := exec.CommandContext(ctx, "controllerbuilder", "prompt",
449+
"--src-dir", opts.branchRepoDir,
450+
"--proto-dir", fmt.Sprintf("%s/.build/third_party/googleapis/", opts.branchRepoDir),
451+
"--input-file", "service_prompt.txt")
452+
service_go.Dir = workDir
453+
var serviceOut strings.Builder
454+
service_go.Stdout = &serviceOut
455+
if err := service_go.Run(); err != nil {
456+
log.Printf("MOCK SERVICE GENERATE error: %q\n", err)
457+
// Currently ignoring error and just basing on if the _http.log was generated.
458+
// log.Fatal(err)
459+
}
460+
stop := time.Now()
461+
diff := stop.Sub(start)
462+
if err := os.WriteFile(serviceFile, []byte(serviceOut.String()), 0755); err != nil {
463+
log.Printf("WRITE MOCK SERVICE %s error: %q\n", serviceFile, err)
464+
}
465+
log.Printf("MOCK SERVICE GENERATE (%v): %q\n", diff, serviceOut.String())
466+
467+
// Check to see if the service go file was created
468+
if _, err := os.Stat(serviceFile); errors.Is(err, os.ErrNotExist) {
469+
log.Printf("SKIPPING %s, %s was not created\r\n", branch.Name, serviceFile)
470+
return
471+
}
472+
} else {
473+
log.Printf("SKIPPING generating service mock go, %s already exists\r\n", serviceFile)
474+
}
475+
476+
// Run the controller builder to generate the resource go file.
477+
resourceFile := filepath.Join(workDir, fmt.Sprintf("mock%s", branch.Group), fmt.Sprintf("%s.go", strings.ToLower(branch.Resource)))
478+
if _, err := os.Stat(resourceFile); errors.Is(err, os.ErrNotExist) {
479+
start := time.Now()
480+
log.Printf("COMMAND: controllerbuilder prompt --src-dir %s --proto-dir %s/.build/third_party/googleapis/ --input-file=resource_prompt.txt", opts.branchRepoDir, opts.branchRepoDir)
481+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
482+
defer cancel()
483+
resource_go := exec.CommandContext(ctx, "controllerbuilder", "prompt",
484+
"--src-dir", opts.branchRepoDir,
485+
"--proto-dir", fmt.Sprintf("%s/.build/third_party/googleapis/", opts.branchRepoDir),
486+
"--input-file", "resource_prompt.txt")
487+
resource_go.Dir = workDir
488+
var resourceOut strings.Builder
489+
resource_go.Stdout = &resourceOut
490+
if err := resource_go.Run(); err != nil {
491+
log.Printf("MOCK RESOURCE GENERATE error: %q\n", err)
492+
// Currently ignoring error and just basing on if the _http.log was generated.
493+
// log.Fatal(err)
494+
}
495+
stop := time.Now()
496+
diff := stop.Sub(start)
497+
if err := os.WriteFile(resourceFile, []byte(resourceOut.String()), 0755); err != nil {
498+
log.Printf("WRITE MOCK RESOURCE %s error: %q\n", resourceFile, err)
499+
}
500+
log.Printf("MOCK RESOURCE GENERATE (%v): %q\n", diff, resourceOut.String())
501+
502+
// Check to see if the service go file was created
503+
if _, err := os.Stat(resourceFile); errors.Is(err, os.ErrNotExist) {
504+
log.Printf("SKIPPING %s, %s was not created\r\n", branch.Name, resourceFile)
505+
return
506+
}
507+
} else {
508+
log.Printf("SKIPPING generating resource mock go, %s already exists\r\n", resourceFile)
509+
}
510+
511+
// Add the new files to the current branch.
512+
log.Printf("COMMAND: git add %s %s\r\n", serviceFile, resourceFile)
513+
gitadd := exec.Command("git", "add", serviceFile, resourceFile)
514+
gitadd.Dir = workDir
515+
gitadd.Stdout = &out
516+
if err := gitadd.Run(); err != nil {
517+
log.Fatal(err)
518+
}
519+
log.Printf("BRANCH ADD: %q\n", out.String())
520+
521+
// Commit the change to the current branch.
522+
log.Printf("COMMAND: git commit -m \"Adding mock service and resource for %s\"\r\n", branch.Name)
523+
gitcommit := exec.Command("git", "commit", "-m", fmt.Sprintf("\"Adding mock service and resource for %s\"", branch.Name))
524+
gitcommit.Dir = workDir
525+
gitcommit.Stdout = &out
526+
if err := gitcommit.Run(); err != nil {
527+
log.Fatal(err)
528+
}
529+
log.Printf("BRANCH COMMIT: %q\n", out.String())
530+
}

‎experiments/conductor/cmd/runner/runner.go

+12-5
Original file line numberDiff line numberDiff line change
@@ -98,10 +98,12 @@ type Branch struct {
9898
Resource string `yaml:"resource"` // model
9999
Controller string `yaml:"controller"` // Unknown
100100

101-
Kind string `yaml:"kind"` // AIModel
102-
Package string `yaml:"package"` // google.ai.generativelanguage.v1beta
103-
Proto string `yaml:"proto"` // Model
104-
ProtoPath string `yaml:"proto-path"` // google.ai.generativelanguage.v1beta.Model
101+
Kind string `yaml:"kind"` // AIModel
102+
Package string `yaml:"package"` // google.ai.generativelanguage.v1beta
103+
Proto string `yaml:"proto"` // Model
104+
ProtoPath string `yaml:"proto-path"` // google.ai.generativelanguage.v1beta.Model
105+
ProtoSvc string `yaml:"proto-service"` // google.ai.generativelanguage.v1beta.ModelService
106+
HostName string `yaml:"host-name"` // generativelanguage.googleapis.com
105107

106108
Notes []string `yaml:"notes"` // Observation goes here
107109
}
@@ -164,9 +166,14 @@ func RunRunner(ctx context.Context, opts *RunnerOptions) error {
164166
}
165167
case 5:
166168
for idx, branch := range branches.Branches {
167-
log.Printf("Catpure HTTP Log: %d name: %s, branch: %s\r\n", idx, branch.Name, branch.Local)
169+
log.Printf("Capture HTTP Log: %d name: %s, branch: %s\r\n", idx, branch.Name, branch.Local)
168170
captureHttpLog(opts, branch)
169171
}
172+
case 6:
173+
for idx, branch := range branches.Branches {
174+
log.Printf("Generate mock Service and Resource go files: %d name: %s, branch: %s\r\n", idx, branch.Name, branch.Local)
175+
generateMockGo(opts, branch)
176+
}
170177
default:
171178
log.Fatalf("unrecognixed command: %d", opts.command)
172179
}

‎experiments/conductor/cmd/runner/utilities.go

+11-2
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,17 @@ func cdRepoBranchDirBash(opts *RunnerOptions, subdir string, stdin io.WriteClose
9191
return msg
9292
}
9393

94+
func checkoutBranch(branch Branch, workDir string, out strings.Builder) {
95+
log.Printf("COMMAND: git checkout %s\r\n", branch.Local)
96+
checkout := exec.Command("git", "checkout", branch.Local)
97+
checkout.Dir = workDir
98+
checkout.Stdout = &out
99+
if err := checkout.Run(); err != nil {
100+
log.Fatal(err)
101+
}
102+
log.Printf("BRANCH CHECKOUT: %q\n", out.String())
103+
}
104+
94105
type closer func()
95106

96107
func setLoggingWriter(opts *RunnerOptions, branch Branch) closer {
@@ -139,7 +150,6 @@ func setLoggingWriter(opts *RunnerOptions, branch Branch) closer {
139150
return func() {
140151
// Initially force out to stdout in case we hit an error we don't
141152
// want to pollute a different runs logs with our logs.
142-
// TODO: Return a log object so we can run in parrellel.
143153
log.SetOutput(os.Stdout)
144154
if err := out.Close(); err != nil {
145155
log.Printf("Failed to clode logging file %s, :%v", logFile, err)
@@ -148,5 +158,4 @@ func setLoggingWriter(opts *RunnerOptions, branch Branch) closer {
148158
}
149159

150160
func noOp() {
151-
return
152161
}

0 commit comments

Comments
 (0)