test.bash: re-write integration test as a Go test

This change:
* upgrades the v2 dependency to the latest version
* copies integration_test.go from v2 and deletes irrelevant code.
* copies internal/cmd/generate-protos from v2 and deletes irrelevant code
and modifies generateLocalProtos accordingly
* deletes protoc-gen-go/testdata since this is covered in v2
* uses import public to alias the v2 version of the well-known types,
plugin proto, and descriptor proto

Change-Id: Ib1d4280afaca8a811b9ff57792e02b15b3b8f0ec
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/167141
Reviewed-by: Herbie Ong <[email protected]>

Author: dsnet

Makefile

@@ -1,3 +1,3 @@
 module github.com/golang/protobuf
 
-require github.com/golang/protobuf/v2 v2.0.0-20190311234237-8692540cce25
+require github.com/golang/protobuf/v2 v2.0.0-20190312230405-4989810018b7

go.mod

@@ -1,5 +1,5 @@
 github.com/golang/protobuf v1.2.1-0.20190311233832-968e039ade03/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf/v2 v2.0.0-20190311234237-8692540cce25 h1:OPot/6pIGOXutoEWwijGIyc0s72oha4VMKjoMKgSD84=
-github.com/golang/protobuf/v2 v2.0.0-20190311234237-8692540cce25/go.mod h1:JBgB92qSNOVlmhtPbF3BkMlBy5KGJj+k0lo1MaNJhxE=
-github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/golang/protobuf/v2 v2.0.0-20190312230405-4989810018b7 h1:aQENkYDAn/68ZfghmurTBhR18Q7/XWx1JpMaqFtP1GM=
+github.com/golang/protobuf/v2 v2.0.0-20190312230405-4989810018b7/go.mod h1:euNorOscCho6jibQUfcx8TAp5HZXU5/+1xMsdtkc8lo=
+github.com/google/go-cmp v0.2.1-0.20190228024137-c81281657ad9 h1:GDk30QdVXDf6i734280OeWAO9UCCjXLeWkLLyHQGUkI=
+github.com/google/go-cmp v0.2.1-0.20190228024137-c81281657ad9/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=

go.sum

@@ -0,0 +1,334 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+)
+
+var (
+	regenerate = flag.Bool("regenerate", false, "regenerate files")
+
+	protobufVersion = "3.7.0"
+	golangVersions  = []string{"1.9.7", "1.10.8", "1.11.5", "1.12"}
+	golangLatest    = golangVersions[len(golangVersions)-1]
+
+	// purgeTimeout determines the maximum age of unused sub-directories.
+	purgeTimeout = 30 * 24 * time.Hour // 1 month
+
+	// Variables initialized by mustInitDeps.
+	goPath     string
+	modulePath string
+)
+
+func Test(t *testing.T) {
+	mustInitDeps(t)
+
+	if *regenerate {
+		t.Run("Generate", func(t *testing.T) {
+			fmt.Print(mustRunCommand(t, ".", "go", "run", "./internal/cmd/generate-protos", "-execute"))
+			files := strings.Split(strings.TrimSpace(mustRunCommand(t, ".", "git", "ls-files", "*.go")), "\n")
+			mustRunCommand(t, ".", append([]string{"gofmt", "-w"}, files...)...)
+		})
+		t.SkipNow()
+	}
+
+	for _, v := range golangVersions {
+		t.Run("Go"+v, func(t *testing.T) {
+			runGo := func(label, workDir string, args ...string) {
+				args[0] += v
+				t.Run(label, func(t *testing.T) {
+					t.Parallel()
+					mustRunCommand(t, workDir, args...)
+				})
+			}
+			workDir := filepath.Join(goPath, "src", modulePath)
+			runGo("Build", workDir, "go", "build", "./...")
+			runGo("TestNormal", workDir, "go", "test", "-race", "./...")
+		})
+	}
+
+	t.Run("GeneratedGoFiles", func(t *testing.T) {
+		diff := mustRunCommand(t, ".", "go", "run", "./internal/cmd/generate-protos")
+		if strings.TrimSpace(diff) != "" {
+			t.Fatalf("stale generated files:\n%v", diff)
+		}
+	})
+	t.Run("FormattedGoFiles", func(t *testing.T) {
+		files := strings.Split(strings.TrimSpace(mustRunCommand(t, ".", "git", "ls-files", "*.go")), "\n")
+		diff := mustRunCommand(t, ".", append([]string{"gofmt", "-d"}, files...)...)
+		if strings.TrimSpace(diff) != "" {
+			t.Fatalf("unformatted source files:\n%v", diff)
+		}
+	})
+	t.Run("CommittedGitChanges", func(t *testing.T) {
+		diff := mustRunCommand(t, ".", "git", "diff", "--no-prefix", "HEAD")
+		if strings.TrimSpace(diff) != "" {
+			t.Fatalf("uncommitted changes:\n%v", diff)
+		}
+	})
+	t.Run("TrackedGitFiles", func(t *testing.T) {
+		diff := mustRunCommand(t, ".", "git", "ls-files", "--others", "--exclude-standard")
+		if strings.TrimSpace(diff) != "" {
+			t.Fatalf("untracked files:\n%v", diff)
+		}
+	})
+}
+
+func mustInitDeps(t *testing.T) {
+	check := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Determine the directory to place the test directory.
+	repoRoot, err := os.Getwd()
+	check(err)
+	testDir := filepath.Join(repoRoot, ".cache")
+	check(os.MkdirAll(testDir, 0775))
+
+	// Travis-CI has a hard-coded timeout where it kills the test after
+	// 10 minutes of a lack of activity on stdout.
+	// We work around this restriction by periodically printing the timestamp.
+	ticker := time.NewTicker(5 * time.Minute)
+	done := make(chan struct{})
+	go func() {
+		now := time.Now()
+		for {
+			select {
+			case t := <-ticker.C:
+				fmt.Printf("\tt=%0.fmin\n", t.Sub(now).Minutes())
+			case <-done:
+				return
+			}
+		}
+	}()
+	defer close(done)
+	defer ticker.Stop()
+
+	// Delete the current directory if non-empty,
+	// which only occurs if a dependency failed to initialize properly.
+	var workingDir string
+	defer func() {
+		if workingDir != "" {
+			os.RemoveAll(workingDir) // best-effort
+		}
+	}()
+
+	// Delete other sub-directories that are no longer relevant.
+	defer func() {
+		subDirs := map[string]bool{"bin": true, "gocache": true, "gopath": true}
+		subDirs["protobuf-"+protobufVersion] = true
+		for _, v := range golangVersions {
+			subDirs["go"+v] = true
+		}
+
+		now := time.Now()
+		fis, _ := ioutil.ReadDir(testDir)
+		for _, fi := range fis {
+			if subDirs[fi.Name()] {
+				os.Chtimes(filepath.Join(testDir, fi.Name()), now, now) // best-effort
+				continue
+			}
+			if now.Sub(fi.ModTime()) < purgeTimeout {
+				continue
+			}
+			fmt.Printf("delete %v\n", fi.Name())
+			os.RemoveAll(filepath.Join(testDir, fi.Name())) // best-effort
+		}
+	}()
+
+	// The bin directory contains symlinks to each tool by version.
+	// It is safe to delete this directory and run the test script from scratch.
+	binPath := filepath.Join(testDir, "bin")
+	check(os.RemoveAll(binPath))
+	check(os.Mkdir(binPath, 0775))
+	check(os.Setenv("PATH", binPath+":"+os.Getenv("PATH")))
+	registerBinary := func(name, path string) {
+		check(os.Symlink(path, filepath.Join(binPath, name)))
+	}
+
+	// Download and build the protobuf toolchain.
+	// We avoid downloading the pre-compiled binaries since they do not contain
+	// the conformance test runner.
+	workingDir = filepath.Join(testDir, "protobuf-"+protobufVersion)
+	if _, err := os.Stat(workingDir); err != nil {
+		fmt.Printf("download %v\n", filepath.Base(workingDir))
+		url := fmt.Sprintf("https://github.com/google/protobuf/releases/download/v%v/protobuf-all-%v.tar.gz", protobufVersion, protobufVersion)
+		downloadArchive(check, workingDir, url, "protobuf-"+protobufVersion)
+
+		fmt.Printf("build %v\n", filepath.Base(workingDir))
+		mustRunCommand(t, workingDir, "./autogen.sh")
+		mustRunCommand(t, workingDir, "./configure")
+		mustRunCommand(t, workingDir, "make")
+		mustRunCommand(t, filepath.Join(workingDir, "conformance"), "make")
+	}
+	patchProtos(check, workingDir)
+	check(os.Setenv("PROTOBUF_ROOT", workingDir)) // for generate-protos
+	registerBinary("conform-test-runner", filepath.Join(workingDir, "conformance", "conformance-test-runner"))
+	registerBinary("protoc", filepath.Join(workingDir, "src", "protoc"))
+	workingDir = ""
+
+	// Download each Go toolchain version.
+	for _, v := range golangVersions {
+		workingDir = filepath.Join(testDir, "go"+v)
+		if _, err := os.Stat(workingDir); err != nil {
+			fmt.Printf("download %v\n", filepath.Base(workingDir))
+			url := fmt.Sprintf("https://dl.google.com/go/go%v.%v-%v.tar.gz", v, runtime.GOOS, runtime.GOARCH)
+			downloadArchive(check, workingDir, url, "go")
+		}
+		registerBinary("go"+v, filepath.Join(workingDir, "bin", "go"))
+	}
+	registerBinary("go", filepath.Join(testDir, "go"+golangLatest, "bin", "go"))
+	registerBinary("gofmt", filepath.Join(testDir, "go"+golangLatest, "bin", "gofmt"))
+	workingDir = ""
+
+	// Travis-CI sets GOROOT, which confuses invocations of the Go toolchain.
+	// Explicitly clear GOROOT, so each toolchain uses their default GOROOT.
+	check(os.Unsetenv("GOROOT"))
+
+	// Set a cache directory within the test directory.
+	check(os.Setenv("GOCACHE", filepath.Join(testDir, "gocache")))
+
+	// Setup GOPATH for pre-module support (i.e., go1.10 and earlier).
+	goPath = filepath.Join(testDir, "gopath")
+	modulePath = strings.TrimSpace(mustRunCommand(t, testDir, "go", "list", "-m", "-f", "{{.Path}}"))
+	check(os.RemoveAll(filepath.Join(goPath, "src")))
+	check(os.MkdirAll(filepath.Join(goPath, "src", filepath.Dir(modulePath)), 0775))
+	check(os.Symlink(repoRoot, filepath.Join(goPath, "src", modulePath)))
+	mustRunCommand(t, repoRoot, "go", "mod", "tidy")
+	mustRunCommand(t, repoRoot, "go", "mod", "vendor")
+	check(os.Setenv("GOPATH", goPath))
+}
+
+func downloadArchive(check func(error), dstPath, srcURL, skipPrefix string) {
+	check(os.RemoveAll(dstPath))
+
+	resp, err := http.Get(srcURL)
+	check(err)
+	defer resp.Body.Close()
+
+	zr, err := gzip.NewReader(resp.Body)
+	check(err)
+
+	tr := tar.NewReader(zr)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			return
+		}
+		check(err)
+
+		// Skip directories or files outside the prefix directory.
+		if len(skipPrefix) > 0 {
+			if !strings.HasPrefix(h.Name, skipPrefix) {
+				continue
+			}
+			if len(h.Name) > len(skipPrefix) && h.Name[len(skipPrefix)] != '/' {
+				continue
+			}
+		}
+
+		path := strings.TrimPrefix(strings.TrimPrefix(h.Name, skipPrefix), "/")
+		path = filepath.Join(dstPath, filepath.FromSlash(path))
+		mode := os.FileMode(h.Mode & 0777)
+		switch h.Typeflag {
+		case tar.TypeReg:
+			b, err := ioutil.ReadAll(tr)
+			check(err)
+			check(ioutil.WriteFile(path, b, mode))
+		case tar.TypeDir:
+			check(os.Mkdir(path, mode))
+		}
+	}
+}
+
+// patchProtos patches proto files with v2 locations of Go packages.
+// TODO: Commit these changes upstream.
+func patchProtos(check func(error), repoRoot string) {
+	javaPackageRx := regexp.MustCompile(`^option\s+java_package\s*=\s*".*"\s*;\s*$`)
+	goPackageRx := regexp.MustCompile(`^option\s+go_package\s*=\s*".*"\s*;\s*$`)
+	files := map[string]string{
+		"src/google/protobuf/any.proto":             "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/api.proto":             "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/duration.proto":        "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/empty.proto":           "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/field_mask.proto":      "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/source_context.proto":  "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/struct.proto":          "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/timestamp.proto":       "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/type.proto":            "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/wrappers.proto":        "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/descriptor.proto":      "github.com/golang/protobuf/v2/types/descriptor;descriptor_proto",
+		"src/google/protobuf/compiler/plugin.proto": "github.com/golang/protobuf/v2/types/plugin;plugin_proto",
+		"conformance/conformance.proto":             "github.com/golang/protobuf/v2/internal/testprotos/conformance;conformance_proto",
+	}
+	for pbpath, gopath := range files {
+		b, err := ioutil.ReadFile(filepath.Join(repoRoot, pbpath))
+		check(err)
+		ss := strings.Split(string(b), "\n")
+
+		// Locate java_package and (possible) go_package options.
+		javaPackageIdx, goPackageIdx := -1, -1
+		for i, s := range ss {
+			if javaPackageIdx < 0 && javaPackageRx.MatchString(s) {
+				javaPackageIdx = i
+			}
+			if goPackageIdx < 0 && goPackageRx.MatchString(s) {
+				goPackageIdx = i
+			}
+		}
+
+		// Ensure the proto file has the correct go_package option.
+		opt := `option go_package = "` + gopath + `";`
+		if goPackageIdx >= 0 {
+			if ss[goPackageIdx] == opt {
+				continue // no changes needed
+			}
+			ss[goPackageIdx] = opt
+		} else {
+			// Insert go_package option before java_package option.
+			ss = append(ss[:javaPackageIdx], append([]string{opt}, ss[javaPackageIdx:]...)...)
+		}
+
+		fmt.Println("patch " + pbpath)
+		b = []byte(strings.Join(ss, "\n"))
+		check(ioutil.WriteFile(filepath.Join(repoRoot, pbpath), b, 0664))
+	}
+}
+
+func mustRunCommand(t *testing.T, dir string, args ...string) string {
+	t.Helper()
+	stdout := new(bytes.Buffer)
+	combined := new(bytes.Buffer)
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Dir = dir
+	cmd.Env = append(os.Environ(), "PWD="+dir)
+	cmd.Stdout = io.MultiWriter(stdout, combined)
+	cmd.Stderr = combined
+	if err := cmd.Run(); err != nil {
+		t.Fatalf("executing (%v): %v\n%s", strings.Join(args, " "), err, combined.String())
+	}
+	return stdout.String()
+}