Introduce configuration knobs for determining the staleness of a lock file.

Author: ubschmidt2

backend/remote-state/gcs/backend.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	"cloud.google.com/go/storage"
 	"github.com/hashicorp/terraform/backend"
@@ -35,6 +36,13 @@ type Backend struct {
 
 	projectID string
 	region    string
+
+	// Time between consecutive heartbeats on the lock file.
+	lockHeartbeatInterval time.Duration
+
+	// The mininum duration that must have passed since the youngest
+	// recorded heartbeat before the lock file is considered stale/orphaned.
+	lockStaleAfter time.Duration
 }
 
 func New() backend.Backend {
@@ -88,6 +96,20 @@ func New() backend.Backend {
 				Description: "Region / location in which to create the bucket",
 				Default:     "",
 			},
+
+			"lock_heartbeat_interval": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "Time between consecutive heartbeats on the lock file as a duration string (cf. https://golang.org/pkg/time/#ParseDuration).",
+				Default:     "1m",
+			},
+
+			"lock_stale_after": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "Mininum duration (cf. https://golang.org/pkg/time/#ParseDuration) that must have passed since the youngest recorded heartbeat before the lock file is considered stale/orphaned.",
+				Default:     "15m",
+			},
 		},
 	}
 
@@ -188,6 +210,16 @@ func (b *Backend) configure(ctx context.Context) error {
 		b.encryptionKey = k
 	}
 
+	b.lockHeartbeatInterval, err = time.ParseDuration(data.Get("lock_heartbeat_interval").(string))
+	if err != nil {
+		return fmt.Errorf("Error parsing lock_heartbeat_interval: %s", err)
+	}
+
+	b.lockStaleAfter, err = time.ParseDuration(data.Get("lock_stale_after").(string))
+	if err != nil {
+		return fmt.Errorf("Error parsing lock_stale_after: %s", err)
+	}
+
 	return nil
 }
 

backend/remote-state/gcs/backend_state.go

@@ -75,12 +75,14 @@ func (b *Backend) client(name string) (*remoteClient, error) {
 	}
 
 	return &remoteClient{
-		storageContext: b.storageContext,
-		storageClient:  b.storageClient,
-		bucketName:     b.bucketName,
-		stateFilePath:  b.stateFile(name),
-		lockFilePath:   b.lockFile(name),
-		encryptionKey:  b.encryptionKey,
+		storageContext:        b.storageContext,
+		storageClient:         b.storageClient,
+		bucketName:            b.bucketName,
+		stateFilePath:         b.stateFile(name),
+		lockFilePath:          b.lockFile(name),
+		encryptionKey:         b.encryptionKey,
+		lockHeartbeatInterval: b.lockHeartbeatInterval,
+		lockStaleAfter:        b.lockStaleAfter,
 	}, nil
 }
 

backend/remote-state/gcs/backend_test.go

@@ -207,15 +207,15 @@ func testStaleLocks(t *testing.T, b1, b2 backend.Backend) {
 	infoB.Who = "clientB"
 
 	// For faster tests, reduce the duration until the lock is considered stale.
-	heartbeatInterval = 5 * time.Second
-	minHeartbeatAgeUntilStale = 20 * time.Second
+	lockerB.(*remote.State).Client.(*remoteClient).lockHeartbeatInterval = 5 * time.Second
+	lockerB.(*remote.State).Client.(*remoteClient).lockStaleAfter = 20 * time.Second
 
 	lockIDA, err := lockerA.Lock(infoA)
 	if err != nil {
 		t.Fatal("unable to get initial lock:", err)
 	}
 
-	// Stop heartbeating on the lock file. It will be considered stale after minHeartbeatAgeUntilStale.
+	// Stop heartbeating on the lock file. It will be considered stale after lockStaleAfter.
 	lockerA.(*remote.State).Client.(*remoteClient).stopHeartbeatCh <- true
 
 	// Lock is still held by A after 10 seconds.

backend/remote-state/gcs/client.go

@@ -10,7 +10,7 @@ import (
 	"time"
 
 	"cloud.google.com/go/storage"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/terraform/state"
 	"github.com/hashicorp/terraform/state/remote"
 	"golang.org/x/net/context"
@@ -22,12 +22,14 @@ import (
 type remoteClient struct {
 	mutex sync.Mutex
 
-	storageContext context.Context
-	storageClient  *storage.Client
-	bucketName     string
-	stateFilePath  string
-	lockFilePath   string
-	encryptionKey  []byte
+	storageContext        context.Context
+	storageClient         *storage.Client
+	bucketName            string
+	stateFilePath         string
+	lockFilePath          string
+	encryptionKey         []byte
+	lockHeartbeatInterval time.Duration
+	lockStaleAfter        time.Duration
 
 	// The initial generation number of the lock file created by this
 	// remoteClient.
@@ -42,18 +44,9 @@ type remoteClient struct {
 // heartbeats on it. This header facilitates a safe migration from previous
 // Terraform versions that do not yet perform any heartbeats on the lock file.
 // A lock file will only be considered stale and force-unlocked if its age
-// exceeds minHeartbeatAgeUntilStale AND this metadata header is present.
+// exceeds lockStaleAfter AND this metadata header is present.
 const metadataHeaderHeartbeatEnabled = "x-goog-meta-heartbeating"
 
-var (
-	// Time between consecutive heartbeats on the lock file.
-	heartbeatInterval = 1 * time.Minute
-
-	// The mininum duration that must have passed since the youngest
-	// recorded heartbeat before the lock file is considered stale/orphaned.
-	minHeartbeatAgeUntilStale = 15 * time.Minute
-)
-
 func (c *remoteClient) Get() (payload *remote.Payload, err error) {
 	stateFileReader, err := c.stateFile().NewReader(c.storageContext)
 	if err != nil {
@@ -153,8 +146,8 @@ func (c *remoteClient) unlockIfStale() bool {
 			log.Printf("[TRACE] Found existing lock file %s from an older client that does not perform heartbeats", c.lockFileURL())
 			return false
 		}
-		age := time.Now().Sub(attrs.Updated)
-		if age > minHeartbeatAgeUntilStale {
+		age := time.Since(attrs.Updated)
+		if age > c.lockStaleAfter {
 			log.Printf("[WARN] Existing lock file %s is considered stale, last heartbeat was %s ago", c.lockFileURL(), age)
 			if err := c.Unlock(strconv.FormatInt(attrs.Generation, 10)); err != nil {
 				log.Printf("[WARN] Failed to release stale lock: %s", err)
@@ -170,9 +163,9 @@ func (c *remoteClient) unlockIfStale() bool {
 // heartbeatLockFile periodically updates the "updated" timestamp of the lock
 // file until the lock is released in Unlock().
 func (c *remoteClient) heartbeatLockFile() {
-	log.Printf("[TRACE] Starting heartbeat on lock file %s, interval is %s", c.lockFileURL(), heartbeatInterval)
+	log.Printf("[TRACE] Starting heartbeat on lock file %s, interval is %s", c.lockFileURL(), c.lockHeartbeatInterval)
 
-	ticker := time.NewTicker(heartbeatInterval)
+	ticker := time.NewTicker(c.lockHeartbeatInterval)
 	defer ticker.Stop()
 
 	defer func() {

website/docs/backends/types/gcs.html.md

@@ -60,3 +60,5 @@ The following configuration options are supported:
  *  `region` / `GOOGLE_REGION` - (Optional) The region in which a new bucket is created.
     For more information, see [Bucket Locations](https://cloud.google.com/storage/docs/bucket-locations).
  *  `encryption_key` / `GOOGLE_ENCRYPTION_KEY` - (Optional) A 32 byte base64 encoded 'customer supplied encryption key' used to encrypt all state. For more information see [Customer Supplied Encryption Keys](https://cloud.google.com/storage/docs/encryption#customer-supplied).
+ *  `lock_heartbeat_interval` - (Optional) The time between consecutive heartbeats on the lock file as a [duration string](https://golang.org/pkg/time/#ParseDuration). Defaults to "1m".
+ *  `lock_stale_after` - (Optional) The mininum duration (as a [duration string](https://golang.org/pkg/time/#ParseDuration)) that must have passed since the youngest recorded heartbeat before the lock file is considered stale/orphaned. Defaults to "15m".