Use MockS3Server instead of seaweedfs for ctest.

fdbbackup/backup.actor.cpp

@@ -2405,7 +2405,8 @@ ACTOR Future<Void> runRestore(Database db,
 				printf(
 				    "No restore target version given, will use maximum restorable version from backup description.\n");
 
-			BackupDescription desc = wait(bc->describeBackup());
+			// For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties
+			BackupDescription desc = wait(bc->describeBackup(false, isBlobstoreUrl(container) ? invalidVersion : 0));
 
 			if (onlyApplyMutationLogs && desc.contiguousLogEnd.present()) {
 				targetVersion = desc.contiguousLogEnd.get() - 1;
@@ -2501,7 +2502,10 @@ ACTOR Future<Void> runFastRestoreTool(Database db,
 		if (performRestore) {
 			if (dbVersion == invalidVersion) {
 				TraceEvent("FastRestoreTool").detail("TargetRestoreVersion", "Largest restorable version");
-				BackupDescription desc = wait(IBackupContainer::openContainer(container, proxy, {})->describeBackup());
+				// For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties
+				BackupDescription desc =
+				    wait(IBackupContainer::openContainer(container, proxy, {})
+				             ->describeBackup(false, isBlobstoreUrl(container) ? invalidVersion : 0));
 				if (!desc.maxRestorableVersion.present()) {
 					fprintf(stderr, "The specified backup is not restorable to any version.\n");
 					throw restore_error();
@@ -2541,7 +2545,9 @@ ACTOR Future<Void> runFastRestoreTool(Database db,
 			restoreVersion = dbVersion;
 		} else {
 			state Reference<IBackupContainer> bc = IBackupContainer::openContainer(container, proxy, {});
-			state BackupDescription description = wait(bc->describeBackup());
+			// For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties
+			state BackupDescription description =
+			    wait(bc->describeBackup(false, isBlobstoreUrl(container) ? invalidVersion : 0));
 
 			if (dbVersion <= 0) {
 				wait(description.resolveVersionTimes(db));

fdbbackup/tests/dir_backup_test.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Test backup and restore from a local directory.
 #

fdbbackup/tests/s3_backup_test.sh

@@ -1,9 +1,9 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Test backup and restore from s3.
 #
 # In the below we start a small FDB cluster, populate it with
-# some data and then start up a seaweedfs instance or use S3
+# some data and then start up MockS3Server or use S3
 # if it is available. We then run a backup to 'S3' and then
 # a restore. We verify the restore is the same as the original.
 #
@@ -27,13 +27,16 @@ function cleanup {
   if type shutdown_fdb_cluster &> /dev/null; then
     shutdown_fdb_cluster
   fi
-  if type shutdown_weed &> /dev/null; then
-    shutdown_weed "${TEST_SCRATCH_DIR}"
+  if type shutdown_mocks3 &> /dev/null; then
+    shutdown_mocks3
   fi
   if type shutdown_aws &> /dev/null; then
     shutdown_aws "${TEST_SCRATCH_DIR}"
   fi
+
+  # Clean up encryption key file
   if [[ -n "${ENCRYPTION_KEY_FILE:-}" ]] && [[ -f "${ENCRYPTION_KEY_FILE}" ]]; then
+    log "Removing encryption key file: ${ENCRYPTION_KEY_FILE}"
     rm -f "${ENCRYPTION_KEY_FILE}"
   fi
 }
@@ -71,9 +74,6 @@ function backup {
   local local_credentials="${4}"
   local local_encryption_key_file="${5:-}"
 
-  # Backup to s3. Without the -k argument in the below, the backup gets
-  # 'No restore target version given, will use maximum restorable version from backup description.'
-  # TODO: Why is -k needed?
   local cmd_args=(
     "-C" "${local_scratch_dir}/loopback_cluster/fdb.cluster"
     "-t" "${TAG}" "-w"
@@ -113,7 +113,7 @@ function restore {
   local cmd_args=(
     "--dest-cluster-file" "${local_scratch_dir}/loopback_cluster/fdb.cluster"
     "-t" "${TAG}" "-w"
-    "-r" "${url}"
+    "-r" "${local_url}"
     "--log" "--logdir=${local_scratch_dir}"
     "--blob-credentials" "${local_credentials}"
   )
@@ -145,17 +145,12 @@ function test_s3_backup_and_restore {
   local local_build_dir="${4}"
   local local_encryption_key_file="${5:-}"
 
-  log "Load data"
-  if ! load_data "${local_build_dir}" "${local_scratch_dir}"; then
-    err "Failed loading data into fdb"
-    return 1
-  fi
   # Edit the url. Backup adds 'data' to the path. Need this url for
   # cleanup of test data.
   local edited_url=$(echo "${local_url}" | sed -e "s/ctest/data\/ctest/" )
   readonly edited_url
   if [[ "${USE_S3}" == "true" ]]; then
-    # Run this rm only if s3. In seaweed, it would fail because
+    # Run this rm only if s3. In MockS3Server, it would fail because
     # bucket doesn't exist yet (they are lazily created).
     local preclear_cmd=("${local_build_dir}/bin/s3client")
     preclear_cmd+=("${KNOBS[@]}")
@@ -169,6 +164,11 @@ function test_s3_backup_and_restore {
       return 1
     fi
   fi
+  log "Load data"
+  if ! load_data "${local_build_dir}" "${local_scratch_dir}"; then
+    err "Failed loading data into fdb"
+    return 1
+  fi
   log "Run s3 backup"
   if ! backup "${local_build_dir}" "${local_scratch_dir}" "${local_url}" "${credentials}" "${local_encryption_key_file}"; then
     err "Failed backup"
@@ -197,7 +197,7 @@ function test_s3_backup_and_restore {
   local cleanup_cmd=("${local_build_dir}/bin/s3client")
   cleanup_cmd+=("${KNOBS[@]}")
 
-  # Only add TLS CA file for real S3, not SeaweedFS
+  # Only add TLS CA file for real S3, not MockS3Server
   if [[ "${USE_S3}" == "true" ]]; then
     cleanup_cmd+=("--tls-ca-file" "${TLS_CA_FILE}")
   fi
@@ -369,7 +369,9 @@ while (( "$#" )); do
 done
 
 # Set positional arguments in their proper place
-set -- "${PARAMS[@]}"
+if [ ${#PARAMS[@]} -ne 0 ]; then
+  set -- "${PARAMS[@]}"
+fi
 
 # Globals
 # TEST_SCRATCH_DIR gets set below. Tests should be their data in here.
@@ -382,17 +384,17 @@ readonly TAG="test_backup"
 # internal apple dev environments where S3 is available).
 readonly USE_S3="${USE_S3:-$( if [[ -n "${OKTETO_NAMESPACE+x}" ]]; then echo "true" ; else echo "false"; fi )}"
 
-# Set KNOBS based on whether we're using real S3 or SeaweedFS
+# Set KNOBS based on whether we're using real S3 or MockS3Server
 if [[ "${USE_S3}" == "true" ]]; then
   # Use AWS KMS encryption for real S3
   KNOBS=("--knob_blobstore_encryption_type=aws:kms" "--knob_http_verbose_level=${HTTP_VERBOSE_LEVEL}")
 else
-  # No encryption for SeaweedFS
+  # No encryption for MockS3Server
   KNOBS=("--knob_http_verbose_level=${HTTP_VERBOSE_LEVEL}")
 fi
 readonly KNOBS
 
-# Set TLS_CA_FILE only when using real S3, not for SeaweedFS
+# Set TLS_CA_FILE only when using real S3, not for MockS3Server
 if [[ "${USE_S3}" == "true" ]]; then
   # Try to find a valid TLS CA file if not explicitly set
   if [[ -z "${TLS_CA_FILE:-}" ]]; then
@@ -406,7 +408,7 @@ if [[ "${USE_S3}" == "true" ]]; then
   fi
   TLS_CA_FILE="${TLS_CA_FILE:-}"
 else
-  # For SeaweedFS, don't use TLS
+  # For MockS3Server, don't use TLS
   TLS_CA_FILE=""
 fi
 readonly TLS_CA_FILE
@@ -471,7 +473,7 @@ else
 fi
 readonly ENCRYPTION_KEY_FILE
 
-# Set host, bucket, and blob_credentials_file whether seaweed or s3.
+# Set host, bucket, and blob_credentials_file whether MockS3Server or s3.
 readonly path_prefix="ctests"
 host=
 query_str=
@@ -502,30 +504,31 @@ if [[ "${USE_S3}" == "true" ]]; then
   export FDB_BLOB_CREDENTIALS="${blob_credentials_file}"
   export FDB_TLS_CA_FILE="${TLS_CA_FILE}"
 else
-  log "Testing against seaweedfs"
-  # Now source in the seaweedfs fixture so we can use its methods in the below.
+  log "Testing against MockS3Server"
+  # Now source in the mocks3 fixture so we can use its methods in the below.
   # shellcheck source=/dev/null
-  if ! source "${cwd}/../../fdbclient/tests/seaweedfs_fixture.sh"; then
-    err "Failed to source seaweedfs_fixture.sh"
+  if ! source "${cwd}/../../fdbclient/tests/mocks3_fixture.sh"; then
+    err "Failed to source mocks3_fixture.sh"
     exit 1
   fi
-  if ! TEST_SCRATCH_DIR=$(create_weed_dir "${scratch_dir}"); then
-    err "Failed create of the weed dir." >&2
-    return 1
+  if ! TEST_SCRATCH_DIR=$(mktemp -d "${scratch_dir}/mocks3_backup_test.XXXXXX"); then
+    err "Failed create of the mocks3 test dir." >&2
+    exit 1
   fi
   readonly TEST_SCRATCH_DIR
-  if ! host=$( run_weed "${scratch_dir}" "${TEST_SCRATCH_DIR}"); then
-    err "Failed to run seaweed"
-    return 1
+  if ! start_mocks3 "${build_dir}"; then
+    err "Failed to start MockS3Server"
+    exit 1
   fi
-  readonly host
-  readonly bucket="${SEAWEED_BUCKET}"
-  readonly region="all_regions"
-  # Reference a non-existent blob file (its ignored by seaweed)
+  readonly host="${MOCKS3_HOST}:${MOCKS3_PORT}"
+  readonly bucket="test-bucket"
+  readonly region="us-east-1"
+  # Create an empty blob credentials file (MockS3Server uses simple auth)
   readonly blob_credentials_file="${TEST_SCRATCH_DIR}/blob_credentials.json"
-  # Let the connection to seaweed be insecure -- not-TLS -- because just awkward to set up.
+  echo '{}' > "${blob_credentials_file}"
+  # Let the connection to MockS3Server be insecure -- not-TLS
   query_str="bucket=${bucket}&region=${region}&secure_connection=0"
-  # Set environment variables for SeaweedFS too
+  # Set environment variables for MockS3Server
   export FDB_BLOB_CREDENTIALS="${blob_credentials_file}"
 fi
 
@@ -551,4 +554,4 @@ log "Backup_agent is up"
 test="test_s3_backup_and_restore"
 url="blobstore://${host}/${path_prefix}/${test}?${query_str}"
 test_s3_backup_and_restore "${url}" "${TEST_SCRATCH_DIR}" "${blob_credentials_file}" "${build_dir}" "${ENCRYPTION_KEY_FILE}"
-log_test_result $? "test_s3_backup_and_restore"
+log_test_result $? "test_s3_backup_and_restore"

fdbclient/FileBackupAgent.actor.cpp

@@ -6624,9 +6624,6 @@ class FileBackupAgentImpl {
 		return Void();
 	}
 
-	// Helper function to check if a URL is a blobstore:// URL
-	static bool isBlobstoreUrl(const std::string& url) { return url.find("blobstore://") == 0; }
-
 	ACTOR static Future<Void> submitParallelRestore(Database cx,
 	                                                Key backupTag,
 	                                                Standalone<VectorRef<KeyRangeRef>> backupRanges,

fdbclient/tests/aws_fixture.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Functions for dealing w/ aws/s3
 #

fdbclient/tests/bulkload_test.sh

@@ -1,10 +1,9 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
-# Test bulkload. Uses S3 or seaweedfs if not available:
-# (https://github.com/seaweedfs/seaweedfs) as substitute.
+# Test bulkload. Uses S3 or MockS3Server if not available.
 #
 # In the below we start a small FDB cluster, populate it with
-# some data and then start up a seaweedfs instance. We
+# some data and then start up MockS3Server. We
 # then run a bulkdump to 'S3' and then a restore. We verify
 # the restore is the same as the original.
 #
@@ -13,7 +12,8 @@
 # so you can manually rerun commands or peruse logs and data
 # under SCRATCH_DIR.
 
-# Make sure cleanup on script exit.
+# Install signal traps. Depends on globals being set.
+# Calls the cleanup function.
 trap "exit 1" HUP INT PIPE QUIT TERM
 trap cleanup  EXIT
 
@@ -22,8 +22,8 @@ function cleanup {
   if type shutdown_fdb_cluster &> /dev/null; then
     shutdown_fdb_cluster
   fi
-  if type shutdown_weed &> /dev/null; then
-    shutdown_weed "${TEST_SCRATCH_DIR}"
+  if type shutdown_mocks3 &> /dev/null; then
+    shutdown_mocks3
   fi
   if type shutdown_aws &> /dev/null; then
     shutdown_aws "${TEST_SCRATCH_DIR}"
@@ -252,17 +252,17 @@ readonly HTTP_VERBOSE_LEVEL=2
 # internal apple dev environments where S3 is available).
 readonly USE_S3="${USE_S3:-$( if [[ -n "${OKTETO_NAMESPACE+x}" ]]; then echo "true" ; else echo "false"; fi )}"
 
-# Set KNOBS based on whether we're using real S3 or SeaweedFS
+# Set KNOBS based on whether we're using real S3 or MockS3Server
 if [[ "${USE_S3}" == "true" ]]; then
   # Use AWS KMS encryption for real S3
   KNOBS=("--knob_blobstore_encryption_type=aws:kms" "--knob_http_verbose_level=${HTTP_VERBOSE_LEVEL}")
 else
-  # No encryption for SeaweedFS
+  # No encryption for MockS3Server
   KNOBS=("--knob_http_verbose_level=${HTTP_VERBOSE_LEVEL}")
 fi
 readonly KNOBS
 
-# Set TLS_CA_FILE only when using real S3, not for SeaweedFS
+# Set TLS_CA_FILE only when using real S3, not for SeaweedFS or MockS3Server
 if [[ "${USE_S3}" == "true" ]]; then
   # Try to find a valid TLS CA file if not explicitly set
   if [[ -z "${TLS_CA_FILE:-}" ]]; then
@@ -276,7 +276,7 @@ if [[ "${USE_S3}" == "true" ]]; then
   fi
   TLS_CA_FILE="${TLS_CA_FILE:-}"
 else
-  # For SeaweedFS, don't use TLS
+  # For SeaweedFS and MockS3Server, don't use TLS
   TLS_CA_FILE=""
 fi
 readonly TLS_CA_FILE
@@ -362,28 +362,29 @@ if [[ "${USE_S3}" == "true" ]]; then
     export FDB_TLS_CA_FILE="${TLS_CA_FILE}"
   fi
 else
-  log "Testing against seaweedfs"
-  # Now source in the seaweedfs fixture so we can use its methods in the below.
+  log "Testing against MockS3Server"
+  # Now source in the mocks3 fixture so we can use its methods in the below.
   # shellcheck source=/dev/null
-  if ! source "${cwd}/../../fdbclient/tests/seaweedfs_fixture.sh"; then
-    err "Failed to source seaweedfs_fixture.sh"
+  if ! source "${cwd}/../../fdbclient/tests/mocks3_fixture.sh"; then
+    err "Failed to source mocks3_fixture.sh"
     exit 1
   fi
-  if ! TEST_SCRATCH_DIR=$(create_weed_dir "${scratch_dir}"); then
-    err "Failed create of the weed dir." >&2
+  if ! TEST_SCRATCH_DIR=$(mktemp -d "${scratch_dir}/mocks3_bulkload_test.XXXXXX"); then
+    err "Failed create of the mocks3 test dir." >&2
     exit 1
   fi
   readonly TEST_SCRATCH_DIR
-  if ! host=$( run_weed "${scratch_dir}" "${TEST_SCRATCH_DIR}"); then
-    err "Failed to run seaweed"
+  if ! start_mocks3 "${build_dir}"; then
+    err "Failed to start MockS3Server"
     exit 1
   fi
-  readonly host
-  readonly bucket="${SEAWEED_BUCKET}"
-  readonly region="all_regions"
-  # Reference a non-existent blob file (its ignored by seaweed)
+  readonly host="${MOCKS3_HOST}:${MOCKS3_PORT}"
+  readonly bucket="test-bucket"
+  readonly region="us-east-1"
+  # Create an empty blob credentials file (MockS3Server uses simple auth)
   readonly blob_credentials_file="${TEST_SCRATCH_DIR}/blob_credentials.json"
-  # Let the connection to seaweed be insecure -- not-TLS -- because just awkward to set up.
+  echo '{}' > "${blob_credentials_file}"
+  # Let the connection to MockS3Server be insecure -- not-TLS
   query_str="bucket=${bucket}&region=${region}&secure_connection=0"
 fi