add support for userProject in java-storage

gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientGrpcTracingInterceptor.java

@@ -17,6 +17,7 @@
 package com.google.cloud.hadoop.gcsio;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.flogger.GoogleLogger;
 import com.google.gson.Gson;
@@ -43,10 +44,14 @@
 public class GoogleCloudStorageClientGrpcTracingInterceptor implements ClientInterceptor {
   private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
   public static final String IDEMPOTENCY_TOKEN_HEADER = "x-goog-gcs-idempotency-token";
+  public static final String USER_PROJECT_HEADER = "x-goog-user-project";
   private static final DateTimeFormatter dtf =
       DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS");
+  private static final String DEFAULT_HEADER_VALUE = "NOT-FOUND";
   private static final Metadata.Key<String> idempotencyKey =
       Metadata.Key.of(IDEMPOTENCY_TOKEN_HEADER, Metadata.ASCII_STRING_MARSHALLER);
+  private static final Metadata.Key<String> userProjectKey =
+      Metadata.Key.of(USER_PROJECT_HEADER, Metadata.ASCII_STRING_MARSHALLER);
 
   @NonNull
   static String fmtProto(@NonNull Object obj) {
@@ -215,15 +220,33 @@
     }
 
     private String getInvocationId() {
-      return headers.get(idempotencyKey);
+      String value = headers.get(idempotencyKey);
+      if (Strings.isNullOrEmpty(value)) {
+        logger.atWarning().log(
+            "Received a null or empty value for invocation Id: %s for rpc method: %s",
+            value, rpcMethod);
+        value = DEFAULT_HEADER_VALUE;
+      }
+      return value;
     }
 
-    private ImmutableMap.Builder<String, Object> getStreamContext() {
+    protected String getRequesterPaysProject() {
+      String value = headers.get(userProjectKey);
+      return value == null ? DEFAULT_HEADER_VALUE : value;
+    }
+
+    private ImmutableMap.Builder<String, Object> getHeaderValues() {
       return new ImmutableMap.Builder<String, Object>()
-          .put(GoogleCloudStorageTracingFields.RPC_METHOD.name, rpcMethod)
+          .put(
+              GoogleCloudStorageTracingFields.REQUESTER_PAYS_PROJECT.name,
+              getRequesterPaysProject())
           .put(GoogleCloudStorageTracingFields.IDEMPOTENCY_TOKEN.name, getInvocationId());
     }
 
+    private ImmutableMap.Builder<String, Object> getStreamContext() {
+      return getHeaderValues().put(GoogleCloudStorageTracingFields.RPC_METHOD.name, rpcMethod);
+    }
+
     private ImmutableMap.Builder<String, Object> getCommonTraceFields() {
       return getStreamContext()
           .put(GoogleCloudStorageTracingFields.CURRENT_TIME.name, dtf.format(LocalDateTime.now()));

gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientImpl.java

@@ -38,6 +38,7 @@
 import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartCleanupStrategy;
 import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartNamingStrategy;
 import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageException;
 import com.google.cloud.storage.StorageOptions;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.MoreObjects;
@@ -74,6 +75,7 @@
 
   private final GoogleCloudStorageOptions storageOptions;
   private final Storage storage;
+  private final RequesterPaysManager requesterPaysManager;
 
   // Error extractor to map APi exception to meaningful ErrorTypes.
   private static final ErrorTypeExtractor errorExtractor = GrpcErrorTypeExtractor.INSTANCE;
@@ -114,6 +116,8 @@
             ? createStorage(
                 credentials, options, gRPCInterceptors, pCUExecutorService, downscopedAccessTokenFn)
             : clientLibraryStorage;
+    this.requesterPaysManager =
+        new RequesterPaysManager(options.getRequesterPaysOptions(), this::shouldRequesterPay);
   }
 
   @Override
@@ -137,7 +141,11 @@
     }
 
     return new GoogleCloudStorageClientWriteChannel(
-        storage, storageOptions, resourceIdWithGeneration, options);
+        storage,
+        storageOptions,
+        resourceIdWithGeneration,
+        options,
+        requesterPaysManager::requesterShouldPay);
   }
 
   @Override
@@ -157,7 +165,8 @@
         itemInfo == null ? getItemInfo(resourceId) : itemInfo,
         readOptions,
         errorExtractor,
-        storageOptions);
+        storageOptions,
+        requesterPaysManager::requesterShouldPay);
   }
 
   @Override
@@ -270,6 +279,15 @@
         .getService();
   }
 
+  private Boolean shouldRequesterPay(String bucketName) {
+    try {
+      storage.testIamPermissions(bucketName, ImmutableList.of("storage.buckets.get"));
+    } catch (StorageException e) {
+      return errorExtractor.userProjectMissing(e);
+    }
+    return false;
+  }
+
   private static ImmutableMap<String, String> getUpdatedHeadersWithUserAgent(
       GoogleCloudStorageOptions storageOptions) {
     ImmutableMap<String, String> httpRequestHeaders =

gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientReadChannel.java

@@ -46,6 +46,7 @@
 import java.nio.channels.SeekableByteChannel;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Function;
 import javax.annotation.Nullable;
 
 /** Provides seekable read access to GCS via java-storage library. */
@@ -64,6 +65,7 @@
   private long objectSize;
   private final ErrorTypeExtractor errorExtractor;
   private ContentReadChannel contentReadChannel;
+  private Function<String, Boolean> requesterShouldPay;
   private boolean gzipEncoded = false;
   private boolean open = true;
 
@@ -76,7 +78,8 @@
       GoogleCloudStorageItemInfo itemInfo,
       GoogleCloudStorageReadOptions readOptions,
       ErrorTypeExtractor errorExtractor,
-      GoogleCloudStorageOptions storageOptions)
+      GoogleCloudStorageOptions storageOptions,
+      Function<String, Boolean> requesterShouldPay)
       throws IOException {
     validate(itemInfo);
     this.storage = storage;
@@ -87,6 +90,7 @@
     this.readOptions = readOptions;
     this.storageOptions = storageOptions;
     this.contentReadChannel = new ContentReadChannel(readOptions, resourceId);
+    this.requesterShouldPay = requesterShouldPay;
     initMetadata(itemInfo.getContentEncoding(), itemInfo.getSize());
   }
 
@@ -582,6 +586,10 @@
         blobReadOptions.add(
             BlobSourceOption.decryptionKey(storageOptions.getEncryptionKey().value()));
       }
+      if (requesterShouldPay.apply((blobId.getBucket()))) {
+        blobReadOptions.add(
+            BlobSourceOption.userProject(storageOptions.getRequesterPaysOptions().getProjectId()));
+      }
       return blobReadOptions.toArray(new BlobSourceOption[blobReadOptions.size()]);
     }
 

gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientWriteChannel.java

@@ -30,6 +30,7 @@
 import java.nio.channels.WritableByteChannel;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Function;
 
 /** Implements WritableByteChannel to provide write access to GCS via java-storage client */
 class GoogleCloudStorageClientWriteChannel implements WritableByteChannel {
@@ -46,11 +47,12 @@ public GoogleCloudStorageClientWriteChannel(
       Storage storage,
       GoogleCloudStorageOptions storageOptions,
       StorageResourceId resourceId,
-      CreateObjectOptions createOptions)
+      CreateObjectOptions createOptions,
+      Function<String, Boolean> requesterShouldPay)
       throws IOException {
     this.resourceId = resourceId;
     BlobWriteSession blobWriteSession =
-        getBlobWriteSession(storage, resourceId, createOptions, storageOptions);
+        getBlobWriteSession(storage, resourceId, createOptions, storageOptions, requesterShouldPay);
     this.writableByteChannel = blobWriteSession.open();
   }
 
@@ -73,16 +75,19 @@ private static BlobWriteSession getBlobWriteSession(
       Storage storage,
       StorageResourceId resourceId,
       CreateObjectOptions createOptions,
-      GoogleCloudStorageOptions storageOptions) {
+      GoogleCloudStorageOptions storageOptions,
+      Function<String, Boolean> requesterShouldPay) {
     return storage.blobWriteSession(
         getBlobInfo(resourceId, createOptions),
-        generateWriteOptions(createOptions, storageOptions));
+        generateWriteOptions(resourceId, createOptions, storageOptions, requesterShouldPay));
   }
 
   private static BlobWriteOption[] generateWriteOptions(
-      CreateObjectOptions createOptions, GoogleCloudStorageOptions storageOptions) {
+      StorageResourceId resourceId,
+      CreateObjectOptions createOptions,
+      GoogleCloudStorageOptions storageOptions,
+      Function<String, Boolean> requesterShouldPay) {
     List<BlobWriteOption> blobWriteOptions = new ArrayList<>();
-
     blobWriteOptions.add(BlobWriteOption.disableGzipContent());
     blobWriteOptions.add(BlobWriteOption.generationMatch());
     if (createOptions.getKmsKeyName() != null) {
@@ -95,6 +100,10 @@ private static BlobWriteOption[] generateWriteOptions(
       blobWriteOptions.add(
           BlobWriteOption.encryptionKey(storageOptions.getEncryptionKey().value()));
     }
+    if (requesterShouldPay.apply(resourceId.getBucketName())) {
+      blobWriteOptions.add(
+          BlobWriteOption.userProject(storageOptions.getRequesterPaysOptions().getProjectId()));
+    }
     return blobWriteOptions.toArray(new BlobWriteOption[blobWriteOptions.size()]);
   }
 

gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageTracingFields.java

@@ -24,6 +24,7 @@ public enum GoogleCloudStorageTracingFields {
   // GRPC specific fields
   RPC_METHOD("rpcMethod"),
   IDEMPOTENCY_TOKEN("idempotency-token"),
+  REQUESTER_PAYS_PROJECT("requesterPaysProject"),
   REQUEST_COUNTER("requestCounter"),
   RESPONSE_COUNTER("responseCounter"),
   REQUEST_MESSAGE_AS_STRING("reqMessageAsString"),

gcsio/src/main/java/com/google/cloud/hadoop/gcsio/RequesterPaysManager.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.hadoop.gcsio;
+
+import com.google.cloud.hadoop.util.RequesterPaysOptions;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import java.time.Duration;
+import java.util.function.Function;
+
+@VisibleForTesting
+public class RequesterPaysManager {
+  private final RequesterPaysOptions requesterPaysOptions;
+  private final Function<String, Boolean> shouldRequesterPay;
+  private final LoadingCache<String, Boolean> autoBuckets =
+      CacheBuilder.newBuilder()
+          .expireAfterWrite(Duration.ofHours(1))
+          .build(
+              new CacheLoader<String, Boolean>() {
+                @Override
+                public Boolean load(String bucketName) {
+                  return shouldRequesterPay.apply(bucketName);
+                }
+              });
+
+  public RequesterPaysManager(
+      RequesterPaysOptions requesterPaysOptions, Function<String, Boolean> shouldRequesterPay) {
+    this.requesterPaysOptions = requesterPaysOptions;
+    this.shouldRequesterPay = shouldRequesterPay;
+  }
+
+  public boolean requesterShouldPay(String bucketName) {
+    if (bucketName == null) {
+      return false;
+    }
+    switch (requesterPaysOptions.getMode()) {
+      case ENABLED:
+        return true;
+      case CUSTOM:
+        return requesterPaysOptions.getBuckets().contains(bucketName);
+      case AUTO:
+        return autoBuckets.getUnchecked(bucketName);
+      default:
+        return false;
+    }
+  }
+}

gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientReadChannelTest.java

@@ -552,6 +552,7 @@ private GoogleCloudStorageClientReadChannel getJavaStorageChannel(
         objectInfo,
         readOptions,
         GrpcErrorTypeExtractor.INSTANCE,
-        GoogleCloudStorageOptions.DEFAULT.toBuilder().build());
+        GoogleCloudStorageOptions.DEFAULT.toBuilder().build(),
+        (bucketName) -> false);
   }
 }

gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientWriteChannelTest.java

@@ -129,7 +129,8 @@ private GoogleCloudStorageClientWriteChannel getJavaStorageChannel() throws IOEx
             .setContentEncoding(CONTENT_ENCODING)
             .setMetadata(GoogleCloudStorageTestHelper.getDecodedMetadata(metadata))
             .setKmsKeyName(KMS_KEY)
-            .build());
+            .build(),
+        (bucketName) -> false);
   }
 
   private static void verifyBlobInfoProperties(