samples(storage transfer): add samples and test cases for storage transfer services

storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs

@@ -0,0 +1,89 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License").
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at 
+//
+// https://www.apache.org/licenses/LICENSE-2.0 
+//
+// Unless required by applicable law or agreed to in writing, software 
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and 
+// limitations under the License.
+
+using Google.Cloud.StorageTransfer.V1;
+using System;
+using Xunit;
+
+namespace StorageTransfer.Samples.Tests;
+
+[Collection(nameof(StorageFixture))]
+public class CheckLatestTransferOperationTest : IDisposable
+{
+    private readonly StorageFixture _fixture;
+    private string _jobName;
+    private readonly string _transferJobName;
+
+    public CheckLatestTransferOperationTest(StorageFixture fixture)
+    {
+        _fixture = fixture;
+        _fixture.BucketNameSource = _fixture.GenerateBucketName();
+        _fixture.BucketNameSink = _fixture.GenerateBucketName();
+        _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource);
+        _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink);
+        _transferJobName = CreateTransferJob();
+    }
+
+    [Fact]
+    public void CheckLatestTransferOperation()
+    {
+        CheckLatestTransferOperationSample checkLatestTransferOperationSample = new CheckLatestTransferOperationSample();
+        var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId, _transferJobName);
+        Assert.Contains("transferJobs/", transferJob.Name);
+        _jobName = transferJob.Name;
+    }
+
+    private string CreateTransferJob()
+    {
+        // Initialize request argument(s)
+        TransferJob transferJob = new TransferJob
+        {
+            ProjectId = _fixture.ProjectId,
+            TransferSpec = new TransferSpec
+            {
+                GcsDataSink = new GcsData { BucketName = _fixture.BucketNameSource },
+                GcsDataSource = new GcsData { BucketName = _fixture.BucketNameSink }
+            },
+            Status = TransferJob.Types.Status.Enabled
+        };
+        CreateTransferJobRequest request = new CreateTransferJobRequest
+        {
+            TransferJob = transferJob
+        };
+        // Make the request
+        TransferJob response = _fixture.Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob });
+        return response.Name;
+    }
+
+    public void Dispose()
+    {
+        try
+        {
+            _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest()
+            {
+                ProjectId = _fixture.ProjectId,
+                JobName = _transferJobName,
+                TransferJob = new TransferJob()
+                {
+                    Name = _transferJobName,
+                    Status = TransferJob.Types.Status.Deleted
+                }
+            });
+        }
+        catch (Exception)
+        {
+            // Do nothing, we delete on a best effort basis.
+        }
+    }
+}

storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs

@@ -0,0 +1,113 @@
+/**
+ * Copyright 2024 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Google.Cloud.PubSub.V1;
+using Google.Cloud.StorageTransfer.V1;
+using System;
+using Xunit;
+
+namespace StorageTransfer.Samples.Tests;
+
+[Collection(nameof(StorageFixture))]
+public class CreateEventDrivenGcsTransferTest : IDisposable
+{
+    private readonly StorageFixture _fixture;
+    private readonly string _pubSubId;
+    private string _transferJobName;
+    private string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString();
+    private string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString();
+    private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create();
+    private PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create();
+
+    public CreateEventDrivenGcsTransferTest(StorageFixture fixture)
+    {
+        _fixture = fixture;
+        _fixture.BucketNameSource = _fixture.GenerateBucketName();
+        _fixture.BucketNameSink = _fixture.GenerateBucketName();
+        _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource);
+        _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink);
+        _pubSubId = $"projects/{_fixture.ProjectId}/subscriptions/{SubscriptionId}";
+        string email = _fixture.Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest()
+        {
+            ProjectId = _fixture.ProjectId
+        }).AccountEmail;
+
+        string memberServiceAccount = "serviceAccount:" + email;
+        // Create subscription name
+        SubscriptionName subscriptionName = new SubscriptionName(_fixture.ProjectId, SubscriptionId);
+        // Create topic name
+        TopicName topicName = new TopicName(_fixture.ProjectId, TopicId);
+        // Create topic
+        PublisherClient.CreateTopic(topicName);
+        // Create subscription.
+        SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500);
+
+        var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy();
+
+        policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount);
+
+        PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest
+        {
+            ResourceAsResourceName = topicName,
+            Policy = policyIamPolicyTopic
+        });
+
+        var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy();
+
+        policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount);
+
+        PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest
+        {
+            ResourceAsResourceName = subscriptionName,
+            Policy = policyIamPolicySubscriber
+        });
+    }
+
+    [Fact]
+    public void CreateEventDrivenGcsTransfer()
+    {
+        CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample();
+        var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _pubSubId);
+        Assert.Contains("transferJobs/", transferJob.Name);
+        _transferJobName = transferJob.Name;
+    }
+
+    public void Dispose()
+    {
+        try
+        {
+            _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest()
+            {
+                ProjectId = _fixture.ProjectId,
+                JobName = _transferJobName,
+                TransferJob = new TransferJob()
+                {
+                    Name = _transferJobName,
+                    Status = TransferJob.Types.Status.Deleted
+                }
+            });
+
+            TopicName topicName = TopicName.FromProjectTopic(_fixture.ProjectId, TopicId);
+            PublisherClient.DeleteTopic(topicName);
+            SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(_fixture.ProjectId, SubscriptionId);
+            SubscriberClient.DeleteSubscription(subscriptionName);
+        }
+        catch (Exception)
+        {
+            // Do nothing, we delete on a best effort basis.
+        }
+    }
+}

storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs

@@ -0,0 +1,83 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License").
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at 
+//
+// https://www.apache.org/licenses/LICENSE-2.0 
+//
+// Unless required by applicable law or agreed to in writing, software 
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and 
+// limitations under the License.
+
+using Google.Cloud.Storage.V1;
+using Google.Cloud.StorageTransfer.V1;
+using System;
+using System.IO;
+using Xunit;
+
+namespace StorageTransfer.Samples.Tests;
+
+[Collection(nameof(StorageFixture))]
+public class DownloadToPosixTest : IDisposable
+{
+    private readonly StorageFixture _fixture;
+    private string _transferJobName;
+    private readonly string _tempDirectory;
+    private readonly string _gcsSourcePath;
+    private readonly string _bucketNamePosixSource;
+
+    public DownloadToPosixTest(StorageFixture fixture)
+    {
+        _fixture = fixture;
+        _bucketNamePosixSource = _fixture.GenerateBucketName();
+        _fixture.CreateBucketAndGrantStsPermissions(_bucketNamePosixSource);
+        _tempDirectory = _fixture.GenerateTempFolderPath();
+        _gcsSourcePath = $"{Guid.NewGuid()}/{Guid.NewGuid()}/";
+        UploadObjectToPosixBucket(_bucketNamePosixSource);
+    }
+
+    [Fact]
+    public void DownloadToPosix()
+    {
+        DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample();
+        Directory.CreateDirectory(_tempDirectory);
+        var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _bucketNamePosixSource, _gcsSourcePath, _tempDirectory);
+        Assert.Contains("transferJobs/", transferJob.Name);
+        Assert.True(Directory.Exists(_tempDirectory));
+        _transferJobName = transferJob.Name;
+    }
+
+    private void UploadObjectToPosixBucket(string bucketName)
+    {
+        byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg");
+        MemoryStream stream = new MemoryStream(byteArray);
+        string fileName = $"{_gcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt";
+        _fixture.Storage.UploadObject(bucketName, fileName, "application/octet-stream", stream);
+    }
+
+    public void Dispose()
+    {
+        try
+        {
+            _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest()
+            {
+                ProjectId = _fixture.ProjectId,
+                JobName = _transferJobName,
+                TransferJob = new TransferJob()
+                {
+                    Name = _transferJobName,
+                    Status = TransferJob.Types.Status.Deleted
+                }
+            });
+            Directory.Delete(_tempDirectory, true);
+            _fixture.Storage.DeleteBucket(_bucketNamePosixSource, new DeleteBucketOptions { DeleteObjects = true });
+        }
+        catch (Exception)
+        {
+            // Do nothing, we delete on a best effort basis.
+        }
+    }
+}

storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs

@@ -1,4 +1,4 @@
-/**
+/**
  * Copyright 2021 Google Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-using System;
 using Google.Cloud.StorageTransfer.V1;
+using System;
 using Xunit;
 
 namespace StorageTransfer.Samples.Tests
@@ -29,6 +29,10 @@ public class QuickstartTest : IDisposable
         public QuickstartTest(StorageFixture fixture)
         {
             _fixture = fixture;
+            _fixture.BucketNameSource = _fixture.GenerateBucketName();
+            _fixture.BucketNameSink = _fixture.GenerateBucketName();
+            _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource);
+            _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink);
         }
 
         [Fact]