added s3 provisioning of bucket

Author: rtisma

.dockerignore

@@ -6,3 +6,4 @@ Makefile
 .*
 !.mvn
 run.sh
+jwt.txt

.gitignore

@@ -3,6 +3,7 @@ target/
 !.mvn/wrapper/maven-wrapper.jar
 !**/src/main/**/target/
 !**/src/test/**/target/
+jwt.txt
 
 ### STS ###
 .apt_generated

README.md

@@ -14,6 +14,23 @@ It is possible to run the dms locally while controlling a remote docker engine.
    DOCKER_HOST=unix:///some/local/path/to/docker.sock
    ```
 
+### Test Score uploads and downloads
+To test the score uploads and downloads, ego, song and score services must be running and healthy. Since the score service responds with presigned s3 urls that use the minio-api as the service name, the urls would not be resolvable outside of the defined `docker-swarm-network`. To fix this, a convienece docker image state was created in the Dockerfile (target is called `genomic-transfer-helper`) which pulls the appropriate song-client and score-client distributions and configures them with the right JWT. The following instructions will allow you to download and upload using this tool:
+1. Log in to the EGO-UI service. The default local url is http://localhost:9002
+2. Log out
+3. By default, new users obtain the role `USER`. In order to proceed, the `ADMIN` role is needed. To do this, run the following command
+```
+docker exec -it $(docker service ps ego-db --no-trunc --format '{{ .Name }}.{{.ID}}' | head -1)  psql -U postgres ego -c "UPDATE egouser set type='ADMIN' where email='<the-email-you-logged-in-with-previously>' and providertype='<one of: GOOGLE,LINKEDIN,FACEBOOK,GITHUB,ORCID>';"
+```
+5. Log in again into EGO-UI (http://localhost:9002) and now you should be able to use the UI
+6. Select `Users` in the side bar, and then select your user record. On the right most pane, click the `Edit` button at the top and then click the `+ Add` button for the `Groups` section, and add your self to the `dcc-admin` group. Then click the `Save` button at the top. This will give you the neccessary permissions to use song and score.
+7. Log out
+8. Start the browsers inspector tool (this will be different for each browser) and filter for XHR requests.
+9. Log in to the EGO-UI. Once logged in, refer to response for the `ego-token?client_id=ego-ui` request. This is the JWT
+10. Copy and paste the JWT from the previous step into the file `./jwt.txt`. 
+11. Run `make start-transfer-shell`. This will automatically run the `genomic-transfer-helper` and load the contents of `jwt.txt` as the JWT to allow authorized access to song and score.
+12. Once logged into the container, you can use the score and song clients. Run `./song-client/bin/sing ping` and `curl $(curl -sL http://score-api:8080/download/ping)` to do a health check.
+
 ## Tips
 ### Checking for OOM messages when a container/service is killed
 Sometimes, if the reserved/limit memory is too low, a container will get killed by the kernel. To find out if this is the case, run

pom.xml

@@ -144,6 +144,13 @@
 			<version>${jline.version}</version>
 		</dependency>
 
+		<!-- AWS S3 -->
+		<dependency>
+			<groupId>software.amazon.awssdk</groupId>
+			<artifactId>s3</artifactId>
+			<version>${aws.java.sdk.version}</version>
+		</dependency>
+
 
 		<!-- Retry -->
 		<dependency>
@@ -388,6 +395,8 @@
 		<okhttp.version>4.9.0</okhttp.version>
 		<spring-cloud-stub-runner.version>3.0.0</spring-cloud-stub-runner.version>
 		<asm.version>8.0.1</asm.version>
+		<aws.java.sdk.version>2.15.69</aws.java.sdk.version>
+
 
 		<!--  Plugins-->
 		<plugin.fmt.version>2.10</plugin.fmt.version>

src/main/java/bio/overture/dms/cli/questionnaire/ScoreQuestionnaire.java

@@ -135,6 +135,7 @@ private ScoreS3Config processScoreS3Config(URL dmsGatewayUrl, ClusterRunModes cl
 
         // https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints
         val awsS3Url = new URL("https://s3." + awsS3Region + ".amazonaws.com");
+        s3Builder.s3Region(awsS3Region);
         s3Builder.url(awsS3Url);
       } else {
         val externalS3Url =

src/main/java/bio/overture/dms/compose/deployment/score/ScoreApiDeployer.java

@@ -1,36 +1,62 @@
 package bio.overture.dms.compose.deployment.score;
 
-import static bio.overture.dms.compose.deployment.SimpleProvisionService.createSimpleProvisionService;
-import static bio.overture.dms.compose.model.ComposeServiceResources.SCORE_API;
-import static bio.overture.dms.compose.model.Constants.DMS_ADMIN_GROUP_NAME;
-
 import bio.overture.dms.compose.deployment.ServiceDeployer;
 import bio.overture.dms.compose.deployment.ego.EgoHelper;
+import bio.overture.dms.compose.model.S3ObjectUploadRequest;
 import bio.overture.dms.core.model.dmsconfig.DmsConfig;
 import bio.overture.dms.core.model.dmsconfig.EgoConfig;
+import bio.overture.dms.core.model.dmsconfig.ScoreConfig;
 import bio.overture.dms.core.model.dmsconfig.ScoreConfig.ScoreApiConfig;
+import bio.overture.dms.core.model.dmsconfig.ScoreConfig.ScoreS3Config;
+import bio.overture.dms.core.model.enums.ClusterRunModes;
+import bio.overture.dms.swarm.properties.DockerProperties;
 import lombok.NonNull;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
+import software.amazon.awssdk.regions.Region;
+
+import java.net.URI;
+
+import static bio.overture.dms.compose.deployment.SimpleProvisionService.createSimpleProvisionService;
+import static bio.overture.dms.compose.deployment.score.s3.S3ServiceFactory.buildS3Service;
+import static bio.overture.dms.compose.model.ComposeServiceResources.MINIO_API;
+import static bio.overture.dms.compose.model.ComposeServiceResources.SCORE_API;
+import static bio.overture.dms.compose.model.Constants.DMS_ADMIN_GROUP_NAME;
+import static bio.overture.dms.core.model.enums.ClusterRunModes.LOCAL;
+import static bio.overture.dms.core.model.enums.ClusterRunModes.PRODUCTION;
+import static java.lang.String.format;
+import static software.amazon.awssdk.regions.Region.US_EAST_1;
 
 @Slf4j
 @Component
 public class ScoreApiDeployer {
 
+
   /** Constants */
   private static final String SCORE_POLICY_NAME = "SCORE";
+  private static final String HELIOGRAPH_CONTENT = "DMS Score Heliograph Object";
+  private static final Region DEFAULT_S3_REGION = US_EAST_1;
+
+  // NOTE: the DEFAULT_OBJECT_PATH is baked into the score-api.yaml.vm
+  private static final String DEFAULT_OBJECT_PATH = "data";
+  // NOTE: the DEFAULT_OBJECT_SENTINEL is baked into the score-api.yaml.vm
+  private static final String DEFAULT_OBJECT_SENTINEL = "heliograph";
+  private static final int MINIO_API_CONTAINER_PORT = 9000;
 
   /** Dependencies */
   private final ServiceDeployer serviceDeployer;
-
   private final EgoHelper egoHelper;
+  private final DockerProperties dockerProperties;
 
   @Autowired
-  public ScoreApiDeployer(@NonNull ServiceDeployer serviceDeployer, @NonNull EgoHelper egoHelper) {
+  public ScoreApiDeployer(@NonNull ServiceDeployer serviceDeployer, @NonNull EgoHelper egoHelper,
+      @NonNull DockerProperties dockerProperties) {
     this.serviceDeployer = serviceDeployer;
     this.egoHelper = egoHelper;
+    this.dockerProperties = dockerProperties;
   }
 
   public void deploy(@NonNull DmsConfig dmsConfig) {
@@ -41,6 +67,47 @@ public void deploy(@NonNull DmsConfig dmsConfig) {
 
   private void provision(DmsConfig dmsConfig) {
     buildEgoScoreProvisioner(dmsConfig.getEgo(), dmsConfig.getScore().getApi()).run();
+    provisionS3Buckets(dmsConfig.getClusterRunMode(), dmsConfig.getScore());
+  }
+
+  private void provisionS3Buckets(ClusterRunModes clusterRunMode, ScoreConfig scoreConfig) {
+    // TODO: May want to create interactive question for this in the future. For now, autocreate
+    // buckets if minio is run
+    val autoCreateBuckets = clusterRunMode == LOCAL;
+    val endpointUri = resolveS3Endpoint(clusterRunMode, scoreConfig.getS3());
+    val region = resolveS3Region(scoreConfig.getS3());
+    val s3Service = buildS3Service(scoreConfig.getS3(), endpointUri, region);
+    s3Service.provisionBucket(scoreConfig.getApi().getStateBucket(), autoCreateBuckets);
+    s3Service.uploadData(
+        S3ObjectUploadRequest.builder()
+            .autoCreateBucket(autoCreateBuckets)
+            .bucketName(scoreConfig.getApi().getObjectBucket())
+            .objectKey(resolveObjectKey())
+            .data(HELIOGRAPH_CONTENT.getBytes())
+            .build());
+  }
+
+  @SneakyThrows
+  private URI resolveS3Endpoint(
+      ClusterRunModes clusterRunMode, ScoreS3Config scoreS3Config) {
+    if (clusterRunMode == PRODUCTION) {
+      return scoreS3Config.getUrl().toURI();
+    } else if (clusterRunMode == LOCAL) {
+      return resolveMinioContainerUri(scoreS3Config);
+    } else {
+      throw new IllegalStateException(
+          format(
+              "The clusterRunMode '%s' is unknown and cannot be processed", clusterRunMode.name()));
+    }
+  }
+
+  @SneakyThrows
+  private URI resolveMinioContainerUri(ScoreS3Config scoreS3Config) {
+    if(dockerProperties.getRunAs()){
+      return new URI("http://" + MINIO_API.toString() + ":" + MINIO_API_CONTAINER_PORT);
+    } else {
+      return scoreS3Config.getUrl().toURI();
+    }
   }
 
   private EgoScoreProvisioner buildEgoScoreProvisioner(
@@ -54,4 +121,17 @@ private EgoScoreProvisioner buildEgoScoreProvisioner(
         .scoreAppCredential(scoreApiConfig.getScoreAppCredential())
         .build();
   }
+
+  private static Region resolveS3Region(ScoreS3Config scoreS3Config){
+    if (scoreS3Config.isS3RegionDefined()){
+      return Region.of(scoreS3Config.getS3Region());
+    } else {
+      return DEFAULT_S3_REGION;
+    }
+  }
+
+  private static String resolveObjectKey() {
+    return DEFAULT_OBJECT_PATH + "/" + DEFAULT_OBJECT_SENTINEL;
+  }
+
 }

src/main/java/bio/overture/dms/compose/deployment/score/s3/S3Service.java

@@ -0,0 +1,74 @@
+package bio.overture.dms.compose.deployment.score.s3;
+
+import static bio.overture.dms.core.util.Exceptions.checkState;
+
+import bio.overture.dms.compose.model.S3ObjectUploadRequest;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import lombok.val;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.Bucket;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+
+@Slf4j
+@RequiredArgsConstructor
+public class S3Service {
+
+  @NonNull private final S3Client s3;
+
+  /** Provisions the request bucket followed by uploading data to the specified object key path */
+  @SneakyThrows
+  public PutObjectResponse uploadData(@NonNull S3ObjectUploadRequest request) {
+    provisionBucket(request.getBucketName(), request.isAutoCreateBucket());
+    log.info(
+        "Uploading data to bucket '{}' with objectKey '{}'",
+        request.getBucketName(),
+        request.getObjectKey());
+    val resp = putObject(request.getBucketName(), request.getObjectKey(), request.getData());
+    log.info("- Done uploading data to bucket");
+    return resp;
+  }
+
+  /**
+   * Creates a bucket if it does not exist and autoCreateBucket is true. If autoCreateBucket is
+   * false, an error is thrown. If the bucket already exists, then bucket creation is skipped.
+   *
+   * @param bucketName to create
+   * @param autoCreateBucket indicated whether the bucket should be created if it does not exist.
+   */
+  public void provisionBucket(@NonNull String bucketName, boolean autoCreateBucket) {
+    if (!autoCreateBucket) {
+      checkBucketExists(bucketName);
+    } else if (!isBucketExist(bucketName)) {
+      log.info("Creating non-existent bucket '{}'", bucketName);
+      createBucket(bucketName);
+      log.info("-  Done");
+    } else {
+      log.info("The bucket '{}' already exists, skipping bucket creation.", bucketName);
+    }
+  }
+
+  private PutObjectResponse putObject(String bucketName, String objectKey, byte[] data) {
+    return s3.putObject(
+        PutObjectRequest.builder().bucket(bucketName).key(objectKey).build(),
+        RequestBody.fromBytes(data));
+  }
+
+  private boolean isBucketExist(@NonNull String bucketName) {
+    return s3.listBuckets().buckets().stream()
+        .map(Bucket::name)
+        .anyMatch(x -> x.equals(bucketName));
+  }
+
+  private void checkBucketExists(String bucketName) {
+    checkState(isBucketExist(bucketName), "The bucket '%s' does not exist", bucketName);
+  }
+
+  private void createBucket(@NonNull String bucketName) {
+    s3.createBucket(x -> x.bucket(bucketName));
+  }
+}

src/main/java/bio/overture/dms/compose/deployment/score/s3/S3ServiceFactory.java

@@ -0,0 +1,56 @@
+package bio.overture.dms.compose.deployment.score.s3;
+
+import static lombok.AccessLevel.PRIVATE;
+import static software.amazon.awssdk.core.client.config.SdkAdvancedClientOption.SIGNER;
+import static software.amazon.awssdk.core.retry.RetryMode.STANDARD;
+import static software.amazon.awssdk.regions.Region.US_EAST_1;
+
+import bio.overture.dms.core.model.dmsconfig.ScoreConfig.ScoreS3Config;
+import java.net.URI;
+import java.time.Duration;
+import java.util.Map;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.SneakyThrows;
+import lombok.val;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
+import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+
+@NoArgsConstructor(access = PRIVATE)
+public class S3ServiceFactory {
+
+  private static final Duration API_CALL_TIMEOUT_DURATION = Duration.ofMillis(15000);
+  public static S3Service buildS3Service(
+      @NonNull ScoreS3Config scoreS3Config, @NonNull URI s3EndpointUri, Region region) {
+    val s3 = buildS3Client(scoreS3Config, s3EndpointUri, region);
+    return new S3Service(s3);
+  }
+
+  @SneakyThrows
+  private static S3Client buildS3Client(ScoreS3Config scoreS3Config, URI s3EndpointUri, Region region) {
+    val b = S3Client.builder();
+    b.region(region);
+    b.endpointOverride(s3EndpointUri);
+    b.credentialsProvider(buildAwsCredentialsProvider(scoreS3Config));
+    b.overrideConfiguration(buildS3Configuration(API_CALL_TIMEOUT_DURATION));
+    return b.build();
+  }
+
+  private static ClientOverrideConfiguration buildS3Configuration(Duration callTimout) {
+    return ClientOverrideConfiguration.builder()
+        .apiCallTimeout(callTimout)
+        .retryPolicy(STANDARD)
+        .advancedOptions(Map.of(SIGNER, AwsS3V4Signer.create()))
+        .build();
+  }
+
+  private static AwsCredentialsProvider buildAwsCredentialsProvider(ScoreS3Config scoreS3Config) {
+    return StaticCredentialsProvider.create(
+        AwsBasicCredentials.create(scoreS3Config.getAccessKey(), scoreS3Config.getSecretKey()));
+  }
+}

src/main/java/bio/overture/dms/compose/model/S3ObjectUploadRequest.java

@@ -0,0 +1,15 @@
+package bio.overture.dms.compose.model;
+
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+
+@Value
+@Builder
+public class S3ObjectUploadRequest {
+
+  private final boolean autoCreateBucket;
+  @NonNull private final String bucketName;
+  @NonNull private final String objectKey;
+  @NonNull private final byte[] data;
+}

src/main/java/bio/overture/dms/core/model/dmsconfig/ScoreConfig.java

@@ -1,5 +1,6 @@
 package bio.overture.dms.core.model.dmsconfig;
 
+import static bio.overture.dms.core.util.Strings.isDefined;
 import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_EMPTY;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
@@ -36,6 +37,11 @@ public static class ScoreS3Config {
     // These are optional
     private boolean useMinio;
     private Integer hostPort;
+    private String s3Region;
+
+    public boolean isS3RegionDefined(){
+      return isDefined(s3Region);
+    }
   }
 
   @Data

src/main/resources/application.yml

@@ -24,7 +24,7 @@ compose:
   network: dms-swarm-network
 
 docker:
-  #NOTE: Indicates if this application is RUNNING-AS a docker container on the host machine, or running on baremetal on the host machine. This is mostly to help resolve the what URLs the DMS uses to interact with the services. If its RUNNING-AS a docker container, then it will be in the dms dswarm network, and so it can use the service names as the domain in the urls, since docker will resolve the IP. If its not, then the url for the service will be used.
+  #NOTE: Indicates if this application is RUNNING-AS a docker container on the host machine. If false, then it is running on bare-metal on the host machine. This is mostly to help resolve the what URLs the DMS uses to interact with the services. If its RUNNING-AS a docker container, then it will be in the dms swarm network, and so it can use the service names as the domain in the urls, since docker will resolve the IP. If its not, then the url for the service will be used.
   run-as: false
-  # If blank, will use the system default docker daemon
+  # If blank, will use the system default docker daemon. Useful for using a remote docker daemon.
   host: ""

src/main/resources/templates/servicespec/score-api.yaml.vm

@@ -16,7 +16,11 @@ TaskTemplate:
       - UPLOAD_PARTSIZE=1073741824
       - UPLOAD_CONNECTION_TIMEOUT=1200000
       - METADATA_URL=http://${composeServiceResources.SONG_API.toString()}:8080
+#if( $dmsConfig.clusterRunMode == 'PRODUCTION')
       - S3_ENDPOINT=$dmsConfig.score.s3.url
+#elseif( $dmsConfig.clusterRunMode == 'LOCAL')
+      - S3_ENDPOINT=http://${composeServiceResources.MINIO_API.toString()}:9000
+#end
       - S3_ACCESSKEY=$dmsConfig.score.s3.accessKey
       - S3_SECRETKEY=$dmsConfig.score.s3.secretKey
       - S3_SIGV4ENABLED=true