diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 538a98c1..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SPHINXPROJ = score -SOURCEDIR = . -CONFDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index 2185f0c3..00000000 --- a/docs/conf.py +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# score documentation build configuration file, created by -# sphinx-quickstart on Thu Sep 27 15:44:32 2018. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = ['sphinx.ext.mathjax', - 'sphinx.ext.viewcode'] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = 'score' -copyright = '2018, andricdu' -author = 'andricdu' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '1.4.0' -# The full version, including alpha/beta/rc tags. -release = '1.4.0' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'sphinx_rtd_theme' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - '**': [ - 'relations.html', # needs 'show_related': True theme option to display - 'searchbox.html', - ] -} - - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = 'scoredoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'score.tex', 'score Documentation', - 'andricdu', 'manual'), -] - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'score', 'score Documentation', - [author], 1) -] - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'score', 'score Documentation', - author, 'score', 'One line description of project.', - 'Miscellaneous'), -] - - - diff --git a/docs/contribution/contribution.md b/docs/contribution/contribution.md new file mode 100644 index 00000000..38a798d7 --- /dev/null +++ b/docs/contribution/contribution.md @@ -0,0 +1,59 @@ + +**Navigation** + +- [Background](../index.md) +- [Operation](../operation/operation.md) + +--- + +# Contribution Guidelines + +We appreciate your interest in contributing to the Overture! Please take the time to read and follow these guidelines before submitting your contributions. By participating in this project, you are expected to abide by the [Code of Conduct](https://github.com/overture-stack/SCORE/blob/readme-update/code_of_conduct.md). Please take the time to read it carefully before contributing. + +## Running score from Source Code + +To contribute to score, you'll need to run the application from the source code. Here's how you can do that: + +1. **Clone the Repository**: First, clone the score repository to your local machine using `git clone https://github.com/overture-stack/SCORE.git`. + +2. **Install Dependencies**: Navigate to the cloned directory and install all required dependencies. + +3. **Run the Application Locally**: After installing the dependencies, start the application locally. This usually involves running a command like `npm start` or `python manage.py runserver`, again depending on the language and framework used. + +4. **Test Your Changes**: Before submitting a pull request, make sure to test your changes thoroughly. Run the automated tests (if available) and perform manual testing to ensure that your contribution does not introduce regressions. + +## Issues and Feature Requests + +Before opening a new issue or feature request, please check if a similar one already exists. If so, please add a comment to the existing issue instead of creating a new one. When submitting an issue or feature request, please include a detailed description of the problem or feature you would like to see, along with any relevant code, error messages, or screenshots. This will help us better understand the issue and respond more efficiently. + +### Pull Requests + +We welcome and encourage pull requests from the community. To submit a pull request, please follow these steps: + +1. **Fork the Repository**: Fork the score repository on GitHub. +2. **Clone Your Fork**: Clone your forked repository to your local machine. +3. **Create a New Branch**: Create a new branch for your changes. +4. **Make Your Changes**: Implement your changes and commit them to your branch. +5. **Push Your Changes**: Push your changes to your forked repository. +6. **Submit a Pull Request**: Open a pull request against the main repository. + +Please ensure your code adheres to the following guidelines before submission: + +- The code should be well-documented and readable. +- The code should be tested. +- Include a clear description of the changes made and the reason for the changes in the pull request. + +For any questions or discussions regarding your pull request, please refer to the GitHub issues system or join our [Slack channel](http://slack.overture.bio/) for further assistance. + +We use GitHub issues and pull requests for communication-related to code changes. For general discussion, feel free to join our [Slack channel](http://slack.overture.bio/). + +Thank you for contributing to Overture-Stack! + +--- + +**Navigation** + +- [Background](../index.md) +- [Operation](../operation/operation.md) + +--- \ No newline at end of file diff --git a/docs/img/azure_policies.png b/docs/img/azure_policies.png deleted file mode 100644 index a06ff45e..00000000 Binary files a/docs/img/azure_policies.png and /dev/null differ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..150835f4 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,51 @@ +# Score Developer Documentation + +--- + +**Navigation** +- [Operation](./operation/operation.md) +- [Contribution](./contribution/contribution.md) + +--- + +# Background + +Score's primary function is to facilitate the secure upload and download of file data to and from an object storage provider. + +Score specializes in data transfer, leaving metadata complexities to its companion application, Song. Song manages metadata validation and tracking, maintaining a separate repository from object storage. Together, Score and Song provide an efficient solution for distributed data organization. + +## High Performance Transfers + +Score offers a high-performance multipart transfer system with several advantages: + +- Enables segmented file downloads, allowing for pause and resume functionality +- Automatically resumes file transfers if interrupted, for instance, due to connection issues +- Utilizes parallelization for efficient and rapid file uploads and downloads + +## Data Integrity + +- Score guarantees file transfer authenticity by conducting MD5 validations on all file uploads and downloads + +## BAM & CRAM Slicing + +- Score client incorporates samtools features, enabling viewing of reads from BAM files +- Provides the capability to slice BAM and CRAM files by genomic regions using command line tools + +## The Score Client + +The Score Client is a user-friendly CLI tool designed to streamline the process of interacting with Score and Song. Key features of the Score Client include: + +- **File Management:** Users can efficiently upload and download files using manifest files retrieved from data portals +- **File Metadata Handling:** The Score Client facilitates the uploading of file metadata, including object IDs and md5 checksums to Song, ensuring that data is consistently tracked and accessible +- **Integration with Object Storage:** Compatible with S3-compliant object storage providers including AWS, Azure, and Google Cloud + +For comprehensive details on the commands and usage of the Score Client, please refer to the [Score Client Reference Documentation](www.overture.bio/documentation/score/user-guide/commands/). + +--- + +**Navigation** + +- [Operation](./operation/operation.md) +- [Contribution](./contribution/contribution.md) + +--- \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 34d422c3..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,17 +0,0 @@ -================================= -Score Documentation -================================= - -.. toctree:: - :maxdepth: 4 - - introduction.rst - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/introduction.rst b/docs/introduction.rst deleted file mode 100644 index 0c55f96c..00000000 --- a/docs/introduction.rst +++ /dev/null @@ -1,89 +0,0 @@ -introduction.rst - -============== -Introduction -============== - - -What is Score? -====================== - -`Score `_ facilitates the transfer and storage of your data seamlessly for cloud-based projects. File bundling, resumable downloads, and BAM/CRAM slicing make data transfer fast and smooth. - -The method with which Score facilitates the transfer of data is through the use of pre-signed URLs. As such, Score can be thought of as a broker between an object storage system and user authorization system, validating user access and generating signed URLs for object access. - - ------------------------------------------ - -.. _introduction_features: - -Features -====================== - -- Multipart Uploads and Downloads (high performance transfers) -- Support for AWS S3, Azure Storage, Google Cloud Storage -- Slicing of BAM and CRAM files by genomic region -- Client includes some samtools functionality such as viewing reads from a BAM -- MD5 validation of uploads and downloads -- ACL security using OAuth2 and scopes based on study codes -- Integrates with the SONG metadata system for data book keeping and consistency -- REST API with swagger docs - - -Projects Using Score -====================== - -1. **Cancer Collaboratory - Toronto**: https://storage.cancercollaboratory.org/swagger-ui.html -2. **AWS - Virginia**: https://virginia.cloud.icgc.org/swagger-ui.html - - -Getting Started -============================ - -Client ---------------- - -The Score Client is available to download as both a docker container or as a Java CLI application. - -The docker container can be pulled from here: - -``docker pull overture/score`` - -For the Java CLI, use the following steps: - -.. code-block:: bash - - wget -O score-client.tar.gz https://artifacts.oicr.on.ca/artifactory/dcc-release/bio/overture/score-client/[RELEASE]/score-client-[RELEASE]-dist.tar.gz - - tar xvzf score-client.tar.gz - - echo export PATH=$(pwd)/score-client-x.x.x/bin:$PATH >> ~/.bashrc - - source ~/.bashrc - - # you will then be able to use it by running the following command anywhere is the system - score-client - - -Server ---------------- - -To be Added... - - -License -============= -Copyright (c) 2018. Ontario Institute for Cancer Research - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index ff176b65..00000000 --- a/docs/make.bat +++ /dev/null @@ -1,37 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set CONFDIR=source -set BUILDDIR=_build -set SPHINXPROJ=score - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd diff --git a/docs/operation/operation.md b/docs/operation/operation.md new file mode 100644 index 00000000..e0b7cc12 --- /dev/null +++ b/docs/operation/operation.md @@ -0,0 +1,236 @@ +# Operational Docs + +Welcome to the Operational Documentation for setting up and managing the Score Server in a development environment. This document provides detailed steps and configurations needed to get your development environment up and running with either a standalone setup or using Docker. This page also includes instructions for integrating Keycloak for authentication and authorization. + +## Table of Contents + +- [Setting up the Development Environment](#setting-up-the-development-environment) + - [Standalone score-server](#standalone-score-server) + - [Clone the Score Repository](#clone-the-score-repository) + - [Build](#build) + - [Start the Server](#start-the-server) + - [Docker for Score](#docker-for-score) + - [Start score-server and all dependencies](#start-score-server-and-all-dependencies) + - [Start the score-server (Apple Silicon Users)](#start-the-score-server-mac-m1-users) + - [Stop score-server and clean up](#stop-score-server-and-clean-up) +- [Integrating Keycloak](#integrating-keycloak) + - [Standalone](#standalone) + - [Docker](#docker) + +## Setting up the Development Environment + +### Standalone score-server + +#### Clone the Score Repository + +Clone the Score repository to your local computer: [https://github.com/overture-stack/score.git](https://github.com/overture-stack/score.git) + +#### Build + +[JDK11](https://www.oracle.com/ca-en/java/technologies/downloads/) and [Maven3](https://maven.apache.org/download.cgi) are required to set up this service. To build the score-server run the following command from the score directory: + +```bash +./mvnw clean install -DskipTests +``` + +#### Start the server + +Before running your score-server, ensure that your local machine is connected and running the following dependent services: + +- Song Server +- Object Storage Service (S3 compatible storage or Azure) +- Authentication/Authorization Service (EGO or Keycloak) + +Set the configuration of above dependent services on `score-server/src/main/resources/application.yml` and make sure to use the profiles according to your needs. + +**Profiles** +| Profile | Description | +| - | - | +| *default* | Required to load default controllers | +| *secure* | Required to load security configuration | +| *collaboratory* or *azure* | Required to choose between S3 compatible or Azure storage | +| *dev* | (Optional) to facilitate dev default configuration | + +Run the following command to start the score-server: + +```bash +cd score-server/ +mvn spring-boot:run -Dspring-boot.run.profiles=default,collaboratory,secure,dev +``` + +>**Warning:** This guide is meant to demonstrate the configuration and usage of Score for development purposes and is not intended for production. If you ignore this warning and use this in any public or production environment, please remember to use Spring profiles accordingly. For production do not use ***dev*** profile. + +### Docker for Score + +Several `make` targets are provided for locally deploying the dependent services using Docker. As the developer, you can replicate a live environment for `score-server` and `score-client`. Using Docker allows you to develop locally, test submissions, create manifests, publish, unpublish and test score uploads/downloads in an isolated environment. + +For more information on the different targets, run `make help` or read the comments above each target for a description. + +> Note: We will need an internet connection for the `make` command, which may take several minutes to build. No external services are required for the `make` command. + +#### Start score-server and all dependencies. + +To start score-server and all dependencies, use the following command: + +```bash +make clean start-score-server +``` + +#### Start the score-server (Apple Silicon Users) + +If using Apple Silicon you must set the Docker BuildKit environment variable to the legacy builder. + +```bash +DOCKER_BUILDKIT=0 make clean start-score-server +``` + +#### Stop score-server and clean up + +To clean everything, including killing all services, maven cleaning, and removing generated files/directories, use the following command: + +```bash +make clean +``` + +> **Warning** Docker for Score is meant to demonstrate the configuration and usage of Score and is not intended for production. If you ignore this warning and use this in any public or production environment, please remember to change the passwords, accessKeys, and secretKeys. + +## Integrating Keycloak + +[Keycloak](https://www.keycloak.org/) is an open-source identity and access management solution that can be used to manage users and application permissions. You can find basic information on integrating Score and Keycloak using docker from our user docs [located here](https://www.overture.bio/documentation/score/docker-install/configuration/authentication/). For a comprehensive guide on installing and configuring Keycloak, refer to the [Keycloak documentation](https://www.keycloak.org/documentation). + +### Standalone: + +If you’re building score using the source code, the following configuration is required in `score-server/src/main/resources/application.yml`. + +```yaml +auth: + server: + url: http://localhost/realms/myrealm/apikey/check_api_key/ + tokenName: apiKey + clientID: score + clientSecret: scoresecret + provider: keycloak + keycloak: + host: http://localhost + realm: "myrealm" + +spring: + security: + oauth2: + resourceserver: + jwt: + jwk-set-uri: http://localhost/realms/myrealm/protocol/openid-connect/certs +``` + +### Docker + +Run the following compose to spin up dependant services: + +```yaml +version: '2.3' +services: + postgresql: + image: docker.io/bitnami/postgresql:16 + environment: + # ALLOW_EMPTY_PASSWORD is recommended only for development. + - ALLOW_EMPTY_PASSWORD=yes + - POSTGRESQL_USERNAME=bn_keycloak + - POSTGRESQL_DATABASE=bitnami_keycloak + volumes: + - 'postgresql_data:/bitnami/postgresql' + + keycloak: + build: . + depends_on: + - postgresql + ports: + - "80:8080" + # remote debugging port is recommended only for development + # - "8787:8787" + environment: + # remote debugging is recommended only for development + # - DEBUG=true + # - DEBUG_PORT=*:8787 + - KC_DB=postgres + - KC_DB_URL=jdbc:postgresql://postgresql/bitnami_keycloak + - KC_DB_USERNAME=bn_keycloak + volumes: + - type: bind + source: ./target/dasniko.keycloak-2fa-sms-authenticator.jar + target: /opt/bitnami/keycloak/providers/keycloak-sms-auth.jar + - type: bind + source: data_import + target: /opt/bitnami/keycloak/data/import + volumes: + postgresql_data: + driver: local + minio: + image: minio/minio:RELEASE.2018-05-11T00-29-24Z + ports: + - 9000:9000 + environment: + MINIO_ACCESS_KEY: minio + MINIO_SECRET_KEY: minio123 + command: server /data +``` + +Create a `.env.score` file with the necessary environment variables: + +```bash +# ============================ +# Spring Run Profiles (Required) +# ============================ + +# Active profiles to determine app behavior & configs +SPRING_PROFILES_ACTIVE=collaboratory,prod,secure + +# Server configuration +SERVER_PORT=8087 +SERVER_SSL_ENABLED=false + +# Logging +LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_WEB=INFO +LOGGING_LEVEL_BIO_OVERTURE_SCORE_SERVER=INFO +LOGGING_LEVEL_ROOT=INFO + +# ============================ +# Server Authentication integration (Required) +# ============================ +AUTH_SERVER_URL=http://host.docker.internal:9082/o/check_api_key/ +AUTH_SERVER_TOKENNAME=apiKey +AUTH_SERVER_CLIENTID=score +AUTH_SERVER_CLIENTSECRET=abc123 +AUTH_SERVER_SCOPE_STUDY_PREFIX=score. +AUTH_SERVER_SCOPE_UPLOAD_SUFFIX=.WRITE +AUTH_SERVER_SCOPE_DOWNLOAD_SUFFIX=.READ +AUTH_SERVER_SCOPE_DOWNLOAD_SYSTEM=score.WRITE +AUTH_SERVER_SCOPE_UPLOAD_SYSTEM=score.READ + + +# ============================ +# Song Integration (Required) +# ============================ +METADATA_URL=http://host.docker.internal:8080 + + + +# ============================ +# Storage Integration (Required) +# ============================ +S3_ENDPOINT=http://localhost:9000/ +S3_ACCESSKEY=minio +S3_SECRETKEY=minio123 +S3_SIGV4ENABLED=true +OBJECT_SENTINEL=heliograph +BUCKET_NAME_OBJECT=oicr.icgc.test +BUCKET_NAME_STATE=oicr.icgc.test +COLLABORATORY_DATA_DIRECTORY=data +UPLOAD_PARTSIZE=1073741824 +UPLOAD_CONNECTION_TIMEOUT=1200000 +``` + +Run the following command: + +```bash +docker run --name score --env-file .env.score -d -p 8087:8087 ghcr.io/overture-stack/score-server:47f006ce +``` diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 04c85770..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -sphinx -sphinx-autobuild -sphinx_rtd_theme diff --git a/docs/start-sphinx-server.sh b/docs/start-sphinx-server.sh deleted file mode 100755 index bfaabab6..00000000 --- a/docs/start-sphinx-server.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -sphinx-autobuild \ --b html \ --H localhost \ ---watch source \ ---ignore "*.swp" \ ---ignore "*.pdf" \ ---ignore "*.log" \ ---ignore "*.out" \ ---ignore "*.toc" \ ---ignore "*.aux" \ ---ignore "*.idx" \ ---ignore "*.ind" \ ---ignore "*.ilg" \ ---ignore "*.tex" \ ---ignore "*.pyc" \ ---ignore "*.py_*" \ ---ignore "*.rst_*" \ ---ignore "*4913*" \ --c source source _build/html \ No newline at end of file