Skip to content

Commit 2a6d7cf

Browse files
leorabaleoraba
authored
Merge pull request #286 from overture-stack/indexing_repositories
Indexing repositories
2 parents e1ec274 + 999dd5e commit 2a6d7cf

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

86 files changed

+3939
-1211
lines changed

‎Dockerfile

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# Global variables
2+
ARG COMMIT=""
3+
ARG APP_USER=maestro
4+
ARG WORKDIR=/usr/src/app
5+
6+
######################
7+
# Configure base image
8+
######################
9+
FROM node:22-alpine AS base
10+
11+
ARG APP_USER
12+
ARG WORKDIR
13+
14+
# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
15+
RUN apk add --no-cache libc6-compat
16+
17+
# install pnpm as root user, before updating node ownership
18+
RUN npm i -g pnpm
19+
20+
# create our own user to run node, don't run node in production as root
21+
ENV APP_UID=9999
22+
ENV APP_GID=9999
23+
RUN addgroup -S -g $APP_GID $APP_USER \
24+
&& adduser -S -u $APP_UID -g $APP_GID $APP_USER \
25+
&& mkdir -p ${WORKDIR}
26+
27+
WORKDIR ${WORKDIR}
28+
29+
RUN chown -R ${APP_USER}:${APP_USER} ${WORKDIR}
30+
31+
USER ${APP_USER}:${APP_USER}
32+
33+
######################
34+
# Configure build image
35+
######################
36+
37+
FROM base as build
38+
39+
ARG APP_USER
40+
ARG WORKDIR
41+
42+
COPY --chown=maestro:maestro . ./
43+
44+
RUN pnpm install --ignore-scripts
45+
46+
RUN pnpm build:all
47+
48+
49+
######################
50+
# Configure prod-deps image
51+
######################
52+
53+
FROM build AS prod-deps
54+
55+
ARG APP_USER
56+
ARG WORKDIR
57+
58+
WORKDIR ${WORKDIR}
59+
60+
USER ${APP_USER}:${APP_USER}
61+
62+
# pnpm will not install any package listed in devDependencies
63+
RUN pnpm install --prod
64+
65+
66+
######################
67+
# Configure server image
68+
######################
69+
FROM base AS server
70+
71+
ARG APP_USER
72+
ARG WORKDIR
73+
74+
USER ${APP_USER}
75+
76+
WORKDIR ${WORKDIR}
77+
78+
COPY --from=prod-deps ${WORKDIR} .
79+
COPY --from=build ${WORKDIR}/apps/server/dist apps/server/dist
80+
81+
EXPOSE 11235
82+
83+
ENV COMMIT_SHA=${COMMIT}
84+
ENV NODE_ENV=production
85+
86+
CMD [ "pnpm", "start:prod" ]

‎Jenkinsfile

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
1-
// @Library(value='jenkins-pipeline-library@master', changelog=false) _
2-
// pipelineOVERTUREMaestro()
3-
1+
@Library(value='jenkins-pipeline-library@maestro-v5', changelog=false) _
2+
pipelineOVERTUREMaestro()

‎Makefile

+11-32
Original file line numberDiff line numberDiff line change
@@ -1,68 +1,47 @@
11
.ONESHELL:
2-
DOCKER_COMPOSE_DEV_FILE = ./apps/server/docker-compose.dev.yml
3-
DOCKER_COMPOSE_FILE = ./apps/server/docker-compose.yml
4-
VERSION=$(shell cat ./.mvn/maven.config | grep revision | cut -d '=' -f2)-SNAPSHOT
2+
DOCKER_COMPOSE_DEV_ES7_FILE = ./apps/server/docker-compose-es7.dev.yml
53
###################
6-
## MAVEN
4+
## PNPM
75
###################
86
compile:
9-
./mvnw clean compile
10-
build:
11-
./mvnw clean install -DskipTests
7+
pnpm i && pnpm run build:all
128
test:
13-
./mvnw clean test
14-
package:
15-
./mvnw clean package -Dmaven.test.skip=true
9+
pnpm run test:all
1610
start:
17-
cd maestro-app
18-
../mvnw spring-boot:run
11+
cd apps/server && pwd && pnpm run start:dev
1912

2013
###################
2114
## DOCKER
2215
###################
23-
docker-push:
24-
docker push overture/maestro:$(VERSION)
25-
26-
docker-build:
27-
docker build -f ci-cd/Dockerfile . -t overture/maestro:$(VERSION)
28-
29-
# use this when you want to run maestro as container
30-
docker-start:
31-
docker-compose -f ${DOCKER_COMPOSE_FILE} up -d
32-
33-
docker-stop:
34-
docker-compose -f ${DOCKER_COMPOSE_FILE} down
35-
3616
# only starts the infrastructure containers needed by maestro
37-
# use this when you run maestro from the ide (not as a container)
3817
docker-start-dev:
39-
docker-compose -f ${DOCKER_COMPOSE_DEV_FILE} up -d
18+
docker-compose -f ${DOCKER_COMPOSE_DEV_ES7_FILE} up -d
4019

4120
docker-stop-dev:
42-
docker-compose -f ${DOCKER_COMPOSE_DEV_FILE} down
21+
docker-compose -f ${DOCKER_COMPOSE_DEV_ES7_FILE} down
4322

4423
###################
4524
## REST API
4625
###################
4726
rest-health:
48-
curl -X GET http://localhost:11235/
27+
curl -X GET http://localhost:11235/health
4928

5029
rest-index-study:
5130
curl -X POST \
52-
http://localhost:11235/index/repository/collab/study/PACA-CA \
31+
http://localhost:11235/index/repository/lyric1/organization/PACA-CA \
5332
-H 'Content-Type: application/json' \
5433
-H 'cache-control: no-cache' \
5534
-d '{}'
5635

5736
rest-index-analysis:
5837
curl -X POST \
59-
http://localhost:11235/index/repository/collab/study/PACA-CA/analysis/ad7cabf8-df45-40f8-9fcb-67d8933f46e6 \
38+
http://localhost:11235/index/repository/lyric1/organization/PACA-CA/id/ad7cabf8-df45-40f8-9fcb-67d8933f46e6 \
6039
-H 'Content-Type: application/json' \
6140
-H 'cache-control: no-cache'
6241

6342
rest-index-repo:
6443
curl -X POST \
65-
http://localhost:11235/index/repository/collab \
44+
http://localhost:11235/index/repository/lyric1 \
6645
-H 'Content-Type: application/json' \
6746
-H 'cache-control: no-cache'
6847

‎README.md

+76-86
Original file line numberDiff line numberDiff line change
@@ -1,116 +1,106 @@
1-
<h1 align="center">Maestro</h1>
2-
<p align="center">Organize geographically distributed data stored in Song and Score, with a single, configurable index.</p>
3-
<p align="center">
4-
<a href="https://github.com/overture-stack/maestro">
5-
<img alt="Under Development"
6-
title="Under Development"
7-
src="http://www.overture.bio/img/progress-horizontal-RC.svg" width="320" />
8-
</a>
9-
</p>
10-
11-
[![Documentation Status](https://readthedocs.org/projects/maestro-overture/badge/?version=latest)](https://maestro-overture.readthedocs.io/en/latest/?badge=latest)
12-
[![Slack](http://slack.overture.bio/badge.svg)](http://slack.overture.bio)
1+
![TypeScript](https://img.shields.io/badge/TypeScript-007ACC?style=for-the-badge&logo=typescript&logoColor=white)
2+
[<img hspace="5" src="https://img.shields.io/badge/chat--with--developers-overture--slack-blue?style=for-the-badge">](http://slack.overture.bio)
133

144
## Documentation:
15-
Documentation is hosted on :
5+
6+
Documentation is hosted on:
7+
168
- github pages: https://overture-stack.github.io/maestro/
17-
- read the docs: https://maestro-overture.readthedocs.io/en/latest/ (using mkdocs https://docs.readthedocs.io/en/stable/intro/getting-started-with-mkdocs.html)
9+
- Overture docs: https://docs.overture.bio/docs/core-software/maestro/overview/
1810

1911
## Introduction
20-
Meastro was created to enable genomic researchers to enhance their Overture SONGs by building indexes, elastic search
21-
by default, that makes searching Analyses and Studies much more powerful and easier.
2212

23-
## TLDR;
24-
Skip down to the How to section it has the steps to get started.
13+
Maestro was created to enable genomic researchers to enhance their Overture SONG/Lyric by building indexes using Elasticsearch (by default), which makes searching Analyses and Studies much easier.
2514

2615
### Features:
27-
- Supports indexing from multiple metadata repositories (SONG).
16+
17+
- Supports indexing from multiple metadata repositories (SONG/Lyric).
2818
- Multiple indexing requests: analysis, study, full repository.
2919
- Event driven indexing.
30-
- Integration with SONG to index published analysis and delete suppressed / unpublished analyses
20+
- Integration with SONG/Lyric to index published analysis and delete suppressed / unpublished analyses
3121
- Ability to Exclude analysis based on different Ids: Study, Analysis, Donor, Sample Or file.
3222
- Slack web hook integration
3323

34-
### Design Goals:
35-
- Reactive
36-
- Event driven
37-
- Elastic
38-
- Resiliency & Fault tolerance
39-
- Failure audit
40-
- Dead letters queue for faulty messages to be retried later and reviewed.
41-
- Human readable Error log
42-
- Runtime configurability
43-
- Extendable: separate domain from infrastructure & configuration
44-
4524
## Technologies & libraries:
46-
- Java 11 - OpenJDK 11
47-
- Maven 3 (YOU NEED MAVEN 3.5+, if you don't want to use the wrapper or your IDE points to older version)
48-
- lombok
49-
- Spring boot 2
50-
- Spring webflux & project reactor
51-
- Spring retry
52-
- Spring Cloud
53-
- Streams (for Kafak integration)
54-
- config client
55-
- Spring boot admin + client
25+
26+
- Node.js v22 or greater
27+
- PNPM package manager
5628
- Elasticsearch 7+
5729
- Apache Kafka
58-
- resilience4j:
59-
- retry module
60-
- vavr
6130
- Testing libraries:
62-
- Junit 5
63-
- Mockito 2
64-
- testcontainers
65-
- Spring cloud contract wiremock
31+
- Mocha
32+
- Chai
33+
- Testcontainers
6634

6735
## Structure
68-
The project is following the ports/adapters architecture, where the domain is completely isolated from external infrastructure
69-
and frameworks.
70-
- Two maven modules:
71-
- maestro domain
72-
- the core features and framework independent logic that is portable and contains the main indexing, rules, notifications
73-
logic as specified by the business features. Has packages like:
74-
- entities : contains POJOs and entities
75-
- api: the logic that fulfills the business features
76-
- ports: contains the interfaces needed by the api to communicate with anything outside the indexing context.
77-
78-
- maestro app:
79-
- The main runnable (spring boot app)
80-
- Contains the infrastructure and adapters (ports implementations) that is needed to connect the domain
81-
with the outside world like elastic search, song web clients, configuration files etc.
82-
It also has the Spring framework configurations here to keep technologies outside of the domain.
83-
36+
37+
The project is organized as a monorepo workspace, where each application and package is located in its own dedicated folder:
38+
39+
```
40+
apps/
41+
├─ server/
42+
packages/
43+
├─ common/
44+
├─ indexer-client/
45+
├─ maestro-provider/
46+
├─ repository/
47+
48+
```
49+
50+
- **Maestro Server:** The main runnable Express Server. Exposes a set of HTTP API routes that serve as the interface between the Maestro provider and external systems.
51+
52+
- **Maestro Common:** Designed to centralize common utilities, reusable functions, and TypeScript type definitions.
53+
54+
- **Maestro Indexer Client:** Abstracts communication with Elasticsearch clients, supporting both version 7 and 8.
55+
56+
- **Maestro Provider:** The core features and provider independent logic that is portable and contains the main indexing, rules, notifications logic as specified by the business features.
57+
58+
- **Maestro Repository:** Designed to manage interactions with data source repositories. It serves as the central interface for retrieving data from various repositories, such as **SONG** and **Lyric**, ensuring a streamlined and consistent approach to data access.
59+
8460
# Dependencies:
61+
8562
To Successfully run Maestro (as is) you need the following services to be deployed and configure it to use them:
63+
8664
- [Elasticsearch](https://www.elastic.co/products/elasticsearch)
8765
- [Apache Kafka](https://kafka.apache.org/)
8866
- [SONG](https://github.com/overture-stack/SONG)
67+
- [Lyric](https://github.com/overture-stack/lyric)
8968

90-
you can check the sample docker compose files under `./run/docker-compose` for containerized versions of elastic & kafka.
91-
for SONG please check the SONG github repo [here](https://github.com/overture-stack/SONG/tree/develop/dev)
92-
on how to run it with docker. Or you can run it as jar.
69+
You can check the sample Docker compose files under `./apps/server/docker-compose-es7.dev.yml` for containerized versions of Elasticsearch 7 & Kafka.
70+
For SONG please check the SONG github repo [here](https://github.com/overture-stack/SONG/tree/develop/dev) on how to run it with docker. Or you can run it as jar.
9371

9472
## How to:
95-
- Swagger API access:
96-
- localhost:11235/api-docs
9773

98-
Note: if you don't/can't use the Makefile, look inside it for the shell commands and replicate them.
99-
- Compile: `make`
74+
- Swagger API access:
75+
- http://localhost:11235/api-docs
76+
77+
> Note: can't use `Make`? replicate the shell commands found within the Makefile.
78+
79+
- Compile: `make`
10080
- Test: `make test`
101-
- Package: `make package`
10281
- Run:
103-
- Source:
104-
- Development:
105-
1. `make docker-start-dev` starts the infrastructure containers
106-
- kafka
107-
- elastic search
108-
- other helper tools if you want like kafka rest proxy
109-
2. `make run` OR start maestro (Maestro.java) from the IDE/cmd as a java application
110-
- Docker:
111-
- Repository: https://hub.docker.com/r/overture/maestro
112-
- `make docker-start` starts maestro from a docker image along with all needed infrastructure
113-
- helm:
114-
- Repository: https://overture-stack.github.io/charts-server/
115-
- see : https://github.com/overture-stack/helm-charts for instructions and the maestro folder for examples
116-
82+
- Source:
83+
- Development:
84+
1. `make docker-start-dev` starts the infrastructure containers
85+
- Kafka
86+
- Elasticsearch
87+
- other helper tools you may want, like a Kafka RESTful proxy
88+
2. `make start` to start application
89+
90+
## Running scripts
91+
92+
This project contains the following scripts for managing the build, testing, and development processes. You can run any of these scripts using the following command: `pnpm run <script-name>`
93+
94+
| Script Name | Description |
95+
| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
96+
| `build:all` | Builds the entire project for all environments (development, production, etc.) |
97+
| `lint` | Runs ESLint to lint all files in the current directory (and subdirectories). It checks for code quality issues, potential errors, and style violations according to the project's ESLint configuration |
98+
| `lint:fix` | Runs ESLint to lint all files in the current directory (and subdirectories), and automatically fixes issues that can be resolved (e.g., formatting, missing semicolons). |
99+
| `test:all` | Runs all tests (unit and integration tests). |
100+
| `test:all:coverage` | Runs all tests (unit and integration tests) and generates a coverage report. |
101+
| `test:unit` | Runs unit tests only. |
102+
| `test:unit:coverage` | Runs unit tests only and generates a coverage report. |
103+
| `test:integration` | Runs integration tests only. |
104+
| `test:integration:coverage` | Runs integration tests only and generates a coverage report. |
105+
| `start:dev` | Starts the development server with live-reloading and debugging enabled. |
106+
| `start:prod` | Starts the production server optimized for performance and stability (The application must be built beforehand). |

0 commit comments

Comments
 (0)