diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e4956d38..5288f03b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,22 +1,85 @@ -name: Build Maestro +name: Build and Publish Maestro + on: push: branches: - '*' tags: - - v*.*.* - - v*.*.*-rc.* + - 'v*.*.*' + - 'v*.*.*-rc.*' pull_request: + workflow_dispatch: jobs: build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + + - name: Setup JDK + uses: actions/setup-java@v4 + with: + distribution: 'zulu' + java-version: '21' + + - name: Setup Gradle + uses: gradle/gradle-build-action@v3 + + - name: Build and Test + run: ./gradlew build -x maestro-server:build -x test + + - name: Upload Build Artifacts + if: success() + uses: actions/upload-artifact@v4 + with: + name: build-artifacts + path: | + **/build/libs/*.jar + + publish-snapshot: + needs: build + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup JDK uses: actions/setup-java@v4 with: distribution: 'zulu' java-version: '21' - - name: Gradle build - run: ./gradlew build + + - name: Setup Gradle + uses: gradle/gradle-build-action@v3 + + - name: Publish Snapshot + run: ./gradlew snapshot -x maestro-server:build -x test + env: + NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.OSSRH_USERNAME }} + NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} + NETFLIX_OSS_SIGNING_KEY: ${{ secrets.SIGNING_KEY }} + NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} + + publish-release: + needs: build + if: startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup JDK + uses: actions/setup-java@v4 + with: + distribution: 'zulu' + java-version: '21' + + - name: Setup Gradle + uses: gradle/gradle-build-action@v3 + + - name: Publish Release + run: ./gradlew -Prelease.useLastTag=true final -x maestro-server:build -x test + env: + NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.OSSRH_USERNAME }} + NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} + NETFLIX_OSS_SIGNING_KEY: ${{ secrets.SIGNING_KEY }} + NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} diff --git a/PUBLISHING.md b/PUBLISHING.md new file mode 100644 index 00000000..0f2c21fd --- /dev/null +++ b/PUBLISHING.md @@ -0,0 +1,111 @@ +# Publishing Maestro to Maven Central + +## Publishing Snapshots + +Snapshots are automatically published when code is pushed to the `main` branch. + +Nebula automatically determines version from git commits (e.g., `1.0.0-dev.5+a1b2c3d`). + +## Publishing Releases + +### Via Git Tag (Recommended) + +1. Create and push a git tag: + ```bash + git tag -a v1.0.0 -m "Release 1.0.0" + git push origin v1.0.0 + ``` +2. GitHub Actions will automatically: + - Detect the tag + - Run `./gradlew final` to create release version + - Build, sign, and publish to Sonatype staging +3. Manual step: Log in to https://s01.oss.sonatype.org/ and release + +**Note:** Nebula automatically determines the version from the git tag. No need to manually update version in gradle.properties. + +### Post-Release Steps + +1. Log in to https://s01.oss.sonatype.org/ +2. Find the staging repository (com.netflix.maestro-XXXX) +3. Click "Close" to validate artifacts +4. Click "Release" to publish to Maven Central +5. Artifacts available in ~15-30 minutes + +## Using Published Artifacts + +### In Netflix Internal Maestro (Gradle) +```gradle +dependencies { + implementation 'com.netflix.maestro:maestro-flow:1.0.0' + implementation 'com.netflix.maestro:maestro-engine:1.0.0' +} +``` + +### Maven +```xml + + com.netflix.maestro + maestro-flow + 1.0.0 + +``` + +## Published Modules + +All 12 library modules are published (maestro-server is excluded as it's an application): +- netflix-sel +- maestro-common +- maestro-dsl +- maestro-engine +- maestro-database +- maestro-flow +- maestro-queue +- maestro-timetrigger +- maestro-signal +- maestro-aws +- maestro-kubernetes +- maestro-http + +## Prerequisites + +Before you can publish releases, you need to: + +1. **Create Sonatype OSSRH Account:** + - Register at https://issues.sonatype.org/ + - Create JIRA ticket to claim `com.netflix.maestro` group ID + - Wait for approval (1-2 business days) + +2. **Generate GPG Key:** + ```bash + # Generate key pair + gpg --gen-key + + # Export public key to key servers + gpg --keyserver keyserver.ubuntu.com --send-keys + gpg --keyserver keys.openpgp.org --send-keys + + # Export secret key for CI/CD (base64 encoded) + gpg --export-secret-keys | base64 + ``` + +3. **Configure GitHub Secrets:** + Add these secrets to the repository: + - `OSSRH_USERNAME` - Sonatype JIRA username + - `OSSRH_PASSWORD` - Sonatype JIRA password/token + - `SIGNING_KEY` - Base64 encoded GPG secret key + - `SIGNING_PASSWORD` - GPG key passphrase + +## Local Testing + +```bash +# Build all modules +./gradlew build + +# Test local publishing (without credentials) +./gradlew publishToMavenLocal + +# Verify artifacts in ~/.m2/repository/com/netflix/maestro/ + +# Check what version Nebula will generate +./gradlew properties | grep version +``` diff --git a/README.md b/README.md index 018882b6..7d9aa388 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,29 @@ You can read more details about it in our series of blog posts - [100X Faster: How We Supercharged Netflix Maestro's Workflow Engine](https://netflixtechblog.com/100x-faster-how-we-supercharged-netflix-maestros-workflow-engine-028e9637f041) - [Incremental Processing using Netflix Maestro and Apache Iceberg](https://netflixtechblog.com/incremental-processing-using-netflix-maestro-and-apache-iceberg-b8ba072ddeeb) +# Using Maestro as a Library + +Maestro modules are published to Maven Central and can be used as dependencies in your projects. + +## Gradle +```gradle +dependencies { + implementation 'com.netflix.maestro:maestro-flow:1.0.0' + implementation 'com.netflix.maestro:maestro-engine:1.0.0' +} +``` + +## Maven +```xml + + com.netflix.maestro + maestro-flow + 1.0.0 + +``` + +See [PUBLISHING.md](PUBLISHING.md) for all available modules and publishing details. + # Get started ## Prerequisite - Git diff --git a/build.gradle b/build.gradle index f0920407..7bfe1542 100644 --- a/build.gradle +++ b/build.gradle @@ -11,6 +11,14 @@ buildscript { } } +plugins { + id "com.netflix.nebula.netflixoss" version "11.6.0" apply false + // The apply false pattern prevents the Spring Boot plugin's buildscript dependencies from polluting + // the root buildscript classpath, avoiding ASM version conflicts with the Nebula plugin. + // Subprojects that need Spring Boot (like maestro-server) inherit this declaration without a version. + id "org.springframework.boot" version "3.+" apply false +} + apply from: "${rootDir}/dependencies.gradle" allprojects { @@ -60,6 +68,16 @@ allprojects { } } +configure(allprojects - project(':maestro-server')) { + apply plugin: 'com.netflix.nebula.netflixoss' + + // Configure javadoc to be lenient with Lombok-annotated code + tasks.withType(Javadoc).configureEach { + options.addStringOption('Xdoclint:none', '-quiet') + failOnError = false + } +} + configure(allprojects - project(':netflix-sel')) { apply plugin: 'checkstyle' apply plugin: 'pmd' diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 00000000..41fbb747 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,2 @@ +netflixossPublishCandidatesToMavenCentral=true + diff --git a/maestro-server/build.gradle b/maestro-server/build.gradle index 0192e23f..bf0e4321 100644 --- a/maestro-server/build.gradle +++ b/maestro-server/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'org.springframework.boot' version '3.+' + id 'org.springframework.boot' // Version inherited from root build.gradle } sourceSets {