Skip to content

Commit 935dd4d

Browse files
ClaudenwClaudenw
authored
Merge branch 'main' into dependabot/maven/org.apache.kafka-kafka-clients-5.3.0-ccs
2 parents 7770274 + aa2bc75 commit 935dd4d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+2147
-565
lines changed

‎.github/workflows/create_release.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,12 +63,12 @@ jobs:
6363
uses: actions/setup-java@v4
6464
with:
6565
distribution: 'adopt'
66-
java-version: 8
66+
java-version: 17
6767
cache: maven
6868

6969
- name: Build
7070
run: |
71-
mvn install -DskipTests
71+
mvn -ntp install -DskipTests
7272
mvn -f kcbq-connector clean package assembly:single@release-artifacts -DskipTests
7373
7474
export tar_file=$(ls ./kcbq-connector/target/ | grep tar)

‎.github/workflows/manual.yml‎

Lines changed: 35 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,15 @@
2323

2424
# Workflow to check pull requests and new commits to main branches
2525
# This checks the source in the state as if after the merge.
26-
name: Manual workflow
26+
name: Manual build
2727
on:
2828
workflow_dispatch:
29+
workflow_call:
30+
secrets:
31+
GCP_CREDENTIALS:
32+
KCBQ_TEST_PROJECT:
33+
KCBQ_TEST_DATASET:
34+
KCBQ_TEST_BUCKET:
2935

3036
permissions:
3137
contents: write
@@ -36,7 +42,7 @@ permissions:
3642
# Disallow concurrent runs for the same PR by cancelling in-progress runs
3743
# when new commits are pushed
3844
concurrency:
39-
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
45+
group: Manual_Build-${{ github.event.pull_request.number || github.ref }}
4046
cancel-in-progress: true
4147

4248
jobs:
@@ -50,61 +56,32 @@ jobs:
5056
env:
5157
GITHUB_CONTEXT: ${{ toJson(github) }}
5258
run: echo "$GITHUB_CONTEXT"
53-
- name: Test
59+
- name: Set up JDK 17
60+
uses: actions/setup-java@v4
61+
with:
62+
distribution: 'adopt'
63+
java-version: 17
64+
cache: maven
65+
- name: Integration tests (Maven)
5466
env:
55-
TEST_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
56-
GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }}
67+
# Necessary for client builder integration tests that run with
68+
# default application credentials
69+
CREDENTIALS_JSON: ${{ secrets.GCP_CREDENTIALS }}
70+
GOOGLE_APPLICATION_CREDENTIALS: /tmp/creds.json
71+
KCBQ_TEST_KEYFILE: /tmp/creds.json
72+
KCBQ_TEST_KEY_SOURCE: FILE
73+
KCBQ_TEST_PROJECT: ${{ secrets.KCBQ_TEST_PROJECT }}
74+
KCBQ_TEST_DATASET: ${{ secrets.KCBQ_TEST_DATASET }}
75+
KCBQ_TEST_BUCKET: ${{ secrets.KCBQ_TEST_BUCKET }}
5776
run: |
58-
echo ${#TEST_GITHUB_TOKEN}
59-
echo ${#GCP_CREDENTIALS}
60-
# - name: Set up JDK 17
61-
# uses: actions/setup-java@v4
62-
# with:
63-
# distribution: 'adopt'
64-
# java-version: 17
65-
# cache: maven
66-
# - name: License header check
67-
# run: |
68-
# mvn -ntp license:remove license:format
69-
# if [[ -n $(git status -s) ]]; then
70-
# echo 1>&2 'Some files do not have the correct license header:'
71-
# git diff --name-only 1>&2
72-
# echo 1>&2 'Please update the license headers for these files by running `mvn license:remove license:format`'
73-
# exit 1
74-
# fi
75-
# - name: Build (Maven)
76-
# run: mvn -P ci --batch-mode clean package -DskipTests
77-
# - name: Unit tests (Maven)
78-
# run: mvn -ntp -P ci --batch-mode test
79-
# - name: "upload build failure reports"
80-
# uses: actions/upload-artifact@v4
81-
# if: failure()
82-
# with:
83-
# name: unit-test-results
84-
# path: |
85-
# **/target/*-reports/**
86-
# retention-days: 1
87-
# - name: Integration tests (Maven)
88-
# env:
89-
# # Necessary for client builder integration tests that run with
90-
# # default application credentials
91-
# CREDENTIALS_JSON: ${{ secrets.GCP_CREDENTIALS }}
92-
# GOOGLE_APPLICATION_CREDENTIALS: /tmp/creds.json
93-
# KCBQ_TEST_KEYFILE: ${{ secrets.GCP_CREDENTIALS }}
94-
# KCBQ_TEST_KEY_SOURCE: JSON
95-
# KCBQ_TEST_PROJECT: ${{ secrets.KCBQ_TEST_PROJECT }}
96-
# KCBQ_TEST_DATASET: ${{ secrets.KCBQ_TEST_DATASET }}
97-
# KCBQ_TEST_BUCKET: ${{ secrets.KCBQ_TEST_BUCKET }}
98-
# run: |
99-
# export
100-
# echo "$CREDENTIALS_JSON" > /tmp/creds.json
101-
# export KCBQ_TEST_TABLE_SUFFIX=_$(date +%s)_$RANDOM
102-
# mvn -ntp -P ci -Dskip.unit.tests=true verify
103-
# - name: Upload integration test results (Maven)
104-
# if: always()
105-
# uses: actions/upload-artifact@v4
106-
# with:
107-
# path: |
108-
# **/target/failsafe-reports/*
109-
# name: integration-test-results
110-
# retention-days: 3
77+
echo "$CREDENTIALS_JSON" > /tmp/creds.json
78+
export KCBQ_TEST_TABLE_SUFFIX=_$(date +%s)_$RANDOM
79+
mvn -ntp -P ci -Dskip.unit.tests=true verify
80+
- name: Upload integration test results (Maven)
81+
if: always()
82+
uses: actions/upload-artifact@v4
83+
with:
84+
path: |
85+
**/target/failsafe-reports/*
86+
name: integration-test-results
87+
retention-days: 1

‎.github/workflows/nightly.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#
2+
# Copyright 2024 Copyright 2022 Aiven Oy and
3+
# bigquery-connector-for-apache-kafka project contributors
4+
#
5+
# This software contains code derived from the Confluent BigQuery
6+
# Kafka Connector, Copyright Confluent, Inc, which in turn
7+
# contains code derived from the WePay BigQuery Kafka Connector,
8+
# Copyright WePay, Inc.
9+
#
10+
# Licensed under the Apache License, Version 2.0 (the "License");
11+
# you may not use this file except in compliance with the License.
12+
# You may obtain a copy of the License at
13+
#
14+
# http://www.apache.org/licenses/LICENSE-2.0
15+
#
16+
# Unless required by applicable law or agreed to in writing,
17+
# software distributed under the License is distributed on an
18+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
19+
# KIND, either express or implied. See the License for the
20+
# specific language governing permissions and limitations
21+
# under the License.
22+
#
23+
24+
# Workflow to check pull requests and new commits to main branches
25+
# This checks the source in the state as if after the merge.
26+
name: Nightly Build
27+
on:
28+
workflow_dispatch:
29+
schedule: ## run GMT 1:17 hours
30+
- cron: '17 1 * * *'
31+
workflow_call:
32+
secrets:
33+
GCP_CREDENTIALS:
34+
KCBQ_TEST_PROJECT:
35+
KCBQ_TEST_DATASET:
36+
KCBQ_TEST_BUCKET:
37+
38+
permissions:
39+
contents: write
40+
pull-requests: write
41+
issues: write
42+
43+
44+
# Disallow concurrent runs for the same PR by cancelling in-progress runs
45+
# when new commits are pushed
46+
#concurrency:
47+
# group: Manual_Build-${{ github.event.pull_request.number || github.ref }}
48+
# cancel-in-progress: true
49+
50+
jobs:
51+
call-workflow-2-in-local-repo:
52+
uses: ./.github/workflows/manual.yml
53+
secrets:
54+
GCP_CREDENTIALS:
55+
KCBQ_TEST_PROJECT:
56+
KCBQ_TEST_DATASET:
57+
KCBQ_TEST_BUCKET:

‎.github/workflows/prs_and_commits.yml‎

Lines changed: 9 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -51,35 +51,22 @@ jobs:
5151
cache: maven
5252
- name: License header check
5353
run: |
54-
mvn license:remove license:format
54+
mvn -ntp license:remove license:format
5555
if [[ -n $(git status -s) ]]; then
5656
echo 1>&2 'Some files do not have the correct license header:'
5757
git diff --name-only 1>&2
5858
echo 1>&2 'Please update the license headers for these files by running `mvn license:remove license:format`'
5959
exit 1
6060
fi
6161
- name: Build (Maven)
62-
run: mvn -P ci --batch-mode clean package -DskipTests
62+
run: mvn -ntp -P ci --batch-mode clean package -DskipTests
6363
- name: Unit tests (Maven)
64-
run: mvn -P ci --batch-mode test
65-
- name: Integration tests (Maven)
66-
env:
67-
# Necessary for client builder integration tests that run with
68-
# default application credentials
69-
CREDENTIALS_JSON: ${{ secrets.GCP_CREDENTIALS }}
70-
GOOGLE_APPLICATION_CREDENTIALS: /tmp/creds.json
71-
KCBQ_TEST_KEYFILE: /tmp/creds.json
72-
KCBQ_TEST_KEY_SOURCE: FILE
73-
KCBQ_TEST_PROJECT: ${{ secrets.KCBQ_TEST_PROJECT }}
74-
KCBQ_TEST_DATASET: ${{ secrets.KCBQ_TEST_DATASET }}
75-
KCBQ_TEST_BUCKET: ${{ secrets.KCBQ_TEST_BUCKET }}
76-
run: |
77-
echo "$CREDENTIALS_JSON" > /tmp/creds.json
78-
export KCBQ_TEST_TABLE_SUFFIX=_$(date +%s)_$RANDOM
79-
mvn -P ci -Dskip.unit.tests=true verify
80-
- name: Upload integration test results (Maven)
81-
if: always()
64+
run: mvn -ntp -P ci --batch-mode test
65+
- name: "Upload build failure reports"
8266
uses: actions/upload-artifact@v4
67+
if: failure()
8368
with:
84-
path: kcbq-connector/target/failsafe-reports/*
85-
name: integration-test-results
69+
name: unit-test-results
70+
path: |
71+
**/target/*-reports/**
72+
retention-days: 1

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ target/
77
*.zip
88

99
.gradle
10+
**/.checkstyle
1011

1112
# Intellij
1213
.idea

‎README.md‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,26 @@ Releases are available in the GitHub release tab.
5151

5252
## Integration test setup
5353

54-
In order to execute the integration tests a clone of this repository must be made and the following
55-
properties added to the repository secrets
54+
In order to execute the integration specific environment variables must be set.
5655

57-
GCP_CREDENTIALS - this should be the contents of a json file that you download when you create GCP account key.
56+
### Local configuration
57+
58+
GOOGLE_APPLICATION_CREDENTIALS - the path to a json file that was download when the GCP account key was created..
59+
60+
KCBQ_TEST_BUCKET - the name of the bucket to use for testing,
61+
62+
KCBQ_TEST_DATASET - the name of the dataset to use for testing,
63+
64+
KCBQ_TEST_KEYFILE - same as the GOOGLE_APPLICATION_CREDENTIALS
65+
66+
KCBQ_TEST_PROJECT - the name of the project to use.
67+
68+
### Github configuration
69+
70+
GCP_CREDENTIALS - the contents of a json file that was download when the GCP account key was created.
5871

5972
KCBQ_TEST_BUCKET - the bucket to use for the tests
6073

6174
KCBQ_TEST_DATASET - the data set to use for the tests.
6275

63-
KBCQ_TEST_PROJECT - the project to use for the tests.
76+
KCBQ_TEST_PROJECT - the project to use for the tests.

‎docs/sink-connector-config-options.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,13 @@ BigQuery Sink Connector Configuration Options
306306
* Default: false
307307
* Importance: low
308308

309+
``preserveKafkaTopicPartitionOffset``
310+
If True and Kafka v3.6 or higher is in use will use the original topic, partition, and offset values as specified before any message transformation occurs.
311+
312+
* Type: boolean
313+
* Default: false
314+
* Importance: low
315+
309316
Common
310317
^^^^^^
311318

‎kcbq-api/pom.xml‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,15 @@
2323
under the License.
2424
2525
-->
26-
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
26+
<project xmlns="http://maven.apache.org/POM/4.0.0"
27+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
28+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
2729
<modelVersion>4.0.0</modelVersion>
2830

2931
<parent>
3032
<groupId>com.wepay.kcbq</groupId>
3133
<artifactId>kcbq-parent</artifactId>
32-
<version>2.6.1-SNAPSHOT</version>
34+
<version>2.9.0-SNAPSHOT</version>
3335
<relativePath>..</relativePath>
3436
</parent>
3537

@@ -45,7 +47,6 @@
4547
<groupId>org.apache.kafka</groupId>
4648
<artifactId>connect-api</artifactId>
4749
</dependency>
48-
4950
</dependencies>
5051

5152
<build>
@@ -58,6 +59,10 @@
5859
<groupId>org.apache.maven.plugins</groupId>
5960
<artifactId>maven-checkstyle-plugin</artifactId>
6061
</plugin>
62+
<plugin>
63+
<groupId>org.apache.maven.plugins</groupId>
64+
<artifactId>maven-jar-plugin</artifactId>
65+
</plugin>
6166
</plugins>
6267
</build>
63-
</project>
68+
</project>

‎kcbq-connector/pom.xml‎

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,20 +23,24 @@
2323
under the License.
2424
2525
-->
26-
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
27-
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
26+
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
27+
xmlns="http://maven.apache.org/POM/4.0.0"
28+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
2829
<modelVersion>4.0.0</modelVersion>
2930

3031
<parent>
3132
<groupId>com.wepay.kcbq</groupId>
3233
<artifactId>kcbq-parent</artifactId>
33-
<version>2.6.1-SNAPSHOT</version>
34+
<version>2.9.0-SNAPSHOT</version>
3435
<relativePath>..</relativePath>
3536
</parent>
36-
37+
<url>https://github.com/Aiven-Open/bigquery-connector-for-apache-kafka/</url>
3738
<artifactId>kcbq-connector</artifactId>
3839
<name>kafka-connect-bigquery</name>
39-
40+
<organization>
41+
<name>Aiven</name>
42+
<url>https://www.aiven.io</url>
43+
</organization>
4044
<properties>
4145
<main.dir>${project.parent.basedir}</main.dir>
4246
</properties>
@@ -179,6 +183,10 @@
179183
<groupId>org.apache.maven.plugins</groupId>
180184
<artifactId>maven-checkstyle-plugin</artifactId>
181185
</plugin>
186+
<plugin>
187+
<groupId>org.apache.maven.plugins</groupId>
188+
<artifactId>maven-jar-plugin</artifactId>
189+
</plugin>
182190
<plugin>
183191
<groupId>org.codehaus.mojo</groupId>
184192
<artifactId>exec-maven-plugin</artifactId>
@@ -216,6 +224,23 @@
216224
<tarLongFileMode>posix</tarLongFileMode>
217225
</configuration>
218226
</plugin>
227+
<plugin>
228+
<groupId>org.apache.maven.plugins</groupId>
229+
<artifactId>maven-jar-plugin</artifactId>
230+
<version>3.4.2</version>
231+
<configuration>
232+
<archive>
233+
<manifest>
234+
<mainClass>io.aiven.kafka.utils.VersionInfo</mainClass>
235+
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
236+
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
237+
</manifest>
238+
<manifestEntries>
239+
<url>${project.url}</url>
240+
</manifestEntries>
241+
</archive>
242+
</configuration>
243+
</plugin>
219244
</plugins>
220245
</build>
221-
</project>
246+
</project>

0 commit comments

Comments
 (0)