Skip to content

Commit e55948d

Browse files
authored
Added S3 sink configuration documentation. (#555)
Closes #515 * Added Scale class to present file sizes and memory sizes in human readable form. * fixed validators to show correct information. * Updated the S3SinkConfigDef so that it contains all the S3SinkConfiguration definition informtion. * Removed S3Sink configuration information out of S3SinkConfig
1 parent d0a913f commit e55948d

File tree

24 files changed

+405
-597
lines changed

24 files changed

+405
-597
lines changed

commons/src/main/java/io/aiven/commons/collections/TimeScale.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,11 @@ public enum TimeScale {
3333
public String format(final long milliseconds) {
3434
return String.format("%s %s", milliseconds, unitName());
3535
}
36-
},
37-
SECONDS(MILLISECONDS.milliseconds * 1000), MINUTES(SECONDS.milliseconds * 60), HOURS(
38-
MINUTES.milliseconds * 60), DAYS(HOURS.milliseconds * 24);
36+
}, //
37+
SECONDS(MILLISECONDS.milliseconds * 1000), //
38+
MINUTES(SECONDS.milliseconds * 60), //
39+
HOURS(MINUTES.milliseconds * 60), //
40+
DAYS(HOURS.milliseconds * 24);
3941

4042
/**
4143
* The Decimal format for the TimeUnit displays.

commons/src/main/java/io/aiven/kafka/connect/common/config/CommonConfig.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,13 +175,14 @@ final Collection<ConfigValue> multiValidateValues(final Map<String, String> prop
175175
}
176176

177177
@Override
178+
@SuppressWarnings("PMD.AvoidCatchingGenericException")
178179
public final List<ConfigValue> validate(final Map<String, String> props) {
179180
final Map<String, ConfigValue> valueMap = validateAll(props);
180181

181182
try {
182183
return new ArrayList<>(multiValidate(valueMap).values());
183-
} catch (RuntimeException e) { // NOPMD AvoidCatchingGenericException
184-
// any exceptions thrown in the above block are accounted for in the super.validate(props) call.
184+
} catch (RuntimeException e) {
185+
// any exceptions thrown in the above block are accounted for in the validateAll(props) call.
185186
return new ArrayList<>(valueMap.values());
186187
}
187188
}

commons/src/main/java/io/aiven/kafka/connect/common/config/FileNameFragment.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,9 @@ public final class FileNameFragment extends ConfigFragment {
5858
@VisibleForTesting
5959
static final String FILE_MAX_RECORDS = "file.max.records";
6060
@VisibleForTesting
61-
static final String FILE_NAME_TIMESTAMP_TIMEZONE = "file.name.timestamp.timezone";
61+
public static final String FILE_NAME_TIMESTAMP_TIMEZONE = "file.name.timestamp.timezone";
6262
@VisibleForTesting
63-
static final String FILE_NAME_TIMESTAMP_SOURCE = "file.name.timestamp.source";
63+
public static final String FILE_NAME_TIMESTAMP_SOURCE = "file.name.timestamp.source";
6464
@VisibleForTesting
6565
public static final String FILE_NAME_TEMPLATE_CONFIG = "file.name.template";
6666
@VisibleForTesting

commons/src/main/java/io/aiven/kafka/connect/common/config/OutputFormatFragment.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
*/
3333
public final class OutputFormatFragment extends ConfigFragment {
3434
@VisibleForTesting
35-
static final String GROUP_FORMAT = "Format";
35+
public static final String GROUP_NAME = "Format";
3636
@VisibleForTesting
3737
static public final String FORMAT_OUTPUT_FIELDS_CONFIG = "format.output.fields";
3838
@VisibleForTesting
@@ -90,25 +90,25 @@ public static int update(final ConfigDef configDef, final OutputFieldType defaul
9090
int formatGroupCounter = 0;
9191

9292
configDef.define(FORMAT_OUTPUT_TYPE_CONFIG, ConfigDef.Type.STRING, FormatType.CSV.name, OUTPUT_TYPE_VALIDATOR,
93-
ConfigDef.Importance.MEDIUM, "The format type of output content.", GROUP_FORMAT, ++formatGroupCounter,
93+
ConfigDef.Importance.MEDIUM, "The format type of output content.", GROUP_NAME, ++formatGroupCounter,
9494
ConfigDef.Width.NONE, FORMAT_OUTPUT_TYPE_CONFIG,
9595
FixedSetRecommender.ofSupportedValues(FormatType.names()));
9696

9797
configDef.define(FORMAT_OUTPUT_FIELDS_CONFIG, ConfigDef.Type.LIST,
9898
Objects.isNull(defaultFieldType) ? null : defaultFieldType.name, // NOPMD NullAssignment
99-
OUTPUT_FIELDS_VALIDATOR, ConfigDef.Importance.MEDIUM, "Fields to put into output files.", GROUP_FORMAT,
99+
OUTPUT_FIELDS_VALIDATOR, ConfigDef.Importance.MEDIUM, "Fields to put into output files.", GROUP_NAME,
100100
++formatGroupCounter, ConfigDef.Width.NONE, FORMAT_OUTPUT_FIELDS_CONFIG,
101101
FixedSetRecommender.ofSupportedValues(OutputFieldType.names()));
102102

103103
configDef.define(FORMAT_OUTPUT_FIELDS_VALUE_ENCODING_CONFIG, ConfigDef.Type.STRING,
104104
OutputFieldEncodingType.BASE64.name, OUTPUT_FIELDS_ENCODING_VALIDATOR, ConfigDef.Importance.MEDIUM,
105105
"The type of encoding for the value field. " + "The supported values are: "
106106
+ OutputFieldEncodingType.SUPPORTED_FIELD_ENCODING_TYPES + ".",
107-
GROUP_FORMAT, ++formatGroupCounter, ConfigDef.Width.NONE, FORMAT_OUTPUT_FIELDS_VALUE_ENCODING_CONFIG,
107+
GROUP_NAME, ++formatGroupCounter, ConfigDef.Width.NONE, FORMAT_OUTPUT_FIELDS_VALUE_ENCODING_CONFIG,
108108
FixedSetRecommender.ofSupportedValues(OutputFieldEncodingType.names()));
109109

110110
configDef.define(FORMAT_OUTPUT_ENVELOPE_CONFIG, ConfigDef.Type.BOOLEAN, true, ConfigDef.Importance.MEDIUM,
111-
"Whether to enable envelope for entries with single field.", GROUP_FORMAT, ++formatGroupCounter,
111+
"Whether to enable envelope for entries with single field.", GROUP_NAME, ++formatGroupCounter,
112112
ConfigDef.Width.SHORT, FORMAT_OUTPUT_ENVELOPE_CONFIG);
113113
return formatGroupCounter;
114114
}

commons/src/main/java/io/aiven/kafka/connect/common/config/validators/OutputFieldsValidator.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,4 +44,9 @@ public void ensureValid(final String name, final Object value) {
4444
}
4545
}
4646

47+
@Override
48+
public String toString() {
49+
return OutputField.SUPPORTED_OUTPUT_FIELDS;
50+
}
51+
4752
}

commons/src/testFixtures/java/io/aiven/kafka/connect/common/config/OutputFormatFragmentFixture.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
public class OutputFormatFragmentFixture {// NOPMD
2020

2121
public enum OutputFormatArgs {
22-
GROUP_FORMAT(OutputFormatFragment.GROUP_FORMAT), FORMAT_OUTPUT_FIELDS_CONFIG(
22+
GROUP_FORMAT(OutputFormatFragment.GROUP_NAME), FORMAT_OUTPUT_FIELDS_CONFIG(
2323
OutputFormatFragment.FORMAT_OUTPUT_FIELDS_CONFIG), FORMAT_OUTPUT_FIELDS_VALUE_ENCODING_CONFIG(
2424
OutputFormatFragment.FORMAT_OUTPUT_FIELDS_VALUE_ENCODING_CONFIG), FORMAT_OUTPUT_ENVELOPE_CONFIG(
2525
OutputFormatFragment.FORMAT_OUTPUT_ENVELOPE_CONFIG), FORMAT_OUTPUT_TYPE_CONFIG(

s3-commons/src/main/java/io/aiven/kafka/connect/config/s3/S3CommonConfig.java

Lines changed: 0 additions & 70 deletions
This file was deleted.

0 commit comments

Comments
 (0)