diff --git a/apis/fluentbit/v1alpha2/plugins/output/azure_blob_types.go b/apis/fluentbit/v1alpha2/plugins/output/azure_blob_types.go index 89f03a691..89f5740fd 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/azure_blob_types.go +++ b/apis/fluentbit/v1alpha2/plugins/output/azure_blob_types.go @@ -29,6 +29,17 @@ type AzureBlob struct { EmulatorMode string `json:"emulatorMode,omitempty"` // HTTP Service of the endpoint (if using EmulatorMode) Endpoint string `json:"endpoint,omitempty"` + // Optional: Enables GZIP compression in the final blockblob file. This option isn't compatible when blob_type = appendblob. + // +kubebuilder:validation:Enum:=on;off + CompressBlob string `json:"compressBlob,omitempty"` + // Enable buffering into disk before ingesting into Azure Blob. + BufferingEnabled *bool `json:"bufferingEnabled,omitempty"` + // Specifies the size of files to be uploaded in MB. Defaults to 200M. + // +kubebuilder:default:="200M" + UploadFileSize string `json:"uploadFileSize,omitempty"` + // Optional. Specify a timeout for uploads. Fluent Bit will start ingesting buffer files which have been created more than x minutes ago and haven't reached upload_file_size limit yet. Defaults to 30m. + // +kubebuilder:default:="30m" + UploadTimeout string `json:"uploadTimeout,omitempty"` // Enable/Disable TLS Encryption. Azure services require TLS to be enabled. *plugins.TLS `json:"tls,omitempty"` // Include fluentbit networking options for this output-plugin @@ -67,6 +78,10 @@ func (o *AzureBlob) Params(sl plugins.SecretLoader) (*params.KVs, error) { plugins.InsertKVString(kvs, "container_name", o.ContainerName) plugins.InsertKVString(kvs, "blob_type", o.BlobType) plugins.InsertKVString(kvs, "auto_create_container", o.AutoCreateContainer) + plugins.InsertKVString(kvs, "compress_blob", o.CompressBlob) + plugins.InsertKVField(kvs, "buffering_enabled", o.BufferingEnabled) + plugins.InsertKVString(kvs, "upload_file_size", o.UploadFileSize) + plugins.InsertKVString(kvs, "upload_timeout", o.UploadTimeout) plugins.InsertKVString(kvs, "path", o.Path) plugins.InsertKVString(kvs, "emulator_mode", o.EmulatorMode) plugins.InsertKVString(kvs, "endpoint", o.Endpoint) diff --git a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go index b54f967c5..ef5f9e382 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go @@ -32,6 +32,11 @@ func (in *AzureBlob) DeepCopyInto(out *AzureBlob) { *out = new(plugins.Secret) (*in).DeepCopyInto(*out) } + if in.BufferingEnabled != nil { + in, out := &in.BufferingEnabled, &out.BufferingEnabled + *out = new(bool) + **out = **in + } if in.TLS != nil { in, out := &in.TLS, &out.TLS *out = new(plugins.TLS) diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml index eaef69623..37b8b05d9 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml @@ -65,6 +65,18 @@ spec: - appendblob - blockblob type: string + bufferingEnabled: + description: Enable buffering into disk before ingesting into + Azure Blob. + type: boolean + compressBlob: + description: 'Optional: Enables GZIP compression in the final + blockblob file. This option isn''t compatible when blob_type + = appendblob.' + enum: + - "on" + - "off" + type: string containerName: description: Name of the container that will contain the blobs type: string @@ -240,6 +252,18 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uploadFileSize: + default: 200M + description: Specifies the size of files to be uploaded in MB. + Defaults to 200M. + type: string + uploadTimeout: + default: 30m + description: Optional. Specify a timeout for uploads. Fluent Bit + will start ingesting buffer files which have been created more + than x minutes ago and haven't reached upload_file_size limit + yet. Defaults to 30m. + type: string required: - accountName - containerName diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml index 6e1222110..6fcdfabaa 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml @@ -65,6 +65,18 @@ spec: - appendblob - blockblob type: string + bufferingEnabled: + description: Enable buffering into disk before ingesting into + Azure Blob. + type: boolean + compressBlob: + description: 'Optional: Enables GZIP compression in the final + blockblob file. This option isn''t compatible when blob_type + = appendblob.' + enum: + - "on" + - "off" + type: string containerName: description: Name of the container that will contain the blobs type: string @@ -240,6 +252,18 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uploadFileSize: + default: 200M + description: Specifies the size of files to be uploaded in MB. + Defaults to 200M. + type: string + uploadTimeout: + default: 30m + description: Optional. Specify a timeout for uploads. Fluent Bit + will start ingesting buffer files which have been created more + than x minutes ago and haven't reached upload_file_size limit + yet. Defaults to 30m. + type: string required: - accountName - containerName diff --git a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml index eaef69623..37b8b05d9 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml @@ -65,6 +65,18 @@ spec: - appendblob - blockblob type: string + bufferingEnabled: + description: Enable buffering into disk before ingesting into + Azure Blob. + type: boolean + compressBlob: + description: 'Optional: Enables GZIP compression in the final + blockblob file. This option isn''t compatible when blob_type + = appendblob.' + enum: + - "on" + - "off" + type: string containerName: description: Name of the container that will contain the blobs type: string @@ -240,6 +252,18 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uploadFileSize: + default: 200M + description: Specifies the size of files to be uploaded in MB. + Defaults to 200M. + type: string + uploadTimeout: + default: 30m + description: Optional. Specify a timeout for uploads. Fluent Bit + will start ingesting buffer files which have been created more + than x minutes ago and haven't reached upload_file_size limit + yet. Defaults to 30m. + type: string required: - accountName - containerName diff --git a/config/crd/bases/fluentbit.fluent.io_outputs.yaml b/config/crd/bases/fluentbit.fluent.io_outputs.yaml index 6e1222110..6fcdfabaa 100644 --- a/config/crd/bases/fluentbit.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_outputs.yaml @@ -65,6 +65,18 @@ spec: - appendblob - blockblob type: string + bufferingEnabled: + description: Enable buffering into disk before ingesting into + Azure Blob. + type: boolean + compressBlob: + description: 'Optional: Enables GZIP compression in the final + blockblob file. This option isn''t compatible when blob_type + = appendblob.' + enum: + - "on" + - "off" + type: string containerName: description: Name of the container that will contain the blobs type: string @@ -240,6 +252,18 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uploadFileSize: + default: 200M + description: Specifies the size of files to be uploaded in MB. + Defaults to 200M. + type: string + uploadTimeout: + default: 30m + description: Optional. Specify a timeout for uploads. Fluent Bit + will start ingesting buffer files which have been created more + than x minutes ago and haven't reached upload_file_size limit + yet. Defaults to 30m. + type: string required: - accountName - containerName diff --git a/docs/plugins/fluentbit/output/azure_blob.md b/docs/plugins/fluentbit/output/azure_blob.md index a12959858..2d6e6181c 100644 --- a/docs/plugins/fluentbit/output/azure_blob.md +++ b/docs/plugins/fluentbit/output/azure_blob.md @@ -13,5 +13,9 @@ Azure Blob is the Azure Blob output plugin, allows to ingest your records into A | path | Optional path to store the blobs. | string | | emulatorMode | Optional toggle to use an Azure emulator | string | | endpoint | HTTP Service of the endpoint (if using EmulatorMode) | string | +| compressBlob | Optional: Enables GZIP compression in the final blockblob file. This option isn't compatible when blob_type = appendblob. | string | +| bufferingEnabled | Enable buffering into disk before ingesting into Azure Blob. | *bool | +| uploadFileSize | Specifies the size of files to be uploaded in MB. Defaults to 200M. | string | +| uploadTimeout | Optional. Specify a timeout for uploads. Fluent Bit will start ingesting buffer files which have been created more than x minutes ago and haven't reached upload_file_size limit yet. Defaults to 30m. | string | | tls | Enable/Disable TLS Encryption. Azure services require TLS to be enabled. | *[plugins.TLS](../tls.md) | | networking | Include fluentbit networking options for this output-plugin | *plugins.Networking | diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 53013ca70..e7f05c397 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -4187,6 +4187,18 @@ spec: - appendblob - blockblob type: string + bufferingEnabled: + description: Enable buffering into disk before ingesting into + Azure Blob. + type: boolean + compressBlob: + description: 'Optional: Enables GZIP compression in the final + blockblob file. This option isn''t compatible when blob_type + = appendblob.' + enum: + - "on" + - "off" + type: string containerName: description: Name of the container that will contain the blobs type: string @@ -4362,6 +4374,18 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uploadFileSize: + default: 200M + description: Specifies the size of files to be uploaded in MB. + Defaults to 200M. + type: string + uploadTimeout: + default: 30m + description: Optional. Specify a timeout for uploads. Fluent Bit + will start ingesting buffer files which have been created more + than x minutes ago and haven't reached upload_file_size limit + yet. Defaults to 30m. + type: string required: - accountName - containerName @@ -33213,6 +33237,18 @@ spec: - appendblob - blockblob type: string + bufferingEnabled: + description: Enable buffering into disk before ingesting into + Azure Blob. + type: boolean + compressBlob: + description: 'Optional: Enables GZIP compression in the final + blockblob file. This option isn''t compatible when blob_type + = appendblob.' + enum: + - "on" + - "off" + type: string containerName: description: Name of the container that will contain the blobs type: string @@ -33388,6 +33424,18 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uploadFileSize: + default: 200M + description: Specifies the size of files to be uploaded in MB. + Defaults to 200M. + type: string + uploadTimeout: + default: 30m + description: Optional. Specify a timeout for uploads. Fluent Bit + will start ingesting buffer files which have been created more + than x minutes ago and haven't reached upload_file_size limit + yet. Defaults to 30m. + type: string required: - accountName - containerName diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 071f7e9a1..323f9e2ac 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -4187,6 +4187,18 @@ spec: - appendblob - blockblob type: string + bufferingEnabled: + description: Enable buffering into disk before ingesting into + Azure Blob. + type: boolean + compressBlob: + description: 'Optional: Enables GZIP compression in the final + blockblob file. This option isn''t compatible when blob_type + = appendblob.' + enum: + - "on" + - "off" + type: string containerName: description: Name of the container that will contain the blobs type: string @@ -4362,6 +4374,18 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uploadFileSize: + default: 200M + description: Specifies the size of files to be uploaded in MB. + Defaults to 200M. + type: string + uploadTimeout: + default: 30m + description: Optional. Specify a timeout for uploads. Fluent Bit + will start ingesting buffer files which have been created more + than x minutes ago and haven't reached upload_file_size limit + yet. Defaults to 30m. + type: string required: - accountName - containerName @@ -33213,6 +33237,18 @@ spec: - appendblob - blockblob type: string + bufferingEnabled: + description: Enable buffering into disk before ingesting into + Azure Blob. + type: boolean + compressBlob: + description: 'Optional: Enables GZIP compression in the final + blockblob file. This option isn''t compatible when blob_type + = appendblob.' + enum: + - "on" + - "off" + type: string containerName: description: Name of the container that will contain the blobs type: string @@ -33388,6 +33424,18 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uploadFileSize: + default: 200M + description: Specifies the size of files to be uploaded in MB. + Defaults to 200M. + type: string + uploadTimeout: + default: 30m + description: Optional. Specify a timeout for uploads. Fluent Bit + will start ingesting buffer files which have been created more + than x minutes ago and haven't reached upload_file_size limit + yet. Defaults to 30m. + type: string required: - accountName - containerName