diff --git a/docker-image/v1.16/debian-stackdriver/Dockerfile b/docker-image/v1.16/debian-stackdriver/Dockerfile index c5fab08d0..fd3a8b4ea 100644 --- a/docker-image/v1.16/debian-stackdriver/Dockerfile +++ b/docker-image/v1.16/debian-stackdriver/Dockerfile @@ -13,14 +13,14 @@ ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.1.0 ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 COPY Gemfile* /fluentd/ -RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev git" \ runtimeDeps="" \ && apt-get update \ && apt-get upgrade -y \ && apt-get install \ -y --no-install-recommends \ - $buildDeps $runtimeDeps net-tools \ - && gem install bundler --version 2.4.17 \ + $buildDeps $runtimeDeps net-tools +RUN gem install bundler --version 2.4.17 \ && bundle config silence_root_warning true \ && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ && SUDO_FORCE_REMOVE=yes \ @@ -29,7 +29,12 @@ RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ $buildDeps \ && rm -rf /var/lib/apt/lists/* \ && gem sources --clear-all \ - && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem\ +RUN gem list +RUN cd /fluentd && git clone https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud.git +RUN cd /fluentd/fluent-plugin-google-cloud \ + && gem build fluent-plugin-google-cloud.gemspec \ + && gem install --local fluent-plugin-google-cloud-0.13.2.gem # Copy configuration files COPY ./conf/fluent.conf /fluentd/etc/ COPY ./conf/systemd.conf /fluentd/etc/ diff --git a/docker-image/v1.16/debian-stackdriver/Gemfile b/docker-image/v1.16/debian-stackdriver/Gemfile index c557b983e..25399b9ff 100644 --- a/docker-image/v1.16/debian-stackdriver/Gemfile +++ b/docker-image/v1.16/debian-stackdriver/Gemfile @@ -10,12 +10,23 @@ gem "fluent-plugin-multi-format-parser", "~> 1.0.0" gem "fluent-plugin-concat", "~> 2.5.0" gem "fluent-plugin-grok-parser", "~> 2.6.2" gem "fluent-plugin-prometheus", "~> 2.1.0" -gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-json-in-json-2", ">= 1.0.2" gem "fluent-plugin-record-modifier", "~> 2.1.0" gem "fluent-plugin-detect-exceptions", "~> 0.0.13" gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" gem "fluent-plugin-parser-cri", "~> 0.1.0" -gem "fluent-plugin-google-cloud", "~> 0.4.10" +# dep fluent-plugin-google-cloud +gem "google-api-client", "= 0.53.0" +gem "googleapis-common-protos", "= 1.4.0" +gem "googleauth", "= 1.3.0" +gem "google-cloud-logging", "= 2.3.2" +gem "google-cloud-monitoring-v3", "= 0.10.0" +gem "google-protobuf", "= 3.22.1" +gem "grpc", "= 1.53.0" +gem "json", "= 2.6.3" +gem "opencensus", "= 0.5.0" +gem "opencensus-stackdriver", "= 0.4.1" +# gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.2.0" gem "ffi" gem "fluent-plugin-systemd", "~> 1.0.5" diff --git a/docker-image/v1.16/debian-stackdriver/conf/fluent.conf b/docker-image/v1.16/debian-stackdriver/conf/fluent.conf index 07b3b4162..2536e0273 100644 --- a/docker-image/v1.16/debian-stackdriver/conf/fluent.conf +++ b/docker-image/v1.16/debian-stackdriver/conf/fluent.conf @@ -9,26 +9,23 @@ @type google_cloud - @id out_google_cloud - - # Set the buffer type to file to improve the reliability and reduce the memory consumption + detect_json true + enable_monitoring true + monitoring_type prometheus + # Allow entries from multiple system logs to be sent in the same request. + split_logs_by_tag false + detect_subservice false buffer_type file - buffer_path /var/log/fluentd-buffers/stackdriver.buffer - # Set queue_full action to block because we want to pause gracefully - # in case of the off-the-limits load instead of throwing an exception + buffer_path /var/log/fluentd-buffers/kubernetes.system.buffer buffer_queue_full_action block - # Set the chunk limit conservatively to avoid exceeding the GCL limit - # of 10MiB per write request. - buffer_chunk_limit 2M - # Cap the combined memory usage of this buffer and the one below to - # 2MiB/chunk * (6 + 2) chunks = 16 MiB - buffer_queue_limit 6 - # Never wait more than 5 seconds before flushing logs in the non-error case. + buffer_chunk_limit 512k + buffer_queue_limit 2 flush_interval 5s - # Never wait longer than 30 seconds between retries. max_retry_wait 30 - # Disable the limit on the number of retries (retry forever). - # disable_retry_limit - # Use multiple threads for processing. + disable_retry_limit num_threads 2 + use_grpc true + # Skip timestamp adjustment as this is in a controlled environment with + # known timestamp format. This helps with CPU usage. + adjust_invalid_timestamps false diff --git a/docker-image/v1.16/debian-stackdriver/list b/docker-image/v1.16/debian-stackdriver/list new file mode 100644 index 000000000..e69de29bb