Devops 1 update fluent plugin google cloud

docker-image/v1.16/debian-stackdriver/Dockerfile

@@ -13,14 +13,14 @@ ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.1.0
 ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
 
 COPY Gemfile* /fluentd/
-RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \
+RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev git" \
   runtimeDeps="" \
       && apt-get update \
      && apt-get upgrade -y \
      && apt-get install \
      -y --no-install-recommends \
-     $buildDeps $runtimeDeps net-tools \
-    && gem install bundler --version 2.4.17 \
+     $buildDeps $runtimeDeps net-tools
+RUN gem install bundler --version 2.4.17 \
     && bundle config silence_root_warning true \
     && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \
     && SUDO_FORCE_REMOVE=yes \
@@ -29,7 +29,12 @@ RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \
                   $buildDeps \
  && rm -rf /var/lib/apt/lists/* \
     && gem sources --clear-all \
-    && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
+    && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem\
+RUN gem list
+RUN cd /fluentd && git clone https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud.git
+RUN cd /fluentd/fluent-plugin-google-cloud \
+    && gem build fluent-plugin-google-cloud.gemspec \
+    && gem install --local fluent-plugin-google-cloud-0.13.2.gem
 # Copy configuration files
 COPY ./conf/fluent.conf /fluentd/etc/
 COPY ./conf/systemd.conf /fluentd/etc/

docker-image/v1.16/debian-stackdriver/Gemfile

@@ -10,12 +10,23 @@ gem "fluent-plugin-multi-format-parser", "~> 1.0.0"
 gem "fluent-plugin-concat", "~> 2.5.0"
 gem "fluent-plugin-grok-parser", "~> 2.6.2"
 gem "fluent-plugin-prometheus", "~> 2.1.0"
-gem 'fluent-plugin-json-in-json-2', ">= 1.0.2"
+gem "fluent-plugin-json-in-json-2", ">= 1.0.2"
 gem "fluent-plugin-record-modifier", "~> 2.1.0"
 gem "fluent-plugin-detect-exceptions", "~> 0.0.13"
 gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0"
 gem "fluent-plugin-parser-cri", "~> 0.1.0"
-gem "fluent-plugin-google-cloud", "~> 0.4.10"
+# dep fluent-plugin-google-cloud
+gem "google-api-client", "= 0.53.0"
+gem "googleapis-common-protos", "= 1.4.0"
+gem "googleauth", "= 1.3.0"
+gem "google-cloud-logging", "= 2.3.2"
+gem "google-cloud-monitoring-v3", "= 0.10.0"
+gem "google-protobuf", "= 3.22.1"
+gem "grpc", "= 1.53.0"
+gem "json", "= 2.6.3"
+gem "opencensus", "= 0.5.0"
+gem "opencensus-stackdriver", "= 0.4.1"
+#
 gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.2.0"
 gem "ffi"
 gem "fluent-plugin-systemd", "~> 1.0.5"

docker-image/v1.16/debian-stackdriver/conf/fluent.conf

@@ -9,26 +9,23 @@
 
 <match **>
   @type google_cloud
-  @id out_google_cloud
-
-  # Set the buffer type to file to improve the reliability and reduce the memory consumption
+  detect_json true
+  enable_monitoring true
+  monitoring_type prometheus
+  # Allow entries from multiple system logs to be sent in the same request.
+  split_logs_by_tag false
+  detect_subservice false
   buffer_type file
-  buffer_path /var/log/fluentd-buffers/stackdriver.buffer
-  # Set queue_full action to block because we want to pause gracefully
-  # in case of the off-the-limits load instead of throwing an exception
+  buffer_path /var/log/fluentd-buffers/kubernetes.system.buffer
   buffer_queue_full_action block
-  # Set the chunk limit conservatively to avoid exceeding the GCL limit
-  # of 10MiB per write request.
-  buffer_chunk_limit 2M
-  # Cap the combined memory usage of this buffer and the one below to
-  # 2MiB/chunk * (6 + 2) chunks = 16 MiB
-  buffer_queue_limit 6
-  # Never wait more than 5 seconds before flushing logs in the non-error case.
+  buffer_chunk_limit 512k
+  buffer_queue_limit 2
   flush_interval 5s
-  # Never wait longer than 30 seconds between retries.
   max_retry_wait 30
-  # Disable the limit on the number of retries (retry forever).
-  # disable_retry_limit
-  # Use multiple threads for processing.
+  disable_retry_limit
   num_threads 2
+  use_grpc true
+  # Skip timestamp adjustment as this is in a controlled environment with
+  # known timestamp format. This helps with CPU usage.
+  adjust_invalid_timestamps false
 </match>