Add transformation test make target and docs

.gitignore

@@ -1,4 +1,12 @@
+# Ignore a file actually called Makefile. The presence of this file interferes
+# with our Linux package building processes. To make a local symlink called
+# "Makefile" for development convenience, use this command:
+# make -f tasks.mak makefile_symlink
 Makefile
 
 # Ignore generated credentials from google-github-actions/auth
 gha-creds-*.json
+
+# This is a folder used to hold local builds of fluent-bit and otelopscol
+# when running transformation tests locally.
+dist/

builds/agent_wrapper.sh

@@ -1,4 +1,18 @@
 #!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -x -e
 DESTDIR=$1
 mkdir -p "$DESTDIR/opt/google-cloud-ops-agent/libexec"

builds/fluent_bit.sh

@@ -1,25 +1,57 @@
 #!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -x -e
-DESTDIR=$1
+
+DESTDIR="$1"
 mkdir -p $DESTDIR
-fluent_bit_dir=/opt/google-cloud-ops-agent/subagents/fluent-bit
-
+
+CMAKE_ARGS=(
+  "-DCMAKE_BUILD_TYPE=RelWithDebInfo"
+  "-DFLB_HTTP_SERVER=ON"
+  "-DFLB_DEBUG=OFF"
+  "-DWITHOUT_HEADERS=ON"
+  # -DFLB_SHARED_LIB=OFF skips building libfluent-bit.so.
+  "-DFLB_SHARED_LIB=OFF"
+  "-DFLB_STREAM_PROCESSOR=OFF"
+  "-DFLB_MSGPACK_TO_JSON_INIT_BUFFER_SIZE=1.5"
+  "-DFLB_MSGPACK_TO_JSON_REALLOC_BUFFER_SIZE=.10"
+  "-DFLB_CONFIG_YAML=OFF"
+)
+
 cd submodules/fluent-bit
 mkdir -p build
 cd build
-# CMAKE_INSTALL_PREFIX here will cause the binary to be put at
-# /usr/lib/google-cloud-ops-agent/bin/fluent-bit
-# Additionally, -DFLB_SHARED_LIB=OFF skips building libfluent-bit.so
-cmake .. -DCMAKE_INSTALL_PREFIX=$fluent_bit_dir \
-  -DFLB_HTTP_SERVER=ON -DFLB_DEBUG=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-  -DWITHOUT_HEADERS=ON -DFLB_SHARED_LIB=OFF -DFLB_STREAM_PROCESSOR=OFF \
-  -DFLB_MSGPACK_TO_JSON_INIT_BUFFER_SIZE=1.5 -DFLB_MSGPACK_TO_JSON_REALLOC_BUFFER_SIZE=.10 \
-  -DFLB_CONFIG_YAML=OFF
-make -j8
-make DESTDIR="$DESTDIR" install
 
-# We don't want fluent-bit's service or configuration, but there are no cmake
-# flags to disable them. Prune after build.
-rm "${DESTDIR}/lib/systemd/system/fluent-bit.service" || true
-rm "${DESTDIR}/usr/lib/systemd/system/fluent-bit.service" || true
-rm -r "${DESTDIR}${fluent_bit_dir}/etc"
+if [ "$LOCAL_ONLY" = "true" ]; then
+  # If this is a local build, we can just build and copy the binary.
+  cmake .. ${CMAKE_ARGS[@]}
+  make -j8
+  cp ./bin/fluent-bit $DESTDIR
+
+# Otherwise, we do a full install.
+else
+  fluent_bit_dir=/opt/google-cloud-ops-agent/subagents/fluent-bit
+  # CMAKE_INSTALL_PREFIX here will cause the binary to be put at
+  # /usr/lib/google-cloud-ops-agent/bin/fluent-bit
+  cmake .. -DCMAKE_INSTALL_PREFIX=$fluent_bit_dir ${CMAKE_ARGS[@]}
+  make -j8
+  make DESTDIR="$DESTDIR" install
+  # We don't want fluent-bit's service or configuration, but there are no cmake
+  # flags to disable them. Prune after build.
+  rm "${DESTDIR}/lib/systemd/system/fluent-bit.service" || true
+  rm "${DESTDIR}/usr/lib/systemd/system/fluent-bit.service" || true
+  rm -r "${DESTDIR}${fluent_bit_dir}/etc"
+fi

builds/ops_agent_diagnostics.sh

@@ -1,4 +1,18 @@
 #!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -x -e
 DESTDIR=$1
 mkdir -p "$DESTDIR/opt/google-cloud-ops-agent/libexec"

builds/otel.sh

@@ -1,26 +1,52 @@
 #!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -x -e
-DESTDIR=$1
+
+ROOTDIR="$1"
 otel_dir=/opt/google-cloud-ops-agent/subagents/opentelemetry-collector
-DESTDIR="${DESTDIR}${otel_dir}"
-
-cd submodules/opentelemetry-java-contrib
-mkdir -p "$DESTDIR"
-./gradlew --no-daemon -Djdk.lang.Process.launchMechanism=vfork :jmx-metrics:build
-cp jmx-metrics/build/libs/opentelemetry-jmx-metrics-*-alpha-SNAPSHOT.jar "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar"
-
-# Rename LICENSE file because it causes issues with file hash consistency due to an unknown
-# issue with the debuild/rpmbuild processes. Something is unzipping the jar in a case-insensitive
-# environment and having a conflict between the LICENSE file and license/ directory, leading to a changed jar file
-mkdir ./META-INF
-unzip -j "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar" "META-INF/LICENSE" -d ./META-INF
-zip -d "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar" "META-INF/LICENSE"
-mv ./META-INF/LICENSE ./META-INF/LICENSE.renamed
-zip -u "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar" "META-INF/LICENSE.renamed"
-
-cd ../opentelemetry-operations-collector
-# Using array assignment to drop the filename from the sha256sum output
-JAR_SHA_256=($(sha256sum "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar"))
-go build -tags=gpu -buildvcs=false -o "$DESTDIR/otelopscol" \
--ldflags "-X github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver.MetricsGathererHash=$JAR_SHA_256" \
-./cmd/otelopscol
+DESTDIR="${ROOTDIR}${otel_dir}"
+
+# Default values for go build arguments.
+BINARYLOCATION="$ROOTDIR/otelopscol"
+LDFLAGS=""
+
+# If this is not a local build, then we build otel java and add it to
+# the ldflags, as well as changing the binary location.
+if [ "$LOCAL_ONLY" != "true" ]; then
+    cd submodules/opentelemetry-java-contrib
+
+    mkdir -p "$DESTDIR"
+    ./gradlew --no-daemon -Djdk.lang.Process.launchMechanism=vfork :jmx-metrics:build
+    cp jmx-metrics/build/libs/opentelemetry-jmx-metrics-*-alpha-SNAPSHOT.jar "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar"
+
+    # Rename LICENSE file because it causes issues with file hash consistency due to an unknown
+    # issue with the debuild/rpmbuild processes. Something is unzipping the jar in a case-insensitive
+    # environment and having a conflict between the LICENSE file and license/ directory, leading to a changed jar file
+    mkdir ./META-INF
+    unzip -j "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar" "META-INF/LICENSE" -d ./META-INF
+    zip -d "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar" "META-INF/LICENSE"
+    mv ./META-INF/LICENSE ./META-INF/LICENSE.renamed
+    zip -u "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar" "META-INF/LICENSE.renamed"
+
+    JAR_SHA_256=($(sha256sum "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar"))
+    LDFLAGS="-X github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver.MetricsGathererHash=$JAR_SHA_256"
+    BINARYLOCATION="$DESTDIR/otelopscol"
+
+    cd ../..
+fi
+
+cd submodules/opentelemetry-operations-collector
+go build -tags=gpu -buildvcs=false -o "$BINARYLOCATION" -ldflags "$LDFLAGS" ./cmd/otelopscol

builds/systemd.sh

@@ -1,4 +1,18 @@
 #!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -x -e
 DESTDIR=$1
 

cloudbuild/new-distro-detector/run.sh

@@ -1,4 +1,18 @@
 #!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 # See go/sdi-new-distro-detector for details on how this script is being used.
 # A short summary is that it is run periodically by GCB to file bugs when

cmd/agent_wrapper/main_test.go

@@ -1,3 +1,17 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

confgenerator/feature_tracking_internal_test.go

@@ -1,3 +1,17 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package confgenerator
 
 import (

dev-docs/makefile.md

@@ -6,14 +6,18 @@ The Makefile `tasks.mak` is a Makefile for the project that is used to quickly p
 
 We are unable to use the actual `Makefile` name in this repo; because we build the Ops Agent into a deb package, `debhelper` tries to autodetect `make` as the build system in most of its `dh_auto` steps. We do not want `debhelper` to pick up the Makfile and try picking up different targets to build and test it.
 
+## Disclaimer
+
+`tasks.mak` is provided without any guarantees or warranty on its targets. It is meant purely for developer convenience, and it is advised not to make any dependency on the targets since they are subject to change at any time.
+
 ## Usage
 
 For convenience it helps to make a symlink from `Makefile` to `tasks.mak` . There is a target to do this, to run it:
 ```
 make -f tasks.mak makefile_symlink
 ```
 
-Now you can call the targets simply with the `make` command without needing to specify the `tasks.mak` file. The file `Makefile` is gitignored, since we don't want to push it to the repo for the reasons stated in the previous section. 
+Now you can call the targets simply with the `make` command without needing to specify the `tasks.mak` file. The file `Makefile` is gitignored, since we don't want to push it to the repo for the reasons stated in the previous section.
 
 To run something from the Makefile, you will need to give `make` the `-f` (file) flag with a path to `tasks.mak`. For example, to run the `test` target:
 ```
@@ -35,8 +39,4 @@ A new target should be added to `tasks.mak` if:
 * There's a new package with Unit Tests
 * There is something that developers may need to run often, especially if it's a complicated command
 * There are new tasks to run in CI
-* There is a new development tool to be installed and used 
-
-## Disclaimer
-
-`tasks.mak` is provided without any guarantees or warranty on its targets. It is meant purely for developer convenience, and it is advised not to make any dependency on the targets since they are subject to change at any time.
+* There is a new development tool to be installed and used

dev-docs/transformation-tests.md

@@ -0,0 +1,43 @@
+# Transformation Tests
+
+The transformation tests are tests that will run Fluent Bit and OpenTelemetry Operations Collector (otelopscol) directly in your environment and verify the results of various logging operations by reading their output from stdout and stderr. This is to check the correctness of log parsing expectations in a more lightweight way than full integration tests.
+
+## Makefile Usage
+
+NOTE: It is assumed that you have followed the [makefile setup instructions](./makefile.md#usage).
+
+For your convenience, there is a Makefile target to run transformation tests:
+```bash
+make transformation_test
+```
+You can also update the transformation test goldens:
+```bash
+make transformation_test_update
+```
+These targets are part of the `precommit` and `precommit_update` targets respectively.
+
+Read on if you are interested in how the transformation tests actually work.
+
+## Usage
+
+The transformation tests take paths to Fluent Bit and otelopscol binaries to execute directly and compare expected output. To run the tests, first you will need to build both binaries. There are scripts to locally build each agent:
+```bash
+sh ./builds/fluent_bit_local.sh
+sh ./builds/otel_local.sh
+```
+After running each script, you will have a `dist` folder that contains each binary.
+
+The paths to each can passed into the test using environment variables or flags.
+```bash
+go test ./transformation_test -flb=$(pwd)/dist/fluent_bit -otelopscol=$(pwd)/dist/otelopscol
+```
+or
+```bash
+FLB=$(pwd)/dist/fluent_bit OTELOPSCOL=$(pwd)/dist/otelopscol go test ./transformation_test
+```
+
+If you have updated the Fluent Bit or otelopscol submodules and need to update the golden testdata, you can add the `-update` flag to the test.
+```bash
+FLB=$(pwd)/dist/fluent_bit OTELOPSCOL=$(pwd)/dist/otelopscol go test ./transformation_test -update
+```
+(Note: this is a flag to the test, not to `go test`. The flag must go after the specified package, not before.)

dockerfiles/compile_test.go

@@ -1,3 +1,17 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

integration_test/gce/zone_picker.go

@@ -1,3 +1,17 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package gce
 
 import (