Skip to content

Commit 0be08e2

Browse files
Hera otel integration (#402)
* changes for otel integration * disable otel unittests as part of code coverage tests --------- Co-authored-by: Rajesh S <[email protected]>
1 parent b94420d commit 0be08e2

File tree

24 files changed

+2930
-61
lines changed

24 files changed

+2930
-61
lines changed

.github/workflows/go.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ jobs:
1616
- name: Set up Go
1717
uses: actions/setup-go@v4
1818
with:
19-
go-version: 1.18.2
19+
go-version: 1.20.14
2020
- name: Install GO Modules
2121
run: |
2222
go mod tidy
@@ -41,7 +41,7 @@ jobs:
4141
- name: Set up Go
4242
uses: actions/setup-go@v4
4343
with:
44-
go-version: 1.18.2
44+
go-version: 1.20.14
4545
- name: Install GO Modules
4646
run: |
4747
go mod tidy

go.mod

+21-2
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,33 @@
11
module github.com/paypal/hera
22

3-
go 1.18
3+
go 1.20
44

55
require (
66
github.com/go-sql-driver/mysql v1.7.1
77
github.com/godror/godror v0.26.3
88
github.com/lib/pq v1.10.3
9+
go.opentelemetry.io/otel v1.24.0
10+
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.24.0
11+
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.24.0
12+
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.24.0
13+
go.opentelemetry.io/otel/metric v1.24.0
14+
go.opentelemetry.io/otel/sdk v1.24.0
15+
go.opentelemetry.io/otel/sdk/metric v1.24.0
16+
go.opentelemetry.io/proto/otlp v1.2.0
17+
google.golang.org/protobuf v1.34.1
918
)
1019

1120
require (
21+
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
1222
github.com/go-logfmt/logfmt v0.5.0 // indirect
13-
google.golang.org/protobuf v1.33.0 // indirect
23+
github.com/go-logr/logr v1.4.2 // indirect
24+
github.com/go-logr/stdr v1.2.2 // indirect
25+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
26+
go.opentelemetry.io/otel/trace v1.24.0 // indirect
27+
golang.org/x/net v0.26.0 // indirect
28+
golang.org/x/sys v0.21.0 // indirect
29+
golang.org/x/text v0.16.0 // indirect
30+
google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect
31+
google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect
32+
google.golang.org/grpc v1.64.0 // indirect
1433
)

go.sum

+47-5
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,61 @@
1+
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
2+
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
3+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
14
github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4=
25
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
6+
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
7+
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
8+
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
9+
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
10+
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
311
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
412
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
513
github.com/godror/godror v0.26.3 h1:V+z+Q/OBGgmmYzuAwyJzpcn4LsPF4Ev0xHAea68V00c=
614
github.com/godror/godror v0.26.3/go.mod h1:1QCn6oXh3r+IlB3DLE8V6qkHXLSHd18a3Hw7szQ9/3Y=
715
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
8-
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
916
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
17+
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
18+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
19+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
1020
github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg=
1121
github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
12-
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
22+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
23+
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
24+
go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
25+
go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
26+
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.24.0 h1:f2jriWfOdldanBwS9jNBdeOKAQN7b4ugAMaNu1/1k9g=
27+
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.24.0/go.mod h1:B+bcQI1yTY+N0vqMpoZbEN7+XU4tNM0DmUiOwebFJWI=
28+
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.24.0 h1:mM8nKi6/iFQ0iqst80wDHU2ge198Ye/TfN0WBS5U24Y=
29+
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.24.0/go.mod h1:0PrIIzDteLSmNyxqcGYRL4mDIo8OTuBAOI/Bn1URxac=
30+
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.24.0 h1:JYE2HM7pZbOt5Jhk8ndWZTUWYOVift2cHjXVMkPdmdc=
31+
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.24.0/go.mod h1:yMb/8c6hVsnma0RpsBMNo0fEiQKeclawtgaIaOp2MLY=
32+
go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
33+
go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
34+
go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
35+
go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
36+
go.opentelemetry.io/otel/sdk/metric v1.24.0 h1:yyMQrPzF+k88/DbH7o4FMAs80puqd+9osbiBrJrz/w8=
37+
go.opentelemetry.io/otel/sdk/metric v1.24.0/go.mod h1:I6Y5FjH6rvEnTTAYQz3Mmv2kl6Ek5IIrmwTLqMrrOE0=
38+
go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
39+
go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
40+
go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94=
41+
go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A=
42+
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
43+
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
1344
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
14-
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
45+
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
46+
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
47+
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
48+
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
49+
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
1550
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
51+
google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ=
52+
google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g=
53+
google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI=
54+
google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
55+
google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY=
56+
google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg=
1657
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
1758
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
18-
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
19-
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
59+
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
60+
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
61+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

lib/config.go

+49-2
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ import (
2323
"github.com/paypal/hera/cal"
2424
"github.com/paypal/hera/config"
2525
"github.com/paypal/hera/utility/logger"
26+
otelconfig "github.com/paypal/hera/utility/logger/otel/config"
27+
2628
"os"
2729
"path/filepath"
2830
"strings"
@@ -236,7 +238,7 @@ func parseMapStrStr(encoded string) map[string]string {
236238
}
237239

238240
// InitConfig initializes the configuration, both the static configuration (from hera.txt) and the dynamic configuration
239-
func InitConfig() error {
241+
func InitConfig(poolName string) error {
240242
currentDir, abserr := filepath.Abs(filepath.Dir(os.Args[0]))
241243

242244
if abserr != nil {
@@ -488,9 +490,39 @@ func InitConfig() error {
488490
gAppConfig.MaxDesiredHealthyWorkerPct = 90
489491
}
490492

493+
//Initialize OTEL configs
494+
initializeOTELConfigs(cdb, poolName)
495+
if logger.GetLogger().V(logger.Info) {
496+
otelconfig.OTelConfigData.Dump()
497+
}
491498
return nil
492499
}
493500

501+
// This function takes care of initialize OTEL configuration
502+
func initializeOTELConfigs(cdb config.Config, poolName string) {
503+
otelconfig.OTelConfigData = &otelconfig.OTelConfig{}
504+
//TODO initialize the values
505+
otelconfig.OTelConfigData.Enabled = cdb.GetOrDefaultBool("enable_otel", false)
506+
otelconfig.OTelConfigData.SkipCalStateLog = cdb.GetOrDefaultBool("skip_cal_statelog", false)
507+
otelconfig.OTelConfigData.MetricNamePrefix = cdb.GetOrDefaultString("otel_metric_prefix", "pp.occ")
508+
otelconfig.OTelConfigData.Host = cdb.GetOrDefaultString("otel_agent_host", "localhost")
509+
otelconfig.OTelConfigData.MetricsPort = cdb.GetOrDefaultInt("otel_agent_metrics_port", 4318)
510+
otelconfig.OTelConfigData.TracePort = cdb.GetOrDefaultInt("otel_agent_trace_port", 4318)
511+
otelconfig.OTelConfigData.OtelMetricGRPC = cdb.GetOrDefaultBool("otel_agent_use_grpc_metric", false)
512+
otelconfig.OTelConfigData.OtelTraceGRPC = cdb.GetOrDefaultBool("otel_agent_use_grpc_trace", false)
513+
otelconfig.OTelConfigData.MetricsURLPath = cdb.GetOrDefaultString("otel_agent_metrics_uri", "")
514+
otelconfig.OTelConfigData.TraceURLPath = cdb.GetOrDefaultString("otel_agent_trace_uri", "")
515+
otelconfig.OTelConfigData.PoolName = poolName
516+
otelconfig.OTelConfigData.UseTls = cdb.GetOrDefaultBool("otel_use_tls", false)
517+
otelconfig.OTelConfigData.TLSCertPath = cdb.GetOrDefaultString("otel_tls_cert_path", "")
518+
otelconfig.OTelConfigData.ResolutionTimeInSec = cdb.GetOrDefaultInt("otel_resolution_time_in_sec", 1)
519+
otelconfig.OTelConfigData.ExporterTimeout = cdb.GetOrDefaultInt("otel_exporter_time_in_sec", 30)
520+
otelconfig.OTelConfigData.EnableRetry = cdb.GetOrDefaultBool("otel_enable_exporter_retry", false)
521+
otelconfig.OTelConfigData.ResourceType = gAppConfig.StateLogPrefix
522+
otelconfig.OTelConfigData.OTelErrorReportingInterval = cdb.GetOrDefaultInt("otel_error_reporting_interval_in_sec", 60)
523+
otelconfig.SetOTelIngestToken(cdb.GetOrDefaultString("otel_ingest_token", ""))
524+
}
525+
494526
func LogOccConfigs() {
495527
whiteListConfigs := map[string]map[string]interface{}{
496528
"BACKLOG": {
@@ -503,6 +535,18 @@ func LogOccConfigs() {
503535
"bouncer_startup_delay": gAppConfig.BouncerStartupDelay,
504536
"bouncer_poll_interval_ms": gAppConfig.BouncerPollInterval,
505537
},
538+
"OTEL": {
539+
"enable_otel": otelconfig.OTelConfigData.Enabled,
540+
"otel_use_tls": otelconfig.OTelConfigData.UseTls,
541+
"skip_cal_statelog": otelconfig.OTelConfigData.SkipCalStateLog,
542+
"otel_agent_host": otelconfig.OTelConfigData.Host,
543+
"otel_agent_metrics_port": otelconfig.OTelConfigData.MetricsPort,
544+
"otel_agent_trace_port": otelconfig.OTelConfigData.TracePort,
545+
"otel_agent_metrics_uri": otelconfig.OTelConfigData.MetricsURLPath,
546+
"otel_agent_trace_uri": otelconfig.OTelConfigData.TraceURLPath,
547+
"otel_resolution_time_in_sec": otelconfig.OTelConfigData.ResolutionTimeInSec,
548+
"otel_error_reporting_interval_in_sec": otelconfig.OTelConfigData.OTelErrorReportingInterval,
549+
},
506550
"PROFILE": {
507551
"enable_profile": gAppConfig.EnableProfile,
508552
"profile_http_port": gAppConfig.ProfileHTTPPort,
@@ -618,6 +662,10 @@ func LogOccConfigs() {
618662
if !gAppConfig.BouncerEnabled {
619663
continue
620664
}
665+
case "OTEL":
666+
if !otelconfig.OTelConfigData.Enabled {
667+
continue
668+
}
621669
case "PROFILE":
622670
if !gAppConfig.EnableProfile {
623671
continue
@@ -679,7 +727,6 @@ func LogOccConfigs() {
679727
}
680728
evt.Completed()
681729
}
682-
683730
}
684731

685732
// CheckOpsConfigChange checks if the ops config file needs to be reloaded and reloads it if necessary.

lib/main.go

+25-5
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,11 @@
1818
package lib
1919

2020
import (
21+
"context"
2122
"flag"
2223
"fmt"
24+
otellogger "github.com/paypal/hera/utility/logger/otel"
25+
otelconfig "github.com/paypal/hera/utility/logger/otel/config"
2326
"math/rand"
2427
"os"
2528
"os/signal"
@@ -36,6 +39,9 @@ func Run() {
3639
signal.Ignore(syscall.SIGPIPE)
3740
mux_process_id := syscall.Getpid()
3841

42+
// Defer release resource in case of any abnormal exit of for application
43+
defer handlePanicAndReleaseResource(mux_process_id)
44+
3945
namePtr := flag.String("name", "", "module name in v$session table")
4046
flag.Parse()
4147

@@ -50,7 +56,7 @@ func Run() {
5056

5157
rand.Seed(time.Now().Unix())
5258

53-
err := InitConfig()
59+
err := InitConfig(*namePtr)
5460
if err != nil {
5561
if logger.GetLogger().V(logger.Alert) {
5662
logger.GetLogger().Log(logger.Alert, "failed to initialize configuration:", err.Error())
@@ -106,7 +112,24 @@ func Run() {
106112
caltxn.SetCorrelationID("runtxn")
107113
caltxn.Completed()
108114

109-
GetStateLog().SetStartTime(time.Now())
115+
//Initialize OTEL
116+
if otelconfig.OTelConfigData.Enabled {
117+
shutdownFunc, err := otellogger.Init(context.Background())
118+
if err != nil {
119+
logger.GetLogger().Log(logger.Alert, fmt.Sprintf("failed to initialize OTEL, err: %v", err))
120+
evt := cal.NewCalEvent("OTEL_INIT", *namePtr, "2", fmt.Sprintf("erro: %v", err))
121+
evt.Completed()
122+
if otelconfig.OTelConfigData.SkipCalStateLog {
123+
logger.GetLogger().Log(logger.Alert, fmt.Sprintf("OTEL initialization failed. Only the OTEL state-log has been enabled. It is not safe to start the server"))
124+
FullShutdown()
125+
}
126+
}
127+
GetStateLog().SetStartTime(time.Now())
128+
defer otellogger.StopMetricCollection() //Stop sending metrics data
129+
defer shutdownFunc(context.Background()) //During exit from mux, this will takecare of OTEL providers clean-up
130+
} else {
131+
GetStateLog().SetStartTime(time.Now())
132+
}
110133

111134
go func() {
112135
sleep := time.Duration(GetConfig().ConfigReloadTimeMs)
@@ -192,9 +215,6 @@ func Run() {
192215
cal.ReleaseCxtResource()
193216
}()
194217

195-
// Defer release resource in case of any abnormal exit of for application
196-
defer handlePanicAndReleaseResource(mux_process_id)
197-
198218
<-GetWorkerBrokerInstance().Stopped()
199219
}
200220

0 commit comments

Comments
 (0)