diff --git a/agent-manager/agent/agent_command.go b/agent-manager/agent/agent_command.go index 73bf8d1c0..09c1fd212 100644 --- a/agent-manager/agent/agent_command.go +++ b/agent-manager/agent/agent_command.go @@ -3,6 +3,7 @@ package agent import ( "context" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/agent-manager/util" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -15,7 +16,7 @@ func (s *Grpc) ListAgentCommands(ctx context.Context, req *ListRequest) (*ListAg commands, total, err := agentCommandService.ListAgentCommands(page, filter) if err != nil { - util.Logger.ErrorF("failed to fetch agents: %v", err) + catcher.Error("failed to fetch agents", err, nil) return nil, status.Errorf(codes.Internal, "failed to fetch agents: %v", err) } diff --git a/agent-manager/agent/agent_imp.go b/agent-manager/agent/agent_imp.go index c88defd34..087f42927 100644 --- a/agent-manager/agent/agent_imp.go +++ b/agent-manager/agent/agent_imp.go @@ -9,6 +9,7 @@ import ( "time" "github.com/google/uuid" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/agent-manager/models" "github.com/utmstack/UTMStack/agent-manager/util" "google.golang.org/grpc/codes" @@ -39,7 +40,7 @@ func (s *Grpc) RegisterAgent(ctx context.Context, req *AgentRequest) (*AuthRespo Key: oldAgent.AgentKey, }, nil } else { - util.Logger.ErrorF("Agent with hostname %s already exists", agent.Hostname) + catcher.Error("Agent already exists", nil, map[string]any{"host_name": agent.Hostname}) return nil, status.Errorf(codes.AlreadyExists, "hostname has already been registered") } } @@ -48,7 +49,7 @@ func (s *Grpc) RegisterAgent(ctx context.Context, req *AgentRequest) (*AuthRespo agent.AgentKey = key err = agentService.Create(agent) if err != nil { - util.Logger.ErrorF("Failed to create agent: %v", err) + catcher.Error("Failed to create agent", err, nil) return nil, err } @@ -58,7 +59,7 @@ func (s *Grpc) RegisterAgent(ctx context.Context, req *AgentRequest) (*AuthRespo err = lastSeenService.Set(key, time.Now()) if err != nil { - util.Logger.ErrorF("Failed to set last seen: %v", err) + catcher.Error("Failed to set last seen", err, nil) return nil, err } res := &AuthResponse{ @@ -66,7 +67,7 @@ func (s *Grpc) RegisterAgent(ctx context.Context, req *AgentRequest) (*AuthRespo Key: key, } - util.Logger.Info("Agent %s with id %d registered correctly", agent.Hostname, agent.ID) + catcher.Info("Agent was registered correctly", map[string]any{"hostname": agent.Hostname, "id": agent.ID}) return res, nil } @@ -88,7 +89,7 @@ func (s *Grpc) UpdateAgent(ctx context.Context, req *AgentRequest) (*AuthRespons agent, err := agentService.FindByID(uint(id)) if err != nil { - util.Logger.ErrorF("Failed to find agent: %v", err) + catcher.Error("Failed to find agent", err, nil) return nil, err } @@ -119,7 +120,7 @@ func (s *Grpc) UpdateAgent(ctx context.Context, req *AgentRequest) (*AuthRespons err = agentService.Update(agent) if err != nil { - util.Logger.ErrorF("Failed to update agent: %v", err) + catcher.Error("Failed to update agent", err, nil) return nil, err } @@ -128,7 +129,7 @@ func (s *Grpc) UpdateAgent(ctx context.Context, req *AgentRequest) (*AuthRespons Key: agent.AgentKey, } - util.Logger.Info("Agent %s with id %d updated correctly", agent.Hostname, agent.ID) + catcher.Info("Agent was updated correctly", map[string]any{"hostname": agent.Hostname, "id": agent.ID}) return res, nil } @@ -146,7 +147,7 @@ func (s *Grpc) DeleteAgent(ctx context.Context, req *AgentDelete) (*AuthResponse id, err := agentService.Delete(uuid.MustParse(key), req.DeletedBy) if err != nil { - util.Logger.ErrorF("Unable to delete agent: %v", err) + catcher.Error("Unable to delete agent", err, nil) return &AuthResponse{}, status.Error(codes.Internal, fmt.Sprintf("unable to delete agent: %v", err.Error())) } @@ -158,7 +159,7 @@ func (s *Grpc) DeleteAgent(ctx context.Context, req *AgentDelete) (*AuthResponse delete(s.AgentStreamMap, key) s.agentStreamMutex.Unlock() - util.Logger.Info("Agent with key %s deleted by %s", key, req.DeletedBy) + catcher.Info("Agent was deleted", map[string]any{"key": key, "deleted_by": req.DeletedBy}) return &AuthResponse{ Id: uint32(id), @@ -173,7 +174,7 @@ func (s *Grpc) ListAgents(ctx context.Context, req *ListRequest) (*ListAgentsRes agents, total, err := agentService.ListAgents(page, filter) if err != nil { - util.Logger.ErrorF("failed to fetch agents: %v", err) + catcher.Error("failed to fetch agents", err, nil) return nil, status.Errorf(codes.Internal, "failed to fetch agents: %v", err) } return convertToAgentResponse(agents, total) @@ -202,7 +203,7 @@ func (s *Grpc) AgentStream(stream AgentService_AgentStreamServer) error { delete(s.AgentStreamMap, agentKey) s.agentStreamMutex.Unlock() - util.Logger.ErrorF("failed to reconnect to client: %v", err) + catcher.Error("failed to reconnect to client", err, nil) return fmt.Errorf("failed to reconnect to client: %v", err) } @@ -226,10 +227,10 @@ func (s *Grpc) AgentStream(stream AgentService_AgentStreamServer) error { switch msg := in.StreamMessage.(type) { case *BidirectionalStream_Command: - util.Logger.Info("Received command: %s", msg.Command.CmdId) + catcher.Info("Received command", map[string]any{"cmd_id": msg.Command.CmdId}) case *BidirectionalStream_Result: - util.Logger.Info("Received command result: %s", msg.Result.CmdId) + catcher.Info("Received command result", map[string]any{"cmd_id": msg.Result.CmdId}) cmdID := msg.Result.GetCmdId() @@ -244,7 +245,7 @@ func (s *Grpc) AgentStream(stream AgentService_AgentStreamServer) error { }, }, }); err != nil { - util.Logger.ErrorF("Failed to send result to server: %v", err) + catcher.Error("Failed to send result to server", err, nil) } s.resultChannelM.Lock() if resultChan, ok := s.ResultChannel[cmdID]; ok { @@ -256,7 +257,7 @@ func (s *Grpc) AgentStream(stream AgentService_AgentStreamServer) error { } } else { - util.Logger.ErrorF("Failed to find result channel for CmdID: %s", cmdID) + catcher.Error("Failed to find result channel for CmdID", nil, map[string]any{"cmd_id": cmdID}) } s.resultChannelM.Unlock() } @@ -346,12 +347,12 @@ func (s *Grpc) ProcessCommand(stream PanelService_ProcessCommandServer) error { func (s *Grpc) UpdateAgentGroup(ctx context.Context, req *AgentGroupUpdate) (*Agent, error) { if req.AgentId == 0 || req.AgentGroup == 0 { - util.Logger.ErrorF("Error in req") + catcher.Error("Error in req", nil, nil) return nil, status.Errorf(codes.FailedPrecondition, "error in req") } agent, err := agentService.UpdateAgentGroup(uint(req.AgentId), uint(req.AgentGroup)) if err != nil { - util.Logger.ErrorF("Unable to update group: %v", err) + catcher.Error("Unable to update group", err, nil) return nil, status.Errorf(codes.Internal, "unable to update group: %v", err) } return parseAgentToProto(agent), nil @@ -359,12 +360,12 @@ func (s *Grpc) UpdateAgentGroup(ctx context.Context, req *AgentGroupUpdate) (*Ag func (s *Grpc) GetAgentByHostname(ctx context.Context, req *Hostname) (*Agent, error) { if req.Hostname == "" { - util.Logger.ErrorF("Error in req") + catcher.Error("Error in req", nil, nil) return nil, status.Errorf(codes.FailedPrecondition, "error in req") } agent, err := agentService.FindByHostname(req.Hostname) if err != nil { - util.Logger.ErrorF("Unable to find agent with hostname: %v", err) + catcher.Error("Unable to find agent with hostname", err, nil) return nil, status.Errorf(codes.NotFound, "unable to find agent with hostname: %v", err) } return parseAgentToProto(*agent), nil @@ -376,7 +377,7 @@ func (s *Grpc) UpdateAgentType(ctx context.Context, req *AgentTypeUpdate) (*Agen } agent, err := agentService.UpdateAgentType(uint(req.AgentId), uint(req.AgentType)) if err != nil { - util.Logger.ErrorF("Unable to update type: %v", err) + catcher.Error("Unable to update type", err, nil) return nil, status.Errorf(codes.Internal, "unable to update type: %v", err) } return parseAgentToProto(agent), nil @@ -387,7 +388,7 @@ func (s *Grpc) LoadAgentCacheFromDatabase() error { // Fill the agentCache map with agentID and agentToken pairs agents, err := agentService.FindAll() if err != nil { - util.Logger.ErrorF("Failed to fetch agents from database: %v", err) + catcher.Error("Failed to fetch agents from database", err, nil) return err } for _, agent := range agents { @@ -403,7 +404,7 @@ func (s *Grpc) ListAgentsWithCommands(ctx context.Context, req *ListRequest) (*L agents, total, err := agentService.ListAgentWithCommands(page, filter) if err != nil { - util.Logger.ErrorF("failed to fetch agents: %v", err) + catcher.Error("failed to fetch agents", err, nil) return nil, status.Errorf(codes.Internal, "failed to fetch agents: %v", err) } @@ -459,14 +460,14 @@ func createHistoryCommand(cmd *UtmCommand, cmdID string) { } err := agentCommandService.Create(cmdHistory) if err != nil { - util.Logger.ErrorF("Unable to create a new command history") + catcher.Error("Unable to create a new command history", err, nil) } } func updateHistoryCommand(cmdResult *CommandResult, cmdID string) { err := agentCommandService.UpdateCommandStatusAndResult(findAgentIdByKey(CacheAgent, cmdResult.AgentKey), cmdID, models.Executed, cmdResult.Result) if err != nil { - util.Logger.ErrorF("Failed to update command status") + catcher.Error("Failed to update command status", nil, nil) } } diff --git a/agent-manager/agent/collector_imp.go b/agent-manager/agent/collector_imp.go index e4708daef..162c438ab 100644 --- a/agent-manager/agent/collector_imp.go +++ b/agent-manager/agent/collector_imp.go @@ -7,6 +7,7 @@ import ( "time" "github.com/google/uuid" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/agent-manager/models" "github.com/utmstack/UTMStack/agent-manager/util" "google.golang.org/grpc/codes" @@ -30,7 +31,7 @@ func (s *Grpc) RegisterCollector(ctx context.Context, req *RegisterRequest) (*Au Key: oldCollector[0].CollectorKey, }, nil } else { - util.Logger.ErrorF("Collector %s(%s) with id %d already registered with different IP", oldCollector[0].Hostname, oldCollector[0].Module, oldCollector[0].ID) + catcher.Error("Collector is already registered with different IP", nil, map[string]any{"hostname": oldCollector[0].Hostname, "module": oldCollector[0].Module, "id": oldCollector[0].ID}) return nil, status.Errorf(codes.AlreadyExists, "hostname has already been registered") } } @@ -39,7 +40,7 @@ func (s *Grpc) RegisterCollector(ctx context.Context, req *RegisterRequest) (*Au collector.CollectorKey = key err = collectorService.Create(collector) if err != nil { - util.Logger.ErrorF("Failed to create collector: %v", err) + catcher.Error("Failed to create collector", err, nil) return nil, err } @@ -49,7 +50,7 @@ func (s *Grpc) RegisterCollector(ctx context.Context, req *RegisterRequest) (*Au err = lastSeenService.Set(key, time.Now()) if err != nil { - util.Logger.ErrorF("Failed to set last seen: %v", err) + catcher.Error("Failed to set last seen", err, nil) return nil, err } res := &AuthResponse{ @@ -57,7 +58,7 @@ func (s *Grpc) RegisterCollector(ctx context.Context, req *RegisterRequest) (*Au Key: key, } - util.Logger.Info("Collector %s(%s) with id %d registered correctly", collector.Hostname, collector.Module, collector.ID) + catcher.Info("Collector was registered correctly", map[string]any{"hostname": collector.Hostname, "module": collector.Module, "id": collector.ID}) return res, nil } @@ -75,7 +76,7 @@ func (s *Grpc) DeleteCollector(ctx context.Context, req *CollectorDelete) (*Auth id, err := collectorService.Delete(uuid.MustParse(key), req.DeletedBy) if err != nil { - util.Logger.ErrorF("unable to delete collector: %v", err) + catcher.Error("unable to delete collector", err, nil) return nil, status.Error(codes.Internal, fmt.Sprintf("unable to delete collector: %v", err.Error())) } @@ -87,7 +88,7 @@ func (s *Grpc) DeleteCollector(ctx context.Context, req *CollectorDelete) (*Auth delete(s.CollectorStreamMap, key) s.collectorStreamMutex.Unlock() - util.Logger.Info("Collector with key %s deleted by %s", key, req.DeletedBy) + catcher.Info("Collector was deleted by", map[string]any{"key": key, "deleted_by": req.DeletedBy}) return &AuthResponse{ Id: uint32(id), @@ -102,7 +103,7 @@ func (s *Grpc) ListCollector(ctx context.Context, req *ListRequest) (*ListCollec collectors, total, err := collectorService.ListCollectors(page, filter) if err != nil { - util.Logger.ErrorF("failed to fetch collectors: %v", err) + catcher.Error("failed to fetch collectors", err, nil) return nil, status.Errorf(codes.Internal, "failed to fetch collectors: %v", err) } return convertToCollectorResponse(collectors, total) @@ -130,7 +131,7 @@ func (s *Grpc) ProcessPendingConfigs() { if ok { collector, err := collectorService.GetByKey(key) if err != nil { - util.Logger.ErrorF("unable to get collector config to send config to stream : %v", err) + catcher.Error("unable to get collector config to send config to stream", err, nil) continue } @@ -140,7 +141,7 @@ func (s *Grpc) ProcessPendingConfigs() { }, }) if err != nil { - util.Logger.ErrorF("failed to send config to collector: %v", err) + catcher.Error("failed to send config to collector", err, nil) } } } @@ -171,7 +172,7 @@ func (s *Grpc) CollectorStream(stream CollectorService_CollectorStreamServer) er delete(s.CollectorStreamMap, collectorKey) s.collectorStreamMutex.Unlock() - util.Logger.ErrorF("failed to reconnect to client: %v", err) + catcher.Error("failed to reconnect to client", err, nil) return fmt.Errorf("failed to reconnect to client: %v", err) } @@ -195,7 +196,7 @@ func (s *Grpc) CollectorStream(stream CollectorService_CollectorStreamServer) er switch msg := in.StreamMessage.(type) { case *CollectorMessages_Result: - util.Logger.Info("Received Knowlodge: %s", msg.Result.RequestId) + catcher.Info("Received Knowledge", map[string]any{"request_id": msg.Result.RequestId}) s.pendingConfigM.Lock() if s.PendingConfigs[collectorKey] == msg.Result.RequestId { @@ -223,7 +224,7 @@ func (s *Grpc) GetCollectorConfig(ctx context.Context, in *ConfigRequest) (*Coll collector, err := collectorService.GetByKey(key) if err != nil { - util.Logger.ErrorF("unable to get collector config: %v", err) + catcher.Error("unable to get collector config", err, nil) return nil, status.Error(codes.Internal, fmt.Sprintf("unable to get collector config: %v", err.Error())) } @@ -251,7 +252,7 @@ func (s *Grpc) RegisterCollectorConfig(ctx context.Context, in *CollectorConfig) err = collectorService.SaveCollectorConfigs(collectorConf, collector.ID) if err != nil { - util.Logger.ErrorF("error saving collector configuration: %v", err) + catcher.Error("error saving collector configuration", err, nil) return nil, status.Errorf(codes.Internal, "error saving collector configuration: %v", err.Error()) } @@ -271,7 +272,7 @@ func (s *Grpc) ListCollectorHostnames(ctx context.Context, req *ListRequest) (*C hostnames, _, err := collectorService.GetHostnames(page, filter) if err != nil { - util.Logger.ErrorF("failed to fetch hostnames: %v", err) + catcher.Error("failed to fetch hostnames", err, nil) return nil, status.Errorf(codes.NotFound, "failed to fetch hostnames: %v", err) } @@ -283,7 +284,7 @@ func (s *Grpc) ListCollectorHostnames(ctx context.Context, req *ListRequest) (*C func (s *Grpc) GetCollectorsByHostnameAndModule(ctx context.Context, filter *FilterByHostAndModule) (*ListCollectorResponse, error) { collectors, err := collectorService.GetCollectorByHostnameAndModule(filter.GetHostname(), filter.GetModule().String()) if err != nil { - util.Logger.ErrorF("unable to get hostname: %v", err) + catcher.Error("unable to get hostname", err, nil) return nil, status.Errorf(codes.NotFound, "unable to get hostname: %v", err) } @@ -293,7 +294,7 @@ func (s *Grpc) GetCollectorsByHostnameAndModule(ctx context.Context, filter *Fil func (s *Grpc) LoadCollectorsCacheFromDatabase() error { collectors, err := collectorService.FindAll() if err != nil { - util.Logger.ErrorF("Failed to fetch collectors from database: %v", err) + catcher.Error("Failed to fetch collectors from database", err, nil) return err } for _, colect := range collectors { diff --git a/agent-manager/agent/ping_imp.go b/agent-manager/agent/ping_imp.go index 73a67a73c..d72948534 100644 --- a/agent-manager/agent/ping_imp.go +++ b/agent-manager/agent/ping_imp.go @@ -4,7 +4,7 @@ import ( "io" "time" - "github.com/utmstack/UTMStack/agent-manager/util" + "github.com/threatwinds/go-sdk/catcher" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -29,7 +29,7 @@ func (s *Grpc) Ping(stream PingService_PingServer) error { } err = lastSeenService.Set(key, time.Now()) if err != nil { - util.Logger.ErrorF("unable to update last seen for: %s with error:%s", key, err) + catcher.Error("unable to update", err, map[string]any{"key": key}) } } } diff --git a/agent-manager/auth/interceptor.go b/agent-manager/auth/interceptor.go index 1b9b7e883..883bf6753 100644 --- a/agent-manager/auth/interceptor.go +++ b/agent-manager/auth/interceptor.go @@ -10,9 +10,9 @@ import ( "strconv" "strings" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/agent-manager/agent" "github.com/utmstack/UTMStack/agent-manager/config" - "github.com/utmstack/UTMStack/agent-manager/util" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -98,7 +98,7 @@ func StreamInterceptor(srv interface{}, ss grpc.ServerStream, info *grpc.StreamS func checkKeyAuth(token string, id uint64, fullMethod string) error { authCache := getAuthCache(fullMethod) if authCache == nil { - util.Logger.ErrorF("unable to resolve auth cache") + catcher.Error("unable to resolve auth cache", nil, nil) return status.Error(codes.Unauthenticated, "unable to resolve auth cache") } @@ -148,19 +148,19 @@ func authenticateRequest(md metadata.MD, authName string) error { authHeader := md.Get(authName) if len(authHeader) == 0 { - util.Logger.ErrorF("%s must be provided", authName) + catcher.Error("must be provided", nil, map[string]any{"authName": authName}) return status.Error(codes.Unauthenticated, fmt.Sprintf("%s must be provided", authName)) } if authName == "connection-key" && authHeader[0] != "" { if !validateToken(authHeader[0]) { - util.Logger.ErrorF("unable to connect with the panel to check the connection-key") + catcher.Error("unable to connect with the panel to check the connection-key", nil, nil) return status.Error(codes.Unauthenticated, "unable to connect with the panel to check the connection-key") } } else if authName == "internal-key" && authHeader[0] != "" { internalKey := os.Getenv(config.UTMSharedKeyEnv) if authHeader[0] != internalKey { - util.Logger.ErrorF("internal key does not match") + catcher.Error("internal key does not match", nil, nil) return status.Error(codes.Unauthenticated, "internal key does not match") } } else { diff --git a/agent-manager/go.mod b/agent-manager/go.mod index 24c22612f..7b388198c 100644 --- a/agent-manager/go.mod +++ b/agent-manager/go.mod @@ -1,30 +1,29 @@ module github.com/utmstack/UTMStack/agent-manager -go 1.23.0 +go 1.24.2 -toolchain go1.24.2 +toolchain go1.24.6 require ( github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0 github.com/gin-contrib/gzip v1.2.3 github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 - github.com/threatwinds/logger v1.2.2 - google.golang.org/grpc v1.73.0 + google.golang.org/grpc v1.74.2 google.golang.org/protobuf v1.36.6 gorm.io/driver/postgres v1.6.0 gorm.io/gorm v1.30.0 ) require ( - github.com/bytedance/sonic v1.13.3 // indirect - github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/bytedance/sonic v1.14.0 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-playground/validator/v10 v10.27.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -33,7 +32,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -41,15 +40,15 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/threatwinds/go-sdk v1.0.45 github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.14 // indirect - golang.org/x/arch v0.18.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sync v0.15.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + github.com/ugorji/go/codec v1.3.0 // indirect + golang.org/x/arch v0.19.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250721164621-a45f3dfb1074 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/agent-manager/go.sum b/agent-manager/go.sum index 063e795a5..da76aa86f 100644 --- a/agent-manager/go.sum +++ b/agent-manager/go.sum @@ -1,10 +1,10 @@ github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0 h1:TBiBl9KCa4i4epY0/q9WSC4ugavL6+6JUkOXWDnMM6I= github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0/go.mod h1:cRhQ3TS/VEfu/z+qaciyuDZdtxgaXgaX8+G6Wa5NzBk= -github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= -github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= +github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= -github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= @@ -20,8 +20,8 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -30,8 +30,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= -github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= +github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -56,11 +56,11 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= @@ -88,48 +88,46 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/threatwinds/logger v1.2.2 h1:sVuT8yhbecPqP4tT8EwHfp1czNC6e1wdkE1ihNnuBdA= -github.com/threatwinds/logger v1.2.2/go.mod h1:Amq0QI1y7fkTpnBUgeGVu2Z/C4u4ys2pNLUOuj3UAAU= +github.com/threatwinds/go-sdk v1.0.45 h1:KZ3s3HviNRrOkg5EqjFnoauANFFzTqjNFyshPLY2SoI= +github.com/threatwinds/go-sdk v1.0.45/go.mod h1:tcWn6r6vqID/W/nL3UKfc5NafA3V/cSkiLvfJnwB58c= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.14 h1:yOQvXCBc3Ij46LRkRoh4Yd5qK6LVOgi0bYOXfb7ifjw= -github.com/ugorji/go/codec v1.2.14/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= +github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= -golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= -golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= +go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= +go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= +go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= +go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +golang.org/x/arch v0.19.0 h1:LmbDQUodHThXE+htjrnmVD73M//D9GTH6wFZjyDkjyU= +golang.org/x/arch v0.19.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= -google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250721164621-a45f3dfb1074 h1:qJW29YvkiJmXOYMu5Tf8lyrTp3dOS+K4z6IixtLaCf8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250721164621-a45f3dfb1074/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= +google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= -gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/agent-manager/main.go b/agent-manager/main.go index 85db62037..f6f5ce586 100644 --- a/agent-manager/main.go +++ b/agent-manager/main.go @@ -4,15 +4,16 @@ import ( "context" "crypto/tls" "net" + "os" _ "net/http/pprof" + "github.com/threatwinds/go-sdk/catcher" pb "github.com/utmstack/UTMStack/agent-manager/agent" "github.com/utmstack/UTMStack/agent-manager/auth" "github.com/utmstack/UTMStack/agent-manager/config" "github.com/utmstack/UTMStack/agent-manager/migration" "github.com/utmstack/UTMStack/agent-manager/updates" - "github.com/utmstack/UTMStack/agent-manager/util" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -22,28 +23,30 @@ import ( ) func main() { - util.Logger.Info("Starting UTMStack Agent Manager") + catcher.Info("Starting UTMStack Agent Manager", nil) defer func() { if r := recover(); r != nil { // Handle the panic here - util.Logger.ErrorF("Panic occurred: %v", r) + catcher.Error("Panic occurred", nil, map[string]any{"message": r}) } }() - util.Logger.Info("Initializing database...") + catcher.Info("Initializing database...", nil) config.InitDb() migration.MigrateDatabase() - util.Logger.Info("[OK] Database initialized") + catcher.Info("[OK] Database initialized", nil) s, err := pb.InitGrpc() if err != nil { - util.Logger.Fatal("Failed to inititialize gRPC: %v", err) + catcher.Error("Failed to initialize gRPC", err, nil) + os.Exit(1) } cert, err := tls.LoadX509KeyPair("/cert/utm.crt", "/cert/utm.key") if err != nil { - util.Logger.Fatal("failed to load server certificates: %v", err) + catcher.Error("failed to load server certificates", err, nil) + os.Exit(1) } tlsConfig := &tls.Config{ @@ -81,12 +84,14 @@ func main() { lis, err := net.Listen("tcp", "0.0.0.0:50051") if err != nil { - util.Logger.Fatal("Failed to listen: %v", err) + catcher.Error("Failed to listen", err, nil) + os.Exit(1) } - util.Logger.Info("Starting gRPC server on 0.0.0.0:50051") + catcher.Info("Starting gRPC server on 0.0.0.0:50051", nil) if err := grpcServer.Serve(lis); err != nil { - util.Logger.Fatal("Failed to serve: %v", err) + catcher.Error("Failed to serve", err, nil) + os.Exit(1) } } @@ -98,7 +103,7 @@ func recoverInterceptor( ) (resp interface{}, err error) { defer func() { if r := recover(); r != nil { - util.Logger.ErrorF("Panic occurred: %v", r) + catcher.Error("Panic occurred", nil, map[string]any{"message": r}) err = status.Errorf(codes.Internal, "Internal server error") } }() diff --git a/agent-manager/migration/migrations.go b/agent-manager/migration/migrations.go index a38780732..fe90ead58 100644 --- a/agent-manager/migration/migrations.go +++ b/agent-manager/migration/migrations.go @@ -4,9 +4,9 @@ import ( "fmt" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/agent-manager/config" "github.com/utmstack/UTMStack/agent-manager/models" - "github.com/utmstack/UTMStack/agent-manager/util" "gorm.io/gorm" ) @@ -21,7 +21,7 @@ func MigrateDatabase() { db := config.GetDB() err := db.AutoMigrate(&Changeset{}) if err != nil { - util.Logger.ErrorF("failed to auto-migrate MigrationRecord table: %v", err) + catcher.Error("failed to auto-migrate MigrationRecord table", err, nil) return } performMigration(db, "performInitialMigrations_15022024_001", "jdieguez89", performInitialMigrations) @@ -47,13 +47,13 @@ func performMigration(db *gorm.DB, migrationName string, executedBy string, migr if result.RowsAffected == 0 { err := migrationFunc(db) if err != nil { - util.Logger.ErrorF("Migration failed (%s): %v\n", migrationName, err) + catcher.Error("Migration failed", err, map[string]any{"migration": migrationName}) return } // Record successful migration db.Create(&Changeset{Name: migrationName, RanAt: time.Now(), ExecutedBy: executedBy}) - util.Logger.Info("Migration executed and recorded: %s\n", migrationName) + catcher.Info("Migration executed and recorded", map[string]any{"migration": migrationName}) } } @@ -61,7 +61,7 @@ func performMigration(db *gorm.DB, migrationName string, executedBy string, migr func executeSQLCommands(db *gorm.DB, sqlCommands []string) error { for _, sql := range sqlCommands { if err := db.Exec(sql).Error; err != nil { - util.Logger.ErrorF("Failed to execute SQL command: %v\n", err) + catcher.Error("Failed to execute SQL command", err, nil) return err } } @@ -76,11 +76,11 @@ func deleteColumnFromTable(db *gorm.DB, table interface{}, columnName string) er if db.Migrator().HasColumn(table, columnName) { err := db.Migrator().DropColumn(table, columnName) if err != nil { - util.Logger.ErrorF("Failed to delete column '%s' from table '%s': %v", columnName, table, err) + catcher.Error("Failed to delete column", err, map[string]any{"column": columnName, "table": table}) return err } } else { - util.Logger.ErrorF("Column '%s' does not exist in table '%s'.", columnName, table) + catcher.Error("Column does not exist", nil, map[string]any{"column": columnName, "table": table}) return nil } return nil @@ -118,11 +118,11 @@ func renameLastSeenTableAndColumnOrCreateTable(db *gorm.DB) error { oldName := "agent_last_seens" if db.Migrator().HasTable(oldName) { if err := db.Migrator().RenameTable("agent_last_seens", newName); err != nil { - util.Logger.ErrorF("Failed to rename table: %v\n", err) + catcher.Error("Failed to rename table", err, nil) return err } if err := db.Migrator().RenameColumn(&models.LastSeen{}, "agent_key", "key"); err != nil { - util.Logger.ErrorF("Failed to rename column: %v\n", err) + catcher.Error("Failed to rename column", err, nil) return err } sqlCommands := []string{ @@ -132,7 +132,7 @@ func renameLastSeenTableAndColumnOrCreateTable(db *gorm.DB) error { } err := executeSQLCommands(db, sqlCommands) if err == nil { - util.Logger.Info("Renamed table and column successfully.") + catcher.Info("Renamed table and column successfully", nil) } return err diff --git a/agent-manager/service/last_seen_service.go b/agent-manager/service/last_seen_service.go index c78f6a830..860b08025 100644 --- a/agent-manager/service/last_seen_service.go +++ b/agent-manager/service/last_seen_service.go @@ -5,9 +5,9 @@ import ( "sync" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/agent-manager/models" "github.com/utmstack/UTMStack/agent-manager/repository" - "github.com/utmstack/UTMStack/agent-manager/util" ) type LastSeenService struct { @@ -31,7 +31,7 @@ func NewLastSeenService() *LastSeenService { func (s *LastSeenService) Start() { pings, err := s.repo.GetAll() if err != nil { - util.Logger.ErrorF("Failed to populate LastSeen cache: %v", err) + catcher.Error("Failed to populate LastSeen cache", err, nil) } else { s.Populate(pings) } @@ -71,7 +71,7 @@ func (s *LastSeenService) flushCachePeriodically() { // Flush the cache to the database err := s.flushCacheToDB() if err != nil { - util.Logger.ErrorF("Failed to flush LastSeen cache to database: %v", err) + catcher.Error("Failed to flush LastSeen cache to database", err, nil) } case <-s.stopCh: return diff --git a/agent-manager/updates/updates.go b/agent-manager/updates/updates.go index 03525e72c..5b9059e70 100644 --- a/agent-manager/updates/updates.go +++ b/agent-manager/updates/updates.go @@ -6,7 +6,7 @@ import ( "github.com/gin-contrib/gzip" "github.com/gin-gonic/gin" - "github.com/utmstack/UTMStack/agent-manager/util" + "github.com/threatwinds/go-sdk/catcher" ) type Version struct { @@ -32,7 +32,7 @@ func ServeDependencies() { cert, err := tls.LoadX509KeyPair("/cert/utm.crt", "/cert/utm.key") if err != nil { - util.Logger.ErrorF("failed to load certificates: %v", err) + catcher.Error("failed to load certificates", err, nil) } tlsConfig := &tls.Config{ @@ -53,10 +53,10 @@ func ServeDependencies() { TLSConfig: tlsConfig, } - util.Logger.Info("Starting HTTP server on port 8080") + catcher.Info("Starting HTTP server on port 8080", nil) err = server.ListenAndServeTLS("", "") if err != nil { - util.Logger.ErrorF("error starting HTTP server: %v", err) + catcher.Error("error starting HTTP server", err, nil) } } diff --git a/agent-manager/util/logger.go b/agent-manager/util/logger.go deleted file mode 100644 index ef198d18f..000000000 --- a/agent-manager/util/logger.go +++ /dev/null @@ -1,31 +0,0 @@ -package util - -import ( - "log" - "os" - "strconv" - - "github.com/threatwinds/logger" -) - -var Logger *logger.Logger - -func init() { - lenv := os.Getenv("LOG_LEVEL") - var level int - var err error - - if lenv != "" && lenv != " " { - level, err = strconv.Atoi(lenv) - if err != nil { - log.Fatalln(err) - } - } else { - level = 200 - } - - Logger = logger.NewLogger(&logger.Config{ - Format: "text", - Level: level, - }) -} diff --git a/agent/modules/syslog.go b/agent/modules/syslog.go index 57bba9539..75e3a8b04 100644 --- a/agent/modules/syslog.go +++ b/agent/modules/syslog.go @@ -9,6 +9,7 @@ import ( "io" "net" "os" + "strconv" "strings" "time" @@ -19,6 +20,20 @@ import ( "github.com/utmstack/UTMStack/agent/utils" ) +const ( + MinBufferSize = 480 + RecommendedBufferSize = 2048 + MaxBufferSize = 8192 + UDPBufferSize = 2048 +) + +type FramingMethod int + +const ( + FramingNewline FramingMethod = iota + FramingOctetCounting +) + type SyslogModule struct { DataType string TCPListener listenerTCP @@ -206,7 +221,7 @@ func (m *SyslogModule) enableUDP() { m.UDPListener.Listener = listener m.UDPListener.CTX, m.UDPListener.Cancel = context.WithCancel(context.Background()) - buffer := make([]byte, 1024) + buffer := make([]byte, UDPBufferSize) msgChannel := make(chan string) go m.handleConnectionUDP(msgChannel) @@ -291,6 +306,88 @@ func (m *SyslogModule) disableUDP() { } } +// detectFramingMethod detects the syslog framing method by peeking at the first byte +func detectFramingMethod(reader *bufio.Reader) (FramingMethod, error) { + firstByte, err := reader.Peek(1) + if err != nil { + utils.Logger.ErrorF("failed to peek first byte for framing detection: %v", err) + return 0, fmt.Errorf("failed to peek first byte: %w", err) + } + + if firstByte[0] >= '0' && firstByte[0] <= '9' { + return FramingOctetCounting, nil + } + + if firstByte[0] == '<' { + return FramingNewline, nil + } + + utils.Logger.ErrorF("unknown framing method detected, first byte: 0x%02x", firstByte[0]) + return 0, fmt.Errorf("unknown framing method, first byte: 0x%02x", firstByte[0]) +} + +// readOctetCountingFrame reads a syslog message using octet counting framing method +func readOctetCountingFrame(reader *bufio.Reader) (string, error) { + lengthStr, err := reader.ReadString(' ') + if err != nil { + utils.Logger.ErrorF("failed to read message length in octet counting frame: %v", err) + return "", fmt.Errorf("failed to read message length: %w", err) + } + + lengthStr = strings.TrimSuffix(lengthStr, " ") + msgLen, err := strconv.Atoi(lengthStr) + if err != nil { + utils.Logger.ErrorF("invalid message length '%s' in octet counting frame: %v", lengthStr, err) + return "", fmt.Errorf("invalid message length '%s': %w", lengthStr, err) + } + + if msgLen < 1 { + utils.Logger.ErrorF("message length %d is too small (minimum 1 byte)", msgLen) + return "", fmt.Errorf("message length %d is too small (minimum 1)", msgLen) + } + if msgLen > MaxBufferSize { + utils.Logger.ErrorF("message length %d exceeds maximum %d bytes", msgLen, MaxBufferSize) + return "", fmt.Errorf("message length %d exceeds maximum %d", msgLen, MaxBufferSize) + } + + msgBytes := make([]byte, msgLen) + _, err = io.ReadFull(reader, msgBytes) + if err != nil { + utils.Logger.ErrorF("failed to read %d byte message body: %v", msgLen, err) + return "", fmt.Errorf("failed to read %d byte message body: %w", msgLen, err) + } + + return string(msgBytes), nil +} + +// readNewlineFrame reads a syslog message using newline-delimited framing method +func readNewlineFrame(reader *bufio.Reader) (string, error) { + message, err := reader.ReadString('\n') + if err != nil { + utils.Logger.ErrorF("failed to read newline-delimited message: %v", err) + return "", fmt.Errorf("failed to read newline-delimited message: %w", err) + } + return message, nil +} + +// readSyslogMessage reads a syslog message with automatic framing detection +func readSyslogMessage(reader *bufio.Reader) (string, error) { + method, err := detectFramingMethod(reader) + if err != nil { + return "", err + } + + switch method { + case FramingOctetCounting: + return readOctetCountingFrame(reader) + case FramingNewline: + return readNewlineFrame(reader) + default: + utils.Logger.ErrorF("unsupported framing method: %d", method) + return "", fmt.Errorf("unsupported framing method: %d", method) + } +} + func (m *SyslogModule) handleConnectionTCP(c net.Conn) { defer c.Close() reader := bufio.NewReader(c) @@ -336,12 +433,17 @@ func (m *SyslogModule) handleConnectionTCP(c net.Conn) { case <-m.TCPListener.CTX.Done(): return default: - message, err := reader.ReadString('\n') + message, err := readSyslogMessage(reader) if err != nil { - if err == io.EOF || err.(net.Error).Timeout() { + if err == io.EOF { + utils.Logger.Info("TCP connection closed by %s", remoteAddr) + return + } + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + utils.Logger.Info("TCP connection timeout from %s", remoteAddr) return } - utils.Logger.ErrorF("error reading tcp data: %v", err) + utils.Logger.ErrorF("error reading syslog message from %s: %v", remoteAddr, err) return } message = config.GetMessageFormated(remoteAddr, message) @@ -396,15 +498,17 @@ func (m *SyslogModule) handleTLSConnection(conn net.Conn) { default: // Set read timeout for each message conn.SetDeadline(time.Now().Add(30 * time.Second)) - message, err := reader.ReadString('\n') + message, err := readSyslogMessage(reader) if err != nil { if err == io.EOF { + utils.Logger.Info("TLS connection closed by %s", remoteAddr) return } if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + utils.Logger.Info("TLS connection timeout from %s", remoteAddr) return } - utils.Logger.ErrorF("error reading TLS data from %s: %v", remoteAddr, err) + utils.Logger.ErrorF("error reading syslog message from %s via TLS: %v", remoteAddr, err) return } message = config.GetMessageFormated(remoteAddr, message) diff --git a/aws/go.mod b/aws/go.mod index 653a8bb5a..52acbf539 100644 --- a/aws/go.mod +++ b/aws/go.mod @@ -1,8 +1,8 @@ module github.com/utmstack/UTMStack/aws -go 1.23.0 +go 1.24.2 -toolchain go1.24.2 +toolchain go1.24.6 require ( github.com/aws/aws-sdk-go v1.55.7 @@ -11,32 +11,33 @@ require ( ) require ( - github.com/bytedance/sonic v1.13.3 // indirect - github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/bytedance/sonic v1.14.0 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/gin-gonic/gin v1.10.1 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-playground/validator/v10 v10.27.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/threatwinds/go-sdk v1.0.45 github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.14 // indirect - golang.org/x/arch v0.18.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect + github.com/ugorji/go/codec v1.3.0 // indirect + golang.org/x/arch v0.19.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/aws/go.sum b/aws/go.sum index 3c81c9b82..739af94f6 100644 --- a/aws/go.sum +++ b/aws/go.sum @@ -2,9 +2,13 @@ github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= +github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= @@ -25,6 +29,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= +github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= @@ -41,6 +47,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= @@ -65,25 +73,39 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/threatwinds/go-sdk v1.0.45 h1:KZ3s3HviNRrOkg5EqjFnoauANFFzTqjNFyshPLY2SoI= +github.com/threatwinds/go-sdk v1.0.45/go.mod h1:tcWn6r6vqID/W/nL3UKfc5NafA3V/cSkiLvfJnwB58c= github.com/threatwinds/logger v1.2.2 h1:sVuT8yhbecPqP4tT8EwHfp1czNC6e1wdkE1ihNnuBdA= github.com/threatwinds/logger v1.2.2/go.mod h1:Amq0QI1y7fkTpnBUgeGVu2Z/C4u4ys2pNLUOuj3UAAU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.14 h1:yOQvXCBc3Ij46LRkRoh4Yd5qK6LVOgi0bYOXfb7ifjw= github.com/ugorji/go/codec v1.2.14/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= +github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/utmstack/config-client-go v1.2.7 h1:JeRdI5JjH1liNzMW3LmyevjuPd67J/yt9MAO3+oJAuM= github.com/utmstack/config-client-go v1.2.7/go.mod h1:kM0KoUizM9ZlcQp0qKviGTWn/+anT5Rfjx3zfZk79nM= golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/arch v0.19.0 h1:LmbDQUodHThXE+htjrnmVD73M//D9GTH6wFZjyDkjyU= +golang.org/x/arch v0.19.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= diff --git a/aws/main.go b/aws/main.go index a95d9bc04..659c40453 100644 --- a/aws/main.go +++ b/aws/main.go @@ -1,10 +1,12 @@ package main import ( + "os" "strings" "sync" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/aws/configuration" "github.com/utmstack/UTMStack/aws/processor" "github.com/utmstack/UTMStack/aws/utils" @@ -14,11 +16,12 @@ import ( ) func main() { - utils.Logger.Info("Starting aws module...") + catcher.Info("Starting aws module...", nil) intKey := configuration.GetInternalKey() panelServ := configuration.GetPanelServiceName() if intKey == "" || panelServ == "" { - utils.Logger.Fatal("Internal key or panel service name is not set. Exiting...") + catcher.Error("Internal key or panel service name is not set. Exiting...", nil, nil) + os.Exit(1) } client := utmconf.NewUTMClient(intKey, "http://"+panelServ) @@ -30,20 +33,20 @@ func main() { for range ticker.C { if err := utils.ConnectionChecker(configuration.URL_CHECK_CONNECTION); err != nil { - utils.Logger.ErrorF("Failed to establish connection: %v", err) + catcher.Error("Failed to establish connection", err, nil) } endTime := time.Now().UTC() - utils.Logger.Info("Syncing logs from %s to %s", startTime, endTime) + catcher.Info("Syncing logs", map[string]any{"start": startTime, "end": endTime}) moduleConfig, err := client.GetUTMConfig(enum.AWS_IAM_USER) if err != nil { if strings.Contains(err.Error(), "invalid character '<'") { - utils.Logger.LogF(100, "error getting configuration of the AWS module: backend is not available") + catcher.Error("error getting configuration of the AWS module: backend is not available", err, nil) } if strings.TrimSpace(err.Error()) != "" { - utils.Logger.ErrorF("error getting configuration of the AWS module: %v", err) + catcher.Error("error getting configuration of the AWS module", err, nil) } continue } @@ -59,7 +62,7 @@ func main() { for _, cnf := range group.Configurations { if strings.TrimSpace(cnf.ConfValue) == "" { - utils.Logger.LogF(100, "program not configured yet for group: %s", group.GroupName) + catcher.Error("program not configured yet for group", nil, map[string]any{"group": group.GroupName}) skip = true break } @@ -73,7 +76,7 @@ func main() { wg.Wait() } - utils.Logger.Info("sync completed from %v to %v, waiting 5 minutes", startTime, endTime) + catcher.Info("sync completed, waiting 5 minutes", map[string]any{"start": startTime, "end": endTime}) startTime = endTime.Add(time.Nanosecond) } } diff --git a/aws/processor/processor.go b/aws/processor/processor.go index 9768618e2..388544e9c 100644 --- a/aws/processor/processor.go +++ b/aws/processor/processor.go @@ -1,14 +1,14 @@ package processor import ( + "fmt" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" - "github.com/threatwinds/logger" - "github.com/utmstack/UTMStack/aws/utils" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/config-client-go/types" ) @@ -33,9 +33,9 @@ func GetAWSProcessor(group types.ModuleGroup) AWSProcessor { return awsPro } -func (p *AWSProcessor) createAWSSession() (*session.Session, *logger.Error) { +func (p *AWSProcessor) createAWSSession() (*session.Session, error) { if p.RegionName == "" { - return nil, utils.Logger.ErrorF("Region is not configured") + return nil, fmt.Errorf("Region is not configured") } sess, err := session.NewSession(&aws.Config{ @@ -48,13 +48,13 @@ func (p *AWSProcessor) createAWSSession() (*session.Session, *logger.Error) { ), }) if err != nil { - return nil, utils.Logger.ErrorF("error creating aws session: %v", err) + return nil, fmt.Errorf("error creating aws session: %v", err) } return sess, nil } -func (p *AWSProcessor) DescribeLogGroups() ([]string, *logger.Error) { +func (p *AWSProcessor) DescribeLogGroups() ([]string, error) { sess, sessionErr := p.createAWSSession() if sessionErr != nil { return nil, sessionErr @@ -70,13 +70,13 @@ func (p *AWSProcessor) DescribeLogGroups() ([]string, *logger.Error) { return !lastPage }) if err != nil { - return nil, utils.Logger.ErrorF("error getting log groups: %v", err) + return nil, fmt.Errorf("error getting log groups: %v", err) } return logGroups, nil } -func (p *AWSProcessor) DescribeLogStreams(logGroup string) ([]string, *logger.Error) { +func (p *AWSProcessor) DescribeLogStreams(logGroup string) ([]string, error) { sess, sessionErr := p.createAWSSession() if sessionErr != nil { return nil, sessionErr @@ -96,13 +96,13 @@ func (p *AWSProcessor) DescribeLogStreams(logGroup string) ([]string, *logger.Er return !lastPage }) if err != nil { - return nil, utils.Logger.ErrorF("error getting log streams: %v", err) + return nil, fmt.Errorf("error getting log streams: %v", err) } return logStreams, nil } -func (p *AWSProcessor) GetLogs(startTime, endTime time.Time, group types.ModuleGroup) ([]TransformedLog, *logger.Error) { +func (p *AWSProcessor) GetLogs(startTime, endTime time.Time, group types.ModuleGroup) ([]TransformedLog, error) { transformedLogs := []TransformedLog{} sess, sessionErr := p.createAWSSession() @@ -124,7 +124,7 @@ func (p *AWSProcessor) GetLogs(startTime, endTime time.Time, group types.ModuleG } for _, stream := range logStreams { - utils.Logger.Info("Processing stream %s from group %s", stream, logGroup) + catcher.Info("Processing stream", map[string]any{"stream": stream, "log_group": logGroup}) params := &cloudwatchlogs.GetLogEventsInput{ LogGroupName: aws.String(logGroup), LogStreamName: aws.String(stream), @@ -140,7 +140,7 @@ func (p *AWSProcessor) GetLogs(startTime, endTime time.Time, group types.ModuleG return !lastPage }) if err != nil { - return nil, utils.Logger.ErrorF("error getting log pages: %v", err) + return nil, catcher.Error("error getting log pages", err, nil) } } } diff --git a/aws/processor/pull.go b/aws/processor/pull.go index 11808b0a2..3516ea88c 100644 --- a/aws/processor/pull.go +++ b/aws/processor/pull.go @@ -3,24 +3,23 @@ package processor import ( "time" - "github.com/threatwinds/logger" - "github.com/utmstack/UTMStack/aws/utils" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/config-client-go/types" ) -func PullLogs(startTime time.Time, endTime time.Time, group types.ModuleGroup) *logger.Error { - utils.Logger.Info("starting log sync for : %s from %s to %s", group.GroupName, startTime, endTime) +func PullLogs(startTime time.Time, endTime time.Time, group types.ModuleGroup) error { + catcher.Info("starting log sync", map[string]any{"group": group.GroupName, "start_time": startTime, "end_time": endTime}) agent := GetAWSProcessor(group) logs, err := agent.GetLogs(startTime, endTime, group) if err != nil { - return err + return catcher.Error("error pulling logs", err, map[string]any{"group": group.GroupName}) } err = SendToLogstash(logs) if err != nil { - return err + return catcher.Error("error sending logs to logstash", err, map[string]any{"group": group.GroupName}) } return nil diff --git a/aws/processor/sendData.go b/aws/processor/sendData.go index 87092dc51..5694eb298 100644 --- a/aws/processor/sendData.go +++ b/aws/processor/sendData.go @@ -8,9 +8,8 @@ import ( "net/http" "time" - "github.com/threatwinds/logger" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/aws/configuration" - "github.com/utmstack/UTMStack/aws/utils" ) var transport = &http.Transport{ @@ -25,15 +24,15 @@ var transport = &http.Transport{ var client = &http.Client{Transport: transport, Timeout: 2 * time.Second} -func SendToLogstash(data []TransformedLog) *logger.Error { +func SendToLogstash(data []TransformedLog) error { for _, str := range data { body, err := json.Marshal(str) if err != nil { - utils.Logger.ErrorF("error encoding log to JSON: %v", err) + catcher.Error("error encoding log to JSON", err, nil) continue } if err := sendLogs(body); err != nil { - utils.Logger.ErrorF("error sending logs to logstach: %v", err) + catcher.Error("error sending logs to logstach", err, nil) continue } } @@ -45,17 +44,17 @@ func sendLogs(log []byte) error { req, err := http.NewRequest("POST", url, bytes.NewBuffer(log)) if err != nil { - return utils.Logger.ErrorF("error creating request: %v", err.Error()) + return catcher.Error("error creating request", err, nil) } resp, err := client.Do(req) if err != nil { - return utils.Logger.ErrorF("error sending logs: %v", err.Error()) + return catcher.Error("error sending logs", err, nil) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return utils.Logger.ErrorF("error sending logs with http code %d", resp.StatusCode) + return catcher.Error("error sending logs with http code", nil, map[string]any{"status_code": resp.StatusCode}) } return nil } diff --git a/aws/utils/check.go b/aws/utils/check.go index f7212deb0..8c06629d3 100644 --- a/aws/utils/check.go +++ b/aws/utils/check.go @@ -3,9 +3,14 @@ package utils import ( "fmt" "net/http" + "strings" "time" + + "github.com/threatwinds/go-sdk/catcher" ) +const wait = 3 * time.Second + func ConnectionChecker(url string) error { checkConn := func() error { if err := checkPanelConnection(url); err != nil { @@ -15,7 +20,7 @@ func ConnectionChecker(url string) error { return nil } - if err := Logger.InfiniteRetryIfXError(checkConn, "connection failed"); err != nil { + if err := infiniteRetryIfXError(checkConn, "connection failed"); err != nil { return err } @@ -40,3 +45,30 @@ func checkPanelConnection(url string) error { return nil } + +func infiniteRetryIfXError(f func() error, exception string) error { + var xErrorWasLogged bool + + for { + err := f() + if err != nil && is(err, exception) { + if !xErrorWasLogged { + _ = catcher.Error("An error occurred (%s), will keep retrying indefinitely...", err, nil) + xErrorWasLogged = true + } + time.Sleep(wait) + continue + } + + return err + } +} + +func is(e error, args ...string) bool { + for _, arg := range args { + if strings.Contains(e.Error(), arg) { + return true + } + } + return false +} diff --git a/aws/utils/env.go b/aws/utils/env.go index 846eaee2a..0332339d2 100644 --- a/aws/utils/env.go +++ b/aws/utils/env.go @@ -1,18 +1,21 @@ package utils import ( - "log" "os" + + "github.com/threatwinds/go-sdk/catcher" ) // Getenv returns the environment variable func Getenv(key string) string { value, defined := os.LookupEnv(key) if !defined { - log.Fatalf("Error loading environment variable: %s: environment variable does not exist\n", key) + catcher.Error("Error loading environment variable, environment variable does not exist", nil, map[string]any{"key": key}) + os.Exit(1) } if (value == "") || (value == " ") { - log.Fatalf("Error loading environment variable: %s: empty environment variable\n", key) + catcher.Error("Error loading environment variable, empty environment variable", nil, map[string]any{"key": key}) + os.Exit(1) } return value } diff --git a/aws/utils/logger.go b/aws/utils/logger.go deleted file mode 100644 index 4332e7f5a..000000000 --- a/aws/utils/logger.go +++ /dev/null @@ -1,31 +0,0 @@ -package utils - -import ( - "log" - "os" - "strconv" - - "github.com/threatwinds/logger" -) - -var Logger *logger.Logger - -func init() { - lenv := os.Getenv("LOG_LEVEL") - var level int - var err error - - if lenv != "" && lenv != " " { - level, err = strconv.Atoi(lenv) - if err != nil { - log.Fatalln(err) - } - } else { - level = 200 - } - - Logger = logger.NewLogger(&logger.Config{ - Format: "text", - Level: level, - }) -} diff --git a/aws/utils/req.go b/aws/utils/req.go index 7d8efcd84..3bc51ca70 100644 --- a/aws/utils/req.go +++ b/aws/utils/req.go @@ -3,18 +3,17 @@ package utils import ( "bytes" "encoding/json" + "fmt" "io" "net/http" - - "github.com/threatwinds/logger" ) -func DoReq[response any](url string, data []byte, method string, headers map[string]string) (response, int, *logger.Error) { +func DoReq[response any](url string, data []byte, method string, headers map[string]string) (response, int, error) { var result response req, err := http.NewRequest(method, url, bytes.NewBuffer(data)) if err != nil { - return result, http.StatusInternalServerError, Logger.ErrorF("%s", err.Error()) + return result, http.StatusInternalServerError, err } for k, v := range headers { @@ -25,22 +24,22 @@ func DoReq[response any](url string, data []byte, method string, headers map[str resp, err := client.Do(req) if err != nil { - return result, http.StatusInternalServerError, Logger.ErrorF("%s", err.Error()) + return result, http.StatusInternalServerError, err } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { - return result, http.StatusInternalServerError, Logger.ErrorF("%s", err.Error()) + return result, http.StatusInternalServerError, err } if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK { - return result, resp.StatusCode, Logger.ErrorF("while sending request to %s received status code: %d and response body: %s", url, resp.StatusCode, body) + return result, resp.StatusCode, fmt.Errorf("while sending request to %s received status code: %d and response body: %s", url, resp.StatusCode, body) } err = json.Unmarshal(body, &result) if err != nil { - return result, http.StatusInternalServerError, Logger.ErrorF("%s", err.Error()) + return result, http.StatusInternalServerError, err } return result, resp.StatusCode, nil diff --git a/backend/pom.xml b/backend/pom.xml index 4b2338e3c..c491a3e65 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -48,6 +48,7 @@ 2.3.3 0.21.0 1.4.2.Final + 1.18.34 3.1.0 3.9.1 @@ -160,6 +161,12 @@ ${mapstruct.version} provided + + org.projectlombok + lombok + ${lombok.version} + provided + org.springframework.boot spring-boot-configuration-processor @@ -466,6 +473,11 @@ spring-boot-configuration-processor ${spring-boot.version} + + org.projectlombok + lombok + ${lombok.version} + org.mapstruct mapstruct-processor diff --git a/backend/src/main/java/com/park/utmstack/advice/GlobalExceptionHandler.java b/backend/src/main/java/com/park/utmstack/advice/GlobalExceptionHandler.java new file mode 100644 index 000000000..ab13fef82 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/advice/GlobalExceptionHandler.java @@ -0,0 +1,68 @@ +package com.park.utmstack.advice; + + +import com.park.utmstack.security.TooMuchLoginAttemptsException; +import com.park.utmstack.service.application_events.ApplicationEventService; +import com.park.utmstack.util.UtilResponse; +import com.park.utmstack.util.exceptions.IncidentAlertConflictException; +import com.park.utmstack.util.exceptions.NoAlertsProvidedException; +import com.park.utmstack.util.exceptions.TfaVerificationException; +import com.park.utmstack.util.exceptions.TooManyRequestsException; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.authentication.BadCredentialsException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +import javax.servlet.http.HttpServletRequest; +import java.util.NoSuchElementException; + +@Slf4j +@RestControllerAdvice +@RequiredArgsConstructor +public class GlobalExceptionHandler { + + private final ApplicationEventService applicationEventService; + + @ExceptionHandler(TfaVerificationException.class) + public ResponseEntity TfaVerificationException(TfaVerificationException e, HttpServletRequest request) { + return UtilResponse.buildErrorResponse(HttpStatus.PRECONDITION_FAILED, e.getMessage()); + } + + @ExceptionHandler(BadCredentialsException.class) + public ResponseEntity handleForbidden(BadCredentialsException e, HttpServletRequest request) { + return UtilResponse.buildUnauthorizedResponse(e.getMessage()); + } + + @ExceptionHandler(TooMuchLoginAttemptsException.class) + public ResponseEntity handleTooManyLoginAttempts(TooMuchLoginAttemptsException e, HttpServletRequest request) { + return UtilResponse.buildLockedResponse(e.getMessage()); + } + + @ExceptionHandler(NoSuchElementException.class) + public ResponseEntity handleNotFound(NoSuchElementException e, HttpServletRequest request) { + return UtilResponse.buildNotFoundResponse(e.getMessage()); + } + + @ExceptionHandler(TooManyRequestsException.class) + public ResponseEntity handleTooManyRequests(TooManyRequestsException e, HttpServletRequest request) { + return UtilResponse.buildErrorResponse(HttpStatus.TOO_MANY_REQUESTS, e.getMessage()); + } + + @ExceptionHandler({NoAlertsProvidedException.class}) + public ResponseEntity handleNoAlertsProvided(Exception e, HttpServletRequest request) { + return UtilResponse.buildErrorResponse(HttpStatus.BAD_REQUEST, e.getMessage()); + } + + @ExceptionHandler(IncidentAlertConflictException.class) + public ResponseEntity handleConflict(IncidentAlertConflictException e, HttpServletRequest request) { + return UtilResponse.buildErrorResponse(HttpStatus.CONFLICT, e.getMessage()); + } + + @ExceptionHandler(Exception.class) + public ResponseEntity handleGenericException(Exception e, HttpServletRequest request) { + return UtilResponse.buildInternalServerErrorResponse(e.getMessage()); + } +} diff --git a/backend/src/main/java/com/park/utmstack/aop/logging/AuditEvent.java b/backend/src/main/java/com/park/utmstack/aop/logging/AuditEvent.java new file mode 100644 index 000000000..133077af4 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/aop/logging/AuditEvent.java @@ -0,0 +1,19 @@ +package com.park.utmstack.aop.logging; + +import com.park.utmstack.domain.application_events.enums.ApplicationEventType; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +public @interface AuditEvent { + ApplicationEventType attemptType(); + String attemptMessage(); + + ApplicationEventType successType(); + String successMessage(); +} + diff --git a/backend/src/main/java/com/park/utmstack/aop/logging/Loggable.java b/backend/src/main/java/com/park/utmstack/aop/logging/Loggable.java new file mode 100644 index 000000000..01bfe5054 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/aop/logging/Loggable.java @@ -0,0 +1,13 @@ +package com.park.utmstack.aop.logging; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +public @interface Loggable { +} + + diff --git a/backend/src/main/java/com/park/utmstack/aop/logging/NoLogException.java b/backend/src/main/java/com/park/utmstack/aop/logging/NoLogException.java new file mode 100644 index 000000000..4325d84e0 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/aop/logging/NoLogException.java @@ -0,0 +1,11 @@ +package com.park.utmstack.aop.logging; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface NoLogException {} + diff --git a/backend/src/main/java/com/park/utmstack/aop/logging/impl/AuditAspect.java b/backend/src/main/java/com/park/utmstack/aop/logging/impl/AuditAspect.java new file mode 100644 index 000000000..ef23f4f8a --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/aop/logging/impl/AuditAspect.java @@ -0,0 +1,90 @@ +package com.park.utmstack.aop.logging.impl; + +import com.park.utmstack.aop.logging.AuditEvent; +import com.park.utmstack.aop.logging.NoLogException; +import com.park.utmstack.domain.application_events.enums.ApplicationEventType; +import com.park.utmstack.domain.shared_types.ApplicationLayer; +import com.park.utmstack.loggin.LogContextBuilder; +import com.park.utmstack.service.application_events.ApplicationEventService; +import com.park.utmstack.service.dto.auditable.AuditableDTO; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.logstash.logback.argument.StructuredArguments; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.reflect.MethodSignature; +import org.slf4j.MDC; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static com.park.utmstack.config.Constants.*; + +@Aspect +@Component +@Slf4j +@RequiredArgsConstructor +public class AuditAspect { + + private final ApplicationEventService applicationEventService; + private final LogContextBuilder logContextBuilder; + + @Around("@annotation(auditEvent)") + public Object logAuditEvent(ProceedingJoinPoint joinPoint, AuditEvent auditEvent) throws Throwable { + return handleAudit(joinPoint, auditEvent.attemptType(), auditEvent.successType(), + auditEvent.attemptMessage(), auditEvent.successMessage()); + } + + private Object handleAudit(ProceedingJoinPoint joinPoint, + ApplicationEventType attemptType, + ApplicationEventType successType, + String attemptMessage, + String successMessage) throws Throwable { + + MethodSignature signature = (MethodSignature) joinPoint.getSignature(); + String context = signature.getDeclaringType().getSimpleName() + "." + signature.getMethod().getName(); + String traceId = UUID.randomUUID().toString(); + + MDC.put(TRACE_ID_KEY, traceId); + MDC.put(CONTEXT_KEY, context); + + Map extra = extractAuditData(joinPoint.getArgs()); + + extra.put(LAYER_KEY, ApplicationLayer.CONTROLLER.getValue()); + + try { + applicationEventService.createEvent(attemptMessage, attemptType, extra); + + Object result = joinPoint.proceed(); + + if (successType != ApplicationEventType.UNDEFINED) { + applicationEventService.createEvent(successMessage, successType, extra); + } + + return result; + + } catch (Exception e) { + if (!e.getClass().isAnnotationPresent(NoLogException.class)) { + String msg = String.format("%s: %s", context, e.getMessage()); + log.error(msg, e, StructuredArguments.keyValue("args", logContextBuilder.buildArgs(e))); + } + + throw e; + } + } + + private Map extractAuditData(Object[] args) { + Map extra = new HashMap<>(); + for (Object arg : args) { + if (arg instanceof AuditableDTO) { + AuditableDTO auditable = (AuditableDTO) arg; + extra.putAll(auditable.toAuditMap()); + } + } + return extra; + } +} + diff --git a/backend/src/main/java/com/park/utmstack/aop/logging/impl/LoggingMethodAspect.java b/backend/src/main/java/com/park/utmstack/aop/logging/impl/LoggingMethodAspect.java new file mode 100644 index 000000000..96c7dba37 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/aop/logging/impl/LoggingMethodAspect.java @@ -0,0 +1,39 @@ +package com.park.utmstack.aop.logging.impl; + +import com.park.utmstack.config.Constants; +import com.park.utmstack.loggin.LogContextBuilder; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.logstash.logback.argument.StructuredArguments; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.slf4j.MDC; +import org.springframework.stereotype.Component; + +@Aspect +@Component +@RequiredArgsConstructor +@Slf4j +public class LoggingMethodAspect { + private final LogContextBuilder logContextBuilder; + + @Around("@annotation(com.park.utmstack.aop.logging.Loggable)") + public Object logExecution(ProceedingJoinPoint joinPoint) throws Throwable { + String traceId = MDC.get(Constants.TRACE_ID_KEY); + String methodName = joinPoint.getSignature().toShortString(); + long start = System.currentTimeMillis(); + + try { + Object result = joinPoint.proceed(); + long duration = System.currentTimeMillis() - start; + String msg = String.format("Method %s executed successfully in %sms", methodName, duration); + log.info( msg, StructuredArguments.keyValue("args", logContextBuilder.buildArgs(methodName, String.valueOf(duration)))); + return result; + } catch (Exception ex) { + String msg = String.format("%s Method %s failed: 5s", traceId, methodName); + log.error(msg, ex, StructuredArguments.keyValue("args", logContextBuilder.buildArgs(ex))); + throw ex; + } + } +} diff --git a/backend/src/main/java/com/park/utmstack/config/Constants.java b/backend/src/main/java/com/park/utmstack/config/Constants.java index 8ef60a7c5..08cad8e12 100644 --- a/backend/src/main/java/com/park/utmstack/config/Constants.java +++ b/backend/src/main/java/com/park/utmstack/config/Constants.java @@ -124,6 +124,23 @@ public final class Constants { public static final String FRONT_BASE_URL = "https://10.21.199.3"; public static final String PDF_SERVICE_URL = "http://web-pdf:8080/generate-pdf"; + // ---------------------------------------------------------------------------------- + // Defines the index pattern for querying Elasticsearch statistics indexes. + // ---------------------------------------------------------------------------------- + public static final String STATISTICS_INDEX_PATTERN = "v11-statistics-*"; + + // Logging + public static final String TRACE_ID_KEY = "traceId"; + public static final String CONTEXT_KEY = "context"; + public static final String USERNAME_KEY = "username"; + public static final String METHOD_KEY = "method"; + public static final String PATH_KEY = "path"; + public static final String REMOTE_ADDR_KEY = "remoteAddr"; + public static final String DURATION_KEY = "duration"; + public static final String CAUSE_KEY = "cause"; + public static final String LAYER_KEY = "layer"; + + private Constants() { } } diff --git a/backend/src/main/java/com/park/utmstack/config/LoggingConfiguration.java b/backend/src/main/java/com/park/utmstack/config/LoggingConfiguration.java index 75b9a14d6..03a9aa9cb 100644 --- a/backend/src/main/java/com/park/utmstack/config/LoggingConfiguration.java +++ b/backend/src/main/java/com/park/utmstack/config/LoggingConfiguration.java @@ -3,8 +3,11 @@ import ch.qos.logback.classic.LoggerContext; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.park.utmstack.loggin.filter.MdcCleanupFilter; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import tech.jhipster.config.JHipsterProperties; @@ -45,4 +48,13 @@ public LoggingConfiguration( addContextListener(context, customFields, loggingProperties); } } + + @Bean + public FilterRegistrationBean mdcCleanupFilter() { + FilterRegistrationBean registrationBean = new FilterRegistrationBean<>(); + registrationBean.setFilter(new MdcCleanupFilter()); + registrationBean.setOrder(Integer.MAX_VALUE); + registrationBean.addUrlPatterns("/*"); + return registrationBean; + } } diff --git a/backend/src/main/java/com/park/utmstack/domain/application_events/enums/ApplicationEventType.java b/backend/src/main/java/com/park/utmstack/domain/application_events/enums/ApplicationEventType.java index 1a901c7ac..9df92c191 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_events/enums/ApplicationEventType.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_events/enums/ApplicationEventType.java @@ -1,7 +1,46 @@ package com.park.utmstack.domain.application_events.enums; public enum ApplicationEventType { + AUTH_ATTEMPT, + AUTH_SUCCESS, + AUTH_FAILURE, + TFA_CODE_SENT, + TFA_CODE_VERIFY_ATTEMPT, + TFA_VERIFIED, + AUTH_LOGOUT, + CONFIG_CHANGED, + USER_MANAGEMENT, + ACCESS_DENIED, + ALERT_UPDATE_ATTEMPT, + ALERT_UPDATE_SUCCESS, + ALERT_STATUS_UPDATE_ATTEMPT, + ALERT_STATUS_UPDATE_SUCCESS, + ALERT_NOTE_UPDATE_ATTEMPT, + ALERT_NOTE_UPDATE_SUCCESS, + ALERT_TAG_UPDATE_ATTEMPT, + ALERT_CONVERT_TO_INCIDENT_ATTEMPT, + ALERT_CONVERT_TO_INCIDENT_SUCCESS, + ALERT_TAG_UPDATE_SUCCESS, + CONFIG_GROUP_CREATE_ATTEMPT, + CONFIG_GROUP_CREATE_SUCCESS, + CONFIG_GROUP_UPDATE_ATTEMPT, + CONFIG_GROUP_UPDATE_SUCCESS, + CONFIG_GROUP_DELETE_ATTEMPT, + CONFIG_GROUP_DELETE_SUCCESS, + CONFIG_GROUP_BULK_DELETE_ATTEMPT, + CONFIG_GROUP_BULK_DELETE_SUCCESS, + CONFIG_UPDATE_ATTEMPT, + CONFIG_UPDATE_SUCCESS, + INCIDENT_CREATION_ATTEMPT, + INCIDENT_CREATED, + INCIDENT_CREATION_SUCCESS, + INCIDENT_ALERTS_ADDED, + INCIDENT_ALERT_ADD_ATTEMPT, + INCIDENT_ALERT_ADD_SUCCESS, + INCIDENT_UPDATE_ATTEMPT, + INCIDENT_UPDATE_SUCCESS, ERROR, WARNING, - INFO + INFO, + MODULE_ACTIVATION_ATTEMPT, MODULE_ACTIVATION_SUCCESS, UNDEFINED } diff --git a/backend/src/main/java/com/park/utmstack/domain/application_events/types/ApplicationEvent.java b/backend/src/main/java/com/park/utmstack/domain/application_events/types/ApplicationEvent.java index ef0195d5b..1ceadc3e4 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_events/types/ApplicationEvent.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_events/types/ApplicationEvent.java @@ -1,88 +1,21 @@ package com.park.utmstack.domain.application_events.types; import com.fasterxml.jackson.annotation.JsonProperty; - +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor public class ApplicationEvent { + @JsonProperty("@timestamp") private String timestamp; + private String source; private String message; private String type; - - public ApplicationEvent() { - } - - public ApplicationEvent(String timestamp, String source, String message, String type) { - this.timestamp = timestamp; - this.source = source; - this.message = message; - this.type = type; - } - - public String getTimestamp() { - return timestamp; - } - - public void setTimestamp(String timestamp) { - this.timestamp = timestamp; - } - - public String getSource() { - return source; - } - - public void setSource(String source) { - this.source = source; - } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder { - private String timestamp; - private String source; - private String message; - private String type; - - public ApplicationEvent build() { - return new ApplicationEvent(timestamp, source, message, type); - } - - public Builder timestamp(String timestamp) { - this.timestamp = timestamp; - return this; - } - - public Builder source(String source) { - this.source = source; - return this; - } - - public Builder message(String message) { - this.message = message; - return this; - } - - public Builder type(String type) { - this.type = type; - return this; - } - } } diff --git a/backend/src/main/java/com/park/utmstack/domain/shared_types/ApplicationLayer.java b/backend/src/main/java/com/park/utmstack/domain/shared_types/ApplicationLayer.java new file mode 100644 index 000000000..16bfac1cb --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/domain/shared_types/ApplicationLayer.java @@ -0,0 +1,13 @@ +package com.park.utmstack.domain.shared_types; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +@Getter +public enum ApplicationLayer { + SERVICE ("SERVICE"), + CONTROLLER ("CONTROLLER"); + + private final String value; +} diff --git a/backend/src/main/java/com/park/utmstack/loggin/LogContextBuilder.java b/backend/src/main/java/com/park/utmstack/loggin/LogContextBuilder.java new file mode 100644 index 000000000..cf9c34167 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/loggin/LogContextBuilder.java @@ -0,0 +1,78 @@ +package com.park.utmstack.loggin; + +import com.park.utmstack.config.Constants; +import com.park.utmstack.security.SecurityUtils; +import com.park.utmstack.util.RequestContextUtils; +import org.slf4j.MDC; +import org.springframework.stereotype.Component; + +import javax.servlet.http.HttpServletRequest; +import java.util.HashMap; +import java.util.Map; + +@Component +public class LogContextBuilder { + + public Map buildArgs(Exception e) { + return RequestContextUtils.getCurrentRequest() + .map(request -> buildArgs(e, request)) + .orElse(buildFallbackArgs(e)); + } + + public Map buildArgs() { + return RequestContextUtils.getCurrentRequest() + .map(this::buildArgs) + .orElse(buildFallbackArgs(null)); + } + + public Map buildArgs(String methodName, String duration) { + Map args = new HashMap<>(); + args.put(Constants.USERNAME_KEY, SecurityUtils.getCurrentUserLogin().orElse("anonymous")); + args.put(Constants.CONTEXT_KEY, methodName); + args.put(Constants.DURATION_KEY, duration); + args.put(Constants.TRACE_ID_KEY, MDC.get(Constants.TRACE_ID_KEY)); + return args; + } + + public Map buildArgs(HttpServletRequest request) { + Map args = new HashMap<>(); + args.put(Constants.USERNAME_KEY, SecurityUtils.getCurrentUserLogin().orElse("anonymous")); + args.put(Constants.METHOD_KEY, request.getMethod()); + args.put(Constants.PATH_KEY, request.getRequestURI()); + args.put(Constants.REMOTE_ADDR_KEY, request.getRemoteAddr()); + args.put(Constants.CONTEXT_KEY, MDC.get(Constants.CONTEXT_KEY)); + args.put(Constants.TRACE_ID_KEY, MDC.get(Constants.TRACE_ID_KEY)); + return args; + } + + public Map buildArgs(Exception e, HttpServletRequest request) { + Map args = buildArgs(request); + if (e != null && e.getCause() != null) { + args.put(Constants.CAUSE_KEY, e.getCause().toString()); + } + return args; + } + + public Map buildArgs(Map extra) { + Map base = buildArgs(); + return mergeArgs(base, extra); + } + + private Map buildFallbackArgs(Exception e) { + Map args = new HashMap<>(); + args.put(Constants.USERNAME_KEY, SecurityUtils.getCurrentUserLogin().orElse("anonymous")); + args.put(Constants.CONTEXT_KEY, MDC.get(Constants.CONTEXT_KEY)); + args.put(Constants.TRACE_ID_KEY, MDC.get(Constants.TRACE_ID_KEY)); + if (e != null && e.getCause() != null) { + args.put(Constants.CAUSE_KEY, e.getCause().toString()); + } + return args; + } + + private Map mergeArgs(Map base, Map extra) { + if (extra != null) { + base.putAll(extra); + } + return base; + } +} diff --git a/backend/src/main/java/com/park/utmstack/loggin/filter/MdcCleanupFilter.java b/backend/src/main/java/com/park/utmstack/loggin/filter/MdcCleanupFilter.java new file mode 100644 index 000000000..1e3f25735 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/loggin/filter/MdcCleanupFilter.java @@ -0,0 +1,27 @@ +package com.park.utmstack.loggin.filter; + +import org.jetbrains.annotations.NotNull; +import org.slf4j.MDC; +import org.springframework.web.filter.OncePerRequestFilter; + +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +public class MdcCleanupFilter extends OncePerRequestFilter { + + @Override + protected void doFilterInternal(@NotNull HttpServletRequest request, + @NotNull HttpServletResponse response, + FilterChain filterChain) + throws ServletException, IOException { + try { + filterChain.doFilter(request, response); + } finally { + MDC.clear(); + } + } +} + diff --git a/backend/src/main/java/com/park/utmstack/service/UserService.java b/backend/src/main/java/com/park/utmstack/service/UserService.java index dabe04022..181314181 100644 --- a/backend/src/main/java/com/park/utmstack/service/UserService.java +++ b/backend/src/main/java/com/park/utmstack/service/UserService.java @@ -213,7 +213,7 @@ public Optional updateUser(UserDTO userDTO) { }).map(UserDTO::new); } - public User updateUserTfaSecret(String userLogin, String tfaSecret) throws Exception { + public User updateUserTfaSecret(String userLogin, String tfaSecret) { final String ctx = CLASS_NAME + ".updateUserTfaSecret"; try { User user = userRepository.findOneByLogin(userLogin) @@ -221,7 +221,7 @@ public User updateUserTfaSecret(String userLogin, String tfaSecret) throws Excep user.setTfaSecret(tfaSecret); return userRepository.save(user); } catch (Exception e) { - throw new Exception(ctx + ": " + e.getMessage()); + throw new RuntimeException(ctx + ": " + e.getMessage()); } } diff --git a/backend/src/main/java/com/park/utmstack/service/application_events/ApplicationEventService.java b/backend/src/main/java/com/park/utmstack/service/application_events/ApplicationEventService.java index 83bb23176..42b185146 100644 --- a/backend/src/main/java/com/park/utmstack/service/application_events/ApplicationEventService.java +++ b/backend/src/main/java/com/park/utmstack/service/application_events/ApplicationEventService.java @@ -3,24 +3,28 @@ import com.park.utmstack.domain.application_events.enums.ApplicationEventSource; import com.park.utmstack.domain.application_events.enums.ApplicationEventType; import com.park.utmstack.domain.application_events.types.ApplicationEvent; +import com.park.utmstack.loggin.LogContextBuilder; import com.park.utmstack.service.elasticsearch.OpensearchClientBuilder; +import lombok.RequiredArgsConstructor; +import net.logstash.logback.argument.StructuredArguments; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import java.time.Instant; +import java.util.Map; @Service +@RequiredArgsConstructor public class ApplicationEventService { private static final String CLASSNAME = "ApplicationEventService"; private final Logger log = LoggerFactory.getLogger(ApplicationEventService.class); private final OpensearchClientBuilder client; + private final LogContextBuilder logContextBuilder; - public ApplicationEventService(OpensearchClientBuilder client) { - this.client = client; - } /** * Create an application event. Can be an error, warning or info @@ -41,4 +45,9 @@ public void createEvent(String message, ApplicationEventType type) { log.error(ctx + ": " + e.getMessage()); } } + + public void createEvent(String message, ApplicationEventType type, Map details) { + String msg = String.format("%s: %s", MDC.get("context"), message); + log.info( msg, StructuredArguments.keyValue("args", logContextBuilder.buildArgs(details))); + } } diff --git a/backend/src/main/java/com/park/utmstack/service/application_modules/UtmModuleQueryService.java b/backend/src/main/java/com/park/utmstack/service/application_modules/UtmModuleQueryService.java index e54de748e..d9961633a 100644 --- a/backend/src/main/java/com/park/utmstack/service/application_modules/UtmModuleQueryService.java +++ b/backend/src/main/java/com/park/utmstack/service/application_modules/UtmModuleQueryService.java @@ -3,7 +3,9 @@ import com.park.utmstack.domain.application_modules.UtmModule; import com.park.utmstack.domain.application_modules.UtmModule_; import com.park.utmstack.repository.application_modules.UtmModuleRepository; +import com.park.utmstack.service.dto.application_modules.ModuleDTO; import com.park.utmstack.service.dto.application_modules.UtmModuleCriteria; +import com.park.utmstack.service.dto.application_modules.UtmModuleMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.data.domain.Page; @@ -29,8 +31,11 @@ public class UtmModuleQueryService extends QueryService { private final UtmModuleRepository utmModuleRepository; - public UtmModuleQueryService(UtmModuleRepository utmModuleRepository) { + private final UtmModuleMapper utmModuleMapper; + + public UtmModuleQueryService(UtmModuleRepository utmModuleRepository, UtmModuleMapper utmModuleMapper) { this.utmModuleRepository = utmModuleRepository; + this.utmModuleMapper = utmModuleMapper; } /** @@ -46,6 +51,15 @@ public List findByCriteria(UtmModuleCriteria criteria) { return utmModuleRepository.findAll(specification); } + @Transactional(readOnly = true) + public ModuleDTO findById(Long id) throws Exception { + UtmModule module = utmModuleRepository.findById(id) + .orElseThrow(() -> new Exception(String.format("Module with ID: %1$s not found", id))); + + return utmModuleMapper.toDto(module, true); + } + + /** * Return a {@link Page} of {@link UtmModule} which matches the criteria from the database * @@ -54,10 +68,12 @@ public List findByCriteria(UtmModuleCriteria criteria) { * @return the matching entities. */ @Transactional(readOnly = true) - public Page findByCriteria(UtmModuleCriteria criteria, Pageable page) { + public Page findByCriteria(UtmModuleCriteria criteria, Pageable page) { log.debug("find by criteria : {}, page: {}", criteria, page); final Specification specification = createSpecification(criteria); - return utmModuleRepository.findAll(specification, page); + Page data = utmModuleRepository.findAll(specification, page); + + return data.map(m -> utmModuleMapper.toDto(m, false)); } /** diff --git a/backend/src/main/java/com/park/utmstack/service/application_modules/UtmModuleService.java b/backend/src/main/java/com/park/utmstack/service/application_modules/UtmModuleService.java index 1f51aebe1..518db27bc 100644 --- a/backend/src/main/java/com/park/utmstack/service/application_modules/UtmModuleService.java +++ b/backend/src/main/java/com/park/utmstack/service/application_modules/UtmModuleService.java @@ -8,6 +8,7 @@ import com.park.utmstack.repository.UtmModuleGroupRepository; import com.park.utmstack.repository.application_modules.UtmModuleRepository; import com.park.utmstack.service.UtmMenuService; +import com.park.utmstack.service.dto.application_modules.ModuleActivationDTO; import com.park.utmstack.service.index_pattern.UtmIndexPatternService; import com.park.utmstack.service.logstash_filter.UtmLogstashFilterService; import org.slf4j.Logger; @@ -53,14 +54,17 @@ public UtmModuleService(UtmModuleRepository moduleRepository, /** * Activate or deactivate the module requested * - * @param nameShort Short name of the module - * @param activationStatus Activation status + * @param moduleActivationDTO The module activation information * @return The current module information - * @throws Exception In case of any error + * @throws NoSuchElementException In case the module definition is not found for the server */ - public UtmModule activateDeactivate(Long serverId, ModuleName nameShort, Boolean activationStatus) throws Exception { + public UtmModule activateDeactivate(ModuleActivationDTO moduleActivationDTO) { final String ctx = CLASSNAME + ".activateDeactivate"; try { + long serverId = moduleActivationDTO.getServerId(); + ModuleName nameShort = moduleActivationDTO.getModuleName(); + boolean activationStatus = moduleActivationDTO.getActivationStatus(); + UtmModule module = moduleRepository.findByServerIdAndModuleName(serverId, nameShort); if (Objects.isNull(module)) @@ -79,7 +83,7 @@ public UtmModule activateDeactivate(Long serverId, ModuleName nameShort, Boolean enableDisableModuleFilter(nameShort, activationStatus); return module; } catch (Exception e) { - throw new Exception(ctx + ": " + e.getMessage()); + throw new RuntimeException(ctx + ": " + e.getMessage()); } } diff --git a/backend/src/main/java/com/park/utmstack/service/collectors/CollectorOpsService.java b/backend/src/main/java/com/park/utmstack/service/collectors/CollectorOpsService.java index 6a747cac6..7e0935aea 100644 --- a/backend/src/main/java/com/park/utmstack/service/collectors/CollectorOpsService.java +++ b/backend/src/main/java/com/park/utmstack/service/collectors/CollectorOpsService.java @@ -28,6 +28,7 @@ import com.park.utmstack.security.SecurityUtils; import com.park.utmstack.service.application_modules.UtmModuleGroupService; import com.park.utmstack.service.application_modules.UtmModuleService; +import com.park.utmstack.service.dto.application_modules.ModuleActivationDTO; import com.park.utmstack.service.dto.collectors.CollectorModuleEnum; import com.park.utmstack.service.dto.collectors.dto.ListCollectorsResponseDTO; import com.park.utmstack.service.dto.collectors.dto.CollectorDTO; @@ -481,7 +482,11 @@ public void deleteCollector(Long id) throws Exception { .collect(Collectors.toList()); if(modules.isEmpty()){ - this.utmModuleService.activateDeactivate(module.getServerId(), module.getModuleName(), false); + this.utmModuleService.activateDeactivate(ModuleActivationDTO.builder() + .serverId(module.getServerId()) + .moduleName(module.getModuleName()) + .activationStatus(false) + .build()); } } } diff --git a/backend/src/main/java/com/park/utmstack/service/dto/alert/ConvertToIncidentRequestBody.java b/backend/src/main/java/com/park/utmstack/service/dto/alert/ConvertToIncidentRequestBody.java new file mode 100644 index 000000000..4f54e071c --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/alert/ConvertToIncidentRequestBody.java @@ -0,0 +1,38 @@ +package com.park.utmstack.service.dto.alert; + +import com.park.utmstack.service.dto.auditable.AuditableDTO; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; +import java.util.List; +import java.util.Map; + +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +public class ConvertToIncidentRequestBody implements AuditableDTO { + @NotNull + private List eventIds; + @NotNull + @Pattern(regexp = "^[^\"]*$", message = "Double quotes are not allowed") + private String incidentName; + @NotNull + private Integer incidentId; + @NotNull + private String incidentSource; + + @Override + public Map toAuditMap() { + return Map.of( + "eventIds", eventIds, + "incidentName", incidentName, + "incidentId", incidentId, + "incidentSource", incidentSource + ); + } +} diff --git a/backend/src/main/java/com/park/utmstack/service/dto/alert/UpdateAlertStatusRequestBody.java b/backend/src/main/java/com/park/utmstack/service/dto/alert/UpdateAlertStatusRequestBody.java new file mode 100644 index 000000000..8740293c6 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/alert/UpdateAlertStatusRequestBody.java @@ -0,0 +1,32 @@ +package com.park.utmstack.service.dto.alert; + +import com.park.utmstack.service.dto.auditable.AuditableDTO; +import lombok.*; + +import javax.validation.constraints.NotNull; +import java.util.List; +import java.util.Map; + +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +public class UpdateAlertStatusRequestBody implements AuditableDTO { + @NotNull + private List alertIds; + private String statusObservation; + @NotNull + private int status; + boolean addFalsePositiveTag; + + + @Override + public Map toAuditMap() { + return Map.of( + "alertIds", alertIds, + "statusObservation", statusObservation, + "status", status, + "addFalsePositiveTag", addFalsePositiveTag + ); + } +} diff --git a/backend/src/main/java/com/park/utmstack/service/dto/alert/UpdateAlertTagsRequestBody.java b/backend/src/main/java/com/park/utmstack/service/dto/alert/UpdateAlertTagsRequestBody.java new file mode 100644 index 000000000..898aef6bf --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/alert/UpdateAlertTagsRequestBody.java @@ -0,0 +1,31 @@ +package com.park.utmstack.service.dto.alert; + +import com.park.utmstack.service.dto.auditable.AuditableDTO; +import lombok.*; + +import javax.validation.constraints.NotNull; +import java.util.List; +import java.util.Map; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class UpdateAlertTagsRequestBody implements AuditableDTO { + + @NotNull + private List alertIds; + + private List tags; + @NotNull + private Boolean createRule; + + @Override + public Map toAuditMap() { + return Map.of( + "alertIds", alertIds, + "tags", tags, + "createRule", createRule + ); + } +} diff --git a/backend/src/main/java/com/park/utmstack/service/dto/application_modules/CheckRequirementsResponse.java b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/CheckRequirementsResponse.java new file mode 100644 index 000000000..c97cfa797 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/CheckRequirementsResponse.java @@ -0,0 +1,20 @@ +package com.park.utmstack.service.dto.application_modules; + +import com.park.utmstack.domain.application_modules.enums.ModuleRequirementStatus; +import com.park.utmstack.domain.application_modules.types.ModuleRequirement; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.util.List; + +@Setter +@Getter +@AllArgsConstructor +@NoArgsConstructor +public class CheckRequirementsResponse { + private ModuleRequirementStatus status; + private List checks; + +} diff --git a/backend/src/main/java/com/park/utmstack/service/dto/application_modules/GroupConfigurationDTO.java b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/GroupConfigurationDTO.java new file mode 100644 index 000000000..cdd146f5f --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/GroupConfigurationDTO.java @@ -0,0 +1,16 @@ +package com.park.utmstack.service.dto.application_modules; + +import com.park.utmstack.domain.application_modules.UtmModuleGroupConfiguration; +import lombok.Data; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import java.util.List; + +@Data +public class GroupConfigurationDTO { + @NotNull + private Long moduleId; + @NotEmpty + private List keys; +} diff --git a/backend/src/main/java/com/park/utmstack/service/dto/application_modules/ModuleActivationDTO.java b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/ModuleActivationDTO.java new file mode 100644 index 000000000..3b836a05f --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/ModuleActivationDTO.java @@ -0,0 +1,27 @@ +package com.park.utmstack.service.dto.application_modules; + +import com.park.utmstack.domain.application_modules.enums.ModuleName; +import com.park.utmstack.service.dto.auditable.AuditableDTO; +import lombok.*; + +import java.util.Map; + +@Getter +@Setter +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ModuleActivationDTO implements AuditableDTO { + private Long serverId; + private ModuleName moduleName; + private Boolean activationStatus; + + @Override + public Map toAuditMap() { + return Map.of( + "serverId", serverId, + "moduleName", moduleName != null ? moduleName.name() : null, + "activationStatus", activationStatus + ); + } +} diff --git a/backend/src/main/java/com/park/utmstack/service/dto/application_modules/ModuleDTO.java b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/ModuleDTO.java new file mode 100644 index 000000000..c6190c0c7 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/ModuleDTO.java @@ -0,0 +1,45 @@ +package com.park.utmstack.service.dto.application_modules; + +import com.park.utmstack.domain.UtmServer; +import com.park.utmstack.domain.application_modules.UtmModuleGroup; +import com.park.utmstack.domain.application_modules.enums.ModuleName; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; + +import java.util.HashSet; +import java.util.Set; + +@Getter +@Setter +@RequiredArgsConstructor +@AllArgsConstructor +public class ModuleDTO { + private Long id; + + private Long serverId; + + private String prettyName; + + private ModuleName moduleName; + + private String moduleDescription; + + private Boolean moduleActive; + + private String moduleIcon; + + private String moduleCategory; + + private Boolean liteVersion; + + private Boolean needsRestart; + + private Boolean isActivatable; + + private UtmServer server; + + private Set moduleGroups = new HashSet<>(); + +} diff --git a/backend/src/main/java/com/park/utmstack/service/dto/application_modules/UtmModuleCriteria.java b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/UtmModuleCriteria.java index 692702328..478cada40 100644 --- a/backend/src/main/java/com/park/utmstack/service/dto/application_modules/UtmModuleCriteria.java +++ b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/UtmModuleCriteria.java @@ -1,6 +1,8 @@ package com.park.utmstack.service.dto.application_modules; import com.park.utmstack.domain.application_modules.enums.ModuleName; +import lombok.Getter; +import lombok.Setter; import tech.jhipster.service.filter.BooleanFilter; import tech.jhipster.service.filter.Filter; import tech.jhipster.service.filter.LongFilter; @@ -16,6 +18,8 @@ * As Spring is unable to properly convert the types, unless specific {@link Filter} class are used, we need to use * fix type specific filters. */ +@Setter +@Getter public class UtmModuleCriteria implements Serializable { /** @@ -36,75 +40,4 @@ public static class ModulesNameShortFilter extends Filter { private BooleanFilter isActivatable; private StringFilter prettyName; - public LongFilter getId() { - return id; - } - - public void setId(LongFilter id) { - this.id = id; - } - - public LongFilter getServerId() { - return serverId; - } - - public void setServerId(LongFilter serverId) { - this.serverId = serverId; - } - - public ModulesNameShortFilter getModuleName() { - return moduleName; - } - - public void setModuleName(ModulesNameShortFilter moduleName) { - this.moduleName = moduleName; - } - - public BooleanFilter getModuleActive() { - return moduleActive; - } - - public void setModuleActive(BooleanFilter moduleActive) { - this.moduleActive = moduleActive; - } - - public StringFilter getModuleCategory() { - return moduleCategory; - } - - public void setModuleCategory(StringFilter moduleCategory) { - this.moduleCategory = moduleCategory; - } - - public BooleanFilter getNeedsRestart() { - return needsRestart; - } - - public void setNeedsRestart(BooleanFilter needsRestart) { - this.needsRestart = needsRestart; - } - - public BooleanFilter getLiteVersion() { - return liteVersion; - } - - public void setLiteVersion(BooleanFilter liteVersion) { - this.liteVersion = liteVersion; - } - - public StringFilter getPrettyName() { - return prettyName; - } - - public void setPrettyName(StringFilter prettyName) { - this.prettyName = prettyName; - } - - public BooleanFilter getIsActivatable() { - return isActivatable; - } - - public void setIsActivatable(BooleanFilter isActivatable) { - this.isActivatable = isActivatable; - } } diff --git a/backend/src/main/java/com/park/utmstack/service/dto/application_modules/UtmModuleGroupConfDTO.java b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/UtmModuleGroupConfDTO.java new file mode 100644 index 000000000..1a4b88ba3 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/UtmModuleGroupConfDTO.java @@ -0,0 +1,15 @@ +package com.park.utmstack.service.dto.application_modules; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class UtmModuleGroupConfDTO { + private String confKey; + private String confValue; +} + diff --git a/backend/src/main/java/com/park/utmstack/service/dto/application_modules/UtmModuleGroupConfWrapperDTO.java b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/UtmModuleGroupConfWrapperDTO.java new file mode 100644 index 000000000..a10923f8b --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/UtmModuleGroupConfWrapperDTO.java @@ -0,0 +1,15 @@ +package com.park.utmstack.service.dto.application_modules; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class UtmModuleGroupConfWrapperDTO { + private List moduleGroupConfigurations; +} + diff --git a/backend/src/main/java/com/park/utmstack/service/dto/application_modules/UtmModuleMapper.java b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/UtmModuleMapper.java new file mode 100644 index 000000000..9dd055335 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/application_modules/UtmModuleMapper.java @@ -0,0 +1,41 @@ +package com.park.utmstack.service.dto.application_modules; + +import com.park.utmstack.domain.application_modules.UtmModule; +import com.park.utmstack.domain.logstash_filter.UtmLogstashFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +@Component +public class UtmModuleMapper { + + private final Logger logger = LoggerFactory.getLogger(UtmModuleMapper.class); + + public ModuleDTO toDto(UtmModule entity, boolean includeDataType) { + ModuleDTO dto = new ModuleDTO(); + dto.setId(entity.getId()); + dto.setServerId(entity.getServerId()); + dto.setPrettyName(entity.getPrettyName()); + dto.setModuleName(entity.getModuleName()); + dto.setModuleDescription(entity.getModuleDescription()); + dto.setModuleActive(entity.getModuleActive()); + dto.setModuleIcon(entity.getModuleIcon()); + dto.setModuleCategory(entity.getModuleCategory()); + dto.setLiteVersion(entity.getLiteVersion()); + dto.setNeedsRestart(entity.getNeedsRestart()); + dto.setIsActivatable(entity.getActivatable()); + dto.setModuleGroups(entity.getModuleGroups()); + + return dto; + } + + public List toListDTO(List modules) { + return modules.stream() + .map((m) -> this.toDto(m, false)) + .collect(Collectors.toList()); + } +} diff --git a/backend/src/main/java/com/park/utmstack/service/dto/auditable/AuditableDTO.java b/backend/src/main/java/com/park/utmstack/service/dto/auditable/AuditableDTO.java new file mode 100644 index 000000000..60f04c0bd --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/auditable/AuditableDTO.java @@ -0,0 +1,7 @@ +package com.park.utmstack.service.dto.auditable; + +import java.util.Map; + +public interface AuditableDTO { + Map toAuditMap(); +} diff --git a/backend/src/main/java/com/park/utmstack/service/dto/auth/JWTToken.java b/backend/src/main/java/com/park/utmstack/service/dto/auth/JWTToken.java new file mode 100644 index 000000000..6a654fab0 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/auth/JWTToken.java @@ -0,0 +1,22 @@ +package com.park.utmstack.service.dto.auth; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class JWTToken { + + private String idToken; + private boolean authenticated; + + @JsonProperty("id_token") + String getIdToken() { + return idToken; + } +} diff --git a/backend/src/main/java/com/park/utmstack/service/dto/collectors/CollectorHostnames.java b/backend/src/main/java/com/park/utmstack/service/dto/collectors/CollectorHostnames.java new file mode 100644 index 000000000..08870817a --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/collectors/CollectorHostnames.java @@ -0,0 +1,15 @@ +package com.park.utmstack.service.dto.collectors; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +@Setter +@NoArgsConstructor +public class CollectorHostnames { + public List hostname = new ArrayList(); +} diff --git a/backend/src/main/java/com/park/utmstack/service/dto/collectors/CollectorModuleEnum.java b/backend/src/main/java/com/park/utmstack/service/dto/collectors/CollectorModuleEnum.java index 8c4bc09a9..14aa6ed2b 100644 --- a/backend/src/main/java/com/park/utmstack/service/dto/collectors/CollectorModuleEnum.java +++ b/backend/src/main/java/com/park/utmstack/service/dto/collectors/CollectorModuleEnum.java @@ -1,5 +1,6 @@ package com.park.utmstack.service.dto.collectors; public enum CollectorModuleEnum { - AS_400; + AS_400, + UTMSTACK } diff --git a/backend/src/main/java/com/park/utmstack/service/dto/collectors/dto/CollectorConfigDTO.java b/backend/src/main/java/com/park/utmstack/service/dto/collectors/dto/CollectorConfigDTO.java deleted file mode 100644 index a37170c12..000000000 --- a/backend/src/main/java/com/park/utmstack/service/dto/collectors/dto/CollectorConfigDTO.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.park.utmstack.service.dto.collectors.dto; - -import com.park.utmstack.web.rest.application_modules.UtmModuleGroupConfigurationResource; - -public class CollectorConfigDTO { - CollectorDTO collector; - - public CollectorDTO getCollector() { - return collector; - } - - public void setCollector(CollectorDTO collector) { - this.collector = collector; - } - - public UtmModuleGroupConfigurationResource.UpdateConfigurationKeysBody getCollectorConfig() { - return collectorConfig; - } - - public void setCollectorConfig(UtmModuleGroupConfigurationResource.UpdateConfigurationKeysBody collectorConfig) { - this.collectorConfig = collectorConfig; - } - - UtmModuleGroupConfigurationResource.UpdateConfigurationKeysBody collectorConfig; - - -} diff --git a/backend/src/main/java/com/park/utmstack/service/dto/collectors/dto/CollectorConfigKeysDTO.java b/backend/src/main/java/com/park/utmstack/service/dto/collectors/dto/CollectorConfigKeysDTO.java new file mode 100644 index 000000000..40af52707 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/collectors/dto/CollectorConfigKeysDTO.java @@ -0,0 +1,39 @@ +package com.park.utmstack.service.dto.collectors.dto; + +import com.park.utmstack.domain.application_modules.UtmModuleGroupConfiguration; + +import javax.validation.constraints.NotNull; +import java.util.List; + +public class CollectorConfigKeysDTO { + @NotNull + CollectorDTO collector; + @NotNull + private Long moduleId; + @NotNull + private List keys; + + public Long getModuleId() { + return moduleId; + } + + public void setModuleId(Long moduleId) { + this.moduleId = moduleId; + } + + public List getKeys() { + return keys; + } + + public void setKeys(List keys) { + this.keys = keys; + } + + public CollectorDTO getCollector() { + return collector; + } + + public void setCollector(CollectorDTO collector) { + this.collector = collector; + } +} diff --git a/backend/src/main/java/com/park/utmstack/service/dto/collectors/dto/ErrorResponse.java b/backend/src/main/java/com/park/utmstack/service/dto/collectors/dto/ErrorResponse.java new file mode 100644 index 000000000..557e40568 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/service/dto/collectors/dto/ErrorResponse.java @@ -0,0 +1,22 @@ +package com.park.utmstack.service.dto.collectors.dto; + +import org.springframework.http.HttpStatus; + +public class ErrorResponse { + private String message; + private HttpStatus status; + + public ErrorResponse(String message, HttpStatus status) { + this.message = message; + this.status = status; + } + + public String getMessage() { + return message; + } + + public HttpStatus getStatus() { + return status; + } +} + diff --git a/backend/src/main/java/com/park/utmstack/service/incident/UtmIncidentService.java b/backend/src/main/java/com/park/utmstack/service/incident/UtmIncidentService.java index 03546f259..94c5ae5d7 100644 --- a/backend/src/main/java/com/park/utmstack/service/incident/UtmIncidentService.java +++ b/backend/src/main/java/com/park/utmstack/service/incident/UtmIncidentService.java @@ -17,6 +17,7 @@ import com.park.utmstack.service.dto.incident.NewIncidentDTO; import com.park.utmstack.service.dto.incident.RelatedIncidentAlertsDTO; import com.park.utmstack.service.incident.util.ResolveIncidentStatus; +import com.park.utmstack.util.exceptions.IncidentAlertConflictException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.data.domain.Page; @@ -27,10 +28,7 @@ import org.springframework.util.CollectionUtils; import javax.validation.Valid; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.Optional; +import java.util.*; import java.util.stream.Collectors; /** @@ -76,7 +74,6 @@ public UtmIncidentService(UtmIncidentRepository utmIncidentRepository, */ public UtmIncident save(UtmIncident utmIncident) { final String ctx = ".save"; - log.debug("Request to save UtmIncident : {}", utmIncident); try { return utmIncidentRepository.save(utmIncident); } catch (Exception e) { @@ -140,6 +137,8 @@ public UtmIncident changeStatus(UtmIncident utmIncident) { public UtmIncident createIncident(NewIncidentDTO newIncidentDTO) { final String ctx = CLASSNAME + ".createIncident"; try { + validateAlertsNotAlreadyLinked(newIncidentDTO.getAlertList(), ctx); + UtmIncident utmIncident = new UtmIncident(); utmIncident.setIncidentName(newIncidentDTO.getIncidentName()); utmIncident.setIncidentDescription(newIncidentDTO.getIncidentDescription()); @@ -177,10 +176,25 @@ public UtmIncident addAlertsIncident(@Valid AddToIncidentDTO addToIncidentDTO) { final String ctx = CLASSNAME + ".addAlertsIncident"; try { log.debug("Request to add alert to UtmIncident : {}", addToIncidentDTO); + + List alertIds = addToIncidentDTO.getAlertList(); + + String alertsIds = alertIds.stream().map(RelatedIncidentAlertsDTO::getAlertId).collect(Collectors.joining(",")); + Map extra = Map.of( + "alertIds", alertsIds, + "source", "service" + ); + String attemptMsg = String.format("Attempt to add %d alerts to incident %d", addToIncidentDTO.getAlertList().size(), addToIncidentDTO.getIncidentId()); + eventService.createEvent(attemptMsg, ApplicationEventType.INCIDENT_ALERT_ADD_ATTEMPT, extra); + + validateAlertsNotAlreadyLinked(addToIncidentDTO.getAlertList(), ctx); UtmIncident utmIncident = utmIncidentRepository.findById(addToIncidentDTO.getIncidentId()).orElseThrow(() -> new RuntimeException(ctx + ": Incident not found")); saveRelatedAlerts(addToIncidentDTO.getAlertList(), utmIncident.getId()); String historyMessage = String.format("New %d alerts added to incident", addToIncidentDTO.getAlertList().size()); utmIncidentHistoryService.createHistory(IncidentHistoryActionEnum.INCIDENT_ALERT_ADD, utmIncident.getId(), "New alerts added to incident", historyMessage); + + eventService.createEvent(historyMessage, ApplicationEventType.INCIDENT_ALERTS_ADDED, extra); + return utmIncident; } catch (Exception e) { String msg = ctx + ": " + e.getMessage(); @@ -284,4 +298,20 @@ private void sendIncidentsEmail(List alertIds, UtmIncident utmIncident) eventService.createEvent(msg, ApplicationEventType.ERROR); } } + + private void validateAlertsNotAlreadyLinked(List alertList, String ctx) { + + List alertIds = alertList.stream() + .map(RelatedIncidentAlertsDTO::getAlertId) + .collect(Collectors.toList()); + + List alertsFound = utmIncidentAlertService.existsAnyAlert(alertIds); + + if (!alertsFound.isEmpty()) { + String alertIdsList = String.join(", ", alertIds); + String msg = "Some alerts are already linked to another incident. Alert IDs: " + alertIdsList + ". Check the related incidents for more details."; + + throw new IncidentAlertConflictException(ctx + ": " + msg); + } + } } diff --git a/backend/src/main/java/com/park/utmstack/service/validators/collector/CollectorValidatorService.java b/backend/src/main/java/com/park/utmstack/service/validators/collector/CollectorValidatorService.java index 3c1866bdb..ffc3156ac 100644 --- a/backend/src/main/java/com/park/utmstack/service/validators/collector/CollectorValidatorService.java +++ b/backend/src/main/java/com/park/utmstack/service/validators/collector/CollectorValidatorService.java @@ -1,7 +1,7 @@ package com.park.utmstack.service.validators.collector; import com.park.utmstack.domain.application_modules.UtmModuleGroupConfiguration; -import com.park.utmstack.web.rest.application_modules.UtmModuleGroupConfigurationResource; +import com.park.utmstack.service.dto.collectors.dto.CollectorConfigKeysDTO; import org.springframework.stereotype.Service; import org.springframework.validation.Errors; import org.springframework.validation.Validator; @@ -13,13 +13,12 @@ public class CollectorValidatorService implements Validator { @Override public boolean supports(Class clazz) { - return UtmModuleGroupConfigurationResource.UpdateConfigurationKeysBody.class.equals(clazz); + return CollectorConfigKeysDTO.class.equals(clazz); } @Override public void validate(Object target, Errors errors) { - UtmModuleGroupConfigurationResource.UpdateConfigurationKeysBody updateConfigurationKeysBody = - (UtmModuleGroupConfigurationResource.UpdateConfigurationKeysBody) target; + CollectorConfigKeysDTO updateConfigurationKeysBody = (CollectorConfigKeysDTO) target; Map hostNames = updateConfigurationKeysBody.getKeys().stream() .filter(config -> config.getConfName().equals("Hostname")) diff --git a/backend/src/main/java/com/park/utmstack/util/RequestContextUtils.java b/backend/src/main/java/com/park/utmstack/util/RequestContextUtils.java new file mode 100644 index 000000000..5e11eddaa --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/util/RequestContextUtils.java @@ -0,0 +1,14 @@ +package com.park.utmstack.util; + +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; + +import javax.servlet.http.HttpServletRequest; +import java.util.Optional; + +public class RequestContextUtils { + public static Optional getCurrentRequest() { + ServletRequestAttributes attrs = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); + return Optional.ofNullable(attrs).map(ServletRequestAttributes::getRequest); + } +} diff --git a/backend/src/main/java/com/park/utmstack/util/UtilResponse.java b/backend/src/main/java/com/park/utmstack/util/UtilResponse.java index d5d1862be..ab51a0c25 100644 --- a/backend/src/main/java/com/park/utmstack/util/UtilResponse.java +++ b/backend/src/main/java/com/park/utmstack/util/UtilResponse.java @@ -32,6 +32,10 @@ public static ResponseEntity buildPreconditionFailedResponse(String msg) return buildErrorResponse(HttpStatus.PRECONDITION_FAILED, msg); } + public static ResponseEntity buildTooManyRequestResponse(String msg) { + return buildErrorResponse(HttpStatus.TOO_MANY_REQUESTS, msg); + } + public static ResponseEntity buildNotFoundResponse(String msg) { return buildErrorResponse(HttpStatus.NOT_FOUND, msg); } diff --git a/backend/src/main/java/com/park/utmstack/util/exceptions/ElasticsearchIndexDocumentUpdateException.java b/backend/src/main/java/com/park/utmstack/util/exceptions/ElasticsearchIndexDocumentUpdateException.java index 195ef2af7..61cad6d0a 100644 --- a/backend/src/main/java/com/park/utmstack/util/exceptions/ElasticsearchIndexDocumentUpdateException.java +++ b/backend/src/main/java/com/park/utmstack/util/exceptions/ElasticsearchIndexDocumentUpdateException.java @@ -1,7 +1,7 @@ package com.park.utmstack.util.exceptions; -public class ElasticsearchIndexDocumentUpdateException extends Exception { +public class ElasticsearchIndexDocumentUpdateException extends RuntimeException { public ElasticsearchIndexDocumentUpdateException(String message) { super(message); } -} +} \ No newline at end of file diff --git a/backend/src/main/java/com/park/utmstack/util/exceptions/IncidentAlertConflictException.java b/backend/src/main/java/com/park/utmstack/util/exceptions/IncidentAlertConflictException.java new file mode 100644 index 000000000..c5aa47586 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/util/exceptions/IncidentAlertConflictException.java @@ -0,0 +1,7 @@ +package com.park.utmstack.util.exceptions; + +public class IncidentAlertConflictException extends RuntimeException { + public IncidentAlertConflictException(String message) { + super(message); + } +} diff --git a/backend/src/main/java/com/park/utmstack/util/exceptions/InvalidConnectionKeyException.java b/backend/src/main/java/com/park/utmstack/util/exceptions/InvalidConnectionKeyException.java index 00a557cd1..6bf45de7f 100644 --- a/backend/src/main/java/com/park/utmstack/util/exceptions/InvalidConnectionKeyException.java +++ b/backend/src/main/java/com/park/utmstack/util/exceptions/InvalidConnectionKeyException.java @@ -1,6 +1,6 @@ package com.park.utmstack.util.exceptions; -public class InvalidConnectionKeyException extends Exception { +public class InvalidConnectionKeyException extends RuntimeException { public InvalidConnectionKeyException(String message) { super(message); } diff --git a/backend/src/main/java/com/park/utmstack/util/exceptions/InvalidTfaStageException.java b/backend/src/main/java/com/park/utmstack/util/exceptions/InvalidTfaStageException.java new file mode 100644 index 000000000..207554ca9 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/util/exceptions/InvalidTfaStageException.java @@ -0,0 +1,8 @@ +package com.park.utmstack.util.exceptions; + +public class InvalidTfaStageException extends RuntimeException { + public InvalidTfaStageException(String message) { + super(message); + } +} + diff --git a/backend/src/main/java/com/park/utmstack/util/exceptions/NoAlertsProvidedException.java b/backend/src/main/java/com/park/utmstack/util/exceptions/NoAlertsProvidedException.java new file mode 100644 index 000000000..9f8c7b22e --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/util/exceptions/NoAlertsProvidedException.java @@ -0,0 +1,7 @@ +package com.park.utmstack.util.exceptions; + +public class NoAlertsProvidedException extends RuntimeException { + public NoAlertsProvidedException(String message) { + super(message); + } +} diff --git a/backend/src/main/java/com/park/utmstack/util/exceptions/TfaVerificationException.java b/backend/src/main/java/com/park/utmstack/util/exceptions/TfaVerificationException.java new file mode 100644 index 000000000..68e73cc58 --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/util/exceptions/TfaVerificationException.java @@ -0,0 +1,8 @@ +package com.park.utmstack.util.exceptions; + +public class TfaVerificationException extends RuntimeException { + public TfaVerificationException(String message) { + super(message); + } +} + diff --git a/backend/src/main/java/com/park/utmstack/util/exceptions/TooManyRequestsException.java b/backend/src/main/java/com/park/utmstack/util/exceptions/TooManyRequestsException.java new file mode 100644 index 000000000..0ca4da9fc --- /dev/null +++ b/backend/src/main/java/com/park/utmstack/util/exceptions/TooManyRequestsException.java @@ -0,0 +1,8 @@ +package com.park.utmstack.util.exceptions; + +public class TooManyRequestsException extends RuntimeException { + public TooManyRequestsException(String message) { + super(message); + } +} + diff --git a/backend/src/main/java/com/park/utmstack/util/exceptions/UtmElasticsearchException.java b/backend/src/main/java/com/park/utmstack/util/exceptions/UtmElasticsearchException.java index 9f9488ee0..693014d73 100644 --- a/backend/src/main/java/com/park/utmstack/util/exceptions/UtmElasticsearchException.java +++ b/backend/src/main/java/com/park/utmstack/util/exceptions/UtmElasticsearchException.java @@ -1,6 +1,6 @@ package com.park.utmstack.util.exceptions; -public class UtmElasticsearchException extends Exception { +public class UtmElasticsearchException extends RuntimeException { public UtmElasticsearchException(String message) { super(message); } diff --git a/backend/src/main/java/com/park/utmstack/web/rest/UserJWTController.java b/backend/src/main/java/com/park/utmstack/web/rest/UserJWTController.java index 9adddc896..e73efc8bd 100644 --- a/backend/src/main/java/com/park/utmstack/web/rest/UserJWTController.java +++ b/backend/src/main/java/com/park/utmstack/web/rest/UserJWTController.java @@ -1,11 +1,13 @@ package com.park.utmstack.web.rest; import com.fasterxml.jackson.annotation.JsonProperty; +import com.park.utmstack.aop.logging.AuditEvent; import com.park.utmstack.config.Constants; import com.park.utmstack.domain.Authority; import com.park.utmstack.domain.User; import com.park.utmstack.domain.application_events.enums.ApplicationEventType; import com.park.utmstack.domain.federation_service.UtmFederationServiceClient; +import com.park.utmstack.loggin.LogContextBuilder; import com.park.utmstack.repository.federation_service.UtmFederationServiceClientRepository; import com.park.utmstack.security.TooMuchLoginAttemptsException; import com.park.utmstack.security.jwt.JWTFilter; @@ -13,20 +15,19 @@ import com.park.utmstack.service.MailService; import com.park.utmstack.service.UserService; import com.park.utmstack.service.application_events.ApplicationEventService; +import com.park.utmstack.service.dto.auth.JWTToken; import com.park.utmstack.service.login_attempts.LoginAttemptService; import com.park.utmstack.service.tfa.TfaService; import com.park.utmstack.util.CipherUtil; -import com.park.utmstack.util.UtilResponse; import com.park.utmstack.util.exceptions.InvalidConnectionKeyException; import com.park.utmstack.web.rest.util.HeaderUtil; import com.park.utmstack.web.rest.vm.LoginVM; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.authentication.AuthenticationManager; -import org.springframework.security.authentication.BadCredentialsException; import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; import org.springframework.security.core.Authentication; import org.springframework.security.core.authority.SimpleGrantedAuthority; @@ -36,20 +37,21 @@ import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.*; +import javax.servlet.http.HttpServletRequest; import javax.validation.Valid; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; /** * Controller to authenticate users. */ +@RequiredArgsConstructor +@Slf4j @RestController @RequestMapping("/api") public class UserJWTController { - private static final String CLASSNAME = "UserJWTController"; - private final Logger log = LoggerFactory.getLogger(UserJWTController.class); - private final TokenProvider tokenProvider; private final AuthenticationManager authenticationManager; private final ApplicationEventService applicationEventService; @@ -58,105 +60,82 @@ public class UserJWTController { private final MailService mailService; private final LoginAttemptService loginAttemptService; private final UtmFederationServiceClientRepository fsClientRepository; - private final PasswordEncoder passwordEncoder; - - public UserJWTController(TokenProvider tokenProvider, - AuthenticationManager authenticationManager, - ApplicationEventService applicationEventService, - UserService userService, - TfaService tfaService, - MailService mailService, - LoginAttemptService loginAttemptService, - UtmFederationServiceClientRepository fsClientRepository, - PasswordEncoder passwordEncoder) { - this.tokenProvider = tokenProvider; - this.authenticationManager = authenticationManager; - this.applicationEventService = applicationEventService; - this.userService = userService; - this.tfaService = tfaService; - this.mailService = mailService; - this.loginAttemptService = loginAttemptService; - this.fsClientRepository = fsClientRepository; - this.passwordEncoder = passwordEncoder; - } + private final LogContextBuilder logContextBuilder; + + @AuditEvent( + attemptType = ApplicationEventType.AUTH_ATTEMPT, + attemptMessage = "Authentication attempt registered", + successType = ApplicationEventType.UNDEFINED, + successMessage = "" + ) @PostMapping("/authenticate") - public ResponseEntity authorize(@Valid @RequestBody LoginVM loginVM) { - final String ctx = CLASSNAME + ".authorize"; - try { - if (loginAttemptService.isBlocked()) - throw new TooMuchLoginAttemptsException(String.format("Client IP %1$s blocked due to too many failed login attempts", loginAttemptService.getClientIP())); + public ResponseEntity authorize(@Valid @RequestBody LoginVM loginVM, HttpServletRequest request) { + + if (loginAttemptService.isBlocked()) { + String ip = loginAttemptService.getClientIP(); + throw new TooMuchLoginAttemptsException(String.format("Authentication blocked: IP %s exceeded login attempt threshold", ip)); + } boolean authenticated = !Boolean.parseBoolean(Constants.CFG.get(Constants.PROP_TFA_ENABLE)); UsernamePasswordAuthenticationToken authenticationToken = - new UsernamePasswordAuthenticationToken(loginVM.getUsername(), loginVM.getPassword()); + new UsernamePasswordAuthenticationToken(loginVM.getUsername(), loginVM.getPassword()); boolean rememberMe = loginVM.isRememberMe() != null && loginVM.isRememberMe(); Authentication authentication = this.authenticationManager.authenticate(authenticationToken); SecurityContextHolder.getContext().setAuthentication(authentication); String jwt = tokenProvider.createToken(authentication, rememberMe, authenticated); + Map args = logContextBuilder.buildArgs(request); if (!authenticated) { String secret = tfaService.generateSecret(); String code = tfaService.generateCode(secret); User user = userService.updateUserTfaSecret(loginVM.getUsername(), secret); + + applicationEventService.createEvent( + "TFA challenge issued for user '" + user.getLogin(), + ApplicationEventType.TFA_CODE_SENT, + args + ); mailService.sendTfaVerificationCode(user, code); + } else { + applicationEventService.createEvent( + "Login successfully completed for user '" + loginVM.getUsername() + "'", + ApplicationEventType.AUTH_SUCCESS, + args); } HttpHeaders httpHeaders = new HttpHeaders(); httpHeaders.add(JWTFilter.AUTHORIZATION_HEADER, "Bearer " + jwt); return new ResponseEntity<>(new JWTToken(jwt, authenticated), httpHeaders, HttpStatus.OK); - } catch (BadCredentialsException e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return UtilResponse.buildUnauthorizedResponse(msg); - } catch (TooMuchLoginAttemptsException e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return UtilResponse.buildLockedResponse(msg); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return UtilResponse.buildInternalServerErrorResponse(msg); - } + } @GetMapping("/check-credentials") public ResponseEntity checkPassword(@Valid @RequestParam String password, @RequestParam String checkUUID) { - final String ctx = CLASSNAME + ".checkPassword"; - try { + User user = userService.getCurrentUserLogin(); UsernamePasswordAuthenticationToken authenticationToken = - new UsernamePasswordAuthenticationToken(user.getLogin(), password); + new UsernamePasswordAuthenticationToken(user.getLogin(), password); Authentication authentication = this.authenticationManager.authenticate(authenticationToken); if (authentication.isAuthenticated()) { return new ResponseEntity<>(checkUUID, HttpStatus.OK); } else { return new ResponseEntity<>(checkUUID, HttpStatus.BAD_REQUEST); } - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers( - HeaderUtil.createFailureAlert("", "", msg)).body(null); - } + } @PostMapping("/authenticateFederationServiceManager") public ResponseEntity authorizeFederationServiceManager(@Valid @RequestBody String token) { - final String ctx = CLASSNAME + ".authorizeFederationServiceManager"; - try { + if (!StringUtils.hasText(token)) throw new InvalidConnectionKeyException("It's needed to provide a connection key"); UtmFederationServiceClient fsToken = fsClientRepository.findByFsClientToken(token) - .orElseThrow(() -> new InvalidConnectionKeyException("Unrecognized connection key")); + .orElseThrow(() -> new InvalidConnectionKeyException("Unrecognized connection key")); String[] tokenInfo = new String(Base64Utils.decodeFromUrlSafeString(fsToken.getFsClientToken())).split("\\|"); @@ -167,7 +146,7 @@ public ResponseEntity authorizeFederationServiceManager(@Valid @Reques throw new InvalidConnectionKeyException("Connection key is corrupt, unrecognized instance");*/ UsernamePasswordAuthenticationToken authenticationToken = - new UsernamePasswordAuthenticationToken(Constants.FS_USER, CipherUtil.decrypt(tokenInfo[1], System.getenv(Constants.ENV_ENCRYPTION_KEY))); + new UsernamePasswordAuthenticationToken(Constants.FS_USER, CipherUtil.decrypt(tokenInfo[1], System.getenv(Constants.ENV_ENCRYPTION_KEY))); Authentication authentication = this.authenticationManager.authenticate(authenticationToken); SecurityContextHolder.getContext().setAuthentication(authentication); @@ -178,30 +157,25 @@ public ResponseEntity authorizeFederationServiceManager(@Valid @Reques httpHeaders.add(JWTFilter.AUTHORIZATION_HEADER, "Bearer " + jwt); return new ResponseEntity<>(new JWTToken(jwt, true), httpHeaders, HttpStatus.OK); - } catch (InvalidConnectionKeyException e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return UtilResponse.buildBadRequestResponse(msg); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return UtilResponse.buildInternalServerErrorResponse(msg); - } + } + @AuditEvent( + attemptType = ApplicationEventType.TFA_CODE_VERIFY_ATTEMPT, + attemptMessage = "Verification attempt for second-factor authentication", + successType = ApplicationEventType.AUTH_SUCCESS, + successMessage = "Login successfully completed" + ) @GetMapping("/tfa/verifyCode") public ResponseEntity verifyCode(@RequestParam String code) { - final String ctx = CLASSNAME + ".verifyCode"; - try { + User user = userService.getCurrentUserLogin(); if (!tfaService.validateCode(user.getTfaSecret(), code)) return ResponseEntity.status(HttpStatus.UNAUTHORIZED).headers( - HeaderUtil.createFailureAlert("", "", "Your secret code is invalid")).body(null); + HeaderUtil.createFailureAlert("", "", "Your secret code is invalid")).body(null); List authorities = user.getAuthorities().stream().map(Authority::getName) - .map(SimpleGrantedAuthority::new).collect(Collectors.toList()); + .map(SimpleGrantedAuthority::new).collect(Collectors.toList()); org.springframework.security.core.userdetails.User principal = new org.springframework.security.core.userdetails.User(user.getLogin(), "", authorities); @@ -212,44 +186,6 @@ public ResponseEntity verifyCode(@RequestParam String code) { HttpHeaders httpHeaders = new HttpHeaders(); httpHeaders.add(JWTFilter.AUTHORIZATION_HEADER, "Bearer " + jwt); return new ResponseEntity<>(new JWTToken(jwt, true), httpHeaders, HttpStatus.OK); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers( - HeaderUtil.createFailureAlert("", "", msg)).body(null); - } } - - /** - * Object to return as body in JWT Authentication. - */ - public static class JWTToken { - - private String idToken; - private boolean authenticated; - - JWTToken(String idToken, boolean authenticated) { - this.idToken = idToken; - this.authenticated = authenticated; - } - - @JsonProperty("id_token") - String getIdToken() { - return idToken; - } - - void setIdToken(String idToken) { - this.idToken = idToken; - } - - public boolean isAuthenticated() { - return authenticated; - } - - public void setAuthenticated(boolean authenticated) { - this.authenticated = authenticated; - } - } } diff --git a/backend/src/main/java/com/park/utmstack/web/rest/UtmAlertResource.java b/backend/src/main/java/com/park/utmstack/web/rest/UtmAlertResource.java index 6c14e09d8..4505208d7 100644 --- a/backend/src/main/java/com/park/utmstack/web/rest/UtmAlertResource.java +++ b/backend/src/main/java/com/park/utmstack/web/rest/UtmAlertResource.java @@ -1,8 +1,12 @@ package com.park.utmstack.web.rest; +import com.park.utmstack.aop.logging.AuditEvent; import com.park.utmstack.domain.application_events.enums.ApplicationEventType; import com.park.utmstack.service.UtmAlertService; import com.park.utmstack.service.application_events.ApplicationEventService; +import com.park.utmstack.service.dto.alert.ConvertToIncidentRequestBody; +import com.park.utmstack.service.dto.alert.UpdateAlertStatusRequestBody; +import com.park.utmstack.service.dto.alert.UpdateAlertTagsRequestBody; import com.park.utmstack.util.AlertUtil; import com.park.utmstack.util.UtilResponse; import com.park.utmstack.web.rest.util.HeaderUtil; @@ -13,9 +17,7 @@ import org.springframework.web.bind.annotation.*; import javax.validation.Valid; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Pattern; -import java.util.List; +import java.io.IOException; /** * REST controller for managing UtmAlert. @@ -40,63 +42,58 @@ public UtmAlertResource(UtmAlertService utmAlertService, } @PostMapping("/utm-alerts/status") - public ResponseEntity updateAlertStatus(@RequestBody UpdateAlertStatusRequestBody rq) { + @AuditEvent( + attemptType = ApplicationEventType.ALERT_UPDATE_ATTEMPT, + attemptMessage = "Attempt to update alert status initiated", + successType = ApplicationEventType.ALERT_UPDATE_SUCCESS, + successMessage = "Alert status updated successfully" + ) + public ResponseEntity updateAlertStatus(@RequestBody UpdateAlertStatusRequestBody rq) throws IOException { final String ctx = CLASSNAME + ".updateAlertStatus"; - try { - utmAlertService.updateStatus(rq.getAlertIds(), rq.getStatus(), rq.getStatusObservation()); - return ResponseEntity.ok().build(); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers( - HeaderUtil.createFailureAlert("", "", msg)).body(null); - } + + utmAlertService.updateStatus(rq.getAlertIds(), rq.getStatus(), rq.getStatusObservation()); + return ResponseEntity.ok().build(); } @PostMapping("/utm-alerts/notes") + @AuditEvent( + attemptType = ApplicationEventType.ALERT_NOTE_UPDATE_ATTEMPT, + attemptMessage = "Attempt to update alert notes initiated", + successType = ApplicationEventType.ALERT_NOTE_UPDATE_SUCCESS, + successMessage = "Alert notes updated successfully" + ) public ResponseEntity updateAlertNotes(@RequestBody(required = false) String notes, @RequestParam String alertId) { final String ctx = CLASSNAME + ".updateAlertNotes"; - try { - utmAlertService.updateNotes(alertId, notes); - return ResponseEntity.ok().build(); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers( - HeaderUtil.createFailureAlert("", "", msg)).body(null); - } + utmAlertService.updateNotes(alertId, notes); + return ResponseEntity.ok().build(); } @PostMapping("/utm-alerts/tags") + @AuditEvent( + attemptType = ApplicationEventType.ALERT_TAG_UPDATE_ATTEMPT, + attemptMessage = "Attempt to update alert tags initiated", + successType = ApplicationEventType.ALERT_TAG_UPDATE_SUCCESS, + successMessage = "Alert tags updated successfully" + ) public ResponseEntity updateAlertTags(@RequestBody @Valid UpdateAlertTagsRequestBody body) { - final String ctx = CLASSNAME + ".updateAlertTags"; - try { - utmAlertService.updateTags(body.getAlertIds(), body.getTags(), body.isCreateRule()); - return ResponseEntity.ok().build(); - } catch (Exception ex) { - String msg = ctx + ": " + ex.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers( - HeaderUtil.createFailureAlert("", "", msg)).body(null); - } + + utmAlertService.updateTags(body.getAlertIds(), body.getTags(), body.getCreateRule()); + return ResponseEntity.ok().build(); } @PostMapping("/utm-alerts/convert-to-incident") + @AuditEvent( + attemptType = ApplicationEventType.ALERT_CONVERT_TO_INCIDENT_ATTEMPT, + attemptMessage = "Attempt to convert alerts to incident initiated", + successType = ApplicationEventType.ALERT_CONVERT_TO_INCIDENT_SUCCESS, + successMessage = "Alerts converted to incident successfully" + ) public ResponseEntity convertToIncident(@RequestBody @Valid ConvertToIncidentRequestBody body) { final String ctx = CLASSNAME + ".convertToIncident"; - try { - utmAlertService.convertToIncident(body.getEventIds(), body.getIncidentName(),body.getIncidentId(), body.getIncidentSource()); - return ResponseEntity.ok().build(); - } catch (Exception ex) { - String msg = ctx + ": " + ex.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers( - HeaderUtil.createFailureAlert("", "", msg)).body(null); - } + + utmAlertService.convertToIncident(body.getEventIds(), body.getIncidentName(),body.getIncidentId(), body.getIncidentSource()); + return ResponseEntity.ok().build(); + } @GetMapping("/utm-alerts/count-open-alerts") @@ -111,135 +108,4 @@ public ResponseEntity countOpenAlerts() { return UtilResponse.buildErrorResponse(HttpStatus.INTERNAL_SERVER_ERROR, msg); } } - - public static class UpdateAlertTagsRequestBody { - @NotNull - private List alertIds; - private List tags; - @NotNull - private Boolean createRule; - - public List getAlertIds() { - return alertIds; - } - - public void setAlertIds(List alertIds) { - this.alertIds = alertIds; - } - - public List getTags() { - return tags; - } - - public void setTags(List tags) { - this.tags = tags; - } - - public Boolean isCreateRule() { - return createRule; - } - - public void setCreateRule(Boolean createRule) { - this.createRule = createRule; - } - } - - public static class UpdateAlertStatusRequestBody { - @NotNull - private List alertIds; - private String statusObservation; - @NotNull - private int status; - - public List getAlertIds() { - return alertIds; - } - - public void setAlertIds(List alertIds) { - this.alertIds = alertIds; - } - - public String getStatusObservation() { - return statusObservation; - } - - public void setStatusObservation(String statusObservation) { - this.statusObservation = statusObservation; - } - - public int getStatus() { - return status; - } - - public void setStatus(int status) { - this.status = status; - } - } - - public static class UpdateAlertSolutionRequestBody { - @NotNull - private String alertName; - @NotNull - private String solution; - - public String getAlertName() { - return alertName; - } - - public void setAlertName(String alertName) { - this.alertName = alertName; - } - - public String getSolution() { - return solution; - } - - public void setSolution(String solution) { - this.solution = solution; - } - } - - public static class ConvertToIncidentRequestBody { - @NotNull - private List eventIds; - @NotNull - @Pattern(regexp = "^[^\"]*$", message = "Double quotes are not allowed") - private String incidentName; - @NotNull - private Integer incidentId; - @NotNull - private String incidentSource; - - public List getEventIds() { - return eventIds; - } - - public void setEventIds(List eventIds) { - this.eventIds = eventIds; - } - - public String getIncidentName() { - return incidentName; - } - - public void setIncidentName(String incidentName) { - this.incidentName = incidentName; - } - - public Integer getIncidentId() { - return incidentId; - } - - public void setIncidentId(Integer incidentId) { - this.incidentId = incidentId; - } - - public String getIncidentSource() { - return incidentSource; - } - - public void setIncidentSource(String incidentSource) { - this.incidentSource = incidentSource; - } - } } diff --git a/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleGroupConfigurationResource.java b/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleGroupConfigurationResource.java index 0b4efe616..8a7026bbd 100644 --- a/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleGroupConfigurationResource.java +++ b/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleGroupConfigurationResource.java @@ -1,10 +1,13 @@ package com.park.utmstack.web.rest.application_modules; +import com.park.utmstack.aop.logging.AuditEvent; import com.park.utmstack.domain.application_events.enums.ApplicationEventType; import com.park.utmstack.domain.application_modules.UtmModuleGroupConfiguration; import com.park.utmstack.service.application_events.ApplicationEventService; import com.park.utmstack.service.application_modules.UtmModuleGroupConfigurationService; +import com.park.utmstack.service.dto.application_modules.GroupConfigurationDTO; import com.park.utmstack.web.rest.util.HeaderUtil; +import lombok.RequiredArgsConstructor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; @@ -12,28 +15,28 @@ import org.springframework.web.bind.annotation.*; import javax.validation.Valid; -import javax.validation.constraints.NotEmpty; -import javax.validation.constraints.NotNull; import java.util.List; @RestController +@RequiredArgsConstructor @RequestMapping("/api") + public class UtmModuleGroupConfigurationResource { private static final String CLASSNAME = "UtmModuleGroupConfigurationResource"; private final Logger log = LoggerFactory.getLogger(UtmModuleGroupConfigurationResource.class); - private final UtmModuleGroupConfigurationService moduleGroupConfigurationService; private final ApplicationEventService applicationEventService; - public UtmModuleGroupConfigurationResource(UtmModuleGroupConfigurationService moduleGroupConfigurationService, - ApplicationEventService applicationEventService) { - this.moduleGroupConfigurationService = moduleGroupConfigurationService; - this.applicationEventService = applicationEventService; - } @PutMapping("/module-group-configurations/update") - public ResponseEntity updateConfiguration(@Valid @RequestBody UpdateConfigurationKeysBody body) { + @AuditEvent( + attemptType = ApplicationEventType.CONFIG_UPDATE_ATTEMPT, + attemptMessage = "Attempt to update configuration keys initiated", + successType = ApplicationEventType.CONFIG_UPDATE_SUCCESS, + successMessage = "Configuration keys updated successfully" + ) + public ResponseEntity updateConfiguration(@Valid @RequestBody GroupConfigurationDTO body) { final String ctx = CLASSNAME + ".updateConfiguration"; try { moduleGroupConfigurationService.updateConfigurationKeys(body.getModuleId(), body.getKeys()); @@ -76,26 +79,4 @@ public ResponseEntity getConfigurationByGroupIdAndC } } - public static class UpdateConfigurationKeysBody { - @NotNull - private Long moduleId; - @NotEmpty - private List keys; - - public Long getModuleId() { - return moduleId; - } - - public void setModuleId(Long moduleId) { - this.moduleId = moduleId; - } - - public List getKeys() { - return keys; - } - - public void setKeys(List keys) { - this.keys = keys; - } - } } diff --git a/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleGroupResource.java b/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleGroupResource.java index 3755de8d0..9f10dbe3c 100644 --- a/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleGroupResource.java +++ b/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleGroupResource.java @@ -1,5 +1,6 @@ package com.park.utmstack.web.rest.application_modules; +import com.park.utmstack.aop.logging.AuditEvent; import com.park.utmstack.domain.application_events.enums.ApplicationEventType; import com.park.utmstack.domain.application_modules.UtmModule; import com.park.utmstack.domain.application_modules.UtmModuleGroup; @@ -58,6 +59,12 @@ public UtmModuleGroupResource(UtmModuleGroupService moduleGroupService, } @PostMapping("/utm-configuration-groups") + @AuditEvent( + attemptType = ApplicationEventType.CONFIG_GROUP_CREATE_ATTEMPT, + attemptMessage = "Attempt to create configuration group initiated", + successType = ApplicationEventType.CONFIG_GROUP_CREATE_SUCCESS, + successMessage = "Configuration group created successfully" + ) public ResponseEntity createConfigurationGroup(@Valid @RequestBody ModuleGroupVM moduleGroupVM) throws URISyntaxException { final String ctx = CLASSNAME + ".createConfigurationGroup"; try { @@ -105,26 +112,19 @@ public ResponseEntity createConfigurationGroup(@Valid @RequestBo * @throws URISyntaxException if the Location URI syntax is incorrect */ @PutMapping("/utm-configuration-groups") - public ResponseEntity updateUtmConfigurationGroup(@Valid @RequestBody UtmModuleGroup utmModuleGroup) throws URISyntaxException { + @AuditEvent( + attemptType = ApplicationEventType.CONFIG_GROUP_UPDATE_ATTEMPT, + attemptMessage = "Attempt to update configuration group initiated", + successType = ApplicationEventType.CONFIG_GROUP_UPDATE_SUCCESS, + successMessage = "Configuration group updated successfully" + ) + public ResponseEntity updateUtmConfigurationGroup(@Valid @RequestBody UtmModuleGroup utmModuleGroup) { final String ctx = CLASSNAME + ".updateUtmConfigurationGroup"; - try { - if (utmModuleGroup.getId() == null) - throw new Exception("Can't update the configuration group because ID is null"); - UtmModuleGroup result = moduleGroupService.save(utmModuleGroup); - return ResponseEntity.ok(result); - } catch (DataIntegrityViolationException e) { - String msg = ctx + ": " + e.getMostSpecificCause().getMessage().replaceAll("\n", ""); - log.error(msg); - eventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers( - HeaderUtil.createFailureAlert("", "", msg)).body(null); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - eventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers( - HeaderUtil.createFailureAlert("", "", msg)).body(null); - } + + if (utmModuleGroup.getId() == null) + throw new RuntimeException("Can't update the configuration group because ID is null"); + UtmModuleGroup result = moduleGroupService.save(utmModuleGroup); + return ResponseEntity.ok(result); } /** @@ -149,45 +149,39 @@ public ResponseEntity> getModuleGroups(@RequestParam Long m @GetMapping("/utm-configuration-groups/{groupId}") public ResponseEntity getConfigurationGroup(@PathVariable Long groupId) { final String ctx = CLASSNAME + ".getConfigurationGroups"; - try { - Optional group = moduleGroupService.findOne(groupId); - return ResponseUtil.wrapOrNotFound(group); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - eventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers( - HeaderUtil.createFailureAlert("", "", msg)).body(null); - } + + Optional group = moduleGroupService.findOne(groupId); + return ResponseUtil.wrapOrNotFound(group); + } @DeleteMapping("/utm-configuration-groups/delete-single-module-group") + @AuditEvent( + attemptType = ApplicationEventType.CONFIG_GROUP_DELETE_ATTEMPT, + attemptMessage = "Attempt to delete single configuration group initiated", + successType = ApplicationEventType.CONFIG_GROUP_DELETE_SUCCESS, + successMessage = "Configuration group deleted successfully" + ) public ResponseEntity deleteSingleModuleGroup(@RequestParam Long groupId) { final String ctx = CLASSNAME + ".deleteSingleModuleGroup"; - try { - moduleGroupService.delete(groupId); - return ResponseEntity.ok().build(); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - eventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers( - HeaderUtil.createFailureAlert("", "", msg)).body(null); - } + + moduleGroupService.delete(groupId); + return ResponseEntity.ok().build(); + } @DeleteMapping("/utm-configuration-groups/delete-all-module-groups") + @AuditEvent( + attemptType = ApplicationEventType.CONFIG_GROUP_BULK_DELETE_ATTEMPT, + attemptMessage = "Attempt to delete all configuration groups for module initiated", + successType = ApplicationEventType.CONFIG_GROUP_BULK_DELETE_SUCCESS, + successMessage = "All configuration groups for module deleted successfully" + ) public ResponseEntity deleteAllModuleGroups(@RequestParam Long moduleId) { final String ctx = CLASSNAME + ".deleteAllModuleGroups"; - try { - moduleGroupService.deleteAllByModuleId(moduleId); - return ResponseEntity.ok().build(); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - eventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers( - HeaderUtil.createFailureAlert("", "", msg)).body(null); - } + + moduleGroupService.deleteAllByModuleId(moduleId); + return ResponseEntity.ok().build(); + } } diff --git a/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleResource.java b/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleResource.java index a6881a989..bef6dfb08 100644 --- a/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleResource.java +++ b/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleResource.java @@ -1,5 +1,6 @@ package com.park.utmstack.web.rest.application_modules; +import com.park.utmstack.aop.logging.AuditEvent; import com.park.utmstack.config.Constants; import com.park.utmstack.domain.application_events.enums.ApplicationEventType; import com.park.utmstack.domain.application_modules.UtmModule; @@ -10,15 +11,17 @@ import com.park.utmstack.domain.application_modules.types.ModuleRequirement; import com.park.utmstack.repository.UtmServerRepository; import com.park.utmstack.security.internalApiKey.InternalApiKeyFilter; -import com.park.utmstack.service.UtmStackService; import com.park.utmstack.service.application_events.ApplicationEventService; import com.park.utmstack.service.application_modules.UtmModuleQueryService; import com.park.utmstack.service.application_modules.UtmModuleService; +import com.park.utmstack.service.dto.application_modules.CheckRequirementsResponse; +import com.park.utmstack.service.dto.application_modules.ModuleActivationDTO; +import com.park.utmstack.service.dto.application_modules.ModuleDTO; import com.park.utmstack.service.dto.application_modules.UtmModuleCriteria; import com.park.utmstack.util.CipherUtil; import com.park.utmstack.util.UtilResponse; -import com.park.utmstack.web.rest.errors.BadRequestAlertException; import com.park.utmstack.web.rest.util.PaginationUtil; +import lombok.RequiredArgsConstructor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.data.domain.Page; @@ -29,19 +32,15 @@ import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.*; -import java.io.IOException; -import java.util.Formattable; import java.util.List; import java.util.Set; -import java.util.logging.FileHandler; -import java.util.logging.Formatter; -import java.util.logging.Level; /** * REST controller for managing UtmModule. */ @RestController @RequestMapping("/api") +@RequiredArgsConstructor public class UtmModuleResource { private static final String CLASSNAME = "UtmModuleResource"; private final Logger log = LoggerFactory.getLogger(UtmModuleResource.class); @@ -50,36 +49,25 @@ public class UtmModuleResource { private final ModuleFactory moduleFactory; private final UtmModuleQueryService utmModuleQueryService; private final ApplicationEventService eventService; - private final UtmStackService utmStackService; private final UtmServerRepository utmServerRepository; - public UtmModuleResource(UtmModuleService moduleService, - ModuleFactory moduleFactory, - UtmModuleQueryService utmModuleQueryService, - ApplicationEventService eventService, - UtmStackService utmStackService, - UtmServerRepository utmServerRepository) { - this.moduleService = moduleService; - this.moduleFactory = moduleFactory; - this.utmModuleQueryService = utmModuleQueryService; - this.eventService = eventService; - this.utmStackService = utmStackService; - this.utmServerRepository = utmServerRepository; - } + + @AuditEvent( + attemptType = ApplicationEventType.MODULE_ACTIVATION_ATTEMPT, + attemptMessage = "Attempt to activate/deactivate module initiated", + successType = ApplicationEventType.MODULE_ACTIVATION_SUCCESS, + successMessage = "Module activated/deactivated successfully" + ) @PutMapping("/utm-modules/activateDeactivate") public ResponseEntity activateDeactivate(@RequestParam Long serverId, @RequestParam ModuleName nameShort, @RequestParam Boolean activationStatus) { - final String ctx = CLASSNAME + ".activateDeactivate"; - try { - return ResponseEntity.ok(moduleService.activateDeactivate(serverId, nameShort, activationStatus)); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - eventService.createEvent(msg, ApplicationEventType.ERROR); - return UtilResponse.buildErrorResponse(HttpStatus.INTERNAL_SERVER_ERROR, msg); - } + return ResponseEntity.ok(moduleService.activateDeactivate(ModuleActivationDTO.builder() + .serverId(serverId) + .moduleName(nameShort) + .activationStatus(activationStatus) + .build())); } /** @@ -90,10 +78,10 @@ public ResponseEntity activateDeactivate(@RequestParam Long serverId, * @return the ResponseEntity with status 200 (OK) and the list of utmModules in body */ @GetMapping("/utm-modules") - public ResponseEntity> getAllUtmModules(UtmModuleCriteria criteria, Pageable pageable) { + public ResponseEntity> getAllUtmModules(UtmModuleCriteria criteria, Pageable pageable) { final String ctx = CLASSNAME + ".getAllUtmModules"; try { - Page page = utmModuleQueryService.findByCriteria(criteria, pageable); + Page page = utmModuleQueryService.findByCriteria(criteria, pageable); HttpHeaders headers = PaginationUtil.generatePaginationHttpHeaders(page, "/api/utm-modules"); return ResponseEntity.ok().headers(headers).body(page.getContent()); } catch (Exception e) { @@ -104,6 +92,19 @@ public ResponseEntity> getAllUtmModules(UtmModuleCriteria criter } } + @GetMapping("/utm-modules/{id}") + public ResponseEntity getModuleById(@PathVariable Long id) { + final String ctx = CLASSNAME + ".getModuleById"; + try { + return ResponseEntity.ok().body(utmModuleQueryService.findById(id)); + } catch (Exception e) { + String msg = ctx + ": " + e.getMessage(); + log.error(msg); + eventService.createEvent(msg, ApplicationEventType.ERROR); + return UtilResponse.buildErrorResponse(HttpStatus.INTERNAL_SERVER_ERROR, msg); + } + } + @GetMapping("/utm-modules/moduleDetails") public ResponseEntity getModuleDetails(@RequestParam Long serverId, @RequestParam ModuleName nameShort) { @@ -200,25 +201,4 @@ public ResponseEntity isActive(@RequestParam ModuleName moduleName) { return UtilResponse.buildErrorResponse(HttpStatus.INTERNAL_SERVER_ERROR, msg); } } - - public static class CheckRequirementsResponse { - private ModuleRequirementStatus status; - private List checks; - - public ModuleRequirementStatus getStatus() { - return status; - } - - public void setStatus(ModuleRequirementStatus status) { - this.status = status; - } - - public List getChecks() { - return checks; - } - - public void setChecks(List checks) { - this.checks = checks; - } - } } diff --git a/backend/src/main/java/com/park/utmstack/web/rest/collectors/UtmCollectorResource.java b/backend/src/main/java/com/park/utmstack/web/rest/collectors/UtmCollectorResource.java index dce71f4f1..7f97bc9b5 100644 --- a/backend/src/main/java/com/park/utmstack/web/rest/collectors/UtmCollectorResource.java +++ b/backend/src/main/java/com/park/utmstack/web/rest/collectors/UtmCollectorResource.java @@ -16,6 +16,7 @@ import com.park.utmstack.service.application_modules.UtmModuleGroupService; import com.park.utmstack.service.collectors.CollectorOpsService; import com.park.utmstack.service.collectors.UtmCollectorService; +import com.park.utmstack.service.dto.collectors.dto.CollectorConfigKeysDTO; import com.park.utmstack.service.dto.collectors.dto.CollectorDTO; import com.park.utmstack.service.dto.collectors.CollectorModuleEnum; import com.park.utmstack.service.dto.collectors.dto.ListCollectorsResponseDTO; @@ -94,7 +95,7 @@ public UtmCollectorResource(CollectorOpsService collectorService, */ @PostMapping("/collector-config") public ResponseEntity upsertCollectorConfig( - @Valid @RequestBody UtmModuleGroupConfigurationResource.UpdateConfigurationKeysBody collectorConfig, + @Valid @RequestBody CollectorConfigKeysDTO collectorConfig, CollectorDTO collectorDTO) { final String ctx = CLASSNAME + ".upsertCollectorConfig"; diff --git a/backend/src/main/java/com/park/utmstack/web/rest/incident/UtmIncidentResource.java b/backend/src/main/java/com/park/utmstack/web/rest/incident/UtmIncidentResource.java index 470e48fed..f7b02d189 100644 --- a/backend/src/main/java/com/park/utmstack/web/rest/incident/UtmIncidentResource.java +++ b/backend/src/main/java/com/park/utmstack/web/rest/incident/UtmIncidentResource.java @@ -2,60 +2,44 @@ import com.park.utmstack.domain.application_events.enums.ApplicationEventType; import com.park.utmstack.domain.incident.UtmIncident; -import com.park.utmstack.service.application_events.ApplicationEventService; import com.park.utmstack.service.dto.incident.*; -import com.park.utmstack.service.incident.UtmIncidentAlertService; import com.park.utmstack.service.incident.UtmIncidentQueryService; import com.park.utmstack.service.incident.UtmIncidentService; -import com.park.utmstack.util.UtilResponse; +import com.park.utmstack.util.exceptions.NoAlertsProvidedException; import com.park.utmstack.web.rest.errors.BadRequestAlertException; import com.park.utmstack.web.rest.util.HeaderUtil; import com.park.utmstack.web.rest.util.PaginationUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.util.CollectionUtils; import org.springframework.web.bind.annotation.*; -import tech.jhipster.web.util.ResponseUtil; +import com.park.utmstack.aop.logging.AuditEvent; import javax.validation.Valid; import java.net.URI; import java.net.URISyntaxException; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; /** * REST controller for managing UtmIncident. */ @RestController +@RequiredArgsConstructor +@Slf4j @RequestMapping("/api") public class UtmIncidentResource { - private final String CLASS_NAME = "UtmIncidentResource"; - private final Logger log = LoggerFactory.getLogger(UtmIncidentResource.class); private static final String ENTITY_NAME = "utmIncident"; private final UtmIncidentService utmIncidentService; - private final UtmIncidentAlertService utmIncidentAlertService; - private final UtmIncidentQueryService utmIncidentQueryService; - private final ApplicationEventService applicationEventService; - - public UtmIncidentResource(UtmIncidentService utmIncidentService, - UtmIncidentAlertService utmIncidentAlertService, - UtmIncidentQueryService utmIncidentQueryService, - ApplicationEventService applicationEventService) { - this.utmIncidentService = utmIncidentService; - this.utmIncidentAlertService = utmIncidentAlertService; - this.utmIncidentQueryService = utmIncidentQueryService; - this.applicationEventService = applicationEventService; - } + /** * Creates a new incident based on the provided details. @@ -75,38 +59,21 @@ public UtmIncidentResource(UtmIncidentService utmIncidentService, * @throws IllegalArgumentException if the input data is invalid. */ @PostMapping("/utm-incidents") + @AuditEvent( + attemptType = ApplicationEventType.INCIDENT_CREATION_ATTEMPT, + attemptMessage = "Attempt to create a new incident initiated", + successType = ApplicationEventType.INCIDENT_CREATION_SUCCESS, + successMessage = "Incident created successfully" + ) public ResponseEntity createUtmIncident(@Valid @RequestBody NewIncidentDTO newIncidentDTO) { final String ctx = ".createUtmIncident"; - try { - if (CollectionUtils.isEmpty(newIncidentDTO.getAlertList())) { - String msg = ctx + ": A new incident has to have at least one alert related"; - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return UtilResponse.buildErrorResponse(HttpStatus.BAD_REQUEST, msg); - } - - List alertIds = newIncidentDTO.getAlertList().stream() - .map(RelatedIncidentAlertsDTO::getAlertId) - .collect(Collectors.toList()); - - List alertsFound = utmIncidentAlertService.existsAnyAlert(alertIds); - - if (!alertsFound.isEmpty()) { - String alertIdsList = String.join(", ", alertIds); - String msg = "Some alerts are already linked to another incident. Alert IDs: " + alertIdsList + ". Check the related incidents for more details."; - log.error(msg); - applicationEventService.createEvent(ctx + ": " + msg , ApplicationEventType.ERROR); - return UtilResponse.buildErrorResponse(HttpStatus.CONFLICT, utmIncidentAlertService.formatAlertMessage(alertsFound)); - } - - - return ResponseEntity.ok(utmIncidentService.createIncident(newIncidentDTO)); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return UtilResponse.buildErrorResponse(HttpStatus.INTERNAL_SERVER_ERROR, msg); + + if (CollectionUtils.isEmpty(newIncidentDTO.getAlertList())) { + String msg = ctx + ": A new incident has to have at least one alert related"; + throw new NoAlertsProvidedException(ctx + ": " + msg); } + + return ResponseEntity.ok(utmIncidentService.createIncident(newIncidentDTO)); } /** @@ -124,36 +91,22 @@ public ResponseEntity createUtmIncident(@Valid @RequestBody NewInci * @throws URISyntaxException if the Location URI syntax is incorrect */ @PostMapping("/utm-incidents/add-alerts") + @AuditEvent( + attemptType = ApplicationEventType.INCIDENT_ALERT_ADD_ATTEMPT, + attemptMessage = "Attempt to add alerts to incident initiated", + successType = ApplicationEventType.INCIDENT_ALERT_ADD_SUCCESS, + successMessage = "Alerts added to incident successfully" + ) public ResponseEntity addAlertsToUtmIncident(@Valid @RequestBody AddToIncidentDTO addToIncidentDTO) throws URISyntaxException { - final String ctx = ".addAlertsToUtmIncident"; - try { - log.debug("REST request to save UtmIncident : {}", addToIncidentDTO); - if (CollectionUtils.isEmpty(addToIncidentDTO.getAlertList())) { - throw new BadRequestAlertException("Add utmIncident cannot already have an empty related alerts", ENTITY_NAME, "alertList"); - } - List alertIds = addToIncidentDTO.getAlertList().stream() - .map(RelatedIncidentAlertsDTO::getAlertId) - .collect(Collectors.toList()); - - List alertsFound = utmIncidentAlertService.existsAnyAlert(alertIds); - - if (!alertsFound.isEmpty()) { - String alertIdsList = String.join(", ", alertIds); - String msg = "Some alerts are already linked to another incident. Alert IDs: " + alertIdsList + ". Check the related incidents for more details."; - log.error(msg); - applicationEventService.createEvent(ctx + ": " + msg , ApplicationEventType.ERROR); - return UtilResponse.buildErrorResponse(HttpStatus.CONFLICT, utmIncidentAlertService.formatAlertMessage(alertsFound)); - } - UtmIncident result = utmIncidentService.addAlertsIncident(addToIncidentDTO); - return ResponseEntity.created(new URI("/api/utm-incidents/add-alerts" + result.getId())) + + if (CollectionUtils.isEmpty(addToIncidentDTO.getAlertList())) { + throw new NoAlertsProvidedException("Add utmIncident cannot already have an empty related alerts"); + } + + UtmIncident result = utmIncidentService.addAlertsIncident(addToIncidentDTO); + return ResponseEntity.created(new URI("/api/utm-incidents/add-alerts" + result.getId())) .headers(HeaderUtil.createEntityCreationAlert(ENTITY_NAME, result.getId().toString())) .body(result); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers(HeaderUtil.createFailureAlert(CLASS_NAME, null, msg)).body(null); - } } /** @@ -166,23 +119,22 @@ public ResponseEntity addAlertsToUtmIncident(@Valid @RequestBody Ad * @throws URISyntaxException if the Location URI syntax is incorrect */ @PutMapping("/utm-incidents/change-status") - public ResponseEntity updateUtmIncident(@Valid @RequestBody UtmIncident utmIncident) throws URISyntaxException { - final String ctx = ".updateUtmIncident"; - try { - log.debug("REST request to update UtmIncident : {}", utmIncident); - if (utmIncident.getId() == null) { - throw new BadRequestAlertException("Invalid id", ENTITY_NAME, "idnull"); - } - UtmIncident result = utmIncidentService.changeStatus(utmIncident); - return ResponseEntity.ok() + @AuditEvent( + attemptType = ApplicationEventType.INCIDENT_UPDATE_ATTEMPT, + attemptMessage = "Attempt to update incident status initiated", + successType = ApplicationEventType.INCIDENT_UPDATE_SUCCESS, + successMessage = "Incident status updated successfully" + ) + public ResponseEntity updateUtmIncident(@Valid @RequestBody UtmIncident utmIncident) { + + if (utmIncident.getId() == null) { + throw new BadRequestAlertException("Invalid id", ENTITY_NAME, "idnull"); + } + UtmIncident result = utmIncidentService.changeStatus(utmIncident); + + return ResponseEntity.ok() .headers(HeaderUtil.createEntityUpdateAlert(ENTITY_NAME, utmIncident.getId().toString())) .body(result); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers(HeaderUtil.createFailureAlert(CLASS_NAME, null, msg)).body(null); - } } /** @@ -194,18 +146,10 @@ public ResponseEntity updateUtmIncident(@Valid @RequestBody UtmInci */ @GetMapping("/utm-incidents") public ResponseEntity> getAllUtmIncidents(UtmIncidentCriteria criteria, Pageable pageable) { - final String ctx = ".getAllUtmIncidents"; - try { - log.debug("REST request to get UtmIncidents by criteria: {}", criteria); - Page page = utmIncidentQueryService.findByCriteria(criteria, pageable); - HttpHeaders headers = PaginationUtil.generatePaginationHttpHeaders(page, "/api/utm-incidents"); - return ResponseEntity.ok().headers(headers).body(page.getContent()); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers(HeaderUtil.createFailureAlert(CLASS_NAME, null, msg)).body(null); - } + + Page page = utmIncidentQueryService.findByCriteria(criteria, pageable); + HttpHeaders headers = PaginationUtil.generatePaginationHttpHeaders(page, "/api/utm-incidents"); + return ResponseEntity.ok().headers(headers).body(page.getContent()); } /** @@ -215,15 +159,7 @@ public ResponseEntity> getAllUtmIncidents(UtmIncidentCriteria */ @GetMapping("/utm-incidents/users-assigned") public ResponseEntity> getAllUserAssigned() { - final String ctx = ".getAllUserAssigned"; - try { - return ResponseEntity.ok().body(utmIncidentQueryService.getAllUsersAssigned()); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers(HeaderUtil.createFailureAlert(CLASS_NAME, null, msg)).body(null); - } + return ResponseEntity.ok().body(utmIncidentQueryService.getAllUsersAssigned()); } /** @@ -234,16 +170,9 @@ public ResponseEntity> getAllUserAssigned() { */ @GetMapping("/utm-incidents/{id}") public ResponseEntity getUtmIncident(@PathVariable Long id) { - final String ctx = ".getUtmIncident"; - try { - log.debug("REST request to get UtmIncident : {}", id); - Optional utmIncident = utmIncidentService.findOne(id); - return ResponseUtil.wrapOrNotFound(utmIncident); - } catch (Exception e) { - String msg = ctx + ": " + e.getMessage(); - log.error(msg); - applicationEventService.createEvent(msg, ApplicationEventType.ERROR); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).headers(HeaderUtil.createFailureAlert(CLASS_NAME, null, msg)).body(null); - } + + Optional utmIncident = utmIncidentService.findOne(id); + return tech.jhipster.web.util.ResponseUtil.wrapOrNotFound(utmIncident); + } } diff --git a/backend/src/main/java/com/park/utmstack/web/rest/vm/LoginVM.java b/backend/src/main/java/com/park/utmstack/web/rest/vm/LoginVM.java index 74c59fd49..bf41765a5 100644 --- a/backend/src/main/java/com/park/utmstack/web/rest/vm/LoginVM.java +++ b/backend/src/main/java/com/park/utmstack/web/rest/vm/LoginVM.java @@ -1,47 +1,36 @@ package com.park.utmstack.web.rest.vm; +import com.park.utmstack.service.dto.auditable.AuditableDTO; +import lombok.Getter; +import lombok.Setter; + import javax.validation.constraints.NotNull; import javax.validation.constraints.Size; +import java.util.HashMap; +import java.util.Map; /** * View Model object for storing a user's credentials. */ -public class LoginVM { +@Setter +public class LoginVM implements AuditableDTO { + @Getter @NotNull @Size(min = 1, max = 50) private String username; + @Getter @NotNull @Size(min = ManagedUserVM.PASSWORD_MIN_LENGTH, max = ManagedUserVM.PASSWORD_MAX_LENGTH) private String password; private Boolean rememberMe; - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - public Boolean isRememberMe() { return rememberMe; } - public void setRememberMe(Boolean rememberMe) { - this.rememberMe = rememberMe; - } - @Override public String toString() { return "LoginVM{" + @@ -49,4 +38,13 @@ public String toString() { ", rememberMe=" + rememberMe + '}'; } + + @Override + public Map toAuditMap() { + Map context = new HashMap<>(); + + context.put("loginAttempt", this.username); + + return context; + } } diff --git a/backend/src/main/resources/config/application-prod.yml b/backend/src/main/resources/config/application-prod.yml index 20d1ca27b..90c98cbba 100644 --- a/backend/src/main/resources/config/application-prod.yml +++ b/backend/src/main/resources/config/application-prod.yml @@ -2,7 +2,7 @@ logging: level: ROOT: error tech.jhipster: error - com.park.utmstack: error + com.park.utmstack: info spring: devtools: diff --git a/backend/src/main/resources/logback-spring.xml b/backend/src/main/resources/logback-spring.xml index 4aa548af3..39ff5894d 100644 --- a/backend/src/main/resources/logback-spring.xml +++ b/backend/src/main/resources/logback-spring.xml @@ -2,30 +2,38 @@ - - - - + + + + timestamp + + + severity + + + msg + + + + + - - 512 - - - - - + + ---> + + @@ -41,8 +49,8 @@ - - + + @@ -55,6 +63,13 @@ + + + + + + + diff --git a/bitdefender/configuration/config.go b/bitdefender/configuration/config.go index 561328117..b2f9c56d7 100644 --- a/bitdefender/configuration/config.go +++ b/bitdefender/configuration/config.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/bitdefender/constants" "github.com/utmstack/UTMStack/bitdefender/utils" "github.com/utmstack/config-client-go/enum" @@ -37,7 +38,7 @@ func ConfigureModules(cnf *types.ConfigurationSection, mutex *sync.Mutex) { time.Sleep(delayCheckConfig) if err := utils.ConnectionChecker(constants.URL_CHECK_CONNECTION); err != nil { - utils.Logger.ErrorF("Failed to establish connection: %v", err) + catcher.Error("Failed to establish connection", err, nil) } tempModuleConfig, err := client.GetUTMConfig(enum.BITDEFENDER) @@ -46,7 +47,7 @@ func ConfigureModules(cnf *types.ConfigurationSection, mutex *sync.Mutex) { continue } if (err.Error() != "") && (err.Error() != " ") { - utils.Logger.ErrorF("error getting configuration of the Bitdefender module: %v", err) + catcher.Error("error getting configuration of the Bitdefender module", err, nil) } continue } @@ -58,18 +59,18 @@ func ConfigureModules(cnf *types.ConfigurationSection, mutex *sync.Mutex) { isNecessaryConfig := compareConfigs(configsSent, group) if isNecessaryConfig { if !araAnyEmpty(group.Configurations[0].ConfValue, group.Configurations[1].ConfValue, group.Configurations[2].ConfValue, group.Configurations[3].ConfValue) { - utils.Logger.Info("new configuration found: groupName: %s, master: %s, CompanyIDs: %s", group.GroupName, group.Configurations[2].ConfValue, group.Configurations[3].ConfValue) + catcher.Info("new configuration found", map[string]any{"groupName": group.GroupName, "master": group.Configurations[2].ConfValue, "CompanyIDs": group.Configurations[3].ConfValue}) if err := confBDGZApiPush(group, "sendConf"); err != nil { - utils.Logger.ErrorF("error sending configuration") + catcher.Error("error sending configuration", err, nil) continue } time.Sleep(15 * time.Second) if err := confBDGZApiPush(group, "getConf"); err != nil { - utils.Logger.ErrorF("error getting configuration") + catcher.Error("error getting configuration", err, nil) continue } if err := confBDGZApiPush(group, "sendTest"); err != nil { - utils.Logger.ErrorF("error sending test event") + catcher.Error("error sending test event", err, nil) continue } @@ -100,20 +101,20 @@ func confBDGZApiPush(config types.ModuleGroup, operation string) error { for i := 0; i < 5; i++ { response, err := fn(config) if err != nil { - utils.Logger.ErrorF("%v", err) + catcher.Error("error sending configuration", err, nil) time.Sleep(1 * time.Minute) continue } defer response.Body.Close() - utils.Logger.Info("Status: %s", response.Status) + catcher.Info("Status", map[string]any{"status": response.Status}) myBody, _ := io.ReadAll(response.Body) - utils.Logger.Info("%s", string(myBody)) + catcher.Info("Response Body", map[string]any{"body": string(myBody)}) if operation == "sendConf" { regex := regexp.MustCompile(`result":true`) match := regex.Match([]byte(string(myBody))) if match { - utils.Logger.Info("Configuration sent correctly") + catcher.Info("Configuration sent correctly", nil) } } return nil @@ -122,33 +123,33 @@ func confBDGZApiPush(config types.ModuleGroup, operation string) error { } func sendPushEventSettings(config types.ModuleGroup) (*http.Response, error) { - utils.Logger.Info("Sending configuration...") + catcher.Info("Sending configuration...", nil) byteTemplate := getTemplateSetPush(config) body, err := json.Marshal(byteTemplate) if err != nil { - utils.Logger.ErrorF("error when marshaling the request body to send the configuration: %v", err) + catcher.Error("error when marshaling the request body to send the configuration", err, nil) return nil, err } return sendRequest(body, config) } func getPushEventSettings(config types.ModuleGroup) (*http.Response, error) { - utils.Logger.Info("Checking configuration...") + catcher.Info("Checking configuration...", nil) byteTemplate := getTemplateGet() body, err := json.Marshal(byteTemplate) if err != nil { - utils.Logger.ErrorF("error when marshaling the request body to send the configuration: %v", err) + catcher.Error("error when marshaling the request body to send the configuration", err, nil) return nil, err } return sendRequest(body, config) } func sendTestPushEvent(config types.ModuleGroup) (*http.Response, error) { - utils.Logger.Info("Sending Event Test...") + catcher.Info("Sending Event Test...", nil) byteTemplate := getTemplateTest() body, err := json.Marshal(byteTemplate) if err != nil { - utils.Logger.ErrorF("error when marshaling the request body to send the configuration: %v", err) + catcher.Error("error when marshaling the request body to send the configuration", err, nil) return nil, err } return sendRequest(body, config) diff --git a/bitdefender/go.mod b/bitdefender/go.mod index 411c2d14f..94d766653 100644 --- a/bitdefender/go.mod +++ b/bitdefender/go.mod @@ -1,8 +1,8 @@ module github.com/utmstack/UTMStack/bitdefender -go 1.23.0 +go 1.24.2 -toolchain go1.24.2 +toolchain go1.24.6 require ( github.com/RackSec/srslog v0.0.0-20180709174129-a4725f04ec91 @@ -12,31 +12,32 @@ require ( ) require ( - github.com/bytedance/sonic v1.13.3 // indirect - github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/bytedance/sonic v1.14.0 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/gin-gonic/gin v1.10.1 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-playground/validator/v10 v10.27.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/threatwinds/go-sdk v1.0.45 github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.14 // indirect - golang.org/x/arch v0.18.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect + github.com/ugorji/go/codec v1.3.0 // indirect + golang.org/x/arch v0.19.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/bitdefender/go.sum b/bitdefender/go.sum index e9b292754..9c771d166 100644 --- a/bitdefender/go.sum +++ b/bitdefender/go.sum @@ -2,9 +2,13 @@ github.com/RackSec/srslog v0.0.0-20180709174129-a4725f04ec91 h1:vX+gnvBc56EbWYrm github.com/RackSec/srslog v0.0.0-20180709174129-a4725f04ec91/go.mod h1:cDLGBht23g0XQdLjzn6xOGXDkLK182YfINAaZEQLCHQ= github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= +github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= @@ -25,6 +29,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= +github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= @@ -39,6 +45,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= @@ -63,25 +71,39 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/threatwinds/go-sdk v1.0.45 h1:KZ3s3HviNRrOkg5EqjFnoauANFFzTqjNFyshPLY2SoI= +github.com/threatwinds/go-sdk v1.0.45/go.mod h1:tcWn6r6vqID/W/nL3UKfc5NafA3V/cSkiLvfJnwB58c= github.com/threatwinds/logger v1.2.2 h1:sVuT8yhbecPqP4tT8EwHfp1czNC6e1wdkE1ihNnuBdA= github.com/threatwinds/logger v1.2.2/go.mod h1:Amq0QI1y7fkTpnBUgeGVu2Z/C4u4ys2pNLUOuj3UAAU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.14 h1:yOQvXCBc3Ij46LRkRoh4Yd5qK6LVOgi0bYOXfb7ifjw= github.com/ugorji/go/codec v1.2.14/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= +github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/utmstack/config-client-go v1.2.7 h1:JeRdI5JjH1liNzMW3LmyevjuPd67J/yt9MAO3+oJAuM= github.com/utmstack/config-client-go v1.2.7/go.mod h1:kM0KoUizM9ZlcQp0qKviGTWn/+anT5Rfjx3zfZk79nM= golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/arch v0.19.0 h1:LmbDQUodHThXE+htjrnmVD73M//D9GTH6wFZjyDkjyU= +golang.org/x/arch v0.19.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= diff --git a/bitdefender/main.go b/bitdefender/main.go index 79168f13b..bfc874c5d 100644 --- a/bitdefender/main.go +++ b/bitdefender/main.go @@ -7,6 +7,7 @@ import ( "sync" "syscall" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/bitdefender/configuration" "github.com/utmstack/UTMStack/bitdefender/server" "github.com/utmstack/UTMStack/bitdefender/utils" @@ -21,18 +22,21 @@ var ( func main() { path, err := utils.GetMyPath() if err != nil { - utils.Logger.Fatal("failed to get current path: %v", err) + catcher.Error("failed to get current path", err, nil) + os.Exit(1) } certsPath := filepath.Join(path, "certs") err = utils.CreatePathIfNotExist(certsPath) if err != nil { - utils.Logger.Fatal("error creating path: %s", err) + catcher.Error("error creating path", err, nil) + os.Exit(1) } err = utils.GenerateCerts(certsPath) if err != nil { - utils.Logger.Fatal("error generating certificates: %v", err) + catcher.Error("error generating certificates", err, nil) + os.Exit(1) } server.ServerUp(&moduleConfig, certsPath) diff --git a/bitdefender/server/server.go b/bitdefender/server/server.go index 3e834b85c..093b4663a 100644 --- a/bitdefender/server/server.go +++ b/bitdefender/server/server.go @@ -8,6 +8,7 @@ import ( "time" "github.com/gorilla/mux" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/bitdefender/constants" "github.com/utmstack/UTMStack/bitdefender/schema" "github.com/utmstack/UTMStack/bitdefender/utils" @@ -19,13 +20,13 @@ var syslogHelper EpsSyslogHelper // GetBDGZLogs gets the Bitdefender Api Push logs and sends them to the syslog server func GetBDGZLogs(config *types.ConfigurationSection) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - utils.Logger.Info("New group of events received") + catcher.Info("New group of events received", nil) // Check if the Bitdefender Module is active if config.ModuleActive { //Check if the authorization exist if r.Header.Get("authorization") == "" { messag := "401 Missing Authorization Header" - utils.Logger.ErrorF("%s", messag) + catcher.Error(messag, nil, nil) j, _ := json.Marshal(messag) w.WriteHeader(http.StatusUnauthorized) w.Write(j) @@ -41,7 +42,7 @@ func GetBDGZLogs(config *types.ConfigurationSection) http.HandlerFunc { } if !isAuth { messag := "401 Invalid Authentication Credentials" - utils.Logger.ErrorF("%s", messag) + catcher.Error(messag, nil, nil) j, _ := json.Marshal(messag) w.WriteHeader(http.StatusUnauthorized) w.Write(j) @@ -52,7 +53,7 @@ func GetBDGZLogs(config *types.ConfigurationSection) http.HandlerFunc { var newBody schema.BodyEvents err := json.NewDecoder(r.Body).Decode(&newBody) if err != nil { - utils.Logger.ErrorF("error to decode body: %v", err) + catcher.Error("error to decode body", err, nil) return } @@ -65,7 +66,7 @@ func GetBDGZLogs(config *types.ConfigurationSection) http.HandlerFunc { w.WriteHeader(http.StatusOK) w.Write(j) } else { - utils.Logger.ErrorF("Bitdefender module disabled") + catcher.Error("Bitdefender module disabled", nil, nil) } } } @@ -95,10 +96,10 @@ func ServerUp(cnf *types.ConfigurationSection, certsPath string) { } go func() { - utils.Logger.Info("Listening in port %s...\n", constants.GetConnectorPort()) + catcher.Info("Listening in port", map[string]any{"port": constants.GetConnectorPort()}) err := server.ListenAndServeTLS(filepath.Join(certsPath, "server.crt"), filepath.Join(certsPath, "server.key")) if err != nil { - utils.Logger.ErrorF("%v", err) + catcher.Error("error starting server", err, nil) } //Close connection with syslogServer syslogHelper.clientSyslog.Close() diff --git a/bitdefender/server/syslog.go b/bitdefender/server/syslog.go index 32c09c9da..5b94c4303 100644 --- a/bitdefender/server/syslog.go +++ b/bitdefender/server/syslog.go @@ -6,8 +6,8 @@ import ( "time" syslog "github.com/RackSec/srslog" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/bitdefender/constants" - "github.com/utmstack/UTMStack/bitdefender/utils" "github.com/utmstack/config-client-go/types" ) @@ -38,16 +38,16 @@ func (g *EpsSyslogHelper) SentToSyslog(config *types.ConfigurationSection, event pattern := "BitdefenderGZCompanyId=" + compID match, err := regexp.MatchString(pattern, syslogMessage) if err != nil { - utils.Logger.ErrorF("error matching pattern: %v", err) + catcher.Error("error matching pattern", err, nil) continue } if match { syslogMessage += " UTM_TENANT=" + cnf.GroupName g.clientSyslog.Warning(syslogMessage) - utils.Logger.Info("message recived: %s", syslogMessage) + catcher.Info("message recived", map[string]any{"message": syslogMessage}) break } else { - utils.Logger.Info("Event received that is not within the configured CompanyId: %s", syslogMessage) + catcher.Info("Event received that is not within the configured CompanyId", map[string]any{"message": syslogMessage}) } } } diff --git a/bitdefender/utils/check.go b/bitdefender/utils/check.go index 2a6780db8..50dabd910 100644 --- a/bitdefender/utils/check.go +++ b/bitdefender/utils/check.go @@ -3,9 +3,14 @@ package utils import ( "fmt" "net/http" + "strings" "time" + + "github.com/threatwinds/go-sdk/catcher" ) +const wait = 3 * time.Second + func ConnectionChecker(url string) error { checkConn := func() error { if err := CheckConnection(url); err != nil { @@ -14,7 +19,7 @@ func ConnectionChecker(url string) error { return nil } - if err := Logger.InfiniteRetryIfXError(checkConn, "connection failed"); err != nil { + if err := infiniteRetryIfXError(checkConn, "connection failed"); err != nil { return err } @@ -39,3 +44,30 @@ func CheckConnection(url string) error { return nil } + +func infiniteRetryIfXError(f func() error, exception string) error { + var xErrorWasLogged bool + + for { + err := f() + if err != nil && is(err, exception) { + if !xErrorWasLogged { + _ = catcher.Error("An error occurred (%s), will keep retrying indefinitely...", err, nil) + xErrorWasLogged = true + } + time.Sleep(wait) + continue + } + + return err + } +} + +func is(e error, args ...string) bool { + for _, arg := range args { + if strings.Contains(e.Error(), arg) { + return true + } + } + return false +} diff --git a/bitdefender/utils/env.go b/bitdefender/utils/env.go index 846eaee2a..0332339d2 100644 --- a/bitdefender/utils/env.go +++ b/bitdefender/utils/env.go @@ -1,18 +1,21 @@ package utils import ( - "log" "os" + + "github.com/threatwinds/go-sdk/catcher" ) // Getenv returns the environment variable func Getenv(key string) string { value, defined := os.LookupEnv(key) if !defined { - log.Fatalf("Error loading environment variable: %s: environment variable does not exist\n", key) + catcher.Error("Error loading environment variable, environment variable does not exist", nil, map[string]any{"key": key}) + os.Exit(1) } if (value == "") || (value == " ") { - log.Fatalf("Error loading environment variable: %s: empty environment variable\n", key) + catcher.Error("Error loading environment variable, empty environment variable", nil, map[string]any{"key": key}) + os.Exit(1) } return value } diff --git a/bitdefender/utils/logger.go b/bitdefender/utils/logger.go deleted file mode 100644 index 4332e7f5a..000000000 --- a/bitdefender/utils/logger.go +++ /dev/null @@ -1,31 +0,0 @@ -package utils - -import ( - "log" - "os" - "strconv" - - "github.com/threatwinds/logger" -) - -var Logger *logger.Logger - -func init() { - lenv := os.Getenv("LOG_LEVEL") - var level int - var err error - - if lenv != "" && lenv != " " { - level, err = strconv.Atoi(lenv) - if err != nil { - log.Fatalln(err) - } - } else { - level = 200 - } - - Logger = logger.NewLogger(&logger.Config{ - Format: "text", - Level: level, - }) -} diff --git a/correlation/Dockerfile b/correlation/Dockerfile index a68677b51..e2ef7958f 100644 --- a/correlation/Dockerfile +++ b/correlation/Dockerfile @@ -14,8 +14,8 @@ RUN wget -O /app/asn-blocks-v6.csv https://cdn.utmstack.com/geoip/asn-blocks-v6. RUN wget -O /app/blocks-v4.csv https://cdn.utmstack.com/geoip/blocks-v4.csv RUN wget -O /app/blocks-v6.csv https://cdn.utmstack.com/geoip/blocks-v6.csv RUN wget -O /app/locations-en.csv https://cdn.utmstack.com/geoip/locations-en.csv -RUN wget -O /app/ip_level1.list.tar.gz https://intelligence.threatwinds.com/api/feeds/v1/download/list/level1/accumulative/ip && cd /app && tar -xf ip_level1.list.tar.gz -RUN wget -O /app/ip_level2.list.tar.gz https://intelligence.threatwinds.com/api/feeds/v1/download/list/level2/accumulative/ip && cd /app && tar -xf ip_level2.list.tar.gz -RUN wget -O /app/ip_level3.list.tar.gz https://intelligence.threatwinds.com/api/feeds/v1/download/list/level3/accumulative/ip && cd /app && tar -xf ip_level3.list.tar.gz +# RUN wget -O /app/ip_level1.list.tar.gz https://intelligence.threatwinds.com/api/feeds/v1/download/list/level1/accumulative/ip && cd /app && tar -xf ip_level1.list.tar.gz +# RUN wget -O /app/ip_level2.list.tar.gz https://intelligence.threatwinds.com/api/feeds/v1/download/list/level2/accumulative/ip && cd /app && tar -xf ip_level2.list.tar.gz +# RUN wget -O /app/ip_level3.list.tar.gz https://intelligence.threatwinds.com/api/feeds/v1/download/list/level3/accumulative/ip && cd /app && tar -xf ip_level3.list.tar.gz RUN mkdir -p /app/rulesets && git clone --depth 1 https://github.com/utmstack/rules.git /app/rulesets/system ENTRYPOINT [ "/run.sh" ] diff --git a/correlation/api/newLogHandler.go b/correlation/api/newLogHandler.go index 3c154183f..c6cc3d065 100644 --- a/correlation/api/newLogHandler.go +++ b/correlation/api/newLogHandler.go @@ -3,12 +3,13 @@ package api import ( "encoding/json" "fmt" - "github.com/utmstack/UTMStack/correlation/ti" "io" - "log" "net/http" "time" + "github.com/threatwinds/go-sdk/catcher" + // "github.com/utmstack/UTMStack/correlation/ti" + "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/tidwall/gjson" @@ -29,7 +30,7 @@ func NewLog(c *gin.Context) { if err != nil { response["status"] = "error" response["error"] = fmt.Sprintf("%v", err) - log.Println(response["error"]) + catcher.Error("Failed to read request body", err, map[string]any{"status": http.StatusBadRequest}) c.JSON(http.StatusBadRequest, response) return } @@ -38,7 +39,7 @@ func NewLog(c *gin.Context) { if err := json.Unmarshal(body, &lo); err != nil { response["status"] = "error" response["error"] = fmt.Sprintf("%v", err) - log.Println(response["error"]) + catcher.Error("Failed to read request body", err, map[string]any{"status": http.StatusBadRequest}) c.JSON(http.StatusBadRequest, response) return } @@ -69,13 +70,16 @@ func NewLog(c *gin.Context) { !gjson.Get(l, "dataSource").Exists() { response["status"] = "error" response["error"] = "The log doesn't have the required fields. Please be sure that you are sending the @timestamp in RFC3339Nano format, the dataType that could be windows, linux, iis, macos, ... and the dataSource that could be the Hostname or IP of the log source." - log.Printf("%s LOG: %s", response["error"], l) + catcher.Error("Log validation failed - missing required fields", nil, map[string]any{ + "status": http.StatusBadRequest, + "log_sample": l, + }) c.JSON(http.StatusBadRequest, response) return } cache.AddToCache(l) - ti.Enqueue(l) + // ti.Enqueue(l) search.AddToQueue(l) response["status"] = "queued" c.JSON(http.StatusOK, response) diff --git a/correlation/cache/cache.go b/correlation/cache/cache.go index da4a80ea9..db97d1eb0 100644 --- a/correlation/cache/cache.go +++ b/correlation/cache/cache.go @@ -1,11 +1,11 @@ package cache import ( - "log" "runtime" "sync" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/tidwall/gjson" "github.com/utmstack/UTMStack/correlation/rules" "github.com/utmstack/UTMStack/correlation/utils" @@ -19,10 +19,10 @@ var storage []string func Status() { for { - log.Printf("Logs in cache: %v", len(storage)) + catcher.Info("Logs in cache", map[string]any{"count": len(storage)}) if len(storage) != 0 { est := gjson.Get(storage[0], "@timestamp").String() - log.Printf("Old document in cache: %s", est) + catcher.Info("Old document in cache", map[string]any{"timestamp": est}) } time.Sleep(60 * time.Second) } @@ -47,7 +47,7 @@ func Search(allOf []rules.AllOf, oneOf []rules.OneOf, seconds int64) []string { est := gjson.Get(storage[i], "@timestamp").String() eit, err := time.Parse(time.RFC3339Nano, est) if err != nil { - log.Printf("Could not parse @timestamp: %v", err) + catcher.Error("Could not parse @timestamp:", err, nil) continue } if eit.Unix() < ait { @@ -85,7 +85,7 @@ var logs = make(chan string, bufferSize) func AddToCache(l string) { if len(logs) == bufferSize { - log.Printf("Buffer is full, you could be lossing events") + catcher.Info("Buffer is full, you could be lossing events", nil) return } logs <- l @@ -116,7 +116,7 @@ func Clean() { old := gjson.Get(storage[0], "@timestamp").String() oldTime, err := time.Parse(time.RFC3339Nano, old) if err != nil { - log.Printf("Could not parse old log timestamp. Cleaning up") + catcher.Error("Could not parse old log timestamp. Cleaning up", err, nil) clean = true } diff --git a/correlation/correlation/finder.go b/correlation/correlation/finder.go index a3ba75ffb..fcb0d1df8 100644 --- a/correlation/correlation/finder.go +++ b/correlation/correlation/finder.go @@ -4,9 +4,9 @@ import ( "bytes" "fmt" "html/template" - "log" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/correlation/cache" "github.com/utmstack/UTMStack/correlation/rules" "github.com/utmstack/UTMStack/correlation/search" @@ -15,13 +15,13 @@ import ( func Finder(rule rules.Rule) { if len(rule.DataTypes) == 0 { - log.Printf("Disabling rule '%s', because dataTypes is empty", rule.Name) + catcher.Info("Disabling rule, because dataTypes is empty", map[string]any{"name": rule.Name}) return } sleep, err := time.ParseDuration(fmt.Sprintf("%ds", rule.Frequency)) if err != nil { - log.Printf("Disabling rule '%s', because of error: '%v", rule.Name, err) + catcher.Error("Disabling rule", err, map[string]any{"name": rule.Name}) return } @@ -52,7 +52,7 @@ func Finder(rule rules.Rule) { continue } - log.Printf("Executing rule: %s", rule.Name) + catcher.Info("Executing rule", map[string]any{"name": rule.Name}) if len(rule.Cache) != 0 { findInCache(rule) @@ -60,7 +60,7 @@ func Finder(rule rules.Rule) { findInSearch(rule) } - log.Printf("Execution of rule '%s' finished", rule.Name) + catcher.Info("Execution of rule finished", map[string]any{"name": rule.Name}) switch sleep { case 0: @@ -84,7 +84,7 @@ func findInSearch(rule rules.Rule) { t := template.Must(template.New("query").Parse(query.Query)) err := t.Execute(&q, fields) if err != nil { - log.Printf("Error while trying to process the query %v of the rule %s: %v", step+1, rule.Name, err) + catcher.Error("Error while trying to process the query", err, map[string]any{"step": step + 1, "rule": rule.Name}) } else { l := search.Search(q.String()) processResponse(l, rule, query.Save, &tmpLogs, len(rule.Search), step, query.MinCount) @@ -108,7 +108,7 @@ func findInCache(rule rules.Rule) { t := template.Must(template.New("allOf").Parse(allOf.Value)) err := t.Execute(&value, fields) if err != nil { - log.Printf("Error while trying to process the query %v of the rule %s: %v", step+1, rule.Name, err) + catcher.Error("Error while trying to process the query", err, map[string]any{"step": step + 1, "rule": rule.Name}) } else { allOfList = append(allOfList, rules.AllOf{Field: allOf.Field, Operator: allOf.Operator, Value: value.String()}) } @@ -120,7 +120,7 @@ func findInCache(rule rules.Rule) { t := template.Must(template.New("oneOf").Parse(oneOf.Value)) err := t.Execute(&value, fields) if err != nil { - log.Printf("Error while trying to process the query %v of the rule %s: %v", step+1, rule.Name, err) + catcher.Error("Error while trying to process the query", err, map[string]any{"step": step + 1, "rule": rule.Name}) } else { oneOfList = append(oneOfList, rules.OneOf{Field: oneOf.Field, Operator: oneOf.Operator, Value: value.String()}) } diff --git a/correlation/correlation/reporter.go b/correlation/correlation/reporter.go index 4b0a43249..c8e8ea7b5 100644 --- a/correlation/correlation/reporter.go +++ b/correlation/correlation/reporter.go @@ -2,15 +2,16 @@ package correlation import ( "encoding/json" + "strconv" + "strings" + "time" + "github.com/google/uuid" "github.com/levigross/grequests" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/correlation/geo" "github.com/utmstack/UTMStack/correlation/search" "github.com/utmstack/UTMStack/correlation/utils" - "log" - "strconv" - "strings" - "time" ) type Host struct { @@ -100,7 +101,7 @@ func Alert(name, severity, description, solution, category, tactic string, refer } } - log.Printf("Reporting alert: %s", name) + catcher.Info("Reporting alert", map[string]any{"name": name}) if !UpdateAlert(name, severity, fields) { NewAlert(name, severity, description, solution, category, tactic, reference, dataType, dataSource, @@ -113,7 +114,7 @@ func UpdateAlert(name, severity string, details map[string]string) bool { index, err := search.IndexBuilder("alert", time.Now().UTC().Format(time.RFC3339Nano)) if err != nil { - log.Printf("Could not build index name: %v", err) + catcher.Error("Could not build index name", err, nil) return true } @@ -208,7 +209,7 @@ func UpdateAlert(name, severity string, details map[string]string) bool { JSON: request, }) if err != nil { - log.Printf("Could not check existent alert: %v", err) + catcher.Error("Could not check existent alert", err, nil) return false } @@ -221,7 +222,7 @@ func UpdateAlert(name, severity string, details map[string]string) bool { err = json.Unmarshal([]byte(resultStr), &resultObj) if err != nil { - log.Printf("Could not check existent alert: %v", err) + catcher.Error("Could not check existent alert", err, nil) return false } @@ -242,7 +243,7 @@ func UpdateAlert(name, severity string, details map[string]string) bool { }, }) if err != nil { - log.Printf("Could not update existent alert: %v", err) + catcher.Error("Could not update existent alert", err, nil) return false } @@ -362,13 +363,13 @@ func NewAlert(name, severity, description, solution, category, tactic string, re url := cnf.Elasticsearch + "/" + index + "/_doc" _, err := utils.DoPost(url, "application/json", body) if err != nil { - log.Printf("Could not send alert to Elasticsearch: %v", err) + catcher.Error("Could not send alert to Elasticsearch", err, nil) } } else { - log.Printf("Could not build index name: %v", err) + catcher.Error("Could not build index name", err, nil) } } else { - log.Printf("Could not encode alert in JSON: %v", err) + catcher.Error("Could not encode alert in JSON", err, nil) } time.Sleep(3 * time.Second) } diff --git a/correlation/geo/bases.go b/correlation/geo/bases.go index 093ed243d..5ee04e558 100644 --- a/correlation/geo/bases.go +++ b/correlation/geo/bases.go @@ -1,15 +1,16 @@ package geo import ( - "github.com/utmstack/UTMStack/correlation/utils" - "log" "net" "path/filepath" "strconv" + + "github.com/threatwinds/go-sdk/catcher" + "github.com/utmstack/UTMStack/correlation/utils" ) func Load() { - log.Printf("Loading GeoIP databases") + catcher.Info("Loading GeoIP databases", nil) var files = []string{ "asn-blocks-v4.csv", @@ -38,10 +39,10 @@ func Load() { } } - log.Printf("asnBlocks rows: %v", len(asnBlocks)) - log.Printf("cityBlocks rows: %v", len(cityBlocks)) - log.Printf("cityLocations rows: %v", len(cityLocations)) - log.Printf("GeoIP databases loaded") + catcher.Info("asnBlocks rows", map[string]any{"count": len(asnBlocks)}) + catcher.Info("cityBlocks rows", map[string]any{"count": len(cityBlocks)}) + catcher.Info("cityLocations rows", map[string]any{"count": len(cityLocations)}) + catcher.Info("GeoIP databases loaded", nil) } func populateASNBlocks(csv [][]string) { @@ -51,13 +52,13 @@ func populateASNBlocks(csv [][]string) { } _, n, err := net.ParseCIDR(line[0]) if err != nil { - log.Printf("Could not get CIDR in populateASNBlocks: %v", err) + catcher.Error("Could not get CIDR in populateASNBlocks", err, nil) continue } asn, err := strconv.Atoi(line[1]) if err != nil { - log.Printf("Could not get ASN in populateASNBlocks: %v", err) + catcher.Error("Could not get ASN in populateASNBlocks", err, nil) continue } @@ -78,7 +79,7 @@ func populateCityBlocks(csv [][]string) { } _, n, err := net.ParseCIDR(line[0]) if err != nil { - log.Printf("Could not parse CIDR in populateCityBlocks: %v", err) + catcher.Error("Could not parse CIDR in populateCityBlocks", err, nil) continue } @@ -88,13 +89,13 @@ func populateCityBlocks(csv [][]string) { geonameID, err := strconv.Atoi(line[1]) if err != nil { - log.Printf("Could not parse geonameID in populateCityBlocks: %v", err) + catcher.Error("Could not parse geonameID in populateCityBlocks", err, nil) continue } isAnonymousProxy, err := strconv.Atoi(line[4]) if err != nil { - log.Printf("Could not parse isAnonymousProxy in populateCityBlocks: %v", err) + catcher.Error("Could not parse isAnonymousProxy in populateCityBlocks", err, nil) continue } @@ -105,7 +106,7 @@ func populateCityBlocks(csv [][]string) { isSatelliteProvider, err := strconv.Atoi(line[5]) if err != nil { - log.Printf("Could not parse isSatelliteProvider in populateCityBlocks: %v", err) + catcher.Error("Could not parse isSatelliteProvider in populateCityBlocks", err, nil) continue } @@ -116,19 +117,19 @@ func populateCityBlocks(csv [][]string) { latitude, err := strconv.ParseFloat(line[7], 64) if err != nil { - log.Printf("Could not parse latitude in populateCityBlocks: %v", err) + catcher.Error("Could not parse latitude in populateCityBlocks", err, nil) continue } longitude, err := strconv.ParseFloat(line[8], 64) if err != nil { - log.Printf("Could not parse longitude in populateCityBlocks: %v", err) + catcher.Error("Could not parse longitude in populateCityBlocks", err, nil) continue } accuracyRadius, err := strconv.Atoi(line[9]) if err != nil { - log.Printf("Could not parse accuracyRadius in populateCityBlocks: %v", err) + catcher.Error("Could not parse accuracyRadius in populateCityBlocks", err, nil) continue } @@ -153,13 +154,13 @@ func populateCityLocations(csv [][]string) { } geonameID, err := strconv.Atoi(line[0]) if err != nil { - log.Printf("Could not parse geonameID in populateCityLocations: %v", err) + catcher.Error("Could not parse geonameID in populateCityLocations", err, nil) continue } isInEuropeanUnion, err := strconv.Atoi(line[13]) if err != nil { - log.Printf("Could not parse isInEuropeanUnion in populateCityLocations: %v", err) + catcher.Error("Could not parse isInEuropeanUnion in populateCityLocations", err, nil) continue } diff --git a/correlation/go.mod b/correlation/go.mod index 6362bb3bf..da8632fef 100644 --- a/correlation/go.mod +++ b/correlation/go.mod @@ -1,8 +1,8 @@ module github.com/utmstack/UTMStack/correlation -go 1.23.0 +go 1.24.2 -toolchain go1.24.2 +toolchain go1.24.6 require ( github.com/fsnotify/fsnotify v1.9.0 @@ -20,8 +20,8 @@ require ( require ( github.com/KyleBanks/depth v1.2.1 // indirect - github.com/bytedance/sonic v1.13.3 // indirect - github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/bytedance/sonic v1.14.0 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect @@ -32,12 +32,12 @@ require ( github.com/go-openapi/swag v0.23.1 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-playground/validator/v10 v10.27.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect github.com/mailru/easyjson v0.9.0 // indirect @@ -46,18 +46,19 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/threatwinds/go-sdk v1.0.45 github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tklauser/go-sysconf v0.3.15 // indirect github.com/tklauser/numcpus v0.10.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.14 // indirect + github.com/ugorji/go/codec v1.3.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - golang.org/x/arch v0.18.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect + golang.org/x/arch v0.19.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect golang.org/x/tools v0.34.0 // indirect google.golang.org/protobuf v1.36.6 // indirect ) diff --git a/correlation/go.sum b/correlation/go.sum index 9adba6ee2..c4afd2643 100644 --- a/correlation/go.sum +++ b/correlation/go.sum @@ -2,9 +2,13 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= +github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= @@ -40,6 +44,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= +github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -57,6 +63,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -105,6 +113,8 @@ github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+z github.com/swaggo/gin-swagger v1.6.0/go.mod h1:BG00cCEy294xtVpyIAHG6+e2Qzj/xKlRdOqDkvq0uzo= github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A= github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg= +github.com/threatwinds/go-sdk v1.0.45 h1:KZ3s3HviNRrOkg5EqjFnoauANFFzTqjNFyshPLY2SoI= +github.com/threatwinds/go-sdk v1.0.45/go.mod h1:tcWn6r6vqID/W/nL3UKfc5NafA3V/cSkiLvfJnwB58c= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -120,15 +130,21 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.14 h1:yOQvXCBc3Ij46LRkRoh4Yd5qK6LVOgi0bYOXfb7ifjw= github.com/ugorji/go/codec v1.2.14/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= +github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/arch v0.19.0 h1:LmbDQUodHThXE+htjrnmVD73M//D9GTH6wFZjyDkjyU= +golang.org/x/arch v0.19.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= @@ -138,6 +154,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= @@ -154,6 +172,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -163,6 +183,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/correlation/main.go b/correlation/main.go index 02437eb9d..0ed5fb784 100644 --- a/correlation/main.go +++ b/correlation/main.go @@ -1,7 +1,7 @@ package main import ( - "github.com/utmstack/UTMStack/correlation/ti" + // "github.com/utmstack/UTMStack/correlation/ti" "os" "os/signal" "syscall" @@ -33,7 +33,7 @@ import ( func main() { sqldb.Connect() geo.Load() - ti.Load() + // ti.Load() rulesL := rules.GetRules() for _, rule := range rulesL { @@ -46,7 +46,7 @@ func main() { go cache.ProcessQueue() go search.ProcessQueue() go statistics.Update() - go ti.IsBlocklisted() + // go ti.IsBlocklisted() go func() { gin.SetMode(gin.ReleaseMode) diff --git a/correlation/rules/rules.go b/correlation/rules/rules.go index 8ed39a382..4767d169c 100644 --- a/correlation/rules/rules.go +++ b/correlation/rules/rules.go @@ -1,22 +1,22 @@ package rules import ( - "log" "os" "path/filepath" "time" "github.com/fsnotify/fsnotify" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/correlation/utils" ) func ListRulesFiles() []string { var files []string cnf := utils.GetConfig() - log.Printf("Listing rules files in %s", cnf.RulesFolder) + catcher.Info("Listing rules files", map[string]any{"folder": cnf.RulesFolder}) err := filepath.Walk(cnf.RulesFolder, func(path string, info os.FileInfo, err error) error { if err != nil { - log.Printf("Could not list rules files: %v", err) + catcher.Error("Could not list rules files", err, nil) } if filepath.Ext(path) == ".yml" { @@ -25,7 +25,7 @@ func ListRulesFiles() []string { return nil }) if err != nil { - log.Printf("Could not list rules files: %v", err) + catcher.Error("Could not list rules files", err, nil) } return files @@ -76,15 +76,15 @@ func GetRules() []Rule { var rules []Rule for _, file := range ListRulesFiles() { - log.Printf("Reading rules from: %s", file) + catcher.Info("Reading rules from", map[string]any{"file": file}) utils.ReadYaml(file, &tmpRules) - log.Printf("%v rule/s found", len(tmpRules)) + catcher.Info("rule/s found", map[string]any{"count": len(tmpRules)}) for _, tr := range tmpRules { n := true for _, r := range rules { if r.Name == tr.Name { n = false - log.Printf("Ignoring rule: '%s' from: %s", r.Name, file) + catcher.Info("Ignoring rule", map[string]any{"name": r.Name, "file": file}) break } } @@ -101,7 +101,7 @@ func Changes(signals chan os.Signal) { cnf := utils.GetConfig() watcher, err := fsnotify.NewWatcher() if err != nil { - log.Printf("Could not create a new watcher: %v", err) + catcher.Error("Could not create a new watcher", err, nil) } defer watcher.Close() @@ -111,16 +111,16 @@ func Changes(signals chan os.Signal) { select { case err, ok := <-watcher.Errors: if !ok { - log.Printf("Could not detect changes in ruleset: %v", err) + catcher.Error("Could not detect changes in ruleset", err, nil) } case event, ok := <-watcher.Events: if !ok { - log.Printf("Error trying to detect changes in ruleset.") + catcher.Error("Error trying to detect changes in ruleset", err, nil) } if event.Op&fsnotify.Write == fsnotify.Write { if event.Name != cnf.RulesFolder+"system/.git/FETCH_HEAD" { - log.Printf("Changes detected in: %s", event.Name) - log.Printf("Restarting correlation engine") + catcher.Info("Changes detected in", map[string]any{"file": event.Name}) + catcher.Info("Restarting correlation engine", nil) signals <- os.Interrupt } } @@ -134,7 +134,7 @@ func Changes(signals chan os.Signal) { for { err := filepath.Walk(cnf.RulesFolder, func(path string, info os.FileInfo, err error) error { if err != nil { - log.Printf("Could not list rules folders: %v", err) + catcher.Error("Could not list rules folders", err, nil) } n := true if info.IsDir() { @@ -147,7 +147,7 @@ func Changes(signals chan os.Signal) { if n { folders = append(folders, path) if err := watcher.Add(path); err != nil { - log.Printf("Could not start watcher for a rules folder: %v", err) + catcher.Error("Could not start watcher for a rules folder", err, nil) } } @@ -155,7 +155,7 @@ func Changes(signals chan os.Signal) { return nil }) if err != nil { - log.Printf("Could not list rules folders: %v", err) + catcher.Error("Could not list rules folders", err, nil) continue } diff --git a/correlation/search/queue.go b/correlation/search/queue.go index e7a952e09..a14d6626f 100644 --- a/correlation/search/queue.go +++ b/correlation/search/queue.go @@ -4,12 +4,13 @@ import ( "bytes" "encoding/json" "fmt" - "log" + "os" "runtime" "strings" "sync" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/tidwall/gjson" "github.com/utmstack/UTMStack/correlation/statistics" "github.com/utmstack/UTMStack/correlation/utils" @@ -39,7 +40,8 @@ func ProcessQueue() { body, err := utils.DoPost(url, "application/x-ndjson", strings.NewReader(tmp)) if err != nil { - log.Fatalf("Could not send logs to Elasticsearch: %v. %s", err, body) + catcher.Error("Could not send logs to Elasticsearch", err, map[string]any{"response": body}) + os.Exit(1) } } time.Sleep(1 * time.Second) @@ -59,7 +61,7 @@ func ProcessQueue() { index, err := IndexBuilder("log-"+dataType, timestamp) if err != nil { - log.Printf("Error trying to build index name: %v", err) + catcher.Error("Error trying to build index name", err, nil) continue } diff --git a/correlation/search/search.go b/correlation/search/search.go index a179561aa..c23058c7c 100644 --- a/correlation/search/search.go +++ b/correlation/search/search.go @@ -2,9 +2,9 @@ package search import ( "fmt" - "log" "strings" + "github.com/threatwinds/go-sdk/catcher" "github.com/tidwall/gjson" "github.com/utmstack/UTMStack/correlation/utils" ) @@ -15,7 +15,7 @@ func Search(query string) []string { url := fmt.Sprintf("%s/log-*/_search", cnf.Elasticsearch) cnn, err := utils.DoPost(url, "application/json", strings.NewReader(query)) if err != nil { - log.Printf("Could not get logs from Elasticsearch: %v", err) + catcher.Error("Could not get logs from Elasticsearch", err, nil) } else { hits := gjson.Get(string(cnn), "hits.hits").Array() for _, hit := range hits { diff --git a/correlation/sqldb/cnn.go b/correlation/sqldb/cnn.go index 365569b2c..105ec7cbb 100644 --- a/correlation/sqldb/cnn.go +++ b/correlation/sqldb/cnn.go @@ -3,9 +3,10 @@ package sqldb import ( "database/sql" "fmt" - "log" + "os" _ "github.com/lib/pq" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/correlation/utils" ) @@ -14,7 +15,7 @@ var err error func Connect() { cnf := utils.GetConfig() - log.Printf("Connecting to Postgres server: %s using port: %v", cnf.Postgres.Server, cnf.Postgres.Port) + catcher.Info("Connecting to Postgres server", map[string]any{"server": cnf.Postgres.Server, "port": cnf.Postgres.Port}) dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%v sslmode=disable", cnf.Postgres.Server, @@ -25,7 +26,8 @@ func Connect() { ) db, err = sql.Open("postgres", dsn) if err != nil { - log.Fatalf("Could not connect to Postgres: %v", err) + catcher.Error("Could not connect to Postgres", err, nil) + os.Exit(1) } ping() @@ -33,6 +35,7 @@ func Connect() { func ping() { if err := db.Ping(); err != nil { - log.Fatalf("Could not reconnect to Postgres: %v", err) + catcher.Error("Could not reconnect to Postgres", err, nil) + os.Exit(1) } } diff --git a/correlation/sqldb/crud.go b/correlation/sqldb/crud.go index c8001b3c6..12faf2a09 100644 --- a/correlation/sqldb/crud.go +++ b/correlation/sqldb/crud.go @@ -1,8 +1,9 @@ package sqldb import ( - "log" "time" + + "github.com/threatwinds/go-sdk/catcher" ) func UpdateStatistics(i, s, t string, c int64) { @@ -16,7 +17,7 @@ func UpdateStatistics(i, s, t string, c int64) { DO UPDATE SET amount = public.utm_asset_metrics.amount + $4`, i, s, t, c) if err != nil { - log.Printf("Error updating statistics for datasource %s: %v", s, err) + catcher.Error("Error updating statistics for datasource", err, map[string]any{"datasource": s}) } timestamp := time.Now().UTC().Unix() @@ -29,6 +30,6 @@ func UpdateStatistics(i, s, t string, c int64) { DO UPDATE SET timestamp=$4, median=$5`, i, s, t, timestamp, int64(10800)) if err != nil { - log.Printf("Error updating status for datasource %s: %v", s, err) + catcher.Error("Error updating status for datasource", err, map[string]any{"datasource": s}) } } diff --git a/correlation/ti/bases.go b/correlation/ti/bases.go index aa3c64deb..5fc2a057a 100644 --- a/correlation/ti/bases.go +++ b/correlation/ti/bases.go @@ -1,53 +1,54 @@ package ti -import ( - "bufio" - "log" - "os" - "path/filepath" -) - -func Load() { - log.Printf("Loading Threat Intelligence Feeds") - - var files = []string{ - "ip_level1.list", - "ip_level2.list", - "ip_level3.list", - } - - for _, file := range files { - var t string - - switch file { - case "ip_level1.list": - t = "Low" - case "ip_level2.list": - t = "Medium" - case "ip_level3.list": - t = "High" - default: - } - - f, err := os.Open(filepath.Join("/app", file)) - if err != nil { - log.Printf("Could not open file: %v", err) - continue - } - - scanner := bufio.NewScanner(f) - - for scanner.Scan() { - element := scanner.Text() - if element == "" { - continue - } - - blockList[element] = t - } - - _ = f.Close() - } - - log.Printf("Threat Intelligence feeds loaded") -} +// import ( +// "bufio" +// "os" +// "path/filepath" + +// "github.com/threatwinds/go-sdk/catcher" +// ) + +// func Load() { +// catcher.Info("Loading Threat Intelligence Feeds", nil) + +// var files = []string{ +// "ip_level1.list", +// "ip_level2.list", +// "ip_level3.list", +// } + +// for _, file := range files { +// var t string + +// switch file { +// case "ip_level1.list": +// t = "Low" +// case "ip_level2.list": +// t = "Medium" +// case "ip_level3.list": +// t = "High" +// default: +// } + +// f, err := os.Open(filepath.Join("/app", file)) +// if err != nil { +// catcher.Error("Could not open file", err, nil) +// continue +// } + +// scanner := bufio.NewScanner(f) + +// for scanner.Scan() { +// element := scanner.Text() +// if element == "" { +// continue +// } + +// blockList[element] = t +// } + +// _ = f.Close() +// } + +// catcher.Info("Threat Intelligence feeds loaded", nil) +// } diff --git a/correlation/ti/ti.go b/correlation/ti/ti.go index 1a90c29de..10787ad74 100644 --- a/correlation/ti/ti.go +++ b/correlation/ti/ti.go @@ -1,56 +1,55 @@ package ti import ( - "net" - "runtime" - "strings" + // "net" + // "runtime" + // "strings" "sync" "time" - - "github.com/tidwall/gjson" - "github.com/utmstack/UTMStack/correlation/correlation" - "github.com/utmstack/UTMStack/correlation/utils" + // "github.com/tidwall/gjson" + // "github.com/utmstack/UTMStack/correlation/correlation" + // "github.com/utmstack/UTMStack/correlation/utils" ) type Cache map[string]bool -var blockList map[string]string +// var blockList map[string]string var channel chan string var cache Cache var cacheLock *sync.RWMutex func init() { - blockList = make(map[string]string, 10000) + // blockList = make(map[string]string, 10000) channel = make(chan string, 10000) cache = make(Cache, 10000) cacheLock = &sync.RWMutex{} cache.AutoClean() } -func blocked(log string) bool { - log = strings.ToLower(log) - - exclusionList := []string{ - "block", - "denied", - "drop", - "reject", - "deny", - "timeout", - "closed", - "close", - "client-rst", - "server-rst", - } - - for _, e := range exclusionList { - if strings.Contains(log, e) { - return true - } - } - - return false -} +// func blocked(log string) bool { +// log = strings.ToLower(log) + +// exclusionList := []string{ +// "block", +// "denied", +// "drop", +// "reject", +// "deny", +// "timeout", +// "closed", +// "close", +// "client-rst", +// "server-rst", +// } + +// for _, e := range exclusionList { +// if strings.Contains(log, e) { +// return true +// } +// } + +// return false +// } func (c *Cache) Add(k string) { cacheLock.Lock() @@ -76,83 +75,83 @@ func (c *Cache) AutoClean() { }() } -func IsBlocklisted() { - saveFields := []utils.SavedField{ - { - Field: "logx.*.proto", - Alias: "Protocol", - }, - { - Field: "logx.*.src_ip", - Alias: "SourceIP", - }, - { - Field: "logx.*.dest_ip", - Alias: "DestinationIP", - }, - { - Field: "logx.*.src_port", - Alias: "SourcePort", - }, - { - Field: "logx.*.dest_port", - Alias: "DestinationPort", - }, - } - - numCPU := runtime.NumCPU() - for i := 0; i < numCPU; i++ { - go func() { - for { - log := <-channel - - sourceIpStr := gjson.Get(log, "logx.*.src_ip") - destinationIpStr := gjson.Get(log, "logx.*.dest_ip") - sourceIp := net.ParseIP(sourceIpStr.String()) - destinationIp := net.ParseIP(destinationIpStr.String()) - - if sourceIp != nil && !cache.IsCached(sourceIp.String()) { - if severity, ok := blockList[sourceIp.String()]; ok && !blocked(log) { - correlation.Alert( - "Connection from a malicious IP", - severity, - "A blocklisted element has been identified in the logs. Further investigation is recommended.", - "", - "Threat Intelligence", - "", - []string{"https://threatwinds.com"}, - gjson.Get(log, "dataType").String(), - gjson.Get(log, "dataSource").String(), - utils.ExtractDetails(saveFields, log), - ) - - } - - cache.Add(sourceIp.String()) - } - - if destinationIp != nil && !cache.IsCached(destinationIp.String()) { - if severity, ok := blockList[destinationIp.String()]; ok && !blocked(log) { - correlation.Alert( - "Connection to a malicious IP", - severity, - "A blocklisted element has been identified in the logs. Further investigation is recommended.", - "", - "Threat Intelligence", - "", - []string{"https://threatwinds.com"}, - gjson.Get(log, "dataType").String(), - gjson.Get(log, "dataSource").String(), - utils.ExtractDetails(saveFields, log), - ) - } - - cache.Add(destinationIp.String()) - } - } - }() - } -} +// func IsBlocklisted() { +// saveFields := []utils.SavedField{ +// { +// Field: "logx.*.proto", +// Alias: "Protocol", +// }, +// { +// Field: "logx.*.src_ip", +// Alias: "SourceIP", +// }, +// { +// Field: "logx.*.dest_ip", +// Alias: "DestinationIP", +// }, +// { +// Field: "logx.*.src_port", +// Alias: "SourcePort", +// }, +// { +// Field: "logx.*.dest_port", +// Alias: "DestinationPort", +// }, +// } + +// numCPU := runtime.NumCPU() +// for i := 0; i < numCPU; i++ { +// go func() { +// for { +// log := <-channel + +// sourceIpStr := gjson.Get(log, "logx.*.src_ip") +// destinationIpStr := gjson.Get(log, "logx.*.dest_ip") +// sourceIp := net.ParseIP(sourceIpStr.String()) +// destinationIp := net.ParseIP(destinationIpStr.String()) + +// if sourceIp != nil && !cache.IsCached(sourceIp.String()) { +// if severity, ok := blockList[sourceIp.String()]; ok && !blocked(log) { +// correlation.Alert( +// "Connection from a malicious IP", +// severity, +// "A blocklisted element has been identified in the logs. Further investigation is recommended.", +// "", +// "Threat Intelligence", +// "", +// []string{"https://threatwinds.com"}, +// gjson.Get(log, "dataType").String(), +// gjson.Get(log, "dataSource").String(), +// utils.ExtractDetails(saveFields, log), +// ) + +// } + +// cache.Add(sourceIp.String()) +// } + +// if destinationIp != nil && !cache.IsCached(destinationIp.String()) { +// if severity, ok := blockList[destinationIp.String()]; ok && !blocked(log) { +// correlation.Alert( +// "Connection to a malicious IP", +// severity, +// "A blocklisted element has been identified in the logs. Further investigation is recommended.", +// "", +// "Threat Intelligence", +// "", +// []string{"https://threatwinds.com"}, +// gjson.Get(log, "dataType").String(), +// gjson.Get(log, "dataSource").String(), +// utils.ExtractDetails(saveFields, log), +// ) +// } + +// cache.Add(destinationIp.String()) +// } +// } +// }() +// } +// } func Enqueue(log string) { if len(channel) >= 10000 { diff --git a/correlation/utils/httpClient.go b/correlation/utils/httpClient.go index 2ca460e3e..9b12c6961 100644 --- a/correlation/utils/httpClient.go +++ b/correlation/utils/httpClient.go @@ -2,14 +2,15 @@ package utils import ( "io" - "log" "net/http" + + "github.com/threatwinds/go-sdk/catcher" ) func DoPost(url, contentType string, body io.Reader) ([]byte, error) { res, err := http.Post(url, contentType, body) if err != nil { - log.Printf("Could not do request to the URL: %v", err) + catcher.Error("Could not do request to the URL", err, nil) return []byte{}, err } @@ -17,7 +18,7 @@ func DoPost(url, contentType string, body io.Reader) ([]byte, error) { response, err := io.ReadAll(res.Body) if err != nil { - log.Printf("Could not read response: %v", err) + catcher.Error("Could not read response", err, nil) return []byte{}, err } return response, nil diff --git a/correlation/utils/readers.go b/correlation/utils/readers.go index 5e4e89732..aa3227ba0 100644 --- a/correlation/utils/readers.go +++ b/correlation/utils/readers.go @@ -2,37 +2,37 @@ package utils import ( "encoding/csv" - "log" "os" "reflect" "strconv" "strings" + "github.com/threatwinds/go-sdk/catcher" "gopkg.in/yaml.v3" ) func ReadYaml(url string, result interface{}) { f, err := os.Open(url) if err != nil { - log.Printf("Could not open file: %v", err) + catcher.Error("Could not open file", err, nil) } defer f.Close() d := yaml.NewDecoder(f) if err := d.Decode(result); err != nil { - log.Printf("Could not decode YAML: %v", err) + catcher.Error("Could not decode YAML", err, nil) } } func ReadCSV(url string) [][]string { f, err := os.Open(url) if err != nil { - log.Printf("Could not open file: %v", err) + catcher.Error("Could not open file", err, nil) } defer f.Close() r := csv.NewReader(f) result, err := r.ReadAll() if err != nil { - log.Printf("Could not read CSV: %v", err) + catcher.Error("Could not read CSV", err, nil) } return result } @@ -51,13 +51,13 @@ func ReadEnvVars(cfg interface{}) { // Check if the environment variable exists envValue, exists := os.LookupEnv(envTag) if !exists { - log.Printf("Environment variable %s not set, skipping...", envTag) + catcher.Error("Environment variable not set", nil, map[string]any{"env_var": envTag}) continue } fieldValue := v.Field(i) if !fieldValue.CanSet() { - log.Printf("Cannot set field %s, skipping...", field.Name) + catcher.Error("Cannot set field", nil, map[string]any{"field": field.Name}) continue } @@ -68,25 +68,25 @@ func ReadEnvVars(cfg interface{}) { if intValue, err := strconv.ParseInt(envValue, 10, fieldValue.Type().Bits()); err == nil { fieldValue.SetInt(intValue) } else { - log.Printf("Failed to convert %s to int for field %s: %v", envValue, field.Name, err) + catcher.Error("Failed to convert to int", err, map[string]any{"value": envValue, "field": field.Name}) } case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: if uintValue, err := strconv.ParseUint(envValue, 10, fieldValue.Type().Bits()); err == nil { fieldValue.SetUint(uintValue) } else { - log.Printf("Failed to convert %s to uint for field %s: %v", envValue, field.Name, err) + catcher.Error("Failed to convert to uint", err, map[string]any{"value": envValue, "field": field.Name}) } case reflect.Float32, reflect.Float64: if floatValue, err := strconv.ParseFloat(envValue, fieldValue.Type().Bits()); err == nil { fieldValue.SetFloat(floatValue) } else { - log.Printf("Failed to convert %s to float for field %s: %v", envValue, field.Name, err) + catcher.Error("Failed to convert to float", err, map[string]any{"value": envValue, "field": field.Name}) } case reflect.Bool: if boolValue, err := strconv.ParseBool(envValue); err == nil { fieldValue.SetBool(boolValue) } else { - log.Printf("Failed to convert %s to bool for field %s: %v", envValue, field.Name, err) + catcher.Error("Failed to convert to bool", err, map[string]any{"value": envValue, "field": field.Name}) } case reflect.Slice: elements := reflect.MakeSlice(fieldValue.Type(), 0, 0) @@ -98,7 +98,7 @@ func ReadEnvVars(cfg interface{}) { ptr := reflect.New(fieldValue.Type().Elem()) fieldValue.Set(ptr) default: - log.Printf("Unsupported field type %s for field %s", fieldValue.Kind(), field.Name) + catcher.Error("Unsupported field type", nil, map[string]any{"field": field.Name, "type": fieldValue.Kind()}) } } } diff --git a/correlation/utils/resources.go b/correlation/utils/resources.go index e19923b08..eee20f6c6 100644 --- a/correlation/utils/resources.go +++ b/correlation/utils/resources.go @@ -1,11 +1,11 @@ package utils import ( - "log" "runtime" "time" "github.com/shirou/gopsutil/v3/mem" + "github.com/threatwinds/go-sdk/catcher" ) var AssignedMemory float32 @@ -29,11 +29,11 @@ func UsedByEngineMemory() uint64 { func Status() { for { usedByEngine := UsedByEngineMemory() - log.Printf("Memory used by engine: %v MB", usedByEngine) - log.Printf("Free memory: %v MB", FreeMemory()) - log.Printf("Physical memory: %v MB", TotalMemory()) + catcher.Info("Memory used by engine", map[string]any{"used_by_engine": usedByEngine}) + catcher.Info("Free memory", map[string]any{"free_memory": FreeMemory()}) + catcher.Info("Physical memory", map[string]any{"physical_memory": TotalMemory()}) AssignedMemory = float32(usedByEngine) / float32(TotalMemory()/4) * 100 - log.Printf("Assigned memory used: %v %%", AssignedMemory) + catcher.Info("Assigned memory used", map[string]any{"assigned_memory_used": AssignedMemory}) time.Sleep(60 * time.Second) } } diff --git a/log-auth-proxy/go.mod b/log-auth-proxy/go.mod index b9e10ba21..9515925e9 100644 --- a/log-auth-proxy/go.mod +++ b/log-auth-proxy/go.mod @@ -1,42 +1,45 @@ module github.com/utmstack/UTMStack/log-auth-proxy -go 1.23.0 +go 1.24.2 -toolchain go1.23.4 +toolchain go1.24.6 require ( github.com/gin-gonic/gin v1.10.1 - github.com/threatwinds/logger v1.2.2 - google.golang.org/grpc v1.73.0 - google.golang.org/protobuf v1.36.6 + google.golang.org/grpc v1.75.1 + google.golang.org/protobuf v1.36.9 ) require ( - github.com/bytedance/sonic v1.13.3 // indirect - github.com/bytedance/sonic/loader v0.2.4 // indirect - github.com/cloudwego/base64x v0.1.5 // indirect - github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/kr/text v0.2.0 // indirect +) + +require ( + github.com/bytedance/sonic v1.14.1 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/gabriel-vasile/mimetype v1.4.10 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-playground/validator/v10 v10.27.0 // indirect github.com/goccy/go-json v0.10.5 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/threatwinds/go-sdk v1.0.45 github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.14 // indirect - golang.org/x/arch v0.18.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + github.com/ugorji/go/codec v1.3.0 // indirect + golang.org/x/arch v0.21.0 // indirect + golang.org/x/crypto v0.42.0 // indirect + golang.org/x/net v0.44.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250908214217-97024824d090 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/log-auth-proxy/go.sum b/log-auth-proxy/go.sum index ef952dfaf..a9f27004a 100644 --- a/log-auth-proxy/go.sum +++ b/log-auth-proxy/go.sum @@ -1,22 +1,23 @@ -github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= -github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= -github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= -github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= -github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.14.1 h1:FBMC0zVz5XUmE4z9wF4Jey0An5FueFvOsTKKKtwIl7w= +github.com/bytedance/sonic v1.14.1/go.mod h1:gi6uhQLMbTdeP0muCnrjHLeCUPyb70ujhnNlhOylAFc= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= -github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/gabriel-vasile/mimetype v1.4.10 h1:zyueNbySn/z8mJZHLt6IPw0KoZsiQNszIpU+bX4+ZK0= +github.com/gabriel-vasile/mimetype v1.4.10/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -25,8 +26,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= -github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= +github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -38,10 +39,12 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -55,56 +58,57 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/threatwinds/logger v1.2.2 h1:sVuT8yhbecPqP4tT8EwHfp1czNC6e1wdkE1ihNnuBdA= -github.com/threatwinds/logger v1.2.2/go.mod h1:Amq0QI1y7fkTpnBUgeGVu2Z/C4u4ys2pNLUOuj3UAAU= +github.com/threatwinds/go-sdk v1.0.45 h1:KZ3s3HviNRrOkg5EqjFnoauANFFzTqjNFyshPLY2SoI= +github.com/threatwinds/go-sdk v1.0.45/go.mod h1:tcWn6r6vqID/W/nL3UKfc5NafA3V/cSkiLvfJnwB58c= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.14 h1:yOQvXCBc3Ij46LRkRoh4Yd5qK6LVOgi0bYOXfb7ifjw= -github.com/ugorji/go/codec v1.2.14/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= +github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= -golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= -golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +golang.org/x/arch v0.21.0 h1:iTC9o7+wP6cPWpDWkivCvQFGAHDQ59SrSxsLPcnkArw= +golang.org/x/arch v0.21.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= -google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250908214217-97024824d090 h1:/OQuEa4YWtDt7uQWHd3q3sUMb+QOLQUg1xa8CEsRv5w= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250908214217-97024824d090/go.mod h1:GmFNa4BdJZ2a8G+wCe9Bg3wwThLrJun751XstdJt5Og= +google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= +google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= +google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= -gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/log-auth-proxy/handlers/http_handler.go b/log-auth-proxy/handlers/http_handler.go index 9c18de307..09cbae622 100644 --- a/log-auth-proxy/handlers/http_handler.go +++ b/log-auth-proxy/handlers/http_handler.go @@ -6,9 +6,9 @@ import ( "net/http" "github.com/gin-gonic/gin" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/log-auth-proxy/config" "github.com/utmstack/UTMStack/log-auth-proxy/logservice" - "github.com/utmstack/UTMStack/log-auth-proxy/utils" ) func HttpLog(logOutputService *logservice.LogOutputService) gin.HandlerFunc { @@ -16,14 +16,14 @@ func HttpLog(logOutputService *logservice.LogOutputService) gin.HandlerFunc { var body map[string]interface{} if err := c.ShouldBindJSON(&body); err != nil { - utils.Logger.ErrorF("Error binding http JSON: %v", err) + catcher.Error("Error binding http JSON", err, nil) c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } logType, source, err := getHeaderAndSource(c) if err != nil { - utils.Logger.ErrorF("Error getting header and source: %v", err) + catcher.Error("Error getting header and source:", err, nil) c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } @@ -36,7 +36,7 @@ func HttpLog(logOutputService *logservice.LogOutputService) gin.HandlerFunc { jsonBytes, err := json.Marshal(body) if err != nil { - utils.Logger.ErrorF("Error marshalling http JSON: %v", err) + catcher.Error("Error marshalling http JSON", err, nil) c.JSON(http.StatusInternalServerError, gin.H{"error": "unable to convert JSON to string"}) return } @@ -54,13 +54,13 @@ func HttpBulkLog(logOutputService *logservice.LogOutputService) gin.HandlerFunc var body []interface{} if err := c.ShouldBindJSON(&body); err != nil { - utils.Logger.ErrorF("Error binding bulk JSON: %v", err) + catcher.Error("Error binding bulk JSON", err, nil) c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } logType, source, err := getHeaderAndSource(c) if err != nil { - utils.Logger.ErrorF("Error getting header and source: %v", err) + catcher.Error("Error getting header and source:", err, nil) c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } @@ -79,7 +79,7 @@ func HttpBulkLog(logOutputService *logservice.LogOutputService) gin.HandlerFunc str, err := json.Marshal(v) if err != nil { - utils.Logger.ErrorF("Error marshalling bulk JSON: %v", err) + catcher.Error("Error marshalling bulk JSON", err, nil) continue } log := string(str) @@ -100,14 +100,14 @@ func HttpGitHubHandler(logOutputService *logservice.LogOutputService) gin.Handle var body interface{} if err := c.ShouldBindJSON(&body); err != nil { - utils.Logger.ErrorF("Error binding github JSON: %v", err) + catcher.Error("Error binding github JSON", err, nil) c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } jsonBytes, err := json.Marshal(body) if err != nil { - utils.Logger.ErrorF("Error marshalling github JSON: %v", err) + catcher.Error("Error marshalling github JSON", err, nil) c.JSON(http.StatusInternalServerError, gin.H{"error": "unable to convert JSON to string"}) return } diff --git a/log-auth-proxy/handlers/tcp_handler.go b/log-auth-proxy/handlers/tcp_handler.go index cdf404a7f..cb4666190 100644 --- a/log-auth-proxy/handlers/tcp_handler.go +++ b/log-auth-proxy/handlers/tcp_handler.go @@ -5,10 +5,10 @@ import ( "net" "strings" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/log-auth-proxy/config" "github.com/utmstack/UTMStack/log-auth-proxy/logservice" "github.com/utmstack/UTMStack/log-auth-proxy/middleware" - "github.com/utmstack/UTMStack/log-auth-proxy/utils" ) func HandleRequest(conn net.Conn, interceptor *middleware.LogAuthInterceptor, logOutputService *logservice.LogOutputService) { @@ -20,7 +20,7 @@ func HandleRequest(conn net.Conn, interceptor *middleware.LogAuthInterceptor, lo parts := strings.Split(message, ",LOG:") if len(parts) != 2 { - utils.Logger.ErrorF("INVALID FORMAT expecting AUTH:,LOG:") + catcher.Error("INVALID FORMAT expecting AUTH:,LOG:", nil, nil) conn.Write([]byte("INVALID FORMAT expecting AUTH:,LOG:\n")) continue } @@ -39,6 +39,6 @@ func HandleRequest(conn net.Conn, interceptor *middleware.LogAuthInterceptor, lo } if err := scanner.Err(); err != nil { - utils.Logger.ErrorF("Error reading from connection: %s", err.Error()) + catcher.Error("Error reading from connection:", err, nil) } } diff --git a/log-auth-proxy/logservice/auth_service.go b/log-auth-proxy/logservice/auth_service.go index d937ed94d..35aa0d98d 100644 --- a/log-auth-proxy/logservice/auth_service.go +++ b/log-auth-proxy/logservice/auth_service.go @@ -7,10 +7,10 @@ import ( "sync" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/log-auth-proxy/agent" "github.com/utmstack/UTMStack/log-auth-proxy/config" "github.com/utmstack/UTMStack/log-auth-proxy/panelservice" - "github.com/utmstack/UTMStack/log-auth-proxy/utils" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" @@ -54,7 +54,8 @@ func (auth *LogAuthService) SyncAuth() { func (auth *LogAuthService) syncKeys(typ agent.ConnectorType) { serverAddress := os.Getenv(config.UTMAgentManagerHostEnv) if serverAddress == "" { - utils.Logger.Fatal("Failed to get the SERVER_ADDRESS ") + catcher.Error("Failed to get the SERVER_ADDRESS", nil, nil) + os.Exit(1) } tlsConfig := &tls.Config{InsecureSkipVerify: true} @@ -63,7 +64,7 @@ func (auth *LogAuthService) syncKeys(typ agent.ConnectorType) { conn, err := grpc.NewClient(serverAddress, opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMessageSize))) if err != nil { - utils.Logger.ErrorF("Failed to connect to gRPC server: %v", err) + catcher.Error("Failed to connect to gRPC server", err, nil) return } defer conn.Close() @@ -82,7 +83,7 @@ func (auth *LogAuthService) syncKeys(typ agent.ConnectorType) { SortBy: "", }) if err != nil { - utils.Logger.ErrorF("Error sync collector keys: %v", err) + catcher.Error("Error sync collector keys", err, nil) return } @@ -104,7 +105,7 @@ func (auth *LogAuthService) syncKeys(typ agent.ConnectorType) { SortBy: "", }) if err != nil { - utils.Logger.ErrorF("Error sync agent keys: %v", err) + catcher.Error("Error sync agent keys", err, nil) return } @@ -121,7 +122,7 @@ func (auth *LogAuthService) syncKeys(typ agent.ConnectorType) { func (auth *LogAuthService) syncConnectionKey() { panelKey, err := panelservice.GetConnectionKey() if err != nil { - utils.Logger.ErrorF("Failed to get connection key: %v", err) + catcher.Error("Failed to get connection key", err, nil) return } auth.Mutex.Lock() diff --git a/log-auth-proxy/logservice/output_service.go b/log-auth-proxy/logservice/output_service.go index 8465048ed..ec0ea14f9 100644 --- a/log-auth-proxy/logservice/output_service.go +++ b/log-auth-proxy/logservice/output_service.go @@ -10,9 +10,9 @@ import ( "sync" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/log-auth-proxy/config" "github.com/utmstack/UTMStack/log-auth-proxy/panelservice" - "github.com/utmstack/UTMStack/log-auth-proxy/utils" ) var transport = &http.Transport{ @@ -45,7 +45,7 @@ func (out *LogOutputService) SendLog(logType config.LogType, logData string) { defer out.Mutex.Unlock() port, err := out.getConnectionPort(logType) if err != nil { - utils.Logger.ErrorF("error getting connection port: %v", err) + catcher.Error("error getting connection port", err, nil) return } singleLog := logData + config.UTMLogSeparator @@ -63,7 +63,7 @@ func (out *LogOutputService) SendBulkLog(logType config.LogType, logDataArray [] port, err := out.getConnectionPort(logType) if err != nil { - utils.Logger.ErrorF("error getting connection port: %v", err) + catcher.Error("error getting connection port", err, nil) return } @@ -90,18 +90,18 @@ func (out *LogOutputService) sendLogsToLogstash(port string, logs string) { url := fmt.Sprintf(config.LogstashPipelinesEndpoint, config.LogstashHost(), port) req, err := http.NewRequest("POST", url, bytes.NewBufferString(logs)) if err != nil { - utils.Logger.ErrorF("error creating request: %v", err) + catcher.Error("error creating request", err, nil) } resp, err := out.Client.Do(req) if err != nil { if !strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") { - utils.Logger.ErrorF("error sending logs with error: %v", err.Error()) + catcher.Error("error sending logs", err, nil) } return } if resp.StatusCode != http.StatusOK { - utils.Logger.ErrorF("error sending logs with http code %d", resp.StatusCode) + catcher.Error("error sending logs with http code", nil, map[string]any{"status_code": resp.StatusCode}) return } } @@ -124,7 +124,7 @@ func (out *LogOutputService) SyncOutputs() { for range out.Ticker.C { serviceMap, err := getServiceMap() if err != nil { - utils.Logger.ErrorF("error getting service map: %v", err) + catcher.Error("error getting service map", err, nil) continue } out.Mutex.Lock() diff --git a/log-auth-proxy/main.go b/log-auth-proxy/main.go index e25436677..bbbfb7363 100644 --- a/log-auth-proxy/main.go +++ b/log-auth-proxy/main.go @@ -4,12 +4,13 @@ import ( "crypto/tls" "net" "net/http" + "os" "github.com/gin-gonic/gin" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/log-auth-proxy/handlers" "github.com/utmstack/UTMStack/log-auth-proxy/logservice" "github.com/utmstack/UTMStack/log-auth-proxy/middleware" - "github.com/utmstack/UTMStack/log-auth-proxy/utils" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/health" @@ -17,7 +18,7 @@ import ( ) func main() { - utils.Logger.Info("Starting Log Auth Proxy...") + catcher.Info("Starting Log Auth Proxy...", nil) autService := logservice.NewLogAuthService() go autService.SyncAuth() authInterceptor := middleware.NewLogAuthInterceptor(autService) @@ -41,7 +42,8 @@ func startHTTPServer(interceptor *middleware.LogAuthInterceptor, logOutputServic cert, err := tls.LoadX509KeyPair("/cert/utm.crt", "/cert/utm.key") if err != nil { - utils.Logger.Fatal("failed to load server certificates: %v", err) + catcher.Error("failed to load server certificates", err, nil) + os.Exit(1) } tlsConfig := &tls.Config{ @@ -55,17 +57,19 @@ func startHTTPServer(interceptor *middleware.LogAuthInterceptor, logOutputServic TLSConfig: tlsConfig, } - utils.Logger.Info("Starting HTTP server on 0.0.0.0:8080") + catcher.Info("Starting HTTP server on 0.0.0.0:8080", nil) err = server.ListenAndServeTLS("", "") if err != nil { - utils.Logger.Fatal("Failed to start HTTP server: %v", err) + catcher.Error("Failed to start HTTP server", err, nil) + os.Exit(1) } } func startGRPCServer(interceptor *middleware.LogAuthInterceptor, logOutputService *logservice.LogOutputService) { cert, err := tls.LoadX509KeyPair("/cert/utm.crt", "/cert/utm.key") if err != nil { - utils.Logger.Fatal("failed to load server certificates: %v", err) + catcher.Error("failed to load server certificates", err, nil) + os.Exit(1) } tlsConfig := &tls.Config{ @@ -93,11 +97,13 @@ func startGRPCServer(interceptor *middleware.LogAuthInterceptor, logOutputServic lis, err := net.Listen("tcp", "0.0.0.0:50051") if err != nil { - utils.Logger.Fatal("failed to listen grpc server: %v", err) + catcher.Error("failed to listen grpc server", err, nil) + os.Exit(1) } - utils.Logger.Info("Starting gRPC server on 0.0.0.0:50051") + catcher.Info("Starting gRPC server on 0.0.0.0:50051", nil) if err := grpcServer.Serve(lis); err != nil { - utils.Logger.Fatal("Failed to serve grpc: %v", err) + catcher.Error("Failed to serve grpc", err, nil) + os.Exit(1) } } diff --git a/log-auth-proxy/middleware/log_auth.go b/log-auth-proxy/middleware/log_auth.go index 2a0329928..87ba874b5 100644 --- a/log-auth-proxy/middleware/log_auth.go +++ b/log-auth-proxy/middleware/log_auth.go @@ -12,9 +12,9 @@ import ( "sync" "github.com/gin-gonic/gin" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/log-auth-proxy/config" "github.com/utmstack/UTMStack/log-auth-proxy/logservice" - "github.com/utmstack/UTMStack/log-auth-proxy/utils" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -65,7 +65,7 @@ func (interceptor *LogAuthInterceptor) GrpcRecoverInterceptor( defer func() { if r := recover(); r != nil { // Handle the panic here - utils.Logger.ErrorF("Panic occurred: %v", r) + catcher.Error("Panic occurred:", nil, map[string]any{"panic": r}) err = status.Errorf(codes.Internal, "Internal server error") } }() @@ -93,7 +93,7 @@ func (interceptor *LogAuthInterceptor) HTTPGitHubAuthInterceptor() gin.HandlerFu return func(c *gin.Context) { body, err := io.ReadAll(c.Request.Body) if err != nil { - utils.Logger.ErrorF("error reading request body: %v", err) + catcher.Error("error reading request body", err, nil) c.AbortWithStatusJSON(http.StatusInternalServerError, "error reading request body") return } diff --git a/log-auth-proxy/utils/logger.go b/log-auth-proxy/utils/logger.go deleted file mode 100644 index 4332e7f5a..000000000 --- a/log-auth-proxy/utils/logger.go +++ /dev/null @@ -1,31 +0,0 @@ -package utils - -import ( - "log" - "os" - "strconv" - - "github.com/threatwinds/logger" -) - -var Logger *logger.Logger - -func init() { - lenv := os.Getenv("LOG_LEVEL") - var level int - var err error - - if lenv != "" && lenv != " " { - level, err = strconv.Atoi(lenv) - if err != nil { - log.Fatalln(err) - } - } else { - level = 200 - } - - Logger = logger.NewLogger(&logger.Config{ - Format: "text", - Level: level, - }) -} diff --git a/office365/go.mod b/office365/go.mod index 3e3b6501e..b9b59aff5 100644 --- a/office365/go.mod +++ b/office365/go.mod @@ -1,8 +1,8 @@ module github.com/utmstack/UTMStack/office365 -go 1.23.0 +go 1.24.2 -toolchain go1.24.2 +toolchain go1.24.6 require ( github.com/threatwinds/logger v1.2.2 @@ -10,31 +10,32 @@ require ( ) require ( - github.com/bytedance/sonic v1.13.3 // indirect - github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/bytedance/sonic v1.14.0 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/gin-gonic/gin v1.10.1 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-playground/validator/v10 v10.27.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/threatwinds/go-sdk v1.0.45 github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.14 // indirect - golang.org/x/arch v0.18.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect + github.com/ugorji/go/codec v1.3.0 // indirect + golang.org/x/arch v0.19.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/office365/go.sum b/office365/go.sum index 18e3ec6ce..22c72acd3 100644 --- a/office365/go.sum +++ b/office365/go.sum @@ -1,8 +1,12 @@ github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= +github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= @@ -23,6 +27,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= +github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= @@ -35,6 +41,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= @@ -59,25 +67,39 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/threatwinds/go-sdk v1.0.45 h1:KZ3s3HviNRrOkg5EqjFnoauANFFzTqjNFyshPLY2SoI= +github.com/threatwinds/go-sdk v1.0.45/go.mod h1:tcWn6r6vqID/W/nL3UKfc5NafA3V/cSkiLvfJnwB58c= github.com/threatwinds/logger v1.2.2 h1:sVuT8yhbecPqP4tT8EwHfp1czNC6e1wdkE1ihNnuBdA= github.com/threatwinds/logger v1.2.2/go.mod h1:Amq0QI1y7fkTpnBUgeGVu2Z/C4u4ys2pNLUOuj3UAAU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.14 h1:yOQvXCBc3Ij46LRkRoh4Yd5qK6LVOgi0bYOXfb7ifjw= github.com/ugorji/go/codec v1.2.14/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= +github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/utmstack/config-client-go v1.2.7 h1:JeRdI5JjH1liNzMW3LmyevjuPd67J/yt9MAO3+oJAuM= github.com/utmstack/config-client-go v1.2.7/go.mod h1:kM0KoUizM9ZlcQp0qKviGTWn/+anT5Rfjx3zfZk79nM= golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/arch v0.19.0 h1:LmbDQUodHThXE+htjrnmVD73M//D9GTH6wFZjyDkjyU= +golang.org/x/arch v0.19.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= diff --git a/office365/main.go b/office365/main.go index b8a002809..77f61a275 100644 --- a/office365/main.go +++ b/office365/main.go @@ -1,10 +1,12 @@ package main import ( + "os" "strings" "sync" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/office365/configuration" "github.com/utmstack/UTMStack/office365/processor" "github.com/utmstack/UTMStack/office365/utils" @@ -14,11 +16,12 @@ import ( ) func main() { - utils.Logger.Info("Starting O365 module") + catcher.Info("Starting O365 module", nil) intKey := configuration.GetInternalKey() panelServ := configuration.GetPanelServiceName() if intKey == "" || panelServ == "" { - utils.Logger.Fatal("Internal key or panel service name is not set. Exiting...") + catcher.Error("Internal key or panel service name is not set. Exiting...", nil, nil) + os.Exit(1) } client := utmconf.NewUTMClient(intKey, "http://"+panelServ) @@ -30,20 +33,20 @@ func main() { for range ticker.C { if err := utils.ConnectionChecker(configuration.LoginUrl); err != nil { - utils.Logger.ErrorF("External connection failure detected: %v", err) + catcher.Error("External connection failure detected", err, nil) } endTime := time.Now().UTC() - utils.Logger.Info("Syncing logs from %s to %s", startTime, endTime) + catcher.Info("Syncing logs", map[string]any{"start_time": startTime, "end_time": endTime}) moduleConfig, err := client.GetUTMConfig(enum.O365) if err != nil { if strings.Contains(err.Error(), "invalid character '<'") { - utils.Logger.LogF(100, "error getting configuration of the O365 module: backend is not available") + catcher.Error("error getting configuration of the O365 module: backend is not available", nil, nil) } if strings.TrimSpace(err.Error()) != "" { - utils.Logger.ErrorF("error getting configuration of the O365 module: %v", err) + catcher.Error("error getting configuration of the O365 module", err, nil) } continue } @@ -59,7 +62,7 @@ func main() { for _, cnf := range group.Configurations { if strings.TrimSpace(cnf.ConfValue) == "" { - utils.Logger.LogF(100, "program not configured yet for group: %s", group.GroupName) + catcher.Error("program not configured yet for group", nil, map[string]any{"group_name": group.GroupName}) skip = true break } @@ -73,7 +76,7 @@ func main() { wg.Wait() } - utils.Logger.Info("sync completed from %v to %v, waiting 5 minutes", startTime, endTime) + catcher.Info("sync completed, waiting 5 minutes", map[string]any{"start_time": startTime, "end_time": endTime}) startTime = endTime.Add(time.Nanosecond) } } diff --git a/office365/processor/processor.go b/office365/processor/processor.go index e0f82e1a9..69209c6c4 100644 --- a/office365/processor/processor.go +++ b/office365/processor/processor.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/office365/configuration" "github.com/utmstack/UTMStack/office365/utils" "github.com/utmstack/config-client-go/types" @@ -71,7 +72,7 @@ func (o *OfficeProcessor) GetAuth() error { func (o *OfficeProcessor) StartSubscriptions() error { for _, subscription := range o.Subscriptions { - utils.Logger.Info("starting subscription: %s...", subscription) + catcher.Info("starting subscription...", map[string]any{"subscription": subscription}) url := configuration.GetStartSubscriptionLink(o.TenantId) + "?contentType=" + subscription headers := map[string]string{ "Content-Type": "application/json", @@ -92,7 +93,7 @@ func (o *OfficeProcessor) StartSubscriptions() error { return fmt.Errorf("failed to unmarshal response: %v", err) } - utils.Logger.Info("starting subscription response: %v", respJson) + catcher.Info("starting subscription response", map[string]any{"response": respJson}) } return nil @@ -136,7 +137,7 @@ func (o *OfficeProcessor) GetLogs(startTime time.Time, endTime time.Time, group for _, subscription := range o.Subscriptions { contentList, err := o.GetContentList(subscription, startTime, endTime, group) if err != nil { - utils.Logger.ErrorF("error getting content list: %v", err) + catcher.Error("error getting content list", err, nil) continue } logsCounter := 0 @@ -144,7 +145,7 @@ func (o *OfficeProcessor) GetLogs(startTime time.Time, endTime time.Time, group for _, log := range contentList { details, err := o.GetContentDetails(log.ContentUri) if err != nil { - utils.Logger.ErrorF("error getting content details: %v", err) + catcher.Error("error getting content details", err, nil) continue } if len(details) > 0 { @@ -152,13 +153,13 @@ func (o *OfficeProcessor) GetLogs(startTime time.Time, endTime time.Time, group cleanLogs := ETLProcess(details, group) err := SendToLogstash(cleanLogs) if err != nil { - utils.Logger.ErrorF("error sending logs to logstash: %v", err) + catcher.Error("error sending logs to logstash", err, nil) continue } } } } - utils.Logger.Info("found: %d new logs in %s for group %s", logsCounter, subscription, group.GroupName) + catcher.Info("new logs were found", map[string]any{"count": logsCounter, "subscription": subscription, "group": group.GroupName}) } } diff --git a/office365/processor/pull.go b/office365/processor/pull.go index d33c9210c..face0c17d 100644 --- a/office365/processor/pull.go +++ b/office365/processor/pull.go @@ -3,24 +3,24 @@ package processor import ( "time" - "github.com/utmstack/UTMStack/office365/utils" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/config-client-go/types" ) func PullLogs(startTime time.Time, endTime time.Time, group types.ModuleGroup) { - utils.Logger.Info("starting log sync for : %s from %s to %s", group.GroupName, startTime, endTime) + catcher.Info("starting log sync", map[string]any{"group_name": group.GroupName, "start_time": startTime, "end_time": endTime}) agent := GetOfficeProcessor(group) err := agent.GetAuth() if err != nil { - utils.Logger.ErrorF("error getting auth token: %v", err) + catcher.Error("error getting auth token", err, nil) return } err = agent.StartSubscriptions() if err != nil { - utils.Logger.ErrorF("error starting subscriptions: %v", err) + catcher.Error("error starting subscriptions", err, nil) return } diff --git a/office365/processor/sendData.go b/office365/processor/sendData.go index 99d92d2af..d7524d687 100644 --- a/office365/processor/sendData.go +++ b/office365/processor/sendData.go @@ -8,9 +8,9 @@ import ( "net/http" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/threatwinds/logger" "github.com/utmstack/UTMStack/office365/configuration" - "github.com/utmstack/UTMStack/office365/utils" ) var transport = &http.Transport{ @@ -29,11 +29,11 @@ func SendToLogstash(data []TransformedLog) *logger.Error { for _, str := range data { body, err := json.Marshal(str) if err != nil { - utils.Logger.ErrorF("error encoding log to JSON: %v", err) + catcher.Error("error encoding log to JSON", err, nil) continue } if err := sendLogs(body); err != nil { - utils.Logger.ErrorF("error sending logs to logstach: %v", err) + catcher.Error("error sending logs to logstach", err, nil) continue } } @@ -45,17 +45,17 @@ func sendLogs(log []byte) error { req, err := http.NewRequest("POST", url, bytes.NewBuffer(log)) if err != nil { - return utils.Logger.ErrorF("error creating request: %v", err.Error()) + return catcher.Error("error creating request", err, nil) } resp, err := client.Do(req) if err != nil { - return utils.Logger.ErrorF("error sending logs: %v", err.Error()) + return catcher.Error("error sending logs: %v", err, nil) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return utils.Logger.ErrorF("error sending logs with http code %d", resp.StatusCode) + return catcher.Error("error sending logs with http code", nil, map[string]any{"status_code": resp.StatusCode}) } return nil } diff --git a/office365/utils/check.go b/office365/utils/check.go index 5ab7c1ccd..bde8e1db4 100644 --- a/office365/utils/check.go +++ b/office365/utils/check.go @@ -4,10 +4,16 @@ import ( "context" "fmt" "net/http" + "strings" "time" + + "github.com/threatwinds/go-sdk/catcher" ) -const connectionTimeout = 5 * time.Second +const ( + connectionTimeout = 5 * time.Second + wait = 3 * time.Second +) func ConnectionChecker(url string) error { checkConn := func() error { @@ -20,7 +26,7 @@ func ConnectionChecker(url string) error { return nil } - if err := Logger.InfiniteRetryIfXError(checkConn, "connection failed"); err != nil { + if err := infiniteRetryIfXError(checkConn, "connection failed"); err != nil { return err } @@ -43,3 +49,30 @@ func checkConnection(url string, ctx context.Context) error { return nil } + +func infiniteRetryIfXError(f func() error, exception string) error { + var xErrorWasLogged bool + + for { + err := f() + if err != nil && is(err, exception) { + if !xErrorWasLogged { + _ = catcher.Error("An error occurred (%s), will keep retrying indefinitely...", err, nil) + xErrorWasLogged = true + } + time.Sleep(wait) + continue + } + + return err + } +} + +func is(e error, args ...string) bool { + for _, arg := range args { + if strings.Contains(e.Error(), arg) { + return true + } + } + return false +} diff --git a/office365/utils/env.go b/office365/utils/env.go index 846eaee2a..0332339d2 100644 --- a/office365/utils/env.go +++ b/office365/utils/env.go @@ -1,18 +1,21 @@ package utils import ( - "log" "os" + + "github.com/threatwinds/go-sdk/catcher" ) // Getenv returns the environment variable func Getenv(key string) string { value, defined := os.LookupEnv(key) if !defined { - log.Fatalf("Error loading environment variable: %s: environment variable does not exist\n", key) + catcher.Error("Error loading environment variable, environment variable does not exist", nil, map[string]any{"key": key}) + os.Exit(1) } if (value == "") || (value == " ") { - log.Fatalf("Error loading environment variable: %s: empty environment variable\n", key) + catcher.Error("Error loading environment variable, empty environment variable", nil, map[string]any{"key": key}) + os.Exit(1) } return value } diff --git a/office365/utils/utils.go b/office365/utils/utils.go deleted file mode 100644 index 4332e7f5a..000000000 --- a/office365/utils/utils.go +++ /dev/null @@ -1,31 +0,0 @@ -package utils - -import ( - "log" - "os" - "strconv" - - "github.com/threatwinds/logger" -) - -var Logger *logger.Logger - -func init() { - lenv := os.Getenv("LOG_LEVEL") - var level int - var err error - - if lenv != "" && lenv != " " { - level, err = strconv.Atoi(lenv) - if err != nil { - log.Fatalln(err) - } - } else { - level = 200 - } - - Logger = logger.NewLogger(&logger.Config{ - Format: "text", - Level: level, - }) -} diff --git a/soc-ai/configurations/config.go b/soc-ai/configurations/config.go index 03de64546..b2c593859 100644 --- a/soc-ai/configurations/config.go +++ b/soc-ai/configurations/config.go @@ -4,6 +4,7 @@ import ( "sync" "time" + "github.com/threatwinds/go-sdk/catcher" UTMStackConfigurationClient "github.com/utmstack/config-client-go" "github.com/utmstack/config-client-go/enum" "github.com/utmstack/soc-ai/utils" @@ -35,21 +36,21 @@ func (c *GPTConfig) UpdateGPTConfigurations() { panelServ := GetPanelServiceName() client := UTMStackConfigurationClient.NewUTMClient(intKey, panelServ) - utils.Logger.Info("Starting to update GPT configurations...") + catcher.Info("Starting to update GPT configurations...", nil) for { if err := utils.ConnectionChecker(GPT_API_ENDPOINT); err != nil { - utils.Logger.ErrorF("Failed to establish internet connection: %v", err) + catcher.Error("Failed to establish internet connection", err, nil) } tempModuleConfig, err := client.GetUTMConfig(enum.SOCAI) if err != nil && err.Error() != "" && err.Error() != " " { - utils.Logger.LogF(100, "Error while getting GPT configuration: %v", err) + catcher.Error("Error while getting GPT configuration", err, nil) time.Sleep(TIME_FOR_GET_CONFIG * time.Second) continue } if tempModuleConfig == nil { - utils.Logger.LogF(100, "Got nil config from server") + catcher.Error("Got nil config from server", nil, nil) time.Sleep(TIME_FOR_GET_CONFIG * time.Second) continue } diff --git a/soc-ai/go.mod b/soc-ai/go.mod index 0e1491e2b..df345207d 100644 --- a/soc-ai/go.mod +++ b/soc-ai/go.mod @@ -1,8 +1,8 @@ module github.com/utmstack/soc-ai -go 1.23.0 +go 1.24.2 -toolchain go1.24.2 +toolchain go1.24.6 require ( github.com/gin-contrib/gzip v1.2.3 @@ -13,32 +13,33 @@ require ( ) require ( - github.com/bytedance/sonic v1.13.3 // indirect - github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/bytedance/sonic v1.14.0 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-playground/validator/v10 v10.27.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/threatwinds/go-sdk v1.0.45 github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.14 // indirect - golang.org/x/arch v0.18.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect + github.com/ugorji/go/codec v1.3.0 // indirect + golang.org/x/arch v0.19.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/soc-ai/go.sum b/soc-ai/go.sum index 6da7c98fc..fc7b0545e 100644 --- a/soc-ai/go.sum +++ b/soc-ai/go.sum @@ -1,8 +1,12 @@ github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= +github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= @@ -27,6 +31,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= +github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -39,6 +45,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= @@ -65,25 +73,39 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/threatwinds/go-sdk v1.0.45 h1:KZ3s3HviNRrOkg5EqjFnoauANFFzTqjNFyshPLY2SoI= +github.com/threatwinds/go-sdk v1.0.45/go.mod h1:tcWn6r6vqID/W/nL3UKfc5NafA3V/cSkiLvfJnwB58c= github.com/threatwinds/logger v1.2.2 h1:sVuT8yhbecPqP4tT8EwHfp1czNC6e1wdkE1ihNnuBdA= github.com/threatwinds/logger v1.2.2/go.mod h1:Amq0QI1y7fkTpnBUgeGVu2Z/C4u4ys2pNLUOuj3UAAU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.14 h1:yOQvXCBc3Ij46LRkRoh4Yd5qK6LVOgi0bYOXfb7ifjw= github.com/ugorji/go/codec v1.2.14/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= +github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/utmstack/config-client-go v1.2.7 h1:JeRdI5JjH1liNzMW3LmyevjuPd67J/yt9MAO3+oJAuM= github.com/utmstack/config-client-go v1.2.7/go.mod h1:kM0KoUizM9ZlcQp0qKviGTWn/+anT5Rfjx3zfZk79nM= golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/arch v0.19.0 h1:LmbDQUodHThXE+htjrnmVD73M//D9GTH6wFZjyDkjyU= +golang.org/x/arch v0.19.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/soc-ai/processor/alertProcessor.go b/soc-ai/processor/alertProcessor.go index 113a0c8a6..e968294a2 100644 --- a/soc-ai/processor/alertProcessor.go +++ b/soc-ai/processor/alertProcessor.go @@ -3,25 +3,25 @@ package processor import ( "fmt" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/soc-ai/elastic" "github.com/utmstack/soc-ai/schema" - "github.com/utmstack/soc-ai/utils" ) func (p *Processor) processAlertsInfo() { for alert := range p.AlertInfoQueue { - utils.Logger.Info("Processing alert info for ID: %s", alert.AlertID) + catcher.Info("Processing alert info for ID", map[string]any{"alert": alert.AlertID}) alertInfo, err := elastic.GetAlertsInfo(alert.AlertID) if err != nil { p.RegisterError(fmt.Sprintf("error while getting alert %s info: %v", alert.AlertID, err), alert.AlertID) continue } - utils.Logger.Info("Alert info retrieved successfully for ID: %s", alert.AlertID) + catcher.Info("Alert info retrieved successfully for ID", map[string]any{"alert": alert.AlertID}) correlation, err := elastic.FindRelatedAlerts(alertInfo) if err != nil { - utils.Logger.ErrorF("error finding related alerts: %v", err) + catcher.Error("error finding related alerts", err, nil) } details := schema.ConvertFromAlertToAlertDB(alertInfo) diff --git a/soc-ai/processor/elastic.go b/soc-ai/processor/elastic.go index 934aa5002..d108d95a8 100644 --- a/soc-ai/processor/elastic.go +++ b/soc-ai/processor/elastic.go @@ -4,10 +4,10 @@ import ( "fmt" "strings" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/soc-ai/configurations" "github.com/utmstack/soc-ai/elastic" "github.com/utmstack/soc-ai/schema" - "github.com/utmstack/soc-ai/utils" ) func (p *Processor) processAlertToElastic() { @@ -30,16 +30,16 @@ func (p *Processor) processAlertToElastic() { if gptConfig.ChangeAlertStatus { err = elastic.ChangeAlertStatus(alert.AlertID, configurations.API_ALERT_COMPLETED_STATUS_CODE, alert.GPTClassification+" - "+alert.GPTReasoning) if err != nil { - utils.Logger.ErrorF("error while changing alert status in elastic: %v", err) + catcher.Error("error while changing alert status in elastic", err, nil) continue } - utils.Logger.Info("alert %s status changed to COMPLETED in Panel", alert.AlertID) + catcher.Info("alert status changed to COMPLETED in Panel", map[string]any{"alert": alert.AlertID}) } if gptConfig.AutomaticIncidentCreation && alert.GPTClassification == "possible incident" { incidentsDetails, err := elastic.GetIncidentsByPattern("Incident in " + alert.DataSource) if err != nil { - utils.Logger.ErrorF("error while getting incidents by pattern: %v", err) + catcher.Error("error while getting incidents by pattern", err, nil) continue } @@ -50,7 +50,7 @@ func (p *Processor) processAlertToElastic() { incidentExists = true err = elastic.AddAlertToIncident(incident.ID, alert) if err != nil { - utils.Logger.ErrorF("error while adding alert to incident: %v", err) + catcher.Error("error while adding alert to incident", err, nil) continue } } @@ -60,14 +60,14 @@ func (p *Processor) processAlertToElastic() { if !incidentExists { err = elastic.CreateNewIncident(alert) if err != nil { - utils.Logger.ErrorF("error while creating incident: %v", err) + catcher.Error("error while creating incident", err, nil) continue } } - utils.Logger.Info("alert %s added to incident in Panel", alert.AlertID) + catcher.Info("alert added to incident in Panel", map[string]any{"alert": alert.AlertID}) } - utils.Logger.Info("alert %s processed correctly", alert.AlertID) + catcher.Info("alert processed correctly", map[string]any{"alert": alert.AlertID}) } } diff --git a/soc-ai/processor/listener.go b/soc-ai/processor/listener.go index 9ccf244c5..ea01c67b2 100644 --- a/soc-ai/processor/listener.go +++ b/soc-ai/processor/listener.go @@ -8,10 +8,10 @@ import ( "github.com/gin-contrib/gzip" "github.com/gin-gonic/gin" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/soc-ai/configurations" "github.com/utmstack/soc-ai/elastic" "github.com/utmstack/soc-ai/schema" - "github.com/utmstack/soc-ai/utils" ) func (p *Processor) restRouter() { @@ -36,20 +36,20 @@ func notFound(c *gin.Context) { func (p *Processor) handleAlerts(c *gin.Context) { err := elastic.CreateIndexIfNotExist(configurations.SOC_AI_INDEX) if err != nil { - utils.Logger.ErrorF("error creating index %s: %v", configurations.SOC_AI_INDEX, err) + catcher.Error("error creating index", err, map[string]any{"index": configurations.SOC_AI_INDEX}) c.JSON(http.StatusInternalServerError, fmt.Sprintf("error creating index %s", configurations.SOC_AI_INDEX)) return } if !configurations.GetGPTConfig().ModuleActive { - utils.Logger.LogF(100, "Droping request to /process, GPT module is not active") + catcher.Error("Droping request to /process, GPT module is not active", nil, nil) c.JSON(http.StatusOK, "GPT module is not active") return } var ids []string if err := c.BindJSON(&ids); err != nil { - utils.Logger.ErrorF("error binding JSON: %v", err) + catcher.Error("error binding JSON", err, nil) c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } @@ -58,7 +58,7 @@ func (p *Processor) handleAlerts(c *gin.Context) { result, err := elastic.ElasticSearch(configurations.SOC_AI_INDEX, "activityId", id) if err != nil { if !strings.Contains(err.Error(), "no such host") { - utils.Logger.ErrorF("error while searching alert %s in elastic: %v", id, err) + catcher.Error("error while searching alert in elastic", err, map[string]any{"alert": id}) c.JSON(http.StatusInternalServerError, fmt.Sprintf("error while searching alert %s in elastic: %v", id, err)) return } @@ -69,7 +69,7 @@ func (p *Processor) handleAlerts(c *gin.Context) { var gptResponses []schema.GPTAlertResponse err = json.Unmarshal(result, &gptResponses) if err != nil { - utils.Logger.ErrorF("error decoding response from elastic: %v", err) + catcher.Error("error decoding response from elastic", err, nil) c.JSON(http.StatusInternalServerError, fmt.Sprintf("error decoding response: %v", err)) return } @@ -77,20 +77,20 @@ func (p *Processor) handleAlerts(c *gin.Context) { if len(gptResponses) == 0 { err = elastic.IndexStatus(id, "Processing", "create") if err != nil { - utils.Logger.ErrorF("error creating doc in index: %v", err) + catcher.Error("error creating doc in index", err, nil) c.JSON(http.StatusInternalServerError, fmt.Sprintf("error creating doc in index: %v", err)) return } } else { err = elastic.IndexStatus(id, "Processing", "update") if err != nil { - utils.Logger.ErrorF("error updating doc in index: %v", err) + catcher.Error("error updating doc in index", err, nil) c.JSON(http.StatusInternalServerError, fmt.Sprintf("error updating doc in index: %v", err)) return } } - utils.Logger.Info("alert %s received", id) + catcher.Info("alert received", map[string]any{"alert": id}) p.AlertInfoQueue <- schema.AlertGPTDetails{AlertID: id} } diff --git a/soc-ai/processor/processor.go b/soc-ai/processor/processor.go index 4d55c59ec..62281e198 100644 --- a/soc-ai/processor/processor.go +++ b/soc-ai/processor/processor.go @@ -6,10 +6,10 @@ import ( "sync" "syscall" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/soc-ai/configurations" "github.com/utmstack/soc-ai/elastic" "github.com/utmstack/soc-ai/schema" - "github.com/utmstack/soc-ai/utils" ) var ( @@ -35,7 +35,7 @@ func NewProcessor() *Processor { } func (p *Processor) ProcessData() { - utils.Logger.Info("Starting SOC-AI Processor...") + catcher.Info("Starting SOC-AI Processor...", nil) go configurations.GetGPTConfig().UpdateGPTConfigurations() go p.restRouter() @@ -55,7 +55,7 @@ func (p *Processor) ProcessData() { func (p *Processor) RegisterError(message, id string) { err := elastic.IndexStatus(id, "Error", "update") if err != nil { - utils.Logger.ErrorF("error while indexing error in elastic: %v", err) + catcher.Error("error while indexing error in elastic", err, nil) } - utils.Logger.ErrorF("%s", message) + catcher.Error(message, nil, nil) } diff --git a/soc-ai/utils/check.go b/soc-ai/utils/check.go index dd0f48632..e838c467c 100644 --- a/soc-ai/utils/check.go +++ b/soc-ai/utils/check.go @@ -3,9 +3,14 @@ package utils import ( "fmt" "net/http" + "strings" "time" + + "github.com/threatwinds/go-sdk/catcher" ) +const wait = 3 * time.Second + func ConnectionChecker(url string) error { checkConn := func() error { if err := checkConnection(url); err != nil { @@ -14,7 +19,7 @@ func ConnectionChecker(url string) error { return nil } - if err := Logger.InfiniteRetryIfXError(checkConn, "connection failed"); err != nil { + if err := infiniteRetryIfXError(checkConn, "connection failed"); err != nil { return err } @@ -39,3 +44,30 @@ func checkConnection(url string) error { return nil } + +func infiniteRetryIfXError(f func() error, exception string) error { + var xErrorWasLogged bool + + for { + err := f() + if err != nil && is(err, exception) { + if !xErrorWasLogged { + _ = catcher.Error("An error occurred (%s), will keep retrying indefinitely...", err, nil) + xErrorWasLogged = true + } + time.Sleep(wait) + continue + } + + return err + } +} + +func is(e error, args ...string) bool { + for _, arg := range args { + if strings.Contains(e.Error(), arg) { + return true + } + } + return false +} diff --git a/soc-ai/utils/logger.go b/soc-ai/utils/logger.go deleted file mode 100644 index 4332e7f5a..000000000 --- a/soc-ai/utils/logger.go +++ /dev/null @@ -1,31 +0,0 @@ -package utils - -import ( - "log" - "os" - "strconv" - - "github.com/threatwinds/logger" -) - -var Logger *logger.Logger - -func init() { - lenv := os.Getenv("LOG_LEVEL") - var level int - var err error - - if lenv != "" && lenv != " " { - level, err = strconv.Atoi(lenv) - if err != nil { - log.Fatalln(err) - } - } else { - level = 200 - } - - Logger = logger.NewLogger(&logger.Config{ - Format: "text", - Level: level, - }) -} diff --git a/sophos/go.mod b/sophos/go.mod index 70a145510..d31dfef9b 100644 --- a/sophos/go.mod +++ b/sophos/go.mod @@ -1,8 +1,8 @@ module github.com/utmstack/UTMStack/sophos -go 1.23.0 +go 1.24.2 -toolchain go1.24.2 +toolchain go1.24.6 require ( github.com/threatwinds/logger v1.2.2 @@ -10,31 +10,32 @@ require ( ) require ( - github.com/bytedance/sonic v1.13.3 // indirect - github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/bytedance/sonic v1.14.0 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/gin-gonic/gin v1.10.1 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-playground/validator/v10 v10.27.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/threatwinds/go-sdk v1.0.45 github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.14 // indirect - golang.org/x/arch v0.18.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect + github.com/ugorji/go/codec v1.3.0 // indirect + golang.org/x/arch v0.19.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sophos/go.sum b/sophos/go.sum index 18e3ec6ce..22c72acd3 100644 --- a/sophos/go.sum +++ b/sophos/go.sum @@ -1,8 +1,12 @@ github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= +github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= @@ -23,6 +27,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= +github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= @@ -35,6 +41,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= @@ -59,25 +67,39 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/threatwinds/go-sdk v1.0.45 h1:KZ3s3HviNRrOkg5EqjFnoauANFFzTqjNFyshPLY2SoI= +github.com/threatwinds/go-sdk v1.0.45/go.mod h1:tcWn6r6vqID/W/nL3UKfc5NafA3V/cSkiLvfJnwB58c= github.com/threatwinds/logger v1.2.2 h1:sVuT8yhbecPqP4tT8EwHfp1czNC6e1wdkE1ihNnuBdA= github.com/threatwinds/logger v1.2.2/go.mod h1:Amq0QI1y7fkTpnBUgeGVu2Z/C4u4ys2pNLUOuj3UAAU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.14 h1:yOQvXCBc3Ij46LRkRoh4Yd5qK6LVOgi0bYOXfb7ifjw= github.com/ugorji/go/codec v1.2.14/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= +github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/utmstack/config-client-go v1.2.7 h1:JeRdI5JjH1liNzMW3LmyevjuPd67J/yt9MAO3+oJAuM= github.com/utmstack/config-client-go v1.2.7/go.mod h1:kM0KoUizM9ZlcQp0qKviGTWn/+anT5Rfjx3zfZk79nM= golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/arch v0.19.0 h1:LmbDQUodHThXE+htjrnmVD73M//D9GTH6wFZjyDkjyU= +golang.org/x/arch v0.19.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= diff --git a/sophos/main.go b/sophos/main.go index ed87dd249..2c306a64c 100644 --- a/sophos/main.go +++ b/sophos/main.go @@ -1,10 +1,12 @@ package main import ( + "os" "strings" "sync" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/sophos/configuration" "github.com/utmstack/UTMStack/sophos/processor" "github.com/utmstack/UTMStack/sophos/utils" @@ -14,11 +16,12 @@ import ( ) func main() { - utils.Logger.Info("Starting sophos central module...") + catcher.Info("Starting sophos central module...", nil) intKey := configuration.GetInternalKey() panelServ := configuration.GetPanelServiceName() if intKey == "" || panelServ == "" { - utils.Logger.Fatal("Internal key or panel service name is not set. Exiting...") + catcher.Error("Internal key or panel service name is not set. Exiting...", nil, nil) + os.Exit(1) } client := utmconf.NewUTMClient(intKey, "http://"+panelServ) @@ -30,22 +33,22 @@ func main() { for range ticker.C { if err := utils.ConnectionChecker(configuration.CHECKCON); err != nil { - utils.Logger.ErrorF("External connection failure detected: %v", err) + catcher.Error("External connection failure detected", err, nil) } endTime := time.Now().UTC() //startTimeStr := startTime.Format(time.RFC3339) //endTimeStr := endTime.Format(time.RFC3339) - utils.Logger.Info("Syncing logs from %s to %s", startTime, endTime) + catcher.Info("Syncing logs", map[string]any{"start_time": startTime, "end_time": endTime}) moduleConfig, err := client.GetUTMConfig(enum.SOPHOS) if err != nil { if strings.Contains(err.Error(), "invalid character '<'") { - utils.Logger.LogF(100, "error getting configuration of the SOPHOS module: backend is not available") + catcher.Error("error getting configuration of the SOPHOS module: backend is not available", nil, nil) } if strings.TrimSpace(err.Error()) != "" { - utils.Logger.ErrorF("error getting configuration of the SOPHOS module: %v", err) + catcher.Error("error getting configuration of the SOPHOS module", err, nil) } continue } @@ -60,7 +63,7 @@ func main() { for _, cnf := range group.Configurations { if strings.TrimSpace(cnf.ConfValue) == "" { - utils.Logger.LogF(100, "program not configured yet for group: %s", group.GroupName) + catcher.Error("program not configured yet for group", nil, map[string]any{"group_name": group.GroupName}) skip = true break } @@ -76,7 +79,7 @@ func main() { wg.Wait() } - utils.Logger.Info("sync completed from %v to %v, waiting 5 minutes", startTime, endTime) + catcher.Info("sync completed, waiting 5 minutes", map[string]any{"start_time": startTime, "end_time": endTime}) startTime = endTime.Add(time.Nanosecond) } } diff --git a/sophos/processor/processor.go b/sophos/processor/processor.go index 0d50aed15..758a4cf4d 100644 --- a/sophos/processor/processor.go +++ b/sophos/processor/processor.go @@ -6,7 +6,7 @@ import ( "net/url" "time" - "github.com/threatwinds/logger" + "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/sophos/configuration" "github.com/utmstack/UTMStack/sophos/utils" "github.com/utmstack/config-client-go/types" @@ -35,7 +35,7 @@ func getSophosCentralProcessor(group types.ModuleGroup) SophosCentralProcessor { return sophosProcessor } -func (p *SophosCentralProcessor) getAccessToken() (string, *logger.Error) { +func (p *SophosCentralProcessor) getAccessToken() (string, error) { data := url.Values{} data.Set("grant_type", "client_credentials") data.Set("client_id", p.ClientID) @@ -48,17 +48,17 @@ func (p *SophosCentralProcessor) getAccessToken() (string, *logger.Error) { response, _, err := utils.DoReq[map[string]any](configuration.AUTHURL, []byte(data.Encode()), http.MethodPost, headers) if err != nil { - return "", utils.Logger.ErrorF("error making auth request: %v", err) + return "", catcher.Error("error making auth request", err, nil) } accessToken, ok := response["access_token"].(string) if !ok || accessToken == "" { - return "", utils.Logger.ErrorF("access_token not found in response") + return "", catcher.Error("access_token not found in response", nil, nil) } expiresIn, ok := response["expires_in"].(float64) if !ok { - return "", utils.Logger.ErrorF("expires_in not found in response") + return "", catcher.Error("expires_in not found in response", nil, nil) } p.AccessToken = accessToken @@ -76,7 +76,7 @@ type ApiHosts struct { DataRegion string `json:"dataRegion"` } -func (p *SophosCentralProcessor) getTenantInfo(accessToken string) *logger.Error { +func (p *SophosCentralProcessor) getTenantInfo(accessToken string) error { headers := map[string]string{ "accept": "application/json", "Authorization": "Bearer " + accessToken, @@ -84,23 +84,23 @@ func (p *SophosCentralProcessor) getTenantInfo(accessToken string) *logger.Error response, _, err := utils.DoReq[WhoamiResponse](configuration.WHOAMIURL, nil, http.MethodGet, headers) if err != nil { - return utils.Logger.ErrorF("error making whoami request: %v", err) + return catcher.Error("error making whoami request", err, nil) } if response.ID == "" { - return utils.Logger.ErrorF("tenant ID not found in whoami response") + return catcher.Error("tenant ID not found in whoami response", nil, nil) } p.TenantID = response.ID if response.ApiHosts.DataRegion == "" { - return utils.Logger.ErrorF("dataRegion not found in whoami response") + return catcher.Error("dataRegion not found in whoami response", nil, nil) } p.DataRegion = response.ApiHosts.DataRegion return nil } -func (p *SophosCentralProcessor) getValidAccessToken() (string, *logger.Error) { +func (p *SophosCentralProcessor) getValidAccessToken() (string, error) { if p.AccessToken != "" && time.Now().Before(p.ExpiresAt) { return p.AccessToken, nil } @@ -119,15 +119,15 @@ type Pages struct { MaxSize int64 `json:"maxSize"` } -func (p *SophosCentralProcessor) getLogs(fromTime int64, nextKey string, group types.ModuleGroup) ([]TransformedLog, string, *logger.Error) { +func (p *SophosCentralProcessor) getLogs(fromTime int64, nextKey string, group types.ModuleGroup) ([]TransformedLog, string, error) { accessToken, err := p.getValidAccessToken() if err != nil { - return nil, "", utils.Logger.ErrorF("error getting access token: %v", err) + return nil, "", catcher.Error("error getting access token", err, nil) } if p.TenantID == "" || p.DataRegion == "" { if err := p.getTenantInfo(accessToken); err != nil { - return nil, "", utils.Logger.ErrorF("error getting tenant information: %v", err) + return nil, "", catcher.Error("error getting tenant information", err, nil) } } @@ -138,7 +138,7 @@ func (p *SophosCentralProcessor) getLogs(fromTime int64, nextKey string, group t for { u, err := p.buildURL(fromTime, currentNextKey) if err != nil { - return nil, "", utils.Logger.ErrorF("error building URL: %v", err) + return nil, "", catcher.Error("error building URL", err, nil) } headers := map[string]string{ @@ -165,11 +165,11 @@ func (p *SophosCentralProcessor) getLogs(fromTime int64, nextKey string, group t return transformedLogs, currentNextKey, nil } -func (p *SophosCentralProcessor) buildURL(fromTime int64, nextKey string) (*url.URL, *logger.Error) { +func (p *SophosCentralProcessor) buildURL(fromTime int64, nextKey string) (*url.URL, error) { baseURL := p.DataRegion + "/siem/v1/events" u, parseErr := url.Parse(baseURL) if parseErr != nil { - return nil, utils.Logger.ErrorF("error parsing url: %v", parseErr) + return nil, catcher.Error("error parsing url", parseErr, make(map[string]any)) } params := url.Values{} diff --git a/sophos/processor/pull.go b/sophos/processor/pull.go index c4d10b482..d8f2d3fad 100644 --- a/sophos/processor/pull.go +++ b/sophos/processor/pull.go @@ -4,7 +4,6 @@ import ( "sync" "time" - "github.com/threatwinds/logger" "github.com/utmstack/config-client-go/types" ) @@ -13,7 +12,7 @@ var ( nextKeysMu sync.RWMutex ) -func PullLogs(group types.ModuleGroup, startTime time.Time) *logger.Error { +func PullLogs(group types.ModuleGroup, startTime time.Time) error { nextKeysMu.RLock() prevKey := nextKeys[group.ModuleID] nextKeysMu.RUnlock() diff --git a/sophos/processor/sendData.go b/sophos/processor/sendData.go index 007a2a845..74ef5fd9e 100644 --- a/sophos/processor/sendData.go +++ b/sophos/processor/sendData.go @@ -8,9 +8,9 @@ import ( "net/http" "time" + "github.com/threatwinds/go-sdk/catcher" "github.com/threatwinds/logger" "github.com/utmstack/UTMStack/sophos/configuration" - "github.com/utmstack/UTMStack/sophos/utils" ) var transport = &http.Transport{ @@ -29,11 +29,11 @@ func SendToLogstash(data []TransformedLog) *logger.Error { for _, str := range data { body, err := json.Marshal(str) if err != nil { - utils.Logger.ErrorF("error encoding log to JSON: %v", err) + catcher.Error("error encoding log to JSON", err, nil) continue } if err := sendLogs(body); err != nil { - utils.Logger.ErrorF("error sending logs to logstach: %v", err) + catcher.Error("error sending logs to logstach", err, nil) continue } } @@ -45,17 +45,17 @@ func sendLogs(log []byte) error { req, err := http.NewRequest("POST", url, bytes.NewBuffer(log)) if err != nil { - return utils.Logger.ErrorF("error creating request: %v", err.Error()) + return catcher.Error("error creating request", err, nil) } resp, err := client.Do(req) if err != nil { - return utils.Logger.ErrorF("error sending logs: %v", err.Error()) + return catcher.Error("error sending logs", err, nil) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return utils.Logger.ErrorF("error sending logs with http code %d", resp.StatusCode) + return catcher.Error("error sending logs with http code", err, map[string]any{"status_code": resp.StatusCode}) } return nil } diff --git a/sophos/utils/check.go b/sophos/utils/check.go index 5ab7c1ccd..bde8e1db4 100644 --- a/sophos/utils/check.go +++ b/sophos/utils/check.go @@ -4,10 +4,16 @@ import ( "context" "fmt" "net/http" + "strings" "time" + + "github.com/threatwinds/go-sdk/catcher" ) -const connectionTimeout = 5 * time.Second +const ( + connectionTimeout = 5 * time.Second + wait = 3 * time.Second +) func ConnectionChecker(url string) error { checkConn := func() error { @@ -20,7 +26,7 @@ func ConnectionChecker(url string) error { return nil } - if err := Logger.InfiniteRetryIfXError(checkConn, "connection failed"); err != nil { + if err := infiniteRetryIfXError(checkConn, "connection failed"); err != nil { return err } @@ -43,3 +49,30 @@ func checkConnection(url string, ctx context.Context) error { return nil } + +func infiniteRetryIfXError(f func() error, exception string) error { + var xErrorWasLogged bool + + for { + err := f() + if err != nil && is(err, exception) { + if !xErrorWasLogged { + _ = catcher.Error("An error occurred (%s), will keep retrying indefinitely...", err, nil) + xErrorWasLogged = true + } + time.Sleep(wait) + continue + } + + return err + } +} + +func is(e error, args ...string) bool { + for _, arg := range args { + if strings.Contains(e.Error(), arg) { + return true + } + } + return false +} diff --git a/sophos/utils/env.go b/sophos/utils/env.go index 846eaee2a..0332339d2 100644 --- a/sophos/utils/env.go +++ b/sophos/utils/env.go @@ -1,18 +1,21 @@ package utils import ( - "log" "os" + + "github.com/threatwinds/go-sdk/catcher" ) // Getenv returns the environment variable func Getenv(key string) string { value, defined := os.LookupEnv(key) if !defined { - log.Fatalf("Error loading environment variable: %s: environment variable does not exist\n", key) + catcher.Error("Error loading environment variable, environment variable does not exist", nil, map[string]any{"key": key}) + os.Exit(1) } if (value == "") || (value == " ") { - log.Fatalf("Error loading environment variable: %s: empty environment variable\n", key) + catcher.Error("Error loading environment variable, empty environment variable", nil, map[string]any{"key": key}) + os.Exit(1) } return value } diff --git a/sophos/utils/logger.go b/sophos/utils/logger.go deleted file mode 100644 index 4332e7f5a..000000000 --- a/sophos/utils/logger.go +++ /dev/null @@ -1,31 +0,0 @@ -package utils - -import ( - "log" - "os" - "strconv" - - "github.com/threatwinds/logger" -) - -var Logger *logger.Logger - -func init() { - lenv := os.Getenv("LOG_LEVEL") - var level int - var err error - - if lenv != "" && lenv != " " { - level, err = strconv.Atoi(lenv) - if err != nil { - log.Fatalln(err) - } - } else { - level = 200 - } - - Logger = logger.NewLogger(&logger.Config{ - Format: "text", - Level: level, - }) -} diff --git a/sophos/utils/req.go b/sophos/utils/req.go index 1650c396f..3bc51ca70 100644 --- a/sophos/utils/req.go +++ b/sophos/utils/req.go @@ -3,18 +3,17 @@ package utils import ( "bytes" "encoding/json" + "fmt" "io" "net/http" - - "github.com/threatwinds/logger" ) -func DoReq[response any](url string, data []byte, method string, headers map[string]string) (response, int, *logger.Error) { +func DoReq[response any](url string, data []byte, method string, headers map[string]string) (response, int, error) { var result response req, err := http.NewRequest(method, url, bytes.NewBuffer(data)) if err != nil { - return result, http.StatusInternalServerError, Logger.ErrorF(err.Error()) + return result, http.StatusInternalServerError, err } for k, v := range headers { @@ -25,22 +24,22 @@ func DoReq[response any](url string, data []byte, method string, headers map[str resp, err := client.Do(req) if err != nil { - return result, http.StatusInternalServerError, Logger.ErrorF(err.Error()) + return result, http.StatusInternalServerError, err } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { - return result, http.StatusInternalServerError, Logger.ErrorF(err.Error()) + return result, http.StatusInternalServerError, err } if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK { - return result, resp.StatusCode, Logger.ErrorF("while sending request to %s received status code: %d and response body: %s", url, resp.StatusCode, body) + return result, resp.StatusCode, fmt.Errorf("while sending request to %s received status code: %d and response body: %s", url, resp.StatusCode, body) } err = json.Unmarshal(body, &result) if err != nil { - return result, http.StatusInternalServerError, Logger.ErrorF(err.Error()) + return result, http.StatusInternalServerError, err } return result, resp.StatusCode, nil