From e48a97bb4bcf25fb717ca1faddc4e1e541e58689 Mon Sep 17 00:00:00 2001 From: Yadian Llada Lopez Date: Wed, 16 Apr 2025 08:54:08 -0400 Subject: [PATCH 01/22] feat: Improve installer to add RHELD support --- installer/cloud.go | 17 +++---- installer/docker.go | 64 +++++++++++++++++-------- installer/main.go | 9 ++-- installer/master.go | 16 +++---- installer/network.go | 79 +++++++++++++++++++++---------- installer/nginx.go | 50 ++++++++++++++----- installer/system.go | 10 +++- installer/templates/interfaces.go | 13 ++++- installer/templates/proxy.go | 53 ++++++++++++++++++++- installer/tools.go | 17 +++++-- installer/utils/requirements.go | 11 +++-- 11 files changed, 250 insertions(+), 89 deletions(-) diff --git a/installer/cloud.go b/installer/cloud.go index 5cb8548a2..46e1f97e4 100644 --- a/installer/cloud.go +++ b/installer/cloud.go @@ -9,7 +9,8 @@ import ( "github.com/utmstack/UTMStack/installer/utils" ) -func Cloud(c *types.Config, update bool) error { +func Cloud(c *types.Config, update bool, distro string) error { + if err := utils.CheckCPU(2); err != nil { return err } @@ -43,7 +44,7 @@ func Cloud(c *types.Config, update bool) error { if utils.GetLock(2, stack.LocksDir) { fmt.Println("Preparing system to run UTMStack") - if err := PrepareSystem(); err != nil { + if err := PrepareSystem(distro); err != nil { return err } @@ -72,11 +73,11 @@ func Cloud(c *types.Config, update bool) error { return err } - if err := InstallVlan(); err != nil { + if err := InstallVlan(distro); err != nil { return err } - if err := ConfigureVLAN(iface); err != nil { + if err := ConfigureVLAN(iface, distro); err != nil { return err } @@ -101,7 +102,7 @@ func Cloud(c *types.Config, update bool) error { if utils.GetLock(3, stack.LocksDir) { fmt.Println("Installing Docker") - if err := InstallDocker(); err != nil { + if err := InstallDocker(distro); err != nil { return err } @@ -145,11 +146,11 @@ func Cloud(c *types.Config, update bool) error { if utils.GetLock(12, stack.LocksDir) || update { fmt.Println("Installing reverse proxy. This may take a while.") - if err := InstallNginx(); err != nil { + if err := InstallNginx(distro); err != nil { return err } - if err := ConfigureNginx(c, stack); err != nil { + if err := ConfigureNginx(c, stack, distro); err != nil { return err } @@ -162,7 +163,7 @@ func Cloud(c *types.Config, update bool) error { if utils.GetLock(5, stack.LocksDir) { fmt.Println("Installing Administration Tools") - if err := InstallTools(); err != nil { + if err := InstallTools(distro); err != nil { return err } diff --git a/installer/docker.go b/installer/docker.go index 437d6d21a..278db7e04 100644 --- a/installer/docker.go +++ b/installer/docker.go @@ -9,33 +9,57 @@ import ( "github.com/utmstack/UTMStack/installer/utils" ) -func InstallDocker() error { - env := []string{"DEBIAN_FRONTEND=noninteractive"} +func InstallDocker(distro string) error { + switch distro { + case "ubuntu": + env := []string{"DEBIAN_FRONTEND=noninteractive"} - if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "apt-transport-https", "ca-certificates", "curl", "gnupg-agent", "software-properties-common"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "apt-transport-https", "ca-certificates", "curl", "gnupg-agent", "software-properties-common"); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "sh", "-c", "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "sh", "-c", "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -"); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "sh", "-c", `add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"`); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "sh", "-c", `add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"`); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "docker-ce", "docker-ce-cli", "containerd.io", "docker-compose"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "docker-ce", "docker-ce-cli", "containerd.io", "docker-compose"); err != nil { + return err + } + case "redhat": + env := []string{"DNF_YUM_AUTO_YES=1"} + + if err := utils.RunEnvCmd(env, "dnf", "install", "-y", "dnf-plugins-core", "ca-certificates", "curl"); err != nil { + return err + } + + if err := utils.RunEnvCmd(env, "dnf", "config-manager", "--add-repo", "https://download.docker.com/linux/centos/docker-ce.repo"); err != nil { + return err + } + if err := utils.RunEnvCmd(env, "dnf", "makecache"); err != nil { + return err + } + + if err := utils.RunEnvCmd(env, "dnf", "install", "-y", "docker-ce", "docker-ce-cli", "containerd.io", "docker-compose-plugin", "docker-buildx-plugin"); err != nil { + return err + } + + if err := utils.RunEnvCmd(env, "systemctl", "enable", "--now", "docker"); err != nil { + return err + } + } return nil } diff --git a/installer/main.go b/installer/main.go index 6a3464649..fe5316376 100644 --- a/installer/main.go +++ b/installer/main.go @@ -4,8 +4,8 @@ import ( "fmt" "os" - "github.com/utmstack/UTMStack/installer/utils" "github.com/utmstack/UTMStack/installer/types" + "github.com/utmstack/UTMStack/installer/utils" ) func main() { @@ -22,7 +22,8 @@ func main() { } } - if err := utils.CheckDistro("ubuntu"); err != nil { + distro, err := utils.CheckDistro() + if err != nil { fmt.Println(err) os.Exit(1) } @@ -77,13 +78,13 @@ func main() { switch config.ServerType { case "aio": - err := Master(config) + err := Master(config, distro) if err != nil { fmt.Println(err) os.Exit(1) } case "cloud": - err := Cloud(config, update) + err := Cloud(config, update, distro) if err != nil { fmt.Println(err) os.Exit(1) diff --git a/installer/master.go b/installer/master.go index 3e6643eb5..792002cfb 100644 --- a/installer/master.go +++ b/installer/master.go @@ -9,7 +9,7 @@ import ( "github.com/utmstack/UTMStack/installer/utils" ) -func Master(c *types.Config) error { +func Master(c *types.Config, distro string) error { if err := utils.CheckCPU(2); err != nil { return err } @@ -43,7 +43,7 @@ func Master(c *types.Config) error { if utils.GetLock(2, stack.LocksDir) { fmt.Println("Preparing system to run UTMStack") - if err := PrepareSystem(); err != nil { + if err := PrepareSystem(distro); err != nil { return err } @@ -72,11 +72,11 @@ func Master(c *types.Config) error { return err } - if err := InstallVlan(); err != nil { + if err := InstallVlan(distro); err != nil { return err } - if err := ConfigureVLAN(iface); err != nil { + if err := ConfigureVLAN(iface, distro); err != nil { return err } @@ -101,7 +101,7 @@ func Master(c *types.Config) error { if utils.GetLock(3, stack.LocksDir) { fmt.Println("Installing Docker") - if err := InstallDocker(); err != nil { + if err := InstallDocker(distro); err != nil { return err } @@ -138,11 +138,11 @@ func Master(c *types.Config) error { fmt.Println("Installing reverse proxy. This may take a while.") - if err := InstallNginx(); err != nil { + if err := InstallNginx(distro); err != nil { return err } - if err := ConfigureNginx(c, stack); err != nil { + if err := ConfigureNginx(c, stack, distro); err != nil { return err } @@ -150,7 +150,7 @@ func Master(c *types.Config) error { if utils.GetLock(5, stack.LocksDir) { fmt.Println("Installing Administration Tools") - if err := InstallTools(); err != nil { + if err := InstallTools(distro); err != nil { return err } diff --git a/installer/network.go b/installer/network.go index 381f7bb0b..d82486eba 100644 --- a/installer/network.go +++ b/installer/network.go @@ -40,40 +40,71 @@ func checkRenderer() (string, error) { return "networkd", nil } -func ConfigureVLAN(mainIface string) error { - renderer, err := checkRenderer() - if err != nil { - return err - } - +func ConfigureVLAN(mainIface string, distro string) error { c := Vlan{ - Renderer: renderer, - Iface: mainIface, + Iface: mainIface, } - log.Println("Generating vlan config") - err = utils.GenerateConfig(c, templates.Vlan, path.Join("/etc", "netplan", "99-vlan.yaml")) - if err != nil { - return err - } + switch distro { + case "ubuntu": + renderer, err := checkRenderer() + if err != nil { + return err + } + c.Renderer = renderer - log.Println("Applying vlan config") - if err := utils.RunCmd("netplan", "apply"); err != nil { - return err - } + log.Println("Generating vlan config") + err = utils.GenerateConfig(c, templates.VlanUbuntu, path.Join("/etc", "netplan", "99-vlan.yaml")) + if err != nil { + return err + } + + log.Println("Applying vlan config") + if err := utils.RunCmd("netplan", "apply"); err != nil { + return err + } + + case "redhat": + err := utils.GenerateConfig(c, templates.VlanRedHat, "/etc/sysconfig/network-scripts/ifcfg-vlan10") + if err != nil { + return err + } + + if err := utils.RunCmd("modprobe", "8021q"); err != nil { + return err + } + + if err := os.WriteFile("/etc/modules-load.d/8021q.conf", []byte("8021q\n"), 0644); err != nil { + return err + } + + if err := utils.RunCmd("systemctl", "restart", "NetworkManager"); err != nil { + return err + } + } return nil } -func InstallVlan() error { - env := []string{"DEBIAN_FRONTEND=noninteractive"} +func InstallVlan(distro string) error { + switch distro { + case "ubuntu": + env := []string{"DEBIAN_FRONTEND=noninteractive"} - if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "update", "-y"); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "vlan"); err != nil { - return err + if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "vlan"); err != nil { + return err + } + + case "redhat": + env := []string{"DNF_YUM_AUTO_YES=1"} + + if err := utils.RunEnvCmd(env, "dnf", "update", "-y"); err != nil { + return err + } } return nil diff --git a/installer/nginx.go b/installer/nginx.go index 6cddc9cf6..3691a5b70 100644 --- a/installer/nginx.go +++ b/installer/nginx.go @@ -8,21 +8,30 @@ import ( "github.com/utmstack/UTMStack/installer/utils" ) -func InstallNginx() error { - env := []string{"DEBIAN_FRONTEND=noninteractive"} +func InstallNginx(distro string) error { + switch distro { + case "ubuntu": + env := []string{"DEBIAN_FRONTEND=noninteractive"} - if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "nginx"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "nginx"); err != nil { + return err + } + + case "redhat": + env := []string{"DNF_YUM_AUTO_YES=1"} + if err := utils.RunEnvCmd(env, "dnf", "install", "-y", "nginx"); err != nil { + return err + } + } return nil } -func ConfigureNginx(conf *types.Config, stack *types.StackConfig) error { +func ConfigureNginx(conf *types.Config, stack *types.StackConfig, distro string) error { c := types.NginxConfig{ SharedKey: conf.InternalKey, } @@ -32,9 +41,26 @@ func ConfigureNginx(conf *types.Config, stack *types.StackConfig) error { return err } - err = utils.GenerateConfig(c, templates.Proxy, path.Join("/", "etc", "nginx", "sites-available", "default")) - if err != nil { - return err + switch distro { + case "ubuntu": + err = utils.GenerateConfig(c, templates.ProxyUbuntu, path.Join("/", "etc", "nginx", "sites-available", "default")) + if err != nil { + return err + } + + case "redhat": + err = utils.GenerateConfig(c, templates.ProxyRHEL, path.Join("/", "etc", "nginx", "nginx.conf")) + if err != nil { + return err + } + + if err := utils.RunCmd("chcon", "-R", "-t", "httpd_sys_content_t", "/utmstack/cert/"); err != nil { + return err + } + + if err := utils.RunCmd("setsebool", "-P", "httpd_read_user_content", "1"); err != nil { + return err + } } if err := utils.RunCmd("systemctl", "restart", "nginx"); err != nil { diff --git a/installer/system.go b/installer/system.go index 10123a13a..00fb3e974 100644 --- a/installer/system.go +++ b/installer/system.go @@ -8,7 +8,15 @@ import ( "github.com/utmstack/UTMStack/installer/utils" ) -func PrepareSystem() error { +func PrepareSystem(distro string) error { + if distro == "redhat" { + if err := utils.RunCmd("systemctl", "disable", "firewalld"); err != nil { + return fmt.Errorf("failed to disable firewalld: %v", err) + } + if err := utils.RunCmd("systemctl", "stop", "firewalld"); err != nil { + return fmt.Errorf("failed to stop firewalld: %v", err) + } + } sysctl := []string{ "vm.max_map_count=262144", "net.ipv6.conf.all.disable_ipv6=1", diff --git a/installer/templates/interfaces.go b/installer/templates/interfaces.go index c22a2f11d..56fec7cb2 100644 --- a/installer/templates/interfaces.go +++ b/installer/templates/interfaces.go @@ -1,6 +1,6 @@ package templates -const Vlan string = ` +const VlanUbuntu string = ` network: version: 2 renderer: {{ .Renderer }} @@ -11,3 +11,14 @@ network: link: {{ .Iface }} addresses: [10.21.199.3/24] ` +const VlanRedHat string = ` +DEVICE=vlan10 +TYPE=Vlan +VLAN_ID=10 +PHYSDEV={{ .Iface }} +BOOTPROTO=none +IPADDR=10.21.199.3 +PREFIX=24 +VLAN=yes +ONBOOT=yes +` diff --git a/installer/templates/proxy.go b/installer/templates/proxy.go index 8170cf87e..9a50b75ad 100644 --- a/installer/templates/proxy.go +++ b/installer/templates/proxy.go @@ -1,6 +1,6 @@ package templates -const Proxy string = `server { +const ProxyUbuntu string = `server { listen 80 default_server; server_name _; return 301 https://$host$request_uri; @@ -34,4 +34,53 @@ server { client_max_body_size 200M; client_body_buffer_size 200M; -}` \ No newline at end of file +}` + +const ProxyRHEL string = `worker_processes auto; + +events { + worker_connections 2048; +} + +http { + include mime.types; + default_type application/octet-stream; + + keepalive_timeout 65; + +server { + listen 80 default_server; + server_name _; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + server_name _; + + set $utmstack http://127.0.0.1:10001; + set $shared_key {{.SharedKey}}; + + location / { + proxy_pass $utmstack; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header x-shared-key $shared_key; + add_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_read_timeout 900; + } + + ssl_certificate /utmstack/cert/utm.crt; + ssl_certificate_key /utmstack/cert/utm.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH'; + ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + chunked_transfer_encoding on; + + client_max_body_size 200M; + client_body_buffer_size 200M; + } +}` diff --git a/installer/tools.go b/installer/tools.go index a3a283981..aca35db06 100644 --- a/installer/tools.go +++ b/installer/tools.go @@ -2,11 +2,18 @@ package main import "github.com/utmstack/UTMStack/installer/utils" -func InstallTools() error { - env := []string{"DEBIAN_FRONTEND=noninteractive"} - err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "cockpit") - if err != nil { - return err +func InstallTools(distro string) error { + switch distro { + case "ubuntu": + env := []string{"DEBIAN_FRONTEND=noninteractive"} + if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "cockpit"); err != nil { + return err + } + case "redhat": + env := []string{"DNF_YUM_AUTO_YES=1"} + if err := utils.RunEnvCmd(env, "dnf", "install", "-y", "cockpit"); err != nil { + return err + } } return nil diff --git a/installer/utils/requirements.go b/installer/utils/requirements.go index 10826cc7d..f6206b1af 100644 --- a/installer/utils/requirements.go +++ b/installer/utils/requirements.go @@ -45,10 +45,13 @@ func CheckCPU(cores int) error { return nil } -func CheckDistro(distro string) error { +func CheckDistro() (string, error) { info, _ := host.Info() - if info.Platform != distro { - return fmt.Errorf("your Linux distribution (%s) is not %s", info.Platform, distro) + + distro := info.Platform + if distro != "ubuntu" && distro != "redhat" { + return "", fmt.Errorf("your Linux distribution (%s) is not supported. Supported distributions are: %s, %s", + distro, "ubuntu", "redhat") } - return nil + return distro, nil } From 91977d87be8405803bcf4f986a0a84ad8cd5d793 Mon Sep 17 00:00:00 2001 From: Yadian Llada Lopez Date: Wed, 16 Apr 2025 09:12:03 -0400 Subject: [PATCH 02/22] fix: Update windows ARM collector --- agent/collectors/windows_arm64.go | 459 ++++++++++++++++++------------ 1 file changed, 282 insertions(+), 177 deletions(-) diff --git a/agent/collectors/windows_arm64.go b/agent/collectors/windows_arm64.go index a71bd68e7..155f1a4e2 100644 --- a/agent/collectors/windows_arm64.go +++ b/agent/collectors/windows_arm64.go @@ -4,215 +4,320 @@ package collectors import ( + "encoding/json" + "encoding/xml" + "fmt" + "log" "os" - "os/exec" - "path/filepath" + "os/signal" + "strconv" "strings" - "time" + "syscall" + "unsafe" + "github.com/threatwinds/go-sdk/plugins" + "github.com/threatwinds/validations" "github.com/utmstack/UTMStack/agent/config" "github.com/utmstack/UTMStack/agent/logservice" - - "github.com/threatwinds/validations" "github.com/utmstack/UTMStack/agent/utils" + "golang.org/x/sys/windows" ) -type Windows struct{} +type Event struct { + XMLName xml.Name `xml:"Event"` + System SystemData `xml:"System"` + EventData []*EventData `xml:"EventData>Data"` +} -func getCollectorsInstances() []Collector { - var collectors []Collector - collectors = append(collectors, Windows{}) - return collectors +type EventData struct { + Key string `xml:"Name,attr"` + Value string `xml:",chardata"` +} + +type ProviderData struct { + ProviderName string `xml:"Name,attr"` + ProviderGUID string `xml:"Guid,attr"` } -const PowerShellScript = ` -<# -.SYNOPSIS - Collects Windows Application, System, and Security logs from the last 5 minutes, then prints them to the console in a compact, single-line JSON format, - emulating the field structure that Winlogbeat typically produces. - -.DESCRIPTION - 1. Retrieves the last 5 minutes of Windows logs (Application, System, Security) using FilterHashtable (no post-fetch filtering). - 2. Maps event properties to a schema similar to Winlogbeat's, including: - - @timestamp - - message - - event.code - - event.provider - - event.kind - - winlog fields (e.g. record_id, channel, activity_id, etc.) - 3. Prints each log record as a single-line JSON object with no indentation/extra spacing. - 4. If no logs are found, the script produces no output at all. -#> - -# Suppress any runtime errors that would clutter the console. -$ErrorActionPreference = 'SilentlyContinue' - -# Calculate the start time for filtering -$startTime = (Get-Date).AddSeconds(-30) - -# Retrieve logs with filter hashtable -$applicationLogs = Get-WinEvent -FilterHashtable @{ LogName='Application'; StartTime=$startTime } -$systemLogs = Get-WinEvent -FilterHashtable @{ LogName='System'; StartTime=$startTime } -$securityLogs = Get-WinEvent -FilterHashtable @{ LogName='Security'; StartTime=$startTime } - -# Safeguard against null results -if (-not $applicationLogs) { $applicationLogs = @() } -if (-not $systemLogs) { $systemLogs = @() } -if (-not $securityLogs) { $securityLogs = @() } - -# Combine them -$recentLogs = $applicationLogs + $systemLogs + $securityLogs - -# If no logs are found, produce no output at all -if (-not $recentLogs) { - return -} - -# Function to convert the raw Properties array to a dictionary-like object under winlog.event_data -function Convert-PropertiesToEventData { - param([Object[]] $Properties) - - # If nothing is there, return an empty hashtable - if (-not $Properties) { return @{} } - - # Winlogbeat places custom fields under winlog.event_data. - # Typically, it tries to parse known keys, but we'll do a simple best-effort approach: - # We'll create paramN = pairs for each array index. - $eventData = [ordered]@{} - - for ($i = 0; $i -lt $Properties.Count; $i++) { - $value = $Properties[$i].Value - - # If the property is itself an object with nested fields, we can flatten or store as-is. - # We'll store as-is for clarity. - # We'll name them param1, param2, param3,... unless you'd like more specific field logic. - $paramName = "param$($i+1)" - - $eventData[$paramName] = $value - } - - return $eventData -} - -# Transform each event into a structure emulating Winlogbeat -foreach ($rawEvent in $recentLogs) { - # Convert TimeCreated to a universal ISO8601 string - $timestamp = $rawEvent.TimeCreated.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') - - # Build the top-level document - $doc = [ordered]@{ - # Matches Winlogbeat's typical top-level timestamp field - '@timestamp' = $timestamp - - # The main message content from the event - 'message' = $rawEvent.Message - - # "event" block: minimal example - 'event' = [ordered]@{ - 'code' = $rawEvent.Id # event_id in Winlogbeat is typically a string or numeric - 'provider' = $rawEvent.ProviderName - 'kind' = 'event' - 'created' = $timestamp # or you could omit if desired - } - - # "winlog" block: tries to mirror Winlogbeat's structure for Windows - 'winlog' = [ordered]@{ - 'record_id' = $rawEvent.RecordId - 'computer_name' = $rawEvent.MachineName - 'channel' = $rawEvent.LogName - 'provider_name' = $rawEvent.ProviderName - 'provider_guid' = $rawEvent.ProviderId - 'process' = [ordered]@{ - 'pid' = $rawEvent.ProcessId - 'thread' = @{ - 'id' = $rawEvent.ThreadId - } - } - 'event_id' = $rawEvent.Id - 'version' = $rawEvent.Version - 'activity_id' = $rawEvent.ActivityId - 'related_activity_id'= $rawEvent.RelatedActivityId - 'task' = $rawEvent.TaskDisplayName - 'opcode' = $rawEvent.OpcodeDisplayName - 'keywords' = $rawEvent.KeywordsDisplayNames - 'time_created' = $timestamp - # Convert "Properties" into a dictionary for event_data - 'event_data' = Convert-PropertiesToEventData $rawEvent.Properties - } - } - - # Convert our object to JSON (with no extra formatting). - $json = $doc | ConvertTo-Json -Depth 20 - - # Remove all newlines and indentation for a single-line representation - $compactJson = $json -replace '(\r?\n\s*)+', '' - - # Output the line - Write-Output $compactJson -} -` +type TimeCreatedData struct { + SystemTime string `xml:"SystemTime,attr"` +} -func (w Windows) Install() error { - return nil +type CorrelationData struct { + ActivityID string `xml:"ActivityID,attr"` } -func (w Windows) SendSystemLogs() { - path := utils.GetMyPath() - collectorPath := filepath.Join(path, "collector.ps1") +type ExecutionData struct { + ProcessID int `xml:"ProcessID,attr"` + ThreadID int `xml:"ThreadID,attr"` +} + +type SecurityData struct{} + +type SystemData struct { + Provider ProviderData `xml:"Provider"` + EventID int `xml:"EventID"` + Version int `xml:"Version"` + Level int `xml:"Level"` + Task int `xml:"Task"` + Opcode int `xml:"Opcode"` + Keywords string `xml:"Keywords"` + TimeCreated TimeCreatedData `xml:"TimeCreated"` + EventRecordID int64 `xml:"EventRecordID"` + Correlation CorrelationData `xml:"Correlation"` + Execution ExecutionData `xml:"Execution"` + Channel string `xml:"Channel"` + Computer string `xml:"Computer"` + Security SecurityData `xml:"Security"` +} + +type EventSubscription struct { + Channel string + Query string + Errors chan error + Callback func(event *Event) + winAPIHandle windows.Handle +} - err := os.WriteFile(collectorPath, []byte(PowerShellScript), 0644) +const ( + EvtSubscribeToFutureEvents = 1 + evtSubscribeActionError = 0 + evtSubscribeActionDeliver = 1 + evtRenderEventXML = 1 +) + +var ( + modwevtapi = windows.NewLazySystemDLL("wevtapi.dll") + procEvtSubscribe = modwevtapi.NewProc("EvtSubscribe") + procEvtRender = modwevtapi.NewProc("EvtRender") + procEvtClose = modwevtapi.NewProc("EvtClose") +) + +func (evtSub *EventSubscription) Create() error { + if evtSub.winAPIHandle != 0 { + return fmt.Errorf("windows_events: subscription has already been created") + } + + winChannel, err := windows.UTF16PtrFromString(evtSub.Channel) if err != nil { - _ = utils.Logger.ErrorF("error writing powershell script: %v", err) - return + return fmt.Errorf("windows_events: invalid channel name: %s", err) } - cmd := exec.Command("Powershell.exe", "-Command", `"Set-ExecutionPolicy -ExecutionPolicy Unrestricted -Scope CurrentUser -Force"`) - err = cmd.Run() + winQuery, err := windows.UTF16PtrFromString(evtSub.Query) if err != nil { - _ = utils.Logger.ErrorF("error setting powershell execution policy: %v", err) - return + return fmt.Errorf("windows_events: invalid query: %s", err) } - for { - select { - case <-time.After(30 * time.Second): - go func() { - cmd := exec.Command("Powershell.exe", "-File", collectorPath) + callback := syscall.NewCallback(evtSub.winAPICallback) - output, err := cmd.Output() - if err != nil { - _ = utils.Logger.ErrorF("error executing powershell script: %v", err) - return - } + log.Printf("Debug - Subscribing to channel: %s", evtSub.Channel) - utils.Logger.LogF(100, "output: %s", string(output)) + handle, _, err := procEvtSubscribe.Call( + 0, + 0, + uintptr(unsafe.Pointer(winChannel)), + uintptr(unsafe.Pointer(winQuery)), + 0, + 0, + callback, + uintptr(EvtSubscribeToFutureEvents), + ) - logLines := strings.Split(string(output), "\n") + if handle == 0 { + return fmt.Errorf("windows_events: failed to subscribe to events: %v", err) + } - validatedLogs := make([]string, 0, len(logLines)) + evtSub.winAPIHandle = windows.Handle(handle) + return nil +} - for _, logLine := range logLines { - validatedLog, _, err := validations.ValidateString(logLine, false) - if err != nil { - utils.Logger.LogF(100, "error validating log: %s: %v", logLine, err) - continue - } +func (evtSub *EventSubscription) Close() error { + if evtSub.winAPIHandle == 0 { + return fmt.Errorf("windows_events: no active subscription to close") + } + ret, _, err := procEvtClose.Call(uintptr(evtSub.winAPIHandle)) + if ret == 0 { + return fmt.Errorf("windows_events: error closing handle: %s", err) + } + evtSub.winAPIHandle = 0 + return nil +} - validatedLogs = append(validatedLogs, validatedLog) +func (evtSub *EventSubscription) winAPICallback(action, userContext, event uintptr) uintptr { + switch action { + case evtSubscribeActionError: + evtSub.Errors <- fmt.Errorf("windows_events: error in callback, code: %x", uint16(event)) + case evtSubscribeActionDeliver: + bufferSize := uint32(4096) + for { + renderSpace := make([]uint16, bufferSize/2) + bufferUsed := uint32(0) + propertyCount := uint32(0) + ret, _, err := procEvtRender.Call( + 0, + event, + evtRenderEventXML, + uintptr(bufferSize), + uintptr(unsafe.Pointer(&renderSpace[0])), + uintptr(unsafe.Pointer(&bufferUsed)), + uintptr(unsafe.Pointer(&propertyCount)), + ) + if ret == 0 { + if err == windows.ERROR_INSUFFICIENT_BUFFER { + bufferSize *= 2 + continue } + evtSub.Errors <- fmt.Errorf("windows_events: failed to render event: %w", err) + return 0 + } + xmlStr := windows.UTF16ToString(renderSpace) + xmlStr = cleanXML(xmlStr) + + dataParsed := new(Event) + if err := xml.Unmarshal([]byte(xmlStr), dataParsed); err != nil { + evtSub.Errors <- fmt.Errorf("windows_events: failed to parse XML: %s", err) + } else { + evtSub.Callback(dataParsed) + } + break + } + default: + evtSub.Errors <- fmt.Errorf("windows_events: unsupported action in callback: %x", uint16(action)) + } + return 0 +} - logservice.LogQueue <- logservice.LogPipe{ - Src: string(config.DataTypeWindowsAgent), - Logs: validatedLogs, - } - }() +func cleanXML(xmlStr string) string { + xmlStr = strings.TrimSpace(xmlStr) + if idx := strings.Index(xmlStr, " 0 { + xmlStr = xmlStr[idx:] + } + xmlStr = strings.Map(func(r rune) rune { + if r < 32 && r != '\n' && r != '\r' && r != '\t' { + return -1 } + return r + }, xmlStr) + return xmlStr +} + +type Windows struct{} + +func getCollectorsInstances() []Collector { + var collectors []Collector + collectors = append(collectors, Windows{}) + return collectors +} + +func (w Windows) SendSystemLogs() { + errorsChan := make(chan error, 10) + + callback := func(event *Event) { + eventJSON, err := convertEventToJSON(event) + if err != nil { + utils.Logger.ErrorF("error converting event to JSON: %v", err) + return + } + host, err := os.Hostname() + if err != nil { + utils.Logger.ErrorF("error getting hostname: %v", err) + host = "unknown" + } + validatedLog, _, err := validations.ValidateString(eventJSON, false) + if err != nil { + utils.Logger.LogF(100, "error validating log: %s: %v", eventJSON, err) + return + } + logservice.LogQueue <- &plugins.Log{ + DataType: string(config.DataTypeWindowsAgent), + DataSource: host, + Raw: validatedLog, + } + } + + channels := []string{"Security", "Application", "System"} + var subscriptions []*EventSubscription + + for _, channel := range channels { + sub := &EventSubscription{ + Channel: channel, + Query: "*", + Errors: errorsChan, + Callback: callback, + } + if err := sub.Create(); err != nil { + utils.Logger.ErrorF("Error subscribing to channel %s: %s", channel, err) + continue + } + subscriptions = append(subscriptions, sub) + utils.Logger.LogF(100, "Subscribed to channel: %s", channel) } + + go func() { + for err := range errorsChan { + utils.Logger.ErrorF("Subscription error: %s", err) + } + }() + + exitChan := make(chan os.Signal, 1) + signal.Notify(exitChan, os.Interrupt) + <-exitChan + utils.Logger.LogF(100, "Interrupt received, closing subscriptions...") + for _, sub := range subscriptions { + if err := sub.Close(); err != nil { + utils.Logger.ErrorF("Error closing subscription for %s: %v", sub.Channel, err) + } + } + utils.Logger.LogF(100, "Agent finished successfully.") +} + +func convertEventToJSON(event *Event) (string, error) { + eventMap := map[string]interface{}{ + "timestamp": event.System.TimeCreated.SystemTime, + "providerName": event.System.Provider.ProviderName, + "providerGuid": event.System.Provider.ProviderGUID, + "eventCode": event.System.EventID, + "version": event.System.Version, + "level": event.System.Level, + "task": event.System.Task, + "opcode": event.System.Opcode, + "keywords": event.System.Keywords, + "timeCreated": event.System.TimeCreated.SystemTime, + "recordId": event.System.EventRecordID, + "correlation": event.System.Correlation, + "execution": event.System.Execution, + "channel": event.System.Channel, + "computer": event.System.Computer, + "data": make(map[string]interface{}), + } + + dataMap := eventMap["data"].(map[string]interface{}) + for _, data := range event.EventData { + if strings.HasPrefix(data.Value, "0x") { + if val, err := strconv.ParseInt(data.Value[2:], 16, 64); err == nil { + dataMap[data.Key] = val + continue + } + } + if data.Key != "" { + value := strings.TrimSpace(data.Value) + if value != "" { + dataMap[data.Key] = value + } + } + } + + jsonBytes, err := json.Marshal(eventMap) + if err != nil { + return "", err + } + return string(jsonBytes), nil +} + +func (w Windows) Install() error { + return nil } func (w Windows) Uninstall() error { - path := utils.GetMyPath() - collectorPath := filepath.Join(path, "collector.ps1") - err := os.Remove(collectorPath) - return err + return nil } From 435cfcadbf0855377cfeceae3d3bd70d5175c5e8 Mon Sep 17 00:00:00 2001 From: Osmany Montero Date: Wed, 16 Apr 2025 19:00:47 +0200 Subject: [PATCH 03/22] wip --- agent/collectors/collectors.go | 4 +- agent/collectors/filebeat_amd64.go | 8 +- agent/collectors/macos_arm64.go | 94 +++++++++++++ agent/collectors/windows_amd64.go | 5 +- agent/collectors/windows_arm64.go | 2 +- agent/config/const.go | 1 + agent/config/macos_arm64.go | 10 ++ .../config/{win_amd64.go => windows_amd64.go} | 0 .../config/{win_arm64.go => windows_arm64.go} | 0 agent/docs/tasks.md | 133 ++++++++++++++++++ agent/logs/utmstack_agent.log | 1 + agent/main.go | 11 +- agent/serv/clean-old.go | 27 ++-- agent/serv/run.go | 2 + agent/updates/dependencies.go | 4 + agent/utils/services.go | 7 +- 16 files changed, 281 insertions(+), 28 deletions(-) create mode 100644 agent/collectors/macos_arm64.go create mode 100644 agent/config/macos_arm64.go rename agent/config/{win_amd64.go => windows_amd64.go} (100%) rename agent/config/{win_arm64.go => windows_arm64.go} (100%) create mode 100644 agent/docs/tasks.md create mode 100644 agent/logs/utmstack_agent.log diff --git a/agent/collectors/collectors.go b/agent/collectors/collectors.go index d49cadfd1..753a655ff 100644 --- a/agent/collectors/collectors.go +++ b/agent/collectors/collectors.go @@ -13,7 +13,7 @@ type CollectorConfig struct { type Collector interface { Install() error - SendSystemLogs() + SendLogs() Uninstall() error } @@ -35,7 +35,7 @@ func InstallCollectors() error { func LogsReader() { collectors := getCollectorsInstances() for _, collector := range collectors { - go collector.SendSystemLogs() + go collector.SendLogs() } } diff --git a/agent/collectors/filebeat_amd64.go b/agent/collectors/filebeat_amd64.go index 290cf64a7..700e02b8d 100644 --- a/agent/collectors/filebeat_amd64.go +++ b/agent/collectors/filebeat_amd64.go @@ -104,7 +104,7 @@ func (f Filebeat) Install() error { return nil } -func (f Filebeat) SendSystemLogs() { +func (f Filebeat) SendLogs() { logLinesChan := make(chan []string) path := utils.GetMyPath() filebLogPath := filepath.Join(path, "beats", "filebeat", "logs") @@ -112,12 +112,16 @@ func (f Filebeat) SendSystemLogs() { parser := parser.GetParser("beats") go utils.WatchFolder("modulescollector", filebLogPath, logLinesChan, config.BatchCapacity) - for logLine := range logLinesChan { + + for { + logLine := <-logLinesChan + beatsData, err := parser.ProcessData(logLine) if err != nil { utils.Logger.ErrorF("error processing beats data: %v", err) continue } + for typ, logB := range beatsData { logservice.LogQueue <- logservice.LogPipe{ Src: typ, diff --git a/agent/collectors/macos_arm64.go b/agent/collectors/macos_arm64.go new file mode 100644 index 000000000..36296a5c5 --- /dev/null +++ b/agent/collectors/macos_arm64.go @@ -0,0 +1,94 @@ +//go:build darwin && arm64 +// +build darwin,arm64 + +package collectors + +import ( + "bufio" + "github.com/threatwinds/validations" + "github.com/utmstack/UTMStack/agent/config" + "github.com/utmstack/UTMStack/agent/logservice" + "github.com/utmstack/UTMStack/agent/utils" + "os/exec" + "path/filepath" +) + +type Darwin struct{} + +func (d Darwin) Install() error { + return nil +} + +func getCollectorsInstances() []Collector { + var collectors []Collector + collectors = append(collectors, Darwin{}) + return collectors +} + +func (d Darwin) SendLogs() { + path := utils.GetMyPath() + collectorPath := filepath.Join(path, "utmstack-collector-mac") + + cmd := exec.Command(collectorPath) + + stdout, err := cmd.StdoutPipe() + if err != nil { + _ = utils.Logger.ErrorF("error creating stdout pipe: %v", err) + return + } + + stderr, err := cmd.StderrPipe() + if err != nil { + _ = utils.Logger.ErrorF("error creating stderr pipe: %v", err) + return + } + + if err := cmd.Start(); err != nil { + _ = utils.Logger.ErrorF("error starting macOS collector: %v", err) + return + } + + go func() { + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + logLine := scanner.Text() + + utils.Logger.LogF(100, "output: %s", logLine) + + validatedLog, _, err := validations.ValidateString(logLine, false) + if err != nil { + utils.Logger.ErrorF("error validating log: %s: %v", logLine, err) + continue + } + + logservice.LogQueue <- logservice.LogPipe{ + Src: string(config.DataTypeMacOSAgent), + Logs: []string{validatedLog}, + } + } + + if err := scanner.Err(); err != nil { + _ = utils.Logger.ErrorF("error reading stdout: %v", err) + } + }() + + go func() { + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + errLine := scanner.Text() + _ = utils.Logger.ErrorF("collector error: %s", errLine) + } + + if err := scanner.Err(); err != nil { + _ = utils.Logger.ErrorF("error reading stderr: %v", err) + } + }() + + if err := cmd.Wait(); err != nil { + _ = utils.Logger.ErrorF("macOS collector process ended with error: %v", err) + } +} + +func (d Darwin) Uninstall() error { + return nil +} diff --git a/agent/collectors/windows_amd64.go b/agent/collectors/windows_amd64.go index c95a1429f..b65b565e7 100644 --- a/agent/collectors/windows_amd64.go +++ b/agent/collectors/windows_amd64.go @@ -70,13 +70,14 @@ func (w Windows) Install() error { return nil } -func (w Windows) SendSystemLogs() { +func (w Windows) SendLogs() { logLinesChan := make(chan []string) path := utils.GetMyPath() winbLogPath := filepath.Join(path, "beats", "winlogbeat", "logs") go utils.WatchFolder("windowscollector", winbLogPath, logLinesChan, config.BatchCapacity) - for logLine := range logLinesChan { + for { + logLine := <-logLinesChan validatedLogs := []string{} for _, log := range logLine { validatedLog, _, err := validations.ValidateString(log, false) diff --git a/agent/collectors/windows_arm64.go b/agent/collectors/windows_arm64.go index a71bd68e7..7f0ba8f2b 100644 --- a/agent/collectors/windows_arm64.go +++ b/agent/collectors/windows_arm64.go @@ -156,7 +156,7 @@ func (w Windows) Install() error { return nil } -func (w Windows) SendSystemLogs() { +func (w Windows) SendLogs() { path := utils.GetMyPath() collectorPath := filepath.Join(path, "collector.ps1") diff --git a/agent/config/const.go b/agent/config/const.go index 287b4842e..40632244f 100644 --- a/agent/config/const.go +++ b/agent/config/const.go @@ -45,6 +45,7 @@ var ( // RedlineServName = "UTMStackRedline" // BatchToSend = 5 + DataTypeMacOSAgent DataType = "macos_agent" DataTypeWindowsAgent DataType = "beats_windows_agent" DataTypeSyslog DataType = "syslog" DataTypeVmware DataType = "vmware" diff --git a/agent/config/macos_arm64.go b/agent/config/macos_arm64.go new file mode 100644 index 000000000..615b84ac0 --- /dev/null +++ b/agent/config/macos_arm64.go @@ -0,0 +1,10 @@ +//go:build darwin && arm64 +// +build darwin,arm64 + +package config + +var ( + UpdaterSelf = "utmstack_updater_self_arm64%s" + ServiceFile = "utmstack_agent_service_arm64%s" + DependFiles = []string{} +) diff --git a/agent/config/win_amd64.go b/agent/config/windows_amd64.go similarity index 100% rename from agent/config/win_amd64.go rename to agent/config/windows_amd64.go diff --git a/agent/config/win_arm64.go b/agent/config/windows_arm64.go similarity index 100% rename from agent/config/win_arm64.go rename to agent/config/windows_arm64.go diff --git a/agent/docs/tasks.md b/agent/docs/tasks.md new file mode 100644 index 000000000..a748637c0 --- /dev/null +++ b/agent/docs/tasks.md @@ -0,0 +1,133 @@ +# UTMStack Agent Improvement Tasks + +This document contains a prioritized list of tasks for improving the UTMStack Agent codebase. Each task is marked with a checkbox that can be checked off when completed. + +## Architecture Improvements + +1. [ ] Implement a more modular plugin architecture to make it easier to add new collectors and data sources +2. [ ] Create a unified configuration management system that centralizes all configuration options +3. [ ] Implement a proper dependency injection system to improve testability and reduce tight coupling +4. [ ] Refactor the service management code to use a common interface across all platforms +5. [ ] Implement a more robust error handling and recovery mechanism for all collectors +6. [ ] Create a standardized logging framework with configurable log levels +7. [ ] Implement a metrics collection system to monitor agent performance +8. [ ] Design a more efficient log batching and transmission system to reduce resource usage +9. [ ] Implement a circuit breaker pattern for external service connections +10. [ ] Create a formal API versioning strategy for communication with the server + +## Code Quality Improvements + +11. [ ] Add comprehensive unit tests for all packages (current coverage appears low) +12. [ ] Implement integration tests for collector functionality +13. [ ] Add proper documentation for all exported functions, types, and interfaces +14. [ ] Standardize error handling across the codebase +15. [ ] Refactor long functions (like in main.go) into smaller, more focused functions +16. [ ] Remove commented-out code in config/const.go and other files +17. [ ] Implement consistent naming conventions across the codebase +18. [ ] Add context support for all long-running operations +19. [ ] Implement proper resource cleanup in error cases +20. [ ] Add validation for all user inputs and configuration values + +## Security Improvements + +21. [ ] Implement secure storage for sensitive configuration (beyond basic encryption) +22. [ ] Add TLS certificate validation by default (currently optional) +23. [ ] Implement proper authentication for all API endpoints +24. [ ] Add integrity verification for downloaded dependencies +25. [ ] Implement secure logging practices (masking sensitive data) +26. [ ] Add rate limiting for authentication attempts +27. [ ] Implement proper permission checking for file operations +28. [ ] Add security scanning in the CI/CD pipeline +29. [ ] Implement secure coding practices training for developers +30. [ ] Create a security incident response plan + +## Performance Improvements + +31. [ ] Profile the application to identify performance bottlenecks +32. [ ] Optimize log collection and processing for high-volume environments +33. [ ] Implement more efficient data structures for log storage +34. [ ] Add caching for frequently accessed configuration +35. [ ] Optimize file operations to reduce disk I/O +36. [ ] Implement connection pooling for network operations +37. [ ] Reduce memory usage in log processing pipelines +38. [ ] Optimize CPU usage during idle periods +39. [ ] Implement more efficient serialization/deserialization +40. [ ] Add performance benchmarks to CI/CD pipeline + +## Platform Support Improvements + +41. [ ] Standardize platform-specific code to reduce duplication +42. [ ] Improve macOS arm64 support (recently added) +43. [ ] Add support for additional Linux distributions +44. [ ] Improve Windows arm64 support +45. [ ] Create a unified build system for all platforms +46. [ ] Implement automated testing on all supported platforms +47. [ ] Add containerization support for easier deployment +48. [ ] Create platform-specific installation documentation +49. [ ] Implement feature parity across all supported platforms +50. [ ] Add support for cloud-native environments + +## User Experience Improvements + +51. [ ] Improve installation process with better user feedback +52. [ ] Create a web-based configuration interface +53. [ ] Implement a status dashboard for monitoring agent health +54. [ ] Add better error messages for common failure scenarios +55. [ ] Improve logging with more context and clarity +56. [ ] Create user-friendly documentation with examples +57. [ ] Implement a troubleshooting guide for common issues +58. [ ] Add a command-line interface for common administrative tasks +59. [ ] Implement a self-diagnostic tool for agent issues +60. [ ] Create a user feedback mechanism + +## Operational Improvements + +61. [ ] Implement a more robust update mechanism with rollback capability +62. [ ] Add health checks for all components +63. [ ] Implement proper service lifecycle management +64. [ ] Create automated deployment scripts for common environments +65. [ ] Implement configuration validation before applying changes +66. [ ] Add support for configuration templates for common scenarios +67. [ ] Implement a backup and restore mechanism for agent configuration +68. [ ] Create operational runbooks for common maintenance tasks +69. [ ] Implement proper service dependencies management +70. [ ] Add support for centralized configuration management + +## Technical Debt Reduction + +71. [ ] Refactor the collector implementation to reduce code duplication +72. [ ] Update deprecated dependencies and APIs +73. [ ] Remove unused code and dependencies +74. [ ] Consolidate duplicate functionality across packages +75. [ ] Implement consistent error handling patterns +76. [ ] Refactor configuration handling to use a single approach +77. [ ] Improve code organization with better package structure +78. [ ] Standardize file naming conventions +79. [ ] Implement a code style guide and enforce with linters +80. [ ] Refactor platform-specific code to improve maintainability + +## Documentation Improvements + +81. [ ] Create comprehensive API documentation +82. [ ] Document the architecture and design decisions +83. [ ] Create developer onboarding documentation +84. [ ] Document the build and release process +85. [ ] Create user guides for all features +86. [ ] Document troubleshooting procedures +87. [ ] Create diagrams for system architecture and data flow +88. [ ] Document security considerations and best practices +89. [ ] Create change management documentation +90. [ ] Document performance tuning recommendations + +## Future Enhancements + +91. [ ] Implement support for additional data sources and integrations +92. [ ] Add machine learning capabilities for anomaly detection +93. [ ] Implement real-time alerting based on log patterns +94. [ ] Create a plugin marketplace for community contributions +95. [ ] Implement advanced data correlation features +96. [ ] Add support for custom log parsing rules +97. [ ] Implement advanced visualization capabilities +98. [ ] Add support for compliance reporting +99. [ ] Implement predictive analytics for security events +100. [ ] Create an ecosystem of complementary tools and integrations \ No newline at end of file diff --git a/agent/logs/utmstack_agent.log b/agent/logs/utmstack_agent.log new file mode 100644 index 000000000..602489253 --- /dev/null +++ b/agent/logs/utmstack_agent.log @@ -0,0 +1 @@ +2025-04-16T13:37:14.854518Z ERROR f18ed04e-83e4-49d7-b82b-b96385b8b0bd /Users/osmany/Projects/UTMStack/agent/serv/service.go 36 error getting config: error reading config file: open /Users/osmany/Projects/UTMStack/agent/config.yml: no such file or directory diff --git a/agent/main.go b/agent/main.go index c733479d6..57bf702d4 100644 --- a/agent/main.go +++ b/agent/main.go @@ -25,6 +25,7 @@ func main() { fmt.Println("Error checking if service is installed: ", err) os.Exit(1) } + if arg != "install" && !isInstalled { fmt.Println("UTMStackAgent service is not installed") os.Exit(1) @@ -62,14 +63,17 @@ func main() { fmt.Println("\nError registering agent: ", err) os.Exit(1) } + if err = config.SaveConfig(cnf); err != nil { fmt.Println("\nError saving config: ", err) os.Exit(1) } + if err = modules.ConfigureCollectorFirstTime(); err != nil { fmt.Println("\nError configuring collector: ", err) os.Exit(1) } + if err = collectors.InstallCollectors(); err != nil { fmt.Println("\nError installing collectors: ", err) os.Exit(1) @@ -80,7 +84,6 @@ func main() { serv.InstallService() fmt.Println("[OK]") fmt.Println("UTMStackAgent service installed correctly") - case "enable-integration", "disable-integration": fmt.Println("Changing integration status ...") integration := os.Args[2] @@ -93,7 +96,6 @@ func main() { } fmt.Printf("Action %s %s %s correctly in port %s\n", arg, integration, proto, port) time.Sleep(5 * time.Second) - case "change-port": fmt.Println("Changing integration port ...") integration := os.Args[2] @@ -107,7 +109,6 @@ func main() { } fmt.Printf("Port changed correctly from %s to %s\n", old, port) time.Sleep(5 * time.Second) - case "uninstall": fmt.Print("Uninstalling UTMStackAgent service ...") @@ -125,10 +126,10 @@ func main() { os.Remove(config.ConfigurationFile) serv.UninstallService() - fmt.Println("[OK]") + fmt.Println("UTMStackAgent service uninstalled correctly") - os.Exit(1) + os.Exit(0) case "help": Help() default: diff --git a/agent/serv/clean-old.go b/agent/serv/clean-old.go index bbe3a53a3..0814da03f 100644 --- a/agent/serv/clean-old.go +++ b/agent/serv/clean-old.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "github.com/utmstack/UTMStack/agent/config" "github.com/utmstack/UTMStack/agent/utils" @@ -14,57 +15,59 @@ func CleanOldServices(cnf *config.Config) { isUpdaterINstalled, err := utils.CheckIfServiceIsInstalled("UTMStackUpdater") if err != nil { - utils.Logger.LogF(100, "Error checking if service is installed: %v", err) + utils.Logger.LogF(100, "error checking if service is installed: %v", err) } if isUpdaterINstalled { oldVersion = true err = utils.StopService("UTMStackUpdater") if err != nil { - utils.Logger.LogF(100, "Error stopping service: %v", err) + utils.Logger.LogF(100, "error stopping service: %v", err) } err = utils.UninstallService("UTMStackUpdater") if err != nil { - utils.Logger.LogF(100, "Error uninstalling service: %v", err) + utils.Logger.LogF(100, "error uninstalling service: %v", err) } } isRedlineInstalled, err := utils.CheckIfServiceIsInstalled("UTMStackRedline") if err != nil { - utils.Logger.LogF(100, "Error checking if service is installed: %v", err) + utils.Logger.LogF(100, "error checking if service is installed: %v", err) } if isRedlineInstalled { oldVersion = true err = utils.StopService("UTMStackRedline") if err != nil { - utils.Logger.LogF(100, "Error stopping service: %v", err) + utils.Logger.LogF(100, "error stopping service: %v", err) } err = utils.UninstallService("UTMStackRedline") if err != nil { - utils.Logger.LogF(100, "Error uninstalling service: %v", err) + utils.Logger.LogF(100, "error uninstalling service: %v", err) } } if oldVersion { - utils.Logger.Info("Old version of agent found, downloading new version") + utils.Logger.Info("old version of agent found, downloading new version") headers := map[string]string{ "key": cnf.AgentKey, "id": fmt.Sprintf("%v", cnf.AgentID), "type": "agent", } - if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, cnf.Server, config.DependenciesPort, fmt.Sprintf(config.UpdaterSelf, "")), headers, fmt.Sprintf(config.UpdaterSelf, "_new"), utils.GetMyPath(), cnf.SkipCertValidation); err != nil { - utils.Logger.LogF(100, "error downloading updater: %v", err) - return + if runtime.GOOS != "darwin" { + if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, cnf.Server, config.DependenciesPort, fmt.Sprintf(config.UpdaterSelf, "")), headers, fmt.Sprintf(config.UpdaterSelf, "_new"), utils.GetMyPath(), cnf.SkipCertValidation); err != nil { + utils.Logger.LogF(100, "error downloading updater: %v", err) + return + } } oldFilePath := filepath.Join(utils.GetMyPath(), fmt.Sprintf(config.UpdaterSelf, "")) newFilePath := filepath.Join(utils.GetMyPath(), fmt.Sprintf(config.UpdaterSelf, "_new")) - utils.Logger.LogF(100, "Renaming %s to %s", newFilePath, oldFilePath) + utils.Logger.LogF(100, "renaming %s to %s", newFilePath, oldFilePath) err := os.Remove(oldFilePath) if err != nil { utils.Logger.LogF(100, "error removing old updater: %v", err) @@ -74,6 +77,6 @@ func CleanOldServices(cnf *config.Config) { utils.Logger.LogF(100, "error renaming updater: %v", err) } } else { - utils.Logger.LogF(100, "No old version of agent found") + utils.Logger.LogF(100, "no old version of agent found") } } diff --git a/agent/serv/run.go b/agent/serv/run.go index d7d864905..b737d6c1e 100644 --- a/agent/serv/run.go +++ b/agent/serv/run.go @@ -8,10 +8,12 @@ import ( func RunService() { svcConfig := GetConfigServ() p := new(program) + newService, err := service.New(p, svcConfig) if err != nil { utils.Logger.Fatal("error creating new service: %v", err) } + err = newService.Run() if err != nil { utils.Logger.Fatal("error running new service: %v", err) diff --git a/agent/updates/dependencies.go b/agent/updates/dependencies.go index a740f3551..a23752828 100644 --- a/agent/updates/dependencies.go +++ b/agent/updates/dependencies.go @@ -12,6 +12,10 @@ import ( ) func DownloadFirstDependencies(address string, authKey string, insecure bool) error { + if runtime.GOOS == "darwin" { + return nil + } + headers := map[string]string{"connection-key": authKey} if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, address, config.DependenciesPort, "version.json"), headers, "version.json", utils.GetMyPath(), insecure); err != nil { diff --git a/agent/utils/services.go b/agent/utils/services.go index f3faf1191..0757f4b16 100644 --- a/agent/utils/services.go +++ b/agent/utils/services.go @@ -53,14 +53,13 @@ func CheckIfServiceIsInstalled(serv string) (bool, error) { err = Execute("sc", path, "query", serv) case "linux": err = Execute("systemctl", path, "status", serv) + case "darwin": + return true, nil default: return false, fmt.Errorf("operative system unknown") } - if err != nil { - return false, nil - } - return true, nil + return err == nil, err } func CreateLinuxService(serviceName string, execStart string) error { From 756066cca91df58e1aeef2fa8254a7730b400f2f Mon Sep 17 00:00:00 2001 From: Yadian Llada Lopez Date: Wed, 16 Apr 2025 15:34:08 -0400 Subject: [PATCH 04/22] feat: Add SELinux configuration for RedHat systems --- installer/system.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/installer/system.go b/installer/system.go index 00fb3e974..e6aa3c95a 100644 --- a/installer/system.go +++ b/installer/system.go @@ -10,6 +10,14 @@ import ( func PrepareSystem(distro string) error { if distro == "redhat" { + if err := utils.RunCmd("setenforce", "0"); err != nil { + return fmt.Errorf("failed to disable SELinux immediately: %v", err) + } + + if err := utils.RunCmd("sed", "-i", "s/^SELINUX=.*/SELINUX=disabled/", "/etc/selinux/config"); err != nil { + return fmt.Errorf("failed to configure permanent SELinux setting: %v", err) + } + if err := utils.RunCmd("systemctl", "disable", "firewalld"); err != nil { return fmt.Errorf("failed to disable firewalld: %v", err) } From 4e505277337cf3d9564502069de8abfe72eb11a5 Mon Sep 17 00:00:00 2001 From: Yorjander Hernandez Vergara Date: Fri, 18 Apr 2025 12:36:12 -0400 Subject: [PATCH 05/22] fix interface agent problem --- agent/collectors/windows_arm64.go | 15 ++++----------- agent/go.mod | 2 +- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/agent/collectors/windows_arm64.go b/agent/collectors/windows_arm64.go index 155f1a4e2..cea5dbda0 100644 --- a/agent/collectors/windows_arm64.go +++ b/agent/collectors/windows_arm64.go @@ -15,7 +15,6 @@ import ( "syscall" "unsafe" - "github.com/threatwinds/go-sdk/plugins" "github.com/threatwinds/validations" "github.com/utmstack/UTMStack/agent/config" "github.com/utmstack/UTMStack/agent/logservice" @@ -209,7 +208,7 @@ func getCollectorsInstances() []Collector { return collectors } -func (w Windows) SendSystemLogs() { +func (w Windows) SendLogs() { errorsChan := make(chan error, 10) callback := func(event *Event) { @@ -218,20 +217,14 @@ func (w Windows) SendSystemLogs() { utils.Logger.ErrorF("error converting event to JSON: %v", err) return } - host, err := os.Hostname() - if err != nil { - utils.Logger.ErrorF("error getting hostname: %v", err) - host = "unknown" - } validatedLog, _, err := validations.ValidateString(eventJSON, false) if err != nil { utils.Logger.LogF(100, "error validating log: %s: %v", eventJSON, err) return } - logservice.LogQueue <- &plugins.Log{ - DataType: string(config.DataTypeWindowsAgent), - DataSource: host, - Raw: validatedLog, + logservice.LogQueue <- logservice.LogPipe{ + Src: string(config.DataTypeWindowsAgent), + Logs: []string{validatedLog}, } } diff --git a/agent/go.mod b/agent/go.mod index e25ed3f7a..ca414eba0 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -12,6 +12,7 @@ require ( github.com/tehmaze/netflow v0.0.0-20240303214733-8c13bb004068 github.com/threatwinds/logger v1.2.1 github.com/threatwinds/validations v1.0.9 + golang.org/x/sys v0.31.0 google.golang.org/grpc v1.71.0 google.golang.org/protobuf v1.36.5 gopkg.in/yaml.v2 v2.4.0 @@ -44,7 +45,6 @@ require ( golang.org/x/arch v0.9.0 // indirect golang.org/x/crypto v0.36.0 // indirect golang.org/x/net v0.37.0 // indirect - golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect From 7b909e91da4999145df696f4b360497cfd3d6e21 Mon Sep 17 00:00:00 2001 From: Yorjander Hernandez Vergara Date: Fri, 18 Apr 2025 12:41:03 -0400 Subject: [PATCH 06/22] add bad gateway page --- installer/nginx.go | 6 +++ installer/templates/nginx.go | 97 ++++++++++++++++++++++++++++++++++++ installer/templates/proxy.go | 12 +++++ installer/utils/os.go | 4 +- 4 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 installer/templates/nginx.go diff --git a/installer/nginx.go b/installer/nginx.go index 3691a5b70..8fc0feea3 100644 --- a/installer/nginx.go +++ b/installer/nginx.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "path" "github.com/utmstack/UTMStack/installer/templates" @@ -41,6 +42,11 @@ func ConfigureNginx(conf *types.Config, stack *types.StackConfig, distro string) return err } + err = utils.WriteToFile(path.Join("/", "etc", "nginx", "html", "custom_502.html"), templates.NginxCustomBadGateway) + if err != nil { + fmt.Printf("Error writing custom 502 page: %v\n", err) + } + switch distro { case "ubuntu": err = utils.GenerateConfig(c, templates.ProxyUbuntu, path.Join("/", "etc", "nginx", "sites-available", "default")) diff --git a/installer/templates/nginx.go b/installer/templates/nginx.go new file mode 100644 index 000000000..e08e46ff1 --- /dev/null +++ b/installer/templates/nginx.go @@ -0,0 +1,97 @@ +package templates + +const NginxCustomBadGateway string = ` + + + + UTMStack - Maintenance + + + + +
+
+ +

We're getting things ready

+

+ UTMStack is currently under maintenance.
Please check back in a few minutes. +

+
+ © UTMStack ©Copyright 2025. All Rights Reserved. +
+
+ + + ` diff --git a/installer/templates/proxy.go b/installer/templates/proxy.go index 9a50b75ad..07dfb4bb0 100644 --- a/installer/templates/proxy.go +++ b/installer/templates/proxy.go @@ -24,6 +24,12 @@ server { proxy_read_timeout 900; } + error_page 502 /custom_502.html; + location = /custom_502.html { + root /etc/nginx/html; + internal; + } + ssl_certificate /utmstack/cert/utm.crt; ssl_certificate_key /utmstack/cert/utm.key; ssl_protocols TLSv1.2 TLSv1.3; @@ -72,6 +78,12 @@ server { proxy_read_timeout 900; } + error_page 502 /custom_502.html; + location = /custom_502.html { + root /etc/nginx/html; + internal; + } + ssl_certificate /utmstack/cert/utm.crt; ssl_certificate_key /utmstack/cert/utm.key; ssl_protocols TLSv1.2 TLSv1.3; diff --git a/installer/utils/os.go b/installer/utils/os.go index b17fd4460..d46d26f6d 100644 --- a/installer/utils/os.go +++ b/installer/utils/os.go @@ -48,7 +48,7 @@ func CreatePathIfNotExist(path string) error { return nil } -func writeToFile(fileName string, body string) error { +func WriteToFile(fileName string, body string) error { file, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.ModePerm) if err != nil { @@ -67,7 +67,7 @@ func WriteYAML(url string, data interface{}) error { return err } - err = writeToFile(url, string(config[:])) + err = WriteToFile(url, string(config[:])) if err != nil { return err } From 1173856fa100d0ad5e2619eaca94b0865defc245 Mon Sep 17 00:00:00 2001 From: Yorjander Hernandez Vergara Date: Sat, 19 Apr 2025 01:15:24 -0400 Subject: [PATCH 07/22] complete macos agent --- agent/agent/incident_response.go | 2 +- agent/collectors/macos_arm64.go | 7 +++--- agent/config/const.go | 4 +-- agent/config/macos_arm64.go | 4 +-- agent/modules/modules.go | 2 +- agent/self/config/macos_arm64.go | 8 ++++++ agent/self/utils/services.go | 34 +++++++++++++++++++++---- agent/updates/dependencies.go | 8 ++---- agent/updates/update.go | 2 +- agent/utils/address.go | 43 +++++++++++++++++++++++++------- agent/utils/os.go | 2 +- agent/utils/services.go | 13 ++++++++-- 12 files changed, 95 insertions(+), 34 deletions(-) create mode 100644 agent/self/config/macos_arm64.go diff --git a/agent/agent/incident_response.go b/agent/agent/incident_response.go index 2ad06f481..478a68102 100644 --- a/agent/agent/incident_response.go +++ b/agent/agent/incident_response.go @@ -119,7 +119,7 @@ func commandProcessor(path string, stream AgentService_AgentStreamClient, cnf *c switch runtime.GOOS { case "windows": result, errB = utils.ExecuteWithResult("cmd.exe", path, "/C", commandPair[0]) - case "linux": + case "linux", "darwin": result, errB = utils.ExecuteWithResult("sh", path, "-c", commandPair[0]) default: utils.Logger.Fatal("unsupported operating system: %s", runtime.GOOS) diff --git a/agent/collectors/macos_arm64.go b/agent/collectors/macos_arm64.go index 36296a5c5..44febabe2 100644 --- a/agent/collectors/macos_arm64.go +++ b/agent/collectors/macos_arm64.go @@ -5,12 +5,13 @@ package collectors import ( "bufio" + "os/exec" + "path/filepath" + "github.com/threatwinds/validations" "github.com/utmstack/UTMStack/agent/config" "github.com/utmstack/UTMStack/agent/logservice" "github.com/utmstack/UTMStack/agent/utils" - "os/exec" - "path/filepath" ) type Darwin struct{} @@ -62,7 +63,7 @@ func (d Darwin) SendLogs() { } logservice.LogQueue <- logservice.LogPipe{ - Src: string(config.DataTypeMacOSAgent), + Src: string(config.DataTypeMacOs), Logs: []string{validatedLog}, } } diff --git a/agent/config/const.go b/agent/config/const.go index 40632244f..550e41197 100644 --- a/agent/config/const.go +++ b/agent/config/const.go @@ -45,7 +45,6 @@ var ( // RedlineServName = "UTMStackRedline" // BatchToSend = 5 - DataTypeMacOSAgent DataType = "macos_agent" DataTypeWindowsAgent DataType = "beats_windows_agent" DataTypeSyslog DataType = "syslog" DataTypeVmware DataType = "vmware" @@ -100,7 +99,6 @@ var ( DataTypeSonicwall: {UDP: "7009", TCP: "7009"}, DataTypeDeceptivebytes: {UDP: "7010", TCP: "7010"}, DataTypeSentinelOne: {UDP: "7012", TCP: "7012"}, - DataTypeMacOs: {UDP: "7015", TCP: "7015"}, DataTypeAix: {UDP: "7016", TCP: "7016"}, DataTypePfsense: {UDP: "7017", TCP: "7017"}, DataTypeFortiweb: {UDP: "7018", TCP: "7018"}, @@ -117,7 +115,7 @@ func GetMessageFormated(host string, msg string) string { func ValidateModuleType(typ string) string { switch DataType(typ) { case DataTypeSyslog, DataTypeVmware, DataTypeEset, DataTypeKaspersky, DataTypeFortinet, DataTypePaloalto, - DataTypeMikrotik, DataTypeSophosXG, DataTypeSonicwall, DataTypeSentinelOne, DataTypeCiscoGeneric, DataTypeMacOs, + DataTypeMikrotik, DataTypeSophosXG, DataTypeSonicwall, DataTypeSentinelOne, DataTypeCiscoGeneric, DataTypeDeceptivebytes, DataTypeAix, DataTypePfsense, DataTypeFortiweb: return "syslog" case DataTypeNetflow: diff --git a/agent/config/macos_arm64.go b/agent/config/macos_arm64.go index 615b84ac0..35533b1ac 100644 --- a/agent/config/macos_arm64.go +++ b/agent/config/macos_arm64.go @@ -4,7 +4,7 @@ package config var ( - UpdaterSelf = "utmstack_updater_self_arm64%s" - ServiceFile = "utmstack_agent_service_arm64%s" + UpdaterSelf = "utmstack_updater_self%s" + ServiceFile = "utmstack_agent_service%s" DependFiles = []string{} ) diff --git a/agent/modules/modules.go b/agent/modules/modules.go index 0ce4eb491..508f2cecc 100644 --- a/agent/modules/modules.go +++ b/agent/modules/modules.go @@ -55,7 +55,7 @@ func StartModules() { if index == -1 { newModule := GetModule(intType) if newModule == nil { - utils.Logger.ErrorF("error getting module %s", intType) + utils.Logger.LogF(100, "error getting module %s", intType) continue } moCache = append(moCache, newModule) diff --git a/agent/self/config/macos_arm64.go b/agent/self/config/macos_arm64.go new file mode 100644 index 000000000..edff9f770 --- /dev/null +++ b/agent/self/config/macos_arm64.go @@ -0,0 +1,8 @@ +//go:build darwin && arm64 +// +build darwin,arm64 + +package config + +var ( + ServiceFile = "utmstack_agent_service%s" +) diff --git a/agent/self/utils/services.go b/agent/self/utils/services.go index b8f273306..b6009d27f 100644 --- a/agent/self/utils/services.go +++ b/agent/self/utils/services.go @@ -16,6 +16,8 @@ func CheckIfServiceIsActive(serv string) (bool, error) { output, errB = ExecuteWithResult("sc", path, "query", serv) case "linux": output, errB = ExecuteWithResult("systemctl", path, "is-active", serv) + case "darwin": + output, errB = ExecuteWithResult("launchctl", path, "list", serv) default: return false, fmt.Errorf("unknown operating system") } @@ -25,13 +27,18 @@ func CheckIfServiceIsActive(serv string) (bool, error) { } serviceStatus := strings.ToLower(strings.TrimSpace(output)) - if runtime.GOOS == "linux" { - return serviceStatus == "active", nil - } else if runtime.GOOS == "windows" { + + switch runtime.GOOS { + case "windows": return strings.Contains(serviceStatus, "running"), nil + case "linux": + return serviceStatus == "active", nil + case "darwin": + // launchctl list returns a JSON-ish block or error.If the service is listed, it's running + return true, nil + default: + return false, fmt.Errorf("unsupported operating system") } - - return false, fmt.Errorf("unsupported operating system") } func RestartService(serv string) error { @@ -66,6 +73,18 @@ func RestartService(serv string) error { return fmt.Errorf("error starting service: %v", err) } } + case "darwin": + plistPath := fmt.Sprintf("/Library/LaunchDaemons/%s.plist", serv) + + if isRunning { + if err := Execute("launchctl", path, "remove", serv); err != nil { + return fmt.Errorf("error stopping macOS service: %v", err) + } + } + + if err := Execute("launchctl", path, "load", plistPath); err != nil { + return fmt.Errorf("error starting macOS service: %v", err) + } } return nil } @@ -83,6 +102,11 @@ func StopService(name string) error { if err != nil { return fmt.Errorf("error stoping service: %v", err) } + case "darwin": + err := Execute("launchctl", path, "remove", name) + if err != nil { + return fmt.Errorf("error stopping macOS service: %v", err) + } } return nil } diff --git a/agent/updates/dependencies.go b/agent/updates/dependencies.go index a23752828..56e59707e 100644 --- a/agent/updates/dependencies.go +++ b/agent/updates/dependencies.go @@ -12,10 +12,6 @@ import ( ) func DownloadFirstDependencies(address string, authKey string, insecure bool) error { - if runtime.GOOS == "darwin" { - return nil - } - headers := map[string]string{"connection-key": authKey} if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, address, config.DependenciesPort, "version.json"), headers, "version.json", utils.GetMyPath(), insecure); err != nil { @@ -44,7 +40,7 @@ func handleDependenciesPostDownload(dependencies []string) error { return fmt.Errorf("error unzipping dependencies: %v", err) } - if runtime.GOOS == "linux" { + if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { if err := utils.Execute("chmod", utils.GetMyPath(), "-R", "777", fmt.Sprintf(config.UpdaterSelf, "")); err != nil { return fmt.Errorf("error executing chmod on %s: %v", fmt.Sprintf(config.UpdaterSelf, ""), err) } @@ -53,7 +49,7 @@ func handleDependenciesPostDownload(dependencies []string) error { if err := os.Remove(filepath.Join(utils.GetMyPath(), file)); err != nil { return fmt.Errorf("error removing file %s: %v", file, err) } - } else if runtime.GOOS == "linux" { + } else if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { if err := utils.Execute("chmod", utils.GetMyPath(), "-R", "777", file); err != nil { return fmt.Errorf("error executing chmod on %s: %v", file, err) } diff --git a/agent/updates/update.go b/agent/updates/update.go index f6ada2f47..e35b09add 100644 --- a/agent/updates/update.go +++ b/agent/updates/update.go @@ -60,7 +60,7 @@ func UpdateDependencies(cnf *config.Config) { continue } - if runtime.GOOS == "linux" { + if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { if err = utils.Execute("chmod", utils.GetMyPath(), "-R", "777", filepath.Join(utils.GetMyPath(), fmt.Sprintf(config.ServiceFile, "_new"))); err != nil { utils.Logger.ErrorF("error executing chmod: %v", err) } diff --git a/agent/utils/address.go b/agent/utils/address.go index ddf0a79db..2c634f808 100644 --- a/agent/utils/address.go +++ b/agent/utils/address.go @@ -3,22 +3,47 @@ package utils import ( "errors" "net" + "strings" ) func GetIPAddress() (string, error) { - addrs, err := net.InterfaceAddrs() - if err != nil { - return "", err + conn, err := net.Dial("udp", "8.8.8.8:80") + if err == nil { + defer conn.Close() + if localAddr, ok := conn.LocalAddr().(*net.UDPAddr); ok { + if ip := localAddr.IP.To4(); ip != nil && !ip.IsLoopback() { + return ip.String(), nil + } + } } - for _, addr := range addrs { - ipNet, ok := addr.(*net.IPNet) - if ok && !ipNet.IP.IsLoopback() { - if ipNet.IP.To4() != nil { - return ipNet.IP.String(), nil + return "", errors.New("failed to get IP address") +} + +func CleanIPAddresses(input string) string { + var clean []string + + addresses := strings.Split(input, ",") + for _, addr := range addresses { + addr = strings.TrimSpace(addr) + + ip := net.ParseIP(addr) + if ip == nil { + if ipNet, _, err := net.ParseCIDR(addr); err == nil { + ip = ipNet } } + + if ip == nil { + continue + } + + if ip.IsLoopback() { + continue + } + + clean = append(clean, ip.String()) } - return "", errors.New("failed to get IP address") + return strings.Join(clean, ",") } diff --git a/agent/utils/os.go b/agent/utils/os.go index 2338880ad..6998f6646 100644 --- a/agent/utils/os.go +++ b/agent/utils/os.go @@ -58,7 +58,7 @@ func GetOsInfo() (*OSInfo, error) { aliases = append(aliases, "") } info.Aliases = strings.Join(aliases, ",") - info.Addresses = strings.Join(hostInfo.Info().IPs, ",") + info.Addresses = CleanIPAddresses(strings.Join(hostInfo.Info().IPs, ",")) return &info, nil } diff --git a/agent/utils/services.go b/agent/utils/services.go index 0757f4b16..dc5565402 100644 --- a/agent/utils/services.go +++ b/agent/utils/services.go @@ -19,6 +19,11 @@ func StopService(name string) error { if err != nil { return fmt.Errorf("error stoping service: %v", err) } + case "darwin": + err := Execute("launchctl", path, "remove", name) + if err != nil { + return fmt.Errorf("error stopping macOS service: %v", err) + } } return nil } @@ -40,11 +45,15 @@ func UninstallService(name string) error { if err != nil { return fmt.Errorf("error uninstalling service: %v", err) } + case "darwin": + Execute("launchctl", path, "remove", name) + Execute("rm", "/Library/LaunchDaemons/"+name+".plist") + Execute("rm", "/Users/"+os.Getenv("USER")+"/Library/LaunchAgents/"+name+".plist") + } return nil } -// CheckIfServiceIsInstalled checks if a service is installed func CheckIfServiceIsInstalled(serv string) (bool, error) { path := GetMyPath() var err error @@ -54,7 +63,7 @@ func CheckIfServiceIsInstalled(serv string) (bool, error) { case "linux": err = Execute("systemctl", path, "status", serv) case "darwin": - return true, nil + err = Execute("launchctl", path, "list", serv) default: return false, fmt.Errorf("operative system unknown") } From c8e3f40f7c5be07861f28fcf95e40c8b5b40de43 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Fri, 18 Apr 2025 14:20:44 -0500 Subject: [PATCH 08/22] feat(module-integration): add SOC AI model selection field --- .../int-generic-group-config.component.html | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/app/app-module/conf/int-generic-group-config/int-generic-group-config.component.html b/frontend/src/app/app-module/conf/int-generic-group-config/int-generic-group-config.component.html index 0f582860f..912732b4f 100644 --- a/frontend/src/app/app-module/conf/int-generic-group-config/int-generic-group-config.component.html +++ b/frontend/src/app/app-module/conf/int-generic-group-config/int-generic-group-config.component.html @@ -102,6 +102,7 @@ Date: Fri, 18 Apr 2025 15:29:59 -0500 Subject: [PATCH 09/22] feat(module-integration): add SOC AI model selection field --- .../int-generic-group-config.component.html | 32 +++++++++++++++++++ .../int-generic-group-config.component.ts | 6 ++++ .../shared/type/utm-module-group-conf.type.ts | 1 + 3 files changed, 39 insertions(+) diff --git a/frontend/src/app/app-module/conf/int-generic-group-config/int-generic-group-config.component.html b/frontend/src/app/app-module/conf/int-generic-group-config/int-generic-group-config.component.html index 912732b4f..7bbe1a976 100644 --- a/frontend/src/app/app-module/conf/int-generic-group-config/int-generic-group-config.component.html +++ b/frontend/src/app/app-module/conf/int-generic-group-config/int-generic-group-config.component.html @@ -99,6 +99,38 @@ + + + + + + + + + + + + + + - - - - - - - - - - - + [ngClass]="{'is-invalid': options.invalid, + 'is-valid': options.valid}"> { - this.groups = response.body; + this.groups = response.body.map(group => { + group.moduleGroupConfigurations.forEach(config => { + if (config.confOptions) { + config.confOptions = JSON.parse(config.confOptions); + } + }); + return group; + }); this.configValidChange.emit(this.tenantGroupConfigValid()); }), switchMap(response => { @@ -356,7 +368,8 @@ export class IntGenericGroupConfigComponent implements OnInit { } } - getOptions(confOptions: string) { - return JSON.parse(confOptions); + getName(name: string): string { + return name.split('.').pop()!; } + } From 4ff0be533163e7e3c4701d0de62a1601fc4a1dd6 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Mon, 21 Apr 2025 09:02:32 -0500 Subject: [PATCH 11/22] feat(module-integration): add SOC AI model selection field --- .../UtmModuleGroupConfiguration.java | 11 +++++ .../factory/impl/ModuleSocAi.java | 29 +++++++++++ .../types/ModuleConfigurationKey.java | 18 +++++++ ...418001_add_options_module_group_config.xml | 49 +++++++++++++++++++ .../resources/config/liquibase/master.xml | 3 ++ 5 files changed, 110 insertions(+) create mode 100644 backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml diff --git a/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java b/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java index 2e42bfaca..19d2b7f3c 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java @@ -46,6 +46,9 @@ public class UtmModuleGroupConfiguration implements Serializable { @Column(name = "conf_required", nullable = false) private Boolean confRequired; + @Column(name = "conf_options", nullable = false) + private String confOptions; + @JsonIgnore @ManyToOne @JoinColumn(name = "group_id", insertable = false, updatable = false) @@ -134,4 +137,12 @@ public UtmModuleGroup getModuleGroup() { public void setModuleGroup(UtmModuleGroup moduleGroup) { this.moduleGroup = moduleGroup; } + + public String getConfOptions() { + return confOptions; + } + + public void setConfOptions(String confOptions) { + this.confOptions = confOptions; + } } diff --git a/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java b/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java index ddb9d8d44..694a9349d 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java @@ -50,6 +50,7 @@ public List getConfigurationKeys(Long groupId) throws Ex .withConfDataType("password") .withConfRequired(true) .build()); + keys.add(ModuleConfigurationKey.builder() .withGroupId(groupId) .withConfKey("utmstack.socai.incidentCreation") @@ -58,6 +59,7 @@ public List getConfigurationKeys(Long groupId) throws Ex .withConfDataType("bool") .withConfRequired(false) .build()); + keys.add(ModuleConfigurationKey.builder() .withGroupId(groupId) .withConfKey("utmstack.socai.changeAlertStatus") @@ -68,6 +70,33 @@ public List getConfigurationKeys(Long groupId) throws Ex .withConfRequired(false) .build()); + keys.add(ModuleConfigurationKey.builder() + .withGroupId(groupId) + .withConfKey("utmstack.socai.model") + .withConfName("Select AI Model") + .withConfDescription("Choose the AI model that SOC AI will use to analyze alerts.") + .withConfDataType("select") + .withConfRequired(true) + .withConfOptions( + "[" + + "{\"value\": \"gpt-4\", \"label\": \"GPT-4\"}," + + "{\"value\": \"gpt-4-0613\", \"label\": \"GPT-4 (0613)\"}," + + "{\"value\": \"gpt-4-32k\", \"label\": \"GPT-4 32K\"}," + + "{\"value\": \"gpt-4-32k-0613\", \"label\": \"GPT-4 32K (0613)\"}," + + "{\"value\": \"gpt-4-turbo\", \"label\": \"GPT-4 Turbo\"}," + + "{\"value\": \"gpt-4o\", \"label\": \"GPT-4 Omni\"}," + + "{\"value\": \"gpt-4o-mini\", \"label\": \"GPT-4 Omni Mini\"}," + + "{\"value\": \"gpt-4.1\", \"label\": \"GPT-4.1\"}," + + "{\"value\": \"gpt-4.1-mini\", \"label\": \"GPT-4.1 Mini\"}," + + "{\"value\": \"gpt-4.1-nano\", \"label\": \"GPT-4.1 Nano\"}," + + "{\"value\": \"gpt-3.5-turbo\", \"label\": \"GPT-3.5 Turbo\"}," + + "{\"value\": \"gpt-3.5-turbo-0613\", \"label\": \"GPT-3.5 Turbo (0613)\"}," + + "{\"value\": \"gpt-3.5-turbo-16k\", \"label\": \"GPT-3.5 Turbo 16K\"}," + + "{\"value\": \"gpt-3.5-turbo-16k-0613\", \"label\": \"GPT-3.5 Turbo 16K (0613)\"}" + + "]" + ) + .build()); + return keys; } } diff --git a/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java b/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java index 765b67cf3..1e546f795 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java @@ -6,6 +6,8 @@ public class ModuleConfigurationKey { private String confName; private String confDescription; private String confDataType; + private String confOptions; + private Boolean confRequired; private ModuleConfigurationKey() { @@ -59,6 +61,14 @@ public void setConfRequired(Boolean confRequired) { this.confRequired = confRequired; } + public String getConfOptions() { + return confOptions; + } + + public void setConfOptions(String confOptions) { + this.confOptions = confOptions; + } + public static Builder builder() { return new Builder(); } @@ -71,6 +81,8 @@ public static class Builder { private String confDataType; private Boolean confRequired; + private String confOptions; + public Builder withGroupId(Long groupId) { this.groupId = groupId; return this; @@ -101,6 +113,11 @@ public Builder withConfRequired(Boolean confRequired) { return this; } + public Builder withConfOptions(String confOptions) { + this.confOptions = confOptions; + return this; + } + public ModuleConfigurationKey build() { ModuleConfigurationKey key = new ModuleConfigurationKey(); key.setGroupId(groupId); @@ -109,6 +126,7 @@ public ModuleConfigurationKey build() { key.setConfDescription(confDescription); key.setConfDataType(confDataType); key.setConfRequired(confRequired); + key.setConfOptions(confOptions); return key; } } diff --git a/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml b/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml new file mode 100644 index 000000000..7ff458451 --- /dev/null +++ b/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml @@ -0,0 +1,49 @@ + + + + + + ALTER TABLE utm_module_group_configuration ADD COLUMN conf_options TEXT; + INSERT INTO public.utm_module_group_configuration ( + id, + group_id, + conf_key, + conf_value, + conf_name, + conf_description, + conf_data_type, + conf_required, + conf_options + ) VALUES ( + default, + 1, + 'utmstack.socai.model', + 'gpt-4-turbo', + 'Select AI Model', + 'Choose the AI model that SOC AI will use to analyze alerts. Models differ in capability, performance, and cost.', + 'select', + true, + '[ + { "value": "gpt-4", "label": "GPT-4 (Default)" }, + { "value": "gpt-4-0613", "label": "GPT-4 (0613)" }, + { "value": "gpt-4-32k", "label": "GPT-4 32K" }, + { "value": "gpt-4-32k-0613", "label": "GPT-4 32K (0613)" }, + { "value": "gpt-4-turbo", "label": "GPT-4 Turbo" }, + { "value": "gpt-4o", "label": "GPT-4 Omni" }, + { "value": "gpt-4o-mini", "label": "GPT-4 Omni Mini" }, + { "value": "gpt-4.1", "label": "GPT-4.1" }, + { "value": "gpt-4.1-mini", "label": "GPT-4.1 Mini" }, + { "value": "gpt-4.1-nano", "label": "GPT-4.1 Nano" }, + { "value": "gpt-3.5-turbo", "label": "GPT-3.5 Turbo" }, + { "value": "gpt-3.5-turbo-0613", "label": "GPT-3.5 Turbo (0613)" }, + { "value": "gpt-3.5-turbo-16k", "label": "GPT-3.5 Turbo 16K" }, + { "value": "gpt-3.5-turbo-16k-0613", "label": "GPT-3.5 Turbo 16K (0613)" } + ]' + ); + + + + \ No newline at end of file diff --git a/backend/src/main/resources/config/liquibase/master.xml b/backend/src/main/resources/config/liquibase/master.xml index 1642ad210..e4508e78c 100644 --- a/backend/src/main/resources/config/liquibase/master.xml +++ b/backend/src/main/resources/config/liquibase/master.xml @@ -89,4 +89,7 @@ + + + From 88def5d5dffcf538aeb6a396dfd2e5e475041857 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Mon, 21 Apr 2025 10:29:48 -0500 Subject: [PATCH 12/22] feat(module-integration): update MacOS guide --- .../guide-linux-agent.component.html | 2 +- .../guide-macos-agent.component.html | 32 +++----------- .../guide-macos-agent.component.ts | 43 ++++++++++++++++++- .../guides/guide-macos-agent/mac.steps.ts | 43 +++---------------- .../guide-winlogbeat.component.html | 2 +- .../agent-install-selector.component.ts | 3 +- 6 files changed, 57 insertions(+), 68 deletions(-) diff --git a/frontend/src/app/app-module/guides/guide-linux-agent/guide-linux-agent.component.html b/frontend/src/app/app-module/guides/guide-linux-agent/guide-linux-agent.component.html index 7aaa173a6..19a394fca 100644 --- a/frontend/src/app/app-module/guides/guide-linux-agent/guide-linux-agent.component.html +++ b/frontend/src/app/app-module/guides/guide-linux-agent/guide-linux-agent.component.html @@ -7,7 +7,7 @@

The Linux agent captures system and application logs and sends them to a UTMStack master server or probe/proxy, - monitors system activity, and executes incident response commands. The UTMStack agents communicate over ports 9000 and 50051. + monitors system activity, and executes incident response commands. The UTMStack agents communicate over ports 9000, 9001 and 50051. Please make sure these ports are open.

diff --git a/frontend/src/app/app-module/guides/guide-macos-agent/guide-macos-agent.component.html b/frontend/src/app/app-module/guides/guide-macos-agent/guide-macos-agent.component.html index bd4730d36..7da832f9c 100644 --- a/frontend/src/app/app-module/guides/guide-macos-agent/guide-macos-agent.component.html +++ b/frontend/src/app/app-module/guides/guide-macos-agent/guide-macos-agent.component.html @@ -1,17 +1,16 @@

- Installing filebeat in MacOS + Installing MacOS agent

-
- This integration requires a UTMStack agent to work properly. Please, make sure you have installed it before you - continue. +
+ This feature is part of a paid subscription. Please contact our support team to receive the appropriate installation dependencies.

- The Syslog daemon (syslog) on MacOS is configured using rsyslog through the /etc/syslog.conf configuration file. - Follow the steps below to send all Syslog messages from an MacOS machine to UTMStack. + The MacOS agent captures system and application logs and sends them to a UTMStack master server or probe/proxy, monitors system activity, and executes incident response commands. + The UTMStack agents communicate over ports 9000, 9001 and 50051. Please make sure these ports are open.

    @@ -22,25 +21,8 @@

    - - - - - - - - - - - - - -
    - - -
    + +     diff --git a/frontend/src/app/app-module/guides/guide-macos-agent/guide-macos-agent.component.ts b/frontend/src/app/app-module/guides/guide-macos-agent/guide-macos-agent.component.ts index ad446ded2..292c51d07 100644 --- a/frontend/src/app/app-module/guides/guide-macos-agent/guide-macos-agent.component.ts +++ b/frontend/src/app/app-module/guides/guide-macos-agent/guide-macos-agent.component.ts @@ -2,6 +2,9 @@ import {Component, Input, OnInit} from '@angular/core'; import {UtmModulesEnum} from '../../shared/enum/utm-module.enum'; import {Step} from '../shared/step'; import {MAC_STEPS} from './mac.steps'; +import { + FederationConnectionService +} from "../../../app-management/connection-key/shared/services/federation-connection.service"; @Component({ selector: 'app-guide-macos-agent', @@ -16,9 +19,45 @@ export class GuideMacosAgentComponent implements OnInit { @Input() serverId: number; steps: Step[] = MAC_STEPS; - constructor() { } + architectures = []; + token: string; + constructor(private federationConnectionService: FederationConnectionService,) { } ngOnInit() { - console.log(this.steps); + this.getToken(); + } + + + getToken() { + this.federationConnectionService.getToken().subscribe(response => { + if (response.body !== null && response.body !== '') { + this.token = response.body; + } else { + this.token = ''; + } + this.loadArchitectures(); + }); + } + + getCommandARM(installerName: string): string { + const ip = window.location.host.includes(':') ? window.location.host.split(':')[0] : window.location.host; + + return `sudo bash -c "./${installerName} ${ip} ${this.token} yes"`; + } + + + getUninstallCommand(installerName: string): string { + return `sudo bash -c "./utmstack_agent_service uninstall"`; + } + + private loadArchitectures() { + this.architectures = [ + { + id: 1, name: 'ARM64', + install: this.getCommandARM('utmstack_agent_service install'), + uninstall: this.getUninstallCommand('utmstack_agent_service install'), + shell: '' + }, + ]; } } diff --git a/frontend/src/app/app-module/guides/guide-macos-agent/mac.steps.ts b/frontend/src/app/app-module/guides/guide-macos-agent/mac.steps.ts index 4f774e847..cdb6558f9 100644 --- a/frontend/src/app/app-module/guides/guide-macos-agent/mac.steps.ts +++ b/frontend/src/app/app-module/guides/guide-macos-agent/mac.steps.ts @@ -1,45 +1,14 @@ import {Step} from '../shared/step'; export const MAC_STEPS: Step[] = [ - {id: '1', name: 'Install Homebrew, using the official documentation\n' + - ' here, if you already installed go to the next step.'}, - {id: '2', name: 'Install rsyslog on MacOS:', - content: { - id: 'stepContent2', - commands: ['brew install rsyslog'] - } + {id: '1', + name: 'Reach out to support to request the installation dependencies for macOS. ' + + 'These are required to proceed with the installation or uninstallation process.', }, - {id: '3', name: 'Open the file /etc/syslog.conf in an editor:\n', + {id: '2', + name: 'Use the following command according to the action you wish to perform (install or uninstall):', content: { - id: 'stepContent3', - commands: ['sudo nano /etc/syslog.conf'] + id: 'stepContent2' } }, - {id: '4', name: 'Append the following line at the end if you want to send over TPC:\n', - content: { - id: 'stepContent4', - commands: ['*.* @@IP_OF_YOUR_UTMSTACK_AGENT:7015'] - } - }, - {id: '5', name: 'Restart the syslog daemon:', - content: { - id: 'stepContent5', - commands: ['sudo launchctl stop /System/Library/LaunchDaemons/com.apple.syslogd.plist', - 'sudo launchctl start /System/Library/LaunchDaemons/com.apple.syslogd.plist'] - } - }, - {id: '6', name: 'Enable log collector.
    ' + - 'To enable the log collector where you have the UTMStack agent installed, ' + - 'follow the instructions below based on your operating system and preferred protocol.\n', - content: { - id: 'stepContent6' - } - }, - {id: '7', name: 'Click on the button shown below, to activate the UTMStack features related to this integration', - content: { - id: 'stepContent7' - } - } ]; diff --git a/frontend/src/app/app-module/guides/guide-winlogbeat/guide-winlogbeat.component.html b/frontend/src/app/app-module/guides/guide-winlogbeat/guide-winlogbeat.component.html index 22459b5ab..4b4189dca 100644 --- a/frontend/src/app/app-module/guides/guide-winlogbeat/guide-winlogbeat.component.html +++ b/frontend/src/app/app-module/guides/guide-winlogbeat/guide-winlogbeat.component.html @@ -13,7 +13,7 @@

    • Compatible with Windows Server version 2016 R2 or higher
      • -
    • The UTMStack agents communicate over ports 9000 and 50051. Please make sure these ports are open.
      • +
    • The UTMStack agents communicate over ports 9000, 9001 and 50051. Please make sure these ports are open.
    • Ensure curl is installed on the system, as it is required for downloading and installing the UTMStack agent.
    diff --git a/frontend/src/app/app-module/guides/shared/components/agent-install-selector.component.ts b/frontend/src/app/app-module/guides/shared/components/agent-install-selector.component.ts index d7425020a..6954ce64c 100644 --- a/frontend/src/app/app-module/guides/shared/components/agent-install-selector.component.ts +++ b/frontend/src/app/app-module/guides/shared/components/agent-install-selector.component.ts @@ -58,11 +58,10 @@ export class AgentInstallSelectorComponent { ]; @Input() platforms = []; - @Input() agent: string; + @Input() _selectedPlatform: any; _selectedProtocol: any; - _selectedPlatform: any; _selectedAction: any; module = UtmModulesEnum; From aa7290859398d3056d27144d2989b00f49344544 Mon Sep 17 00:00:00 2001 From: Osmany Montero Date: Wed, 16 Apr 2025 19:00:47 +0200 Subject: [PATCH 13/22] wip --- agent/config/const.go | 1 + agent/updates/dependencies.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/agent/config/const.go b/agent/config/const.go index 550e41197..6e41c4ae7 100644 --- a/agent/config/const.go +++ b/agent/config/const.go @@ -45,6 +45,7 @@ var ( // RedlineServName = "UTMStackRedline" // BatchToSend = 5 + DataTypeMacOSAgent DataType = "macos_agent" DataTypeWindowsAgent DataType = "beats_windows_agent" DataTypeSyslog DataType = "syslog" DataTypeVmware DataType = "vmware" diff --git a/agent/updates/dependencies.go b/agent/updates/dependencies.go index 56e59707e..9ec17c6a8 100644 --- a/agent/updates/dependencies.go +++ b/agent/updates/dependencies.go @@ -12,6 +12,10 @@ import ( ) func DownloadFirstDependencies(address string, authKey string, insecure bool) error { + if runtime.GOOS == "darwin" { + return nil + } + headers := map[string]string{"connection-key": authKey} if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, address, config.DependenciesPort, "version.json"), headers, "version.json", utils.GetMyPath(), insecure); err != nil { From ad97c50a8f6b063af794127e74cc201c0fb92b3c Mon Sep 17 00:00:00 2001 From: Yorjander Hernandez Vergara Date: Sat, 19 Apr 2025 01:15:24 -0400 Subject: [PATCH 14/22] complete macos agent --- agent/config/const.go | 1 - agent/updates/dependencies.go | 4 ---- 2 files changed, 5 deletions(-) diff --git a/agent/config/const.go b/agent/config/const.go index 6e41c4ae7..550e41197 100644 --- a/agent/config/const.go +++ b/agent/config/const.go @@ -45,7 +45,6 @@ var ( // RedlineServName = "UTMStackRedline" // BatchToSend = 5 - DataTypeMacOSAgent DataType = "macos_agent" DataTypeWindowsAgent DataType = "beats_windows_agent" DataTypeSyslog DataType = "syslog" DataTypeVmware DataType = "vmware" diff --git a/agent/updates/dependencies.go b/agent/updates/dependencies.go index 9ec17c6a8..56e59707e 100644 --- a/agent/updates/dependencies.go +++ b/agent/updates/dependencies.go @@ -12,10 +12,6 @@ import ( ) func DownloadFirstDependencies(address string, authKey string, insecure bool) error { - if runtime.GOOS == "darwin" { - return nil - } - headers := map[string]string{"connection-key": authKey} if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, address, config.DependenciesPort, "version.json"), headers, "version.json", utils.GetMyPath(), insecure); err != nil { From e189f5d6aa0a9d3681363cc948c23d8b1dd21879 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Mon, 21 Apr 2025 09:02:32 -0500 Subject: [PATCH 15/22] feat(module-integration): add SOC AI model selection field --- .../UtmModuleGroupConfiguration.java | 11 +++++ .../factory/impl/ModuleSocAi.java | 29 +++++++++++ .../types/ModuleConfigurationKey.java | 18 +++++++ ...418001_add_options_module_group_config.xml | 49 +++++++++++++++++++ .../resources/config/liquibase/master.xml | 3 ++ 5 files changed, 110 insertions(+) create mode 100644 backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml diff --git a/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java b/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java index 2e42bfaca..19d2b7f3c 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java @@ -46,6 +46,9 @@ public class UtmModuleGroupConfiguration implements Serializable { @Column(name = "conf_required", nullable = false) private Boolean confRequired; + @Column(name = "conf_options", nullable = false) + private String confOptions; + @JsonIgnore @ManyToOne @JoinColumn(name = "group_id", insertable = false, updatable = false) @@ -134,4 +137,12 @@ public UtmModuleGroup getModuleGroup() { public void setModuleGroup(UtmModuleGroup moduleGroup) { this.moduleGroup = moduleGroup; } + + public String getConfOptions() { + return confOptions; + } + + public void setConfOptions(String confOptions) { + this.confOptions = confOptions; + } } diff --git a/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java b/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java index ddb9d8d44..694a9349d 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java @@ -50,6 +50,7 @@ public List getConfigurationKeys(Long groupId) throws Ex .withConfDataType("password") .withConfRequired(true) .build()); + keys.add(ModuleConfigurationKey.builder() .withGroupId(groupId) .withConfKey("utmstack.socai.incidentCreation") @@ -58,6 +59,7 @@ public List getConfigurationKeys(Long groupId) throws Ex .withConfDataType("bool") .withConfRequired(false) .build()); + keys.add(ModuleConfigurationKey.builder() .withGroupId(groupId) .withConfKey("utmstack.socai.changeAlertStatus") @@ -68,6 +70,33 @@ public List getConfigurationKeys(Long groupId) throws Ex .withConfRequired(false) .build()); + keys.add(ModuleConfigurationKey.builder() + .withGroupId(groupId) + .withConfKey("utmstack.socai.model") + .withConfName("Select AI Model") + .withConfDescription("Choose the AI model that SOC AI will use to analyze alerts.") + .withConfDataType("select") + .withConfRequired(true) + .withConfOptions( + "[" + + "{\"value\": \"gpt-4\", \"label\": \"GPT-4\"}," + + "{\"value\": \"gpt-4-0613\", \"label\": \"GPT-4 (0613)\"}," + + "{\"value\": \"gpt-4-32k\", \"label\": \"GPT-4 32K\"}," + + "{\"value\": \"gpt-4-32k-0613\", \"label\": \"GPT-4 32K (0613)\"}," + + "{\"value\": \"gpt-4-turbo\", \"label\": \"GPT-4 Turbo\"}," + + "{\"value\": \"gpt-4o\", \"label\": \"GPT-4 Omni\"}," + + "{\"value\": \"gpt-4o-mini\", \"label\": \"GPT-4 Omni Mini\"}," + + "{\"value\": \"gpt-4.1\", \"label\": \"GPT-4.1\"}," + + "{\"value\": \"gpt-4.1-mini\", \"label\": \"GPT-4.1 Mini\"}," + + "{\"value\": \"gpt-4.1-nano\", \"label\": \"GPT-4.1 Nano\"}," + + "{\"value\": \"gpt-3.5-turbo\", \"label\": \"GPT-3.5 Turbo\"}," + + "{\"value\": \"gpt-3.5-turbo-0613\", \"label\": \"GPT-3.5 Turbo (0613)\"}," + + "{\"value\": \"gpt-3.5-turbo-16k\", \"label\": \"GPT-3.5 Turbo 16K\"}," + + "{\"value\": \"gpt-3.5-turbo-16k-0613\", \"label\": \"GPT-3.5 Turbo 16K (0613)\"}" + + "]" + ) + .build()); + return keys; } } diff --git a/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java b/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java index 765b67cf3..1e546f795 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java @@ -6,6 +6,8 @@ public class ModuleConfigurationKey { private String confName; private String confDescription; private String confDataType; + private String confOptions; + private Boolean confRequired; private ModuleConfigurationKey() { @@ -59,6 +61,14 @@ public void setConfRequired(Boolean confRequired) { this.confRequired = confRequired; } + public String getConfOptions() { + return confOptions; + } + + public void setConfOptions(String confOptions) { + this.confOptions = confOptions; + } + public static Builder builder() { return new Builder(); } @@ -71,6 +81,8 @@ public static class Builder { private String confDataType; private Boolean confRequired; + private String confOptions; + public Builder withGroupId(Long groupId) { this.groupId = groupId; return this; @@ -101,6 +113,11 @@ public Builder withConfRequired(Boolean confRequired) { return this; } + public Builder withConfOptions(String confOptions) { + this.confOptions = confOptions; + return this; + } + public ModuleConfigurationKey build() { ModuleConfigurationKey key = new ModuleConfigurationKey(); key.setGroupId(groupId); @@ -109,6 +126,7 @@ public ModuleConfigurationKey build() { key.setConfDescription(confDescription); key.setConfDataType(confDataType); key.setConfRequired(confRequired); + key.setConfOptions(confOptions); return key; } } diff --git a/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml b/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml new file mode 100644 index 000000000..7ff458451 --- /dev/null +++ b/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml @@ -0,0 +1,49 @@ + + + + + + ALTER TABLE utm_module_group_configuration ADD COLUMN conf_options TEXT; + INSERT INTO public.utm_module_group_configuration ( + id, + group_id, + conf_key, + conf_value, + conf_name, + conf_description, + conf_data_type, + conf_required, + conf_options + ) VALUES ( + default, + 1, + 'utmstack.socai.model', + 'gpt-4-turbo', + 'Select AI Model', + 'Choose the AI model that SOC AI will use to analyze alerts. Models differ in capability, performance, and cost.', + 'select', + true, + '[ + { "value": "gpt-4", "label": "GPT-4 (Default)" }, + { "value": "gpt-4-0613", "label": "GPT-4 (0613)" }, + { "value": "gpt-4-32k", "label": "GPT-4 32K" }, + { "value": "gpt-4-32k-0613", "label": "GPT-4 32K (0613)" }, + { "value": "gpt-4-turbo", "label": "GPT-4 Turbo" }, + { "value": "gpt-4o", "label": "GPT-4 Omni" }, + { "value": "gpt-4o-mini", "label": "GPT-4 Omni Mini" }, + { "value": "gpt-4.1", "label": "GPT-4.1" }, + { "value": "gpt-4.1-mini", "label": "GPT-4.1 Mini" }, + { "value": "gpt-4.1-nano", "label": "GPT-4.1 Nano" }, + { "value": "gpt-3.5-turbo", "label": "GPT-3.5 Turbo" }, + { "value": "gpt-3.5-turbo-0613", "label": "GPT-3.5 Turbo (0613)" }, + { "value": "gpt-3.5-turbo-16k", "label": "GPT-3.5 Turbo 16K" }, + { "value": "gpt-3.5-turbo-16k-0613", "label": "GPT-3.5 Turbo 16K (0613)" } + ]' + ); + + + + \ No newline at end of file diff --git a/backend/src/main/resources/config/liquibase/master.xml b/backend/src/main/resources/config/liquibase/master.xml index 1642ad210..e4508e78c 100644 --- a/backend/src/main/resources/config/liquibase/master.xml +++ b/backend/src/main/resources/config/liquibase/master.xml @@ -89,4 +89,7 @@ + + + From a479f61cf874d19ec0d17d0a29ed3e5dc6da59a7 Mon Sep 17 00:00:00 2001 From: Yorjander Hernandez Vergara Date: Tue, 22 Apr 2025 09:28:31 -0400 Subject: [PATCH 16/22] fix custom bad gateway page creation --- installer/utils/os.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/installer/utils/os.go b/installer/utils/os.go index d46d26f6d..272ab89fc 100644 --- a/installer/utils/os.go +++ b/installer/utils/os.go @@ -49,6 +49,11 @@ func CreatePathIfNotExist(path string) error { } func WriteToFile(fileName string, body string) error { + filePath := filepath.Dir(fileName) + if err := CreatePathIfNotExist(filePath); err != nil { + return fmt.Errorf("error creating directory for file %s: %v", fileName, err) + } + file, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.ModePerm) if err != nil { From 4e56cb04ccda0f6d4790387ae0699a0ed0534c60 Mon Sep 17 00:00:00 2001 From: Yorjander Hernandez Vergara Date: Tue, 22 Apr 2025 09:29:29 -0400 Subject: [PATCH 17/22] not contains operator grammar(Jose Angel) --- correlation/cache/operators.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/correlation/cache/operators.go b/correlation/cache/operators.go index c0322a14c..2c9e2634b 100644 --- a/correlation/cache/operators.go +++ b/correlation/cache/operators.go @@ -88,7 +88,7 @@ func compare(operator, val1, val2 string) bool { return !lowerEqual(val1, val2) case "contains": return contain(val1, val2) - case "not contain": + case "not contain", "not contains": return !contain(val1, val2) case "in": return in(val1, val2) From 98779e773828f44d424a6ce8fc3ff4f62d09466b Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Tue, 22 Apr 2025 09:39:30 -0500 Subject: [PATCH 18/22] fix(agent-details): show correct agent version and format OS version --- .../util/utm-agent-detail/utm-agent-detail.component.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/shared/components/utm/util/utm-agent-detail/utm-agent-detail.component.html b/frontend/src/app/shared/components/utm/util/utm-agent-detail/utm-agent-detail.component.html index 9fda0af24..8b4b26264 100644 --- a/frontend/src/app/shared/components/utm/util/utm-agent-detail/utm-agent-detail.component.html +++ b/frontend/src/app/shared/components/utm/util/utm-agent-detail/utm-agent-detail.component.html @@ -56,9 +56,13 @@

- OS Version:  + Agent Version:  {{agent.version}}
+
+ OS Version:  + {{agent.osMajorVersion + '.' + agent.osMinorVersion}} +
Last seen:  {{agent.lastSeen}} From 7387814a4c073ec28dff2f39b002f6806fb16b67 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Tue, 22 Apr 2025 11:32:51 -0500 Subject: [PATCH 19/22] feat(module-integration): add SOC AI model selection field --- ...418001_add_options_module_group_config.xml | 61 ++++++++++++++----- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml b/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml index 7ff458451..9df2d404a 100644 --- a/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml +++ b/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml @@ -16,8 +16,8 @@ conf_description, conf_data_type, conf_required, - conf_options - ) VALUES ( + conf_options) + VALUES ( default, 1, 'utmstack.socai.model', @@ -27,20 +27,49 @@ 'select', true, '[ - { "value": "gpt-4", "label": "GPT-4 (Default)" }, - { "value": "gpt-4-0613", "label": "GPT-4 (0613)" }, - { "value": "gpt-4-32k", "label": "GPT-4 32K" }, - { "value": "gpt-4-32k-0613", "label": "GPT-4 32K (0613)" }, - { "value": "gpt-4-turbo", "label": "GPT-4 Turbo" }, - { "value": "gpt-4o", "label": "GPT-4 Omni" }, - { "value": "gpt-4o-mini", "label": "GPT-4 Omni Mini" }, - { "value": "gpt-4.1", "label": "GPT-4.1" }, - { "value": "gpt-4.1-mini", "label": "GPT-4.1 Mini" }, - { "value": "gpt-4.1-nano", "label": "GPT-4.1 Nano" }, - { "value": "gpt-3.5-turbo", "label": "GPT-3.5 Turbo" }, - { "value": "gpt-3.5-turbo-0613", "label": "GPT-3.5 Turbo (0613)" }, - { "value": "gpt-3.5-turbo-16k", "label": "GPT-3.5 Turbo 16K" }, - { "value": "gpt-3.5-turbo-16k-0613", "label": "GPT-3.5 Turbo 16K (0613)" } + { "value": "o3-mini", "label": "O3 Mini Model" }, + { "value": "o3-mini-2025-01-31", "label": "O3 Mini Model (2025-01-31)" }, + { "value": "o1", "label": "O1 Model" }, + { "value": "o1-2024-12-17", "label": "O1 Model (2024-12-17)" }, + { "value": "o1-preview", "label": "O1 Preview Model" }, + { "value": "o1-preview-2024-09-12", "label": "O1 Preview Model (2024-09-12)" }, + { "value": "o1-mini", "label": "O1 Mini Model" }, + { "value": "o1-mini-2024-09-12", "label": "O1 Mini Model (2024-09-12)" }, + { "value": "gpt-4o", "label": "GPT-4 Omni Model" }, + { "value": "gpt-4o-2024-11-20", "label": "GPT-4 Omni Model (2024-11-20)" }, + { "value": "gpt-4o-2024-08-06", "label": "GPT-4 Omni Model (2024-08-06)" }, + { "value": "gpt-4o-2024-05-13", "label": "GPT-4 Omni Model (2024-05-13)" }, + { "value": "gpt-4o-audio-preview", "label": "GPT-4 Omni Audio Preview" }, + { "value": "gpt-4o-audio-preview-2024-10-01", "label": "GPT-4 Omni Audio Preview (2024-10-01)" }, + { "value": "gpt-4o-audio-preview-2024-12-17", "label": "GPT-4 Omni Audio Preview (2024-12-17)" }, + { "value": "gpt-4o-mini-audio-preview", "label": "GPT-4 Omni Mini Audio Preview" }, + { "value": "gpt-4o-mini-audio-preview-2024-12-17", "label": "GPT-4 Omni Mini Audio Preview (2024-12-17)" }, + { "value": "gpt-4o-search-preview", "label": "GPT-4 Omni Search Preview" }, + { "value": "gpt-4o-mini-search-preview", "label": "GPT-4 Omni Mini Search Preview" }, + { "value": "gpt-4o-search-preview-2025-03-11", "label": "GPT-4 Omni Search Preview (2025-03-11)" }, + { "value": "gpt-4o-mini-search-preview-2025-03-11", "label": "GPT-4 Omni Mini Search Preview (2025-03-11)" }, + { "value": "chatgpt-4o-latest", "label": "ChatGPT-4 Omni Latest Model" }, + { "value": "gpt-4o-mini", "label": "GPT-4 Omni Mini Model" }, + { "value": "gpt-4o-mini-2024-07-18", "label": "GPT-4 Omni Mini Model (2024-07-18)" }, + { "value": "gpt-4-turbo", "label": "GPT-4 Turbo Model" }, + { "value": "gpt-4-turbo-2024-04-09", "label": "GPT-4 Turbo Model (2024-04-09)" }, + { "value": "gpt-4-0125-preview", "label": "GPT-4 Model (0125 Preview)" }, + { "value": "gpt-4-turbo-preview", "label": "GPT-4 Turbo Model (Preview)" }, + { "value": "gpt-4-1106-preview", "label": "GPT-4 Model (1106 Preview)" }, + { "value": "gpt-4-vision-preview", "label": "GPT-4 Vision Model Preview" }, + { "value": "gpt-4", "label": "GPT-4 Model" }, + { "value": "gpt-4-0314", "label": "GPT-4 Model (0314)" }, + { "value": "gpt-4-0613", "label": "GPT-4 Model (0613)" }, + { "value": "gpt-4-32k", "label": "GPT-4 Model (32K)" }, + { "value": "gpt-4-32k-0314", "label": "GPT-4 Model (32K 0314)" }, + { "value": "gpt-4-32k-0613", "label": "GPT-4 Model (32K 0613)" }, + { "value": "gpt-3.5-turbo", "label": "GPT-3.5 Turbo Model" }, + { "value": "gpt-3.5-turbo-16k", "label": "GPT-3.5 Turbo Model (16K)" }, + { "value": "gpt-3.5-turbo-0301", "label": "GPT-3.5 Turbo Model (0301)" }, + { "value": "gpt-3.5-turbo-0613", "label": "GPT-3.5 Turbo Model (0613)" }, + { "value": "gpt-3.5-turbo-1106", "label": "GPT-3.5 Turbo Model (1106)" }, + { "value": "gpt-3.5-turbo-0125", "label": "GPT-3.5 Turbo Model (0125)" }, + { "value": "gpt-3.5-turbo-16k-0613", "label": "GPT-3.5 Turbo Model (16K 0613)" } ]' ); From 87a92822bb1de38ef2f8aa21cb5cbcd19f8d95f5 Mon Sep 17 00:00:00 2001 From: Yorjander Hernandez Vergara Date: Tue, 22 Apr 2025 14:07:08 -0400 Subject: [PATCH 20/22] Update and publish SOC-AI models --- .github/workflows/principal-multi-env.yml | 2 +- .github/workflows/used-runner.yml | 2 +- installer/types/compose.go | 2 +- soc-ai/Dockerfile | 13 ++ soc-ai/configurations/config.go | 71 ++++++++ soc-ai/configurations/const.go | 89 ++++++++++ soc-ai/elastic/alerts.go | 67 ++++++++ soc-ai/elastic/incidents.go | 102 +++++++++++ soc-ai/elastic/index.go | 98 +++++++++++ soc-ai/go.mod | 46 +++++ soc-ai/go.sum | 145 ++++++++++++++++ soc-ai/gpt/client.go | 81 +++++++++ soc-ai/gpt/counter.go | 44 +++++ soc-ai/main.go | 10 ++ soc-ai/processor/alertProcessor.go | 21 +++ soc-ai/processor/cleaner.go | 27 +++ soc-ai/processor/elastic.go | 73 ++++++++ soc-ai/processor/gpt.go | 57 +++++++ soc-ai/processor/listener.go | 98 +++++++++++ soc-ai/processor/processor.go | 61 +++++++ soc-ai/schema/convert.go | 161 ++++++++++++++++++ soc-ai/schema/models.go | 61 +++++++ soc-ai/schema/schema.go | 195 ++++++++++++++++++++++ soc-ai/utils/check.go | 41 +++++ soc-ai/utils/convert.go | 25 +++ soc-ai/utils/env.go | 21 +++ soc-ai/utils/files.go | 35 ++++ soc-ai/utils/logger.go | 31 ++++ soc-ai/utils/request.go | 63 +++++++ 29 files changed, 1739 insertions(+), 3 deletions(-) create mode 100644 soc-ai/Dockerfile create mode 100644 soc-ai/configurations/config.go create mode 100644 soc-ai/configurations/const.go create mode 100644 soc-ai/elastic/alerts.go create mode 100644 soc-ai/elastic/incidents.go create mode 100644 soc-ai/elastic/index.go create mode 100644 soc-ai/go.mod create mode 100644 soc-ai/go.sum create mode 100644 soc-ai/gpt/client.go create mode 100644 soc-ai/gpt/counter.go create mode 100644 soc-ai/main.go create mode 100644 soc-ai/processor/alertProcessor.go create mode 100644 soc-ai/processor/cleaner.go create mode 100644 soc-ai/processor/elastic.go create mode 100644 soc-ai/processor/gpt.go create mode 100644 soc-ai/processor/listener.go create mode 100644 soc-ai/processor/processor.go create mode 100644 soc-ai/schema/convert.go create mode 100644 soc-ai/schema/models.go create mode 100644 soc-ai/schema/schema.go create mode 100644 soc-ai/utils/check.go create mode 100644 soc-ai/utils/convert.go create mode 100644 soc-ai/utils/env.go create mode 100644 soc-ai/utils/files.go create mode 100644 soc-ai/utils/logger.go create mode 100644 soc-ai/utils/request.go diff --git a/.github/workflows/principal-multi-env.yml b/.github/workflows/principal-multi-env.yml index 3561be85a..0c7e934b3 100644 --- a/.github/workflows/principal-multi-env.yml +++ b/.github/workflows/principal-multi-env.yml @@ -88,7 +88,7 @@ jobs: strategy: fail-fast: false matrix: - service: ['aws', 'backend', 'correlation', 'frontend', 'bitdefender', 'mutate', 'office365', 'log-auth-proxy', 'sophos', 'user-auditor', 'web-pdf'] + service: ['aws', 'backend', 'correlation', 'frontend', 'bitdefender', 'mutate', 'office365', 'log-auth-proxy', 'soc-ai', 'sophos', 'user-auditor', 'web-pdf'] uses: ./.github/workflows/used-runner.yml with: microservice: ${{ matrix.service }} diff --git a/.github/workflows/used-runner.yml b/.github/workflows/used-runner.yml index cea5e73f8..18d6e730c 100644 --- a/.github/workflows/used-runner.yml +++ b/.github/workflows/used-runner.yml @@ -22,7 +22,7 @@ jobs: id: get_tech run: | folder_changed="${{inputs.microservice}}" - if [[ "$folder_changed" == "aws" || "$folder_changed" == "correlation" || "$folder_changed" == "bitdefender" || "$folder_changed" == "office365" || "$folder_changed" == "sophos" || "$folder_changed" == "log-auth-proxy" ]]; then + if [[ "$folder_changed" == "aws" || "$folder_changed" == "correlation" || "$folder_changed" == "bitdefender" || "$folder_changed" == "office365" || "$folder_changed" == "soc-ai" || "$folder_changed" == "sophos" || "$folder_changed" == "log-auth-proxy" ]]; then tech="golang" elif [[ "$folder_changed" == "backend" ]]; then tech="java-11" diff --git a/installer/types/compose.go b/installer/types/compose.go index 5dbd6a0a3..28b1d33c0 100644 --- a/installer/types/compose.go +++ b/installer/types/compose.go @@ -507,7 +507,7 @@ func (c *Compose) Populate(conf *Config, stack *StackConfig) *Compose { socAIMem := stack.ServiceResources["socai"].AssignedMemory c.Services["socai"] = Service{ - Image: utils.Str("ghcr.io/utmstack/soc-ai/soc-ai:" + conf.Branch), + Image: utils.Str("ghcr.io/utmstack/utmstack/soc-ai:" + conf.Branch), DependsOn: []string{ "node1", "backend", diff --git a/soc-ai/Dockerfile b/soc-ai/Dockerfile new file mode 100644 index 000000000..bb8839947 --- /dev/null +++ b/soc-ai/Dockerfile @@ -0,0 +1,13 @@ +FROM ubuntu:24.04 + +COPY soc-ai /app/ + +RUN apt-get update && \ + apt-get install -y ca-certificates jq wget && \ + update-ca-certificates && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +EXPOSE 8080 + +CMD ["/app/soc-ai"] \ No newline at end of file diff --git a/soc-ai/configurations/config.go b/soc-ai/configurations/config.go new file mode 100644 index 000000000..cbb66b05f --- /dev/null +++ b/soc-ai/configurations/config.go @@ -0,0 +1,71 @@ +package configurations + +import ( + "time" + + UTMStackConfigurationClient "github.com/utmstack/config-client-go" + "github.com/utmstack/config-client-go/enum" + "github.com/utmstack/soc-ai/utils" +) + +var ( + gptConfig GPTConfig +) + +type GPTConfig struct { + APIKey string + ChangeAlertStatus bool + AutomaticIncidentCreation bool + Model string + ModuleActive bool +} + +func GetGPTConfig() *GPTConfig { + return &gptConfig +} + +func UpdateGPTConfigurations() { + intKey := GetInternalKey() + panelServ := GetPanelServiceName() + client := UTMStackConfigurationClient.NewUTMClient(intKey, panelServ) + + for { + if err := utils.ConnectionChecker(GPT_API_ENDPOINT); err != nil { + utils.Logger.ErrorF("Failed to establish internet connection: %v", err) + } + + tempModuleConfig, err := client.GetUTMConfig(enum.SOCAI) + if err != nil && err.Error() != "" && err.Error() != " " { + utils.Logger.LogF(100, "Error while getting GPT configuration: %v", err) + time.Sleep(TIME_FOR_GET_CONFIG * time.Second) + continue + } + + gptConfig.ModuleActive = tempModuleConfig.ModuleActive + + if gptConfig.ModuleActive && tempModuleConfig != nil && len(tempModuleConfig.ConfigurationGroups) > 0 { + for _, config := range tempModuleConfig.ConfigurationGroups[0].Configurations { + switch config.ConfKey { + case "utmstack.socai.key": + if config.ConfValue != "" && config.ConfValue != " " { + gptConfig.APIKey = config.ConfValue + } + case "utmstack.socai.incidentCreation": + if config.ConfValue != "" && config.ConfValue != " " { + gptConfig.AutomaticIncidentCreation = config.ConfValue == "true" + } + case "utmstack.socai.changeAlertStatus": + if config.ConfValue != "" && config.ConfValue != " " { + gptConfig.ChangeAlertStatus = config.ConfValue == "true" + } + case "utmstack.socai.model": + if config.ConfValue != "" && config.ConfValue != " " { + gptConfig.Model = config.ConfValue + } + } + } + } + + time.Sleep(TIME_FOR_GET_CONFIG * time.Second) + } +} diff --git a/soc-ai/configurations/const.go b/soc-ai/configurations/const.go new file mode 100644 index 000000000..842acfec3 --- /dev/null +++ b/soc-ai/configurations/const.go @@ -0,0 +1,89 @@ +package configurations + +import ( + "path/filepath" + + "github.com/utmstack/soc-ai/utils" +) + +const ( + SOC_AI_SERVER_PORT = "8080" + SOC_AI_SERVER_ENDPOINT = "/process" + API_ALERT_ENDPOINT = "/api/elasticsearch/search" + API_ALERT_STATUS_ENDPOINT = "/api/utm-alerts/status" + API_INCIDENT_ENDPOINT = "/api/utm-incidents" + API_INCIDENT_ADD_NEW_ALERT_ENDPOINT = "/api/utm-incidents/add-alerts" + API_ALERT_COMPLETED_STATUS_CODE = 5 + API_ALERT_INFO_PARAMS = "?page=0&size=25&top=10000&indexPattern=" + ELASTIC_DOC_ENDPOINT = "/_doc/" + ELASTIC_UPDATE_BY_QUERY_ENDPOINT = "/_update_by_query" + ALERT_INDEX_PATTERN = "alert-*" + LOGS_INDEX_PATTERN = "log-*" + SOC_AI_INDEX = "soc-ai" + GPT_API_ENDPOINT = "https://api.openai.com/v1/chat/completions" + TIME_FOR_GET_CONFIG = 10 + CLEANER_DELAY = 10 + MAX_ATTEMPS_TO_GPT = 3 + GPT_RESPONSE_TOKENS = 10 + HTTP_GPT_TIMEOUT = 90 + HTTP_TIMEOUT = 30 + LOGS_SEPARATOR = "[utm-logs-separator]" +) + +var ( + AllowedGPTModels = map[string]int{ + "gpt-4.1": 1047576, + "gpt-4.1-mini": 1047576, + "gpt-4.1-nano": 1047576, + "gpt-4o": 128000, + "gpt-4o-mini": 128000, + "gpt-4-turbo": 128000, + "gpt-4-0614": 8192, + "gpt-4-0125-preview": 128000, + "gpt-3.5-turbo": 16385, + "gpt-3.5-turbo-instruct": 4096, + "gpt-3.5-turbo-1106": 16385, + "o1": 200000, + "o1-pro": 200000, + "o3": 200000, + "o3-mini": 200000, + "o4-mini": 200000, + // "gpt-4-0314": 8192, // Removed 2024-06-13 + // "gpt-4-1106-preview": 128000, // Removed 2024-12-06 + } +) + +type SensitivePattern struct { + Regexp string + FakeValue string +} + +var ( + SensitivePatterns = map[string]SensitivePattern{ + "email": {Regexp: `([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})`, FakeValue: "jhondoe@gmail.com"}, + //"ipv4": `(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)`, + } + GPT_INSTRUCTION = "You are an expert security engineer. Perform a deep analysis of an alert created by a SIEM and the logs related to it. Determine if the alert could be an actual potential threat or not and explain why. Provide a description that shows a deep understanding of the alert based on a deep analysis of its logs and estimate the risk to the systems affected. Classify the alert in the following manner: if the alert information is sufficient to determine that the security, availability, confidentiality, or integrity of the systems has being compromised, then classify it as \"possible incident\". If the alert does not pose a security risk to the organization or has no security relevance, classify it as \"possible false positive\". If the alert does not pose an imminent risk to the systems, requires no urgent action from an administrator, or requires not urgent review by an administrator, it should be classified as a \"standard alert\". You will also provide context-specific instructions for remediation, mitigation, or further investigation, related to the alert and logs analyzed. Your answer should be provided using the following JSON format and the total number of characters in your answer must not exceed 1500 words. Your entire answer must be inside this json format. {\"activity_id\":\"\",\"classification\":\"\",\"reasoning\":[\"\"],\"nextSteps\":[{\"step\":1,\"action\":\"\",\"details\":\"\"},{\"step\":2,\"action\":\"\",\"details\":\"\"},{\"step\":3,\"action\":\"\"]}Ensure that your entire answer adheres to the provided JSON format. The response should be valid JSON syntax and schema." + GPT_FALSE_POSITIVE = "This alert is categorized as a potential false positive due to two key factors. Firstly, it originates from an automated system, which may occasionally produce alerts without direct human validation. Additionally, the absence of any correlated logs further raises suspicion, as a genuine incident typically leaves a trail of relevant log entries. Hence, the combination of its system-generated nature and the lack of associated logs suggests a likelihood of being a false positive rather than a genuine security incident." +) + +func GetInternalKey() string { + return utils.Getenv("INTERNAL_KEY", true) +} + +func GetPanelServiceName() string { + return utils.Getenv("PANEL_SERV_NAME", true) +} + +func GetOpenSearchHost() string { + return "http://" + utils.Getenv("OPENSEARCH_HOST", true) +} + +func GetOpenSearchPort() string { + return utils.Getenv("OPENSEARCH_PORT", true) +} + +func GetAlertsDBPath() string { + path, _ := utils.GetMyPath() + return filepath.Join(path, "database", "alerts.sqlite3") +} diff --git a/soc-ai/elastic/alerts.go b/soc-ai/elastic/alerts.go new file mode 100644 index 000000000..678608e8a --- /dev/null +++ b/soc-ai/elastic/alerts.go @@ -0,0 +1,67 @@ +package elastic + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/utmstack/soc-ai/configurations" + "github.com/utmstack/soc-ai/schema" + "github.com/utmstack/soc-ai/utils" +) + +func GetAlertsInfo(id string) (schema.Alert, error) { + result, err := ElasticSearch(configurations.ALERT_INDEX_PATTERN, "id", id) + if err != nil { + return schema.Alert{}, fmt.Errorf("error while getting alert %s info: %v", id, err) + } + + var alertDetails schema.AlertDetails + err = json.Unmarshal(result, &alertDetails) + if err != nil { + return schema.Alert{}, fmt.Errorf("error decoding response: %v", err) + } + + if len(alertDetails) == 0 { + return schema.Alert{}, fmt.Errorf("no alert found for id") + } + + if len(alertDetails[0].Logs) > 0 { + var logs []string + if len(alertDetails[0].Logs) > 3 { + logs = alertDetails[0].Logs[:3] + } else { + logs = alertDetails[0].Logs + } + + for i, log := range logs { + resp, err := ElasticSearch(configurations.LOGS_INDEX_PATTERN, "id", log) + if err != nil { + continue + } + alertDetails[0].Logs[i] = string(resp) + } + } + return alertDetails[0], nil +} + +func ChangeAlertStatus(id string, status int, observations string) error { + url := configurations.GetPanelServiceName() + configurations.API_ALERT_STATUS_ENDPOINT + headers := map[string]string{ + "Content-Type": "application/json", + "Utm-Internal-Key": configurations.GetInternalKey(), + } + + body := schema.ChangeAlertStatus{AlertIDs: []string{id}, Status: status, StatusObservation: observations} + bodyBytes, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("error marshalling body: %v", err) + } + + resp, statusCode, err := utils.DoReq(url, bodyBytes, "POST", headers, configurations.HTTP_TIMEOUT) + if err != nil || statusCode != http.StatusOK { + return fmt.Errorf("error while doing request: %v, status: %d, response: %v", err, statusCode, string(resp)) + } + + return nil +} diff --git a/soc-ai/elastic/incidents.go b/soc-ai/elastic/incidents.go new file mode 100644 index 000000000..f385b024a --- /dev/null +++ b/soc-ai/elastic/incidents.go @@ -0,0 +1,102 @@ +package elastic + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/utmstack/soc-ai/configurations" + "github.com/utmstack/soc-ai/schema" + "github.com/utmstack/soc-ai/utils" +) + +func CreateNewIncident(alertDetails schema.AlertGPTDetails) error { + url := configurations.GetPanelServiceName() + configurations.API_INCIDENT_ENDPOINT + headers := map[string]string{ + "Content-Type": "application/json", + "Utm-Internal-Key": configurations.GetInternalKey(), + } + + t := time.Now() + body := schema.CreateNewIncidentRequest{ + IncidentName: fmt.Sprintf("INC-%s%s Incident in %s", t.Format("2006010215"), t.Format("04"), alertDetails.DataSource), + IncidentDescription: fmt.Sprintf("AI GENERATED ANALYSIS: Multiple related alerts were detected and grouped in the %s datasource during the last 24 hours. Artificial intelligence classified this grouping as a possible incident", alertDetails.DataSource), + IncidentAssignedTo: "None", + AlertList: schema.AlertList{{ + AlertID: alertDetails.AlertID, + AlertName: alertDetails.Name, + AlertStatus: alertDetails.Status, + AlertSeverity: alertDetails.Severity, + }}, + } + + bodyBytes, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("error marshalling body: %v", err) + } + + resp, statusCode, err := utils.DoReq(url, bodyBytes, "POST", headers, configurations.HTTP_TIMEOUT) + if err != nil || statusCode != http.StatusOK { + return fmt.Errorf("error while doing request: %v, status: %d, response: %v", err, statusCode, string(resp)) + } + + return nil +} + +func AddAlertToIncident(incidentId int, alertDetails schema.AlertGPTDetails) error { + url := configurations.GetPanelServiceName() + configurations.API_INCIDENT_ADD_NEW_ALERT_ENDPOINT + headers := map[string]string{ + "Content-Type": "application/json", + "Utm-Internal-Key": configurations.GetInternalKey(), + } + + body := schema.AddNewAlertToIncidentRequest{ + IncidenId: incidentId, + AlertList: schema.AlertList{{ + AlertID: alertDetails.AlertID, + AlertName: alertDetails.Name, + AlertStatus: alertDetails.Status, + AlertSeverity: alertDetails.Severity, + }}, + } + + bodyBytes, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("error marshalling body: %v", err) + } + + resp, statusCode, err := utils.DoReq(url, bodyBytes, "POST", headers, configurations.HTTP_TIMEOUT) + if err != nil || (statusCode != http.StatusOK && statusCode != http.StatusCreated) { + return fmt.Errorf("error while doing request: %v, status: %d, response: %v", err, statusCode, string(resp)) + } + + return nil +} + +func GetIncidentsByPattern(pattern string) ([]schema.IncidentResp, error) { + pattern = strings.ReplaceAll(pattern, " ", "%20") + tnow := time.Now().UTC() + t24hAfter := tnow.Add(24 * time.Hour) + t24hBefore := tnow.Add(-24 * time.Hour) + + url := configurations.GetPanelServiceName() + configurations.API_INCIDENT_ENDPOINT + "?incidentName.contains=" + pattern + "&incidentCreatedDate.greaterThanOrEqual=" + t24hBefore.Format(time.RFC3339) + "&incidentCreatedDate.lessThanOrEqual=" + t24hAfter.Format(time.RFC3339) + "&incidentStatus.in=IN_REVIEW,OPEN&page=0&size=100" + headers := map[string]string{ + "Content-Type": "application/json", + "Utm-Internal-Key": configurations.GetInternalKey(), + } + + resp, statusCode, err := utils.DoReq(url, nil, "GET", headers, configurations.HTTP_TIMEOUT) + if err != nil || statusCode != http.StatusOK { + return nil, fmt.Errorf("error while doing request: %v, status: %d, response: %v", err, statusCode, string(resp)) + } + + var incidents []schema.IncidentResp + err = json.Unmarshal(resp, &incidents) + if err != nil { + return nil, fmt.Errorf("error while unmarshalling response: %v", err) + } + + return incidents, nil +} diff --git a/soc-ai/elastic/index.go b/soc-ai/elastic/index.go new file mode 100644 index 000000000..d1cee0c98 --- /dev/null +++ b/soc-ai/elastic/index.go @@ -0,0 +1,98 @@ +package elastic + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/utmstack/soc-ai/configurations" + "github.com/utmstack/soc-ai/schema" + "github.com/utmstack/soc-ai/utils" +) + +func ElasticQuery(index string, query interface{}, op string) error { + var endp string + switch op { + case "create": + endp = configurations.ELASTIC_DOC_ENDPOINT + case "update": + endp = configurations.ELASTIC_UPDATE_BY_QUERY_ENDPOINT + } + url := configurations.GetOpenSearchHost() + ":" + configurations.GetOpenSearchPort() + "/" + index + endp + headers := map[string]string{ + "Content-Type": "application/json", + } + + queryBytes, err := json.Marshal(query) + if err != nil { + return fmt.Errorf("error marshalling query: %v", err) + } + + resp, statusCode, err := utils.DoReq(url, queryBytes, "POST", headers, configurations.HTTP_TIMEOUT) + if err != nil || (statusCode != http.StatusOK && statusCode != http.StatusCreated) { + return fmt.Errorf("error while doing request: %v, status: %d, response: %v", err, statusCode, string(resp)) + } + + return nil +} + +func ElasticSearch(index, field, value string) ([]byte, error) { + url := configurations.GetPanelServiceName() + configurations.API_ALERT_ENDPOINT + configurations.API_ALERT_INFO_PARAMS + index + headers := map[string]string{ + "Content-Type": "application/json", + "Utm-Internal-Key": configurations.GetInternalKey(), + } + + body := schema.SearchDetailsRequest{{Field: field, Operator: "IS", Value: value}} + bodyBytes, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("error marshalling body: %v", err) + } + + resp, statusCode, err := utils.DoReq(url, bodyBytes, "POST", headers, configurations.HTTP_TIMEOUT) + if err != nil || statusCode != http.StatusOK { + return nil, fmt.Errorf("error while doing request for get Alert Details: %v: %s", err, string(resp)) + } + + return resp, nil +} + +func IndexStatus(id, status, op string) error { + doc := schema.GPTAlertResponse{ + Timestamp: time.Now().UTC().Format("2006-01-02T15:04:05.999999Z07:00"), + Status: status, + ActivityID: id, + } + + if op == "update" { + query, err := schema.ConvertGPTResponseToUpdateQuery(doc) + if err != nil { + return fmt.Errorf("error while converting response to update query: %v", err) + } + return ElasticQuery(configurations.SOC_AI_INDEX, query, op) + } else { + return ElasticQuery(configurations.SOC_AI_INDEX, doc, op) + } +} + +func CreateIndexIfNotExist(index string) error { + url := configurations.GetOpenSearchHost() + ":" + configurations.GetOpenSearchPort() + "/" + index + headers := map[string]string{ + "Content-Type": "application/json", + } + + resp, statusCode, err := utils.DoReq(url, nil, "HEAD", headers, configurations.HTTP_TIMEOUT) + if err != nil { + return fmt.Errorf("error while doing request: %v, status: %d, response: %v", err, statusCode, string(resp)) + } + + if statusCode == 404 { + resp, statusCode, err = utils.DoReq(url, nil, "PUT", headers, configurations.HTTP_TIMEOUT) + if err != nil || (statusCode != http.StatusOK && statusCode != http.StatusCreated) { + return fmt.Errorf("error while doing request: %v, status: %d, response: %v", err, statusCode, string(resp)) + } + } + + return nil +} diff --git a/soc-ai/go.mod b/soc-ai/go.mod new file mode 100644 index 000000000..f5af25a6d --- /dev/null +++ b/soc-ai/go.mod @@ -0,0 +1,46 @@ +module github.com/utmstack/soc-ai + +go 1.22.4 + +toolchain go1.23.4 + +require ( + github.com/gin-contrib/gzip v0.0.6 + github.com/gin-gonic/gin v1.10.0 + github.com/pkoukk/tiktoken-go v0.1.6 + github.com/threatwinds/logger v1.2.1 + github.com/utmstack/config-client-go v1.2.4 +) + +require ( + github.com/bytedance/sonic v1.12.1 // indirect + github.com/bytedance/sonic/loader v0.2.0 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/dlclark/regexp2 v1.10.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.5 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.22.0 // indirect + github.com/goccy/go-json v0.10.3 // indirect + github.com/google/go-cmp v0.5.8 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + golang.org/x/arch v0.9.0 // indirect + golang.org/x/crypto v0.32.0 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/soc-ai/go.sum b/soc-ai/go.sum new file mode 100644 index 000000000..64651d345 --- /dev/null +++ b/soc-ai/go.sum @@ -0,0 +1,145 @@ +github.com/bytedance/sonic v1.12.1 h1:jWl5Qz1fy7X1ioY74WqO0KjAMtAGQs4sYnjiEBiyX24= +github.com/bytedance/sonic v1.12.1/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= +github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= +github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= +github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= +github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= +github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= +github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4Bx7ia+JlgcnOao= +github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw= +github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/threatwinds/logger v1.2.1 h1:uN7efZaHobMX3DRi6GOPtxESPxt5xj0bNflnmgklwII= +github.com/threatwinds/logger v1.2.1/go.mod h1:eevkhjP9wSpRekRIgi4ZAq7DMdlUMy+Shwx/QNDvOHg= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/utmstack/config-client-go v1.2.4 h1:q/if6a/qGMkUj53MGHmF6ZucixfoqHuM5CG2pHNpWcY= +github.com/utmstack/config-client-go v1.2.4/go.mod h1:5B1phBmxHZmnggujCsNVAHK2oFXIy89mLCc9ixrDcCo= +golang.org/x/arch v0.9.0 h1:ub9TgUInamJ8mrZIGlBG6/4TqWeMszd4N8lNorbrr6k= +golang.org/x/arch v0.9.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/soc-ai/gpt/client.go b/soc-ai/gpt/client.go new file mode 100644 index 000000000..edff874ee --- /dev/null +++ b/soc-ai/gpt/client.go @@ -0,0 +1,81 @@ +package gpt + +import ( + "encoding/json" + "fmt" + "sync" + + "github.com/utmstack/soc-ai/configurations" + "github.com/utmstack/soc-ai/schema" + "github.com/utmstack/soc-ai/utils" +) + +type GPTClient struct{} + +var ( + client *GPTClient + clientOnce sync.Once +) + +func GetGPTClient() *GPTClient { + clientOnce.Do(func() { + client = &GPTClient{} + }) + return client +} + +func (c *GPTClient) Request(alert schema.AlertGPTDetails) (string, error) { + content := configurations.GPT_INSTRUCTION + if alert.Logs == "" || alert.Logs == " " { + content += content + ". " + configurations.GPT_FALSE_POSITIVE + } + jsonContent, err := json.Marshal(alert) + if err != nil { + return "", fmt.Errorf("error marshalling alert: %v", err) + } + + req := schema.GPTRequest{ + Model: configurations.GetGPTConfig().Model, + Messages: []schema.GPTMessage{ + { + Role: "system", + Content: content, + }, + { + Role: "user", + Content: string(jsonContent), + }, + }, + } + + requestJson, error := json.Marshal(req) + if error != nil { + return "", fmt.Errorf("error marshalling request: %v", error) + } + + headers := map[string]string{ + "Authorization": "Bearer " + configurations.GetGPTConfig().APIKey, + "Content-Type": "application/json", + } + + response, status, err := utils.DoParseReq[schema.GPTResponse](configurations.GPT_API_ENDPOINT, requestJson, "POST", headers, configurations.HTTP_GPT_TIMEOUT) + if err != nil { + if status == 401 { + return "", fmt.Errorf("invalid api-key") + } + return "", fmt.Errorf("error making request to GPT: %v", err) + } + + return response.Choices[0].Message.Content, nil +} + +func (c *GPTClient) ProcessResponse(response string) (schema.GPTAlertResponse, error) { + alertRespProcessed := schema.GPTAlertResponse{} + + err := json.Unmarshal([]byte(response), &alertRespProcessed) + if err != nil { + return schema.GPTAlertResponse{}, fmt.Errorf("error while unmarshalling response: %v", err) + } + + return alertRespProcessed, nil +} diff --git a/soc-ai/gpt/counter.go b/soc-ai/gpt/counter.go new file mode 100644 index 000000000..606c72c44 --- /dev/null +++ b/soc-ai/gpt/counter.go @@ -0,0 +1,44 @@ +package gpt + +import ( + "fmt" + "strings" + + "github.com/pkoukk/tiktoken-go" +) + +func NumTokensFromMessage(message string, model string) (numTokens int, err error) { + tkm, err := tiktoken.EncodingForModel(model) + if err != nil { + return -1, fmt.Errorf("encoding for model: %v", err) + } + + var tokensPerMessage, tokensPerName int + switch model { + case "gpt-3.5-turbo-0613", + "gpt-3.5-turbo-16k-0613", + "gpt-4-0314", + "gpt-4-32k-0314", + "gpt-4-0613", + "gpt-4-32k-0613": + tokensPerMessage = 3 + tokensPerName = 1 + case "gpt-3.5-turbo-0301": + tokensPerMessage = 4 + tokensPerName = -1 + default: + if strings.Contains(model, "gpt-3.5-turbo") { + return NumTokensFromMessage(message, "gpt-3.5-turbo-0613") + } else if strings.Contains(model, "gpt-4") { + return NumTokensFromMessage(message, "gpt-4-0613") + } else { + return -1, fmt.Errorf("num_tokens_from_messages() is not implemented for model %s. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens", model) + } + } + + numTokens += tokensPerMessage + numTokens += len(tkm.Encode(message, nil, nil)) + numTokens += tokensPerName + + return numTokens, nil +} diff --git a/soc-ai/main.go b/soc-ai/main.go new file mode 100644 index 000000000..66e51900e --- /dev/null +++ b/soc-ai/main.go @@ -0,0 +1,10 @@ +package main + +import ( + "github.com/utmstack/soc-ai/processor" +) + +func main() { + processor := processor.NewProcessor() + processor.ProcessData() +} diff --git a/soc-ai/processor/alertProcessor.go b/soc-ai/processor/alertProcessor.go new file mode 100644 index 000000000..2a4146c0d --- /dev/null +++ b/soc-ai/processor/alertProcessor.go @@ -0,0 +1,21 @@ +package processor + +import ( + "fmt" + + "github.com/utmstack/soc-ai/elastic" + "github.com/utmstack/soc-ai/schema" +) + +func (p *Processor) processAlertsInfo() { + for alert := range p.AlertInfoQueue { + alertInfo, err := elastic.GetAlertsInfo(alert.AlertID) + if err != nil { + p.RegisterError(fmt.Sprintf("error while getting alert %s info: %v", alert.AlertID, err), alert.AlertID) + continue + } + + details := schema.ConvertFromAlertToAlertDB(alertInfo) + p.GPTQueue <- cleanAlerts(&details) + } +} diff --git a/soc-ai/processor/cleaner.go b/soc-ai/processor/cleaner.go new file mode 100644 index 000000000..6dbd89f2d --- /dev/null +++ b/soc-ai/processor/cleaner.go @@ -0,0 +1,27 @@ +package processor + +import ( + "reflect" + "regexp" + + "github.com/utmstack/soc-ai/configurations" + "github.com/utmstack/soc-ai/schema" +) + +func cleanAlerts(alertDetails *schema.AlertGPTDetails) schema.AlertGPTDetails { + v := reflect.ValueOf(alertDetails).Elem() + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + if field.Kind() == reflect.String { + str := field.String() + for _, pattern := range configurations.SensitivePatterns { + re := regexp.MustCompile(pattern.Regexp) + if re.MatchString(str) { + str = re.ReplaceAllString(str, pattern.FakeValue) + } + } + field.SetString(str) + } + } + return *alertDetails +} diff --git a/soc-ai/processor/elastic.go b/soc-ai/processor/elastic.go new file mode 100644 index 000000000..1b19974b4 --- /dev/null +++ b/soc-ai/processor/elastic.go @@ -0,0 +1,73 @@ +package processor + +import ( + "fmt" + "strings" + + "github.com/utmstack/soc-ai/configurations" + "github.com/utmstack/soc-ai/elastic" + "github.com/utmstack/soc-ai/schema" + "github.com/utmstack/soc-ai/utils" +) + +func (p *Processor) processAlertToElastic() { + for alert := range p.ElasticQueue { + gptConfig := configurations.GetGPTConfig() + + resp := schema.ConvertFromAlertDBToGPTResponse(alert) + resp.Status = "Completed" + query, err := schema.ConvertGPTResponseToUpdateQuery(resp) + if err != nil { + p.RegisterError(fmt.Sprintf("error converting gpt response to update query: %v", err), alert.AlertID) + continue + } + err = elastic.ElasticQuery(configurations.SOC_AI_INDEX, query, "update") + if err != nil { + p.RegisterError(fmt.Sprintf("error indexing gpt response in elastic: %v", err), alert.AlertID) + continue + } + + if gptConfig.ChangeAlertStatus && alert.GPTClassification == "possible false positive" { + err = elastic.ChangeAlertStatus(alert.AlertID, configurations.API_ALERT_COMPLETED_STATUS_CODE, alert.GPTClassification+" - "+alert.GPTReasoning) + if err != nil { + utils.Logger.ErrorF("error while changing alert status in elastic: %v", err) + continue + } + utils.Logger.Info("alert %s status changed to COMPLETED in Panel", alert.AlertID) + } + + if gptConfig.AutomaticIncidentCreation && alert.GPTClassification == "possible incident" { + incidentsDetails, err := elastic.GetIncidentsByPattern("Incident in " + alert.DataSource) + if err != nil { + utils.Logger.ErrorF("error while getting incidents by pattern: %v", err) + continue + } + + incidentExists := false + if len(incidentsDetails) != 0 { + for _, incident := range incidentsDetails { + if strings.HasSuffix(incident.IncidentName, "Incident in "+alert.DataSource) { + incidentExists = true + err = elastic.AddAlertToIncident(incident.ID, alert) + if err != nil { + utils.Logger.ErrorF("error while adding alert to incident: %v", err) + continue + } + } + } + } + + if !incidentExists { + err = elastic.CreateNewIncident(alert) + if err != nil { + utils.Logger.ErrorF("error while creating incident: %v", err) + continue + } + } + utils.Logger.Info("alert %s added to incident in Panel", alert.AlertID) + } + + utils.Logger.Info("alert %s processed correctly", alert.AlertID) + + } +} diff --git a/soc-ai/processor/gpt.go b/soc-ai/processor/gpt.go new file mode 100644 index 000000000..298095cd2 --- /dev/null +++ b/soc-ai/processor/gpt.go @@ -0,0 +1,57 @@ +package processor + +import ( + "fmt" + "strings" + "time" + + "github.com/utmstack/soc-ai/configurations" + "github.com/utmstack/soc-ai/gpt" + "github.com/utmstack/soc-ai/schema" + "github.com/utmstack/soc-ai/utils" +) + +func (p *Processor) processGPTRequests() { + for alert := range p.GPTQueue { + response, err := gpt.GetGPTClient().Request(alert) + if err != nil { + p.RegisterError(fmt.Sprintf("error while making request to GPT: %v", err), alert.AlertID) + continue + } + + response, err = clearJson(response) + if err != nil { + p.RegisterError(fmt.Sprintf("error while cleaning json response: %v", err), alert.AlertID) + continue + } + + alertResponse, err := utils.ConvertFromJsonToStruct[schema.GPTAlertResponse](response) + if err != nil { + p.RegisterError(fmt.Sprintf("error while converting response to struct: %v", err), alert.AlertID) + continue + } + + nextSteps := []string{} + for _, step := range alertResponse.NextSteps { + nextSteps = append(nextSteps, fmt.Sprintf("%s:: %s", step.Action, step.Details)) + } + + alert.GPTTimestamp = time.Now().UTC().Format("2006-01-02T15:04:05.999999Z07:00") + alert.GPTClassification = alertResponse.Classification + alert.GPTReasoning = strings.Join(alertResponse.Reasoning, configurations.LOGS_SEPARATOR) + alert.GPTNextSteps = strings.Join(nextSteps, "\n") + + p.ElasticQueue <- alert + } +} + +func clearJson(s string) (string, error) { + start := strings.Index(s, "{") + end := strings.LastIndex(s, "}") + + if start == -1 || end == -1 { + return "", fmt.Errorf("no valid json found in gpt response") + } + + return s[start : end+1], nil +} diff --git a/soc-ai/processor/listener.go b/soc-ai/processor/listener.go new file mode 100644 index 000000000..9ccf244c5 --- /dev/null +++ b/soc-ai/processor/listener.go @@ -0,0 +1,98 @@ +package processor + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/gin-contrib/gzip" + "github.com/gin-gonic/gin" + "github.com/utmstack/soc-ai/configurations" + "github.com/utmstack/soc-ai/elastic" + "github.com/utmstack/soc-ai/schema" + "github.com/utmstack/soc-ai/utils" +) + +func (p *Processor) restRouter() { + gin.SetMode(gin.ReleaseMode) + r := gin.New() + r.Use( + gin.Recovery(), + gzip.Gzip(gzip.DefaultCompression), + ) + + r.NoRoute(notFound) + + r.POST("/process", p.handleAlerts) + + r.Run(":" + configurations.SOC_AI_SERVER_PORT) +} + +func notFound(c *gin.Context) { + c.JSON(http.StatusBadRequest, "url not found") +} + +func (p *Processor) handleAlerts(c *gin.Context) { + err := elastic.CreateIndexIfNotExist(configurations.SOC_AI_INDEX) + if err != nil { + utils.Logger.ErrorF("error creating index %s: %v", configurations.SOC_AI_INDEX, err) + c.JSON(http.StatusInternalServerError, fmt.Sprintf("error creating index %s", configurations.SOC_AI_INDEX)) + return + } + + if !configurations.GetGPTConfig().ModuleActive { + utils.Logger.LogF(100, "Droping request to /process, GPT module is not active") + c.JSON(http.StatusOK, "GPT module is not active") + return + } + + var ids []string + if err := c.BindJSON(&ids); err != nil { + utils.Logger.ErrorF("error binding JSON: %v", err) + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + for _, id := range ids { + result, err := elastic.ElasticSearch(configurations.SOC_AI_INDEX, "activityId", id) + if err != nil { + if !strings.Contains(err.Error(), "no such host") { + utils.Logger.ErrorF("error while searching alert %s in elastic: %v", id, err) + c.JSON(http.StatusInternalServerError, fmt.Sprintf("error while searching alert %s in elastic: %v", id, err)) + return + } + c.JSON(http.StatusInternalServerError, fmt.Sprintf("error while searching alert %s in elastic", id)) + return + } + + var gptResponses []schema.GPTAlertResponse + err = json.Unmarshal(result, &gptResponses) + if err != nil { + utils.Logger.ErrorF("error decoding response from elastic: %v", err) + c.JSON(http.StatusInternalServerError, fmt.Sprintf("error decoding response: %v", err)) + return + } + + if len(gptResponses) == 0 { + err = elastic.IndexStatus(id, "Processing", "create") + if err != nil { + utils.Logger.ErrorF("error creating doc in index: %v", err) + c.JSON(http.StatusInternalServerError, fmt.Sprintf("error creating doc in index: %v", err)) + return + } + } else { + err = elastic.IndexStatus(id, "Processing", "update") + if err != nil { + utils.Logger.ErrorF("error updating doc in index: %v", err) + c.JSON(http.StatusInternalServerError, fmt.Sprintf("error updating doc in index: %v", err)) + return + } + } + + utils.Logger.Info("alert %s received", id) + p.AlertInfoQueue <- schema.AlertGPTDetails{AlertID: id} + } + + c.JSON(http.StatusOK, "alerts received successfully") +} diff --git a/soc-ai/processor/processor.go b/soc-ai/processor/processor.go new file mode 100644 index 000000000..518c804a0 --- /dev/null +++ b/soc-ai/processor/processor.go @@ -0,0 +1,61 @@ +package processor + +import ( + "os" + "os/signal" + "sync" + "syscall" + + "github.com/utmstack/soc-ai/configurations" + "github.com/utmstack/soc-ai/elastic" + "github.com/utmstack/soc-ai/schema" + "github.com/utmstack/soc-ai/utils" +) + +var ( + processor *Processor + processorOnce sync.Once +) + +type Processor struct { + AlertInfoQueue chan schema.AlertGPTDetails + GPTQueue chan schema.AlertGPTDetails + ElasticQueue chan schema.AlertGPTDetails +} + +func NewProcessor() *Processor { + processorOnce.Do(func() { + processor = &Processor{ + AlertInfoQueue: make(chan schema.AlertGPTDetails, 1000), + GPTQueue: make(chan schema.AlertGPTDetails, 1000), + ElasticQueue: make(chan schema.AlertGPTDetails, 1000), + } + }) + return processor +} + +func (p *Processor) ProcessData() { + utils.Logger.Info("Starting SOC-AI Processor...") + + go configurations.UpdateGPTConfigurations() + go p.restRouter() + go p.processAlertsInfo() + + for i := 0; i < 10; i++ { + go p.processGPTRequests() + } + + go p.processAlertToElastic() + + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + <-signals +} + +func (p *Processor) RegisterError(message, id string) { + err := elastic.IndexStatus(id, "Error", "update") + if err != nil { + utils.Logger.ErrorF("error while indexing error in elastic: %v", err) + } + utils.Logger.ErrorF("%s", message) +} diff --git a/soc-ai/schema/convert.go b/soc-ai/schema/convert.go new file mode 100644 index 000000000..a97ed08ef --- /dev/null +++ b/soc-ai/schema/convert.go @@ -0,0 +1,161 @@ +package schema + +import ( + "fmt" + "reflect" + "strconv" + "strings" + "time" + + "github.com/utmstack/soc-ai/configurations" +) + +func ConvertFromAlertToAlertDB(alert Alert) AlertGPTDetails { + return AlertGPTDetails{ + AlertID: alert.ID, + Severity: alert.Severity, + TagRulesApplied: alert.TagRulesApplied, + SeverityLabel: alert.SeverityLabel, + Notes: alert.Notes, + DataType: alert.DataType, + Description: alert.Description, + StatusLabel: alert.StatusLabel, + Tactic: alert.Tactic, + Tags: strings.Join(alert.Tags, ","), + Reference: strings.Join(alert.Reference, ","), + Protocol: alert.Protocol, + Timestamp: alert.Timestamp, + Solution: alert.Solution, + StatusObservation: alert.StatusObservation, + Name: alert.Name, + IsIncident: alert.IsIncident, + Category: alert.Category, + DataSource: alert.DataSource, + Logs: strings.Join(alert.Logs, configurations.LOGS_SEPARATOR), + Status: alert.Status, + DestinationCountry: alert.Destination.Country, + DestinationAccuracyRadius: alert.Destination.AccuracyRadius, + DestinationCity: alert.Destination.City, + DestinationIP: alert.Destination.IP, + DestinationPort: alert.Destination.Port, + DestinationCountryCode: alert.Destination.CountryCode, + DestinationIsAnonymousProxy: alert.Destination.IsAnonymousProxy, + DestinationHost: alert.Destination.Host, + DestinationCoordinates: strings.Join(convertFloatSliceToStringSlice(alert.Destination.Coordinates), ","), + DestinationIsSatelliteProvider: alert.Destination.IsSatelliteProvider, + DestinationAso: alert.Destination.Aso, + DestinationAsn: alert.Destination.Asn, + DestinationUser: alert.Destination.User, + SourceCountry: alert.Source.Country, + SourceAccuracyRadius: alert.Source.AccuracyRadius, + SourceCity: alert.Source.City, + SourceIP: alert.Source.IP, + SourceCoordinates: strings.Join(convertFloatSliceToStringSlice(alert.Source.Coordinates), ","), + SourcePort: alert.Source.Port, + SourceCountryCode: alert.Source.CountryCode, + SourceIsAnonymousProxy: alert.Source.IsAnonymousProxy, + SourceIsSatelliteProvider: alert.Source.IsSatelliteProvider, + SourceHost: alert.Source.Host, + SourceAso: alert.Source.Aso, + SourceAsn: alert.Source.Asn, + SourceUser: alert.Source.User, + IncidentCreatedBy: alert.IncidentDetails.CreatedBy, + IncidentObservation: alert.IncidentDetails.Observation, + IncidentSource: alert.IncidentDetails.Source, + IncidentCreationDate: alert.IncidentDetails.CreationDate, + IncidentName: alert.IncidentDetails.IncidentName, + IncidentID: alert.IncidentDetails.IncidentID, + GPTTimestamp: time.Now().UTC().Format("2006-01-02T15:04:05.999999Z07:00"), + GPTClassification: "", + GPTReasoning: "", + GPTNextSteps: "", + } +} + +func convertFloatSliceToStringSlice(floatSlice []float64) []string { + strSlice := make([]string, len(floatSlice)) + for i, num := range floatSlice { + strSlice[i] = strconv.FormatFloat(num, 'f', -1, 64) + } + return strSlice +} + +func ConvertFromAlertDBToGPTResponse(alertDetails AlertGPTDetails) GPTAlertResponse { + resp := GPTAlertResponse{ + Timestamp: time.Now().UTC().Format("2006-01-02T15:04:05.999999Z07:00"), + Severity: alertDetails.Severity, + Category: alertDetails.Category, + AlertName: alertDetails.Name, + ActivityID: alertDetails.AlertID, + Classification: alertDetails.GPTClassification, + Reasoning: strings.Split(alertDetails.GPTReasoning, configurations.LOGS_SEPARATOR), + NextSteps: []NextStep{}, + } + + nextSteps := strings.Split(alertDetails.GPTNextSteps, "\n") + for i, step := range nextSteps { + actionAndDetails := strings.Split(step, "::") + if len(actionAndDetails) < 2 { + continue + } + resp.NextSteps = append(resp.NextSteps, NextStep{ + Step: i + 1, + Action: actionAndDetails[0], + Details: actionAndDetails[1], + }) + } + + return resp +} + +func ConvertGPTResponseToUpdateQuery(gptResp GPTAlertResponse) (UpdateDocRequest, error) { + source, err := BuildScriptString(gptResp) + if err != nil { + return UpdateDocRequest{}, err + } + + return UpdateDocRequest{ + Query: Query{ + Bool: Bool{ + Must: []Must{ + {Match{ + ActivityID: gptResp.ActivityID, + }}, + }, + }, + }, + Script: Script{ + Source: source, + Lang: "painless", + Params: gptResp, + }, + }, nil +} + +func BuildScriptString(alert GPTAlertResponse) (string, error) { + v := reflect.ValueOf(alert) + typeOfAlert := v.Type() + + source := "" + for i := 0; i < v.NumField(); i++ { + jsonTag := typeOfAlert.Field(i).Tag.Get("json") + jsonFieldName := strings.Split(jsonTag, ",")[0] + fieldValue := v.Field(i).Interface() + + switch reflect.TypeOf(fieldValue).Kind() { + case reflect.String, reflect.Int, reflect.Struct: + if fieldValue != reflect.Zero(reflect.TypeOf(fieldValue)).Interface() { + source += fmt.Sprintf("ctx._source['%s'] = params['%s']; ", jsonFieldName, jsonFieldName) + } + case reflect.Slice: + s := reflect.ValueOf(fieldValue) + if s.Len() > 0 { + source += fmt.Sprintf("ctx._source['%s'] = params.%s; ", jsonFieldName, jsonFieldName) + } + default: + return "", fmt.Errorf("unsupported type: %v", reflect.TypeOf(fieldValue).Kind()) + } + } + + return source, nil +} diff --git a/soc-ai/schema/models.go b/soc-ai/schema/models.go new file mode 100644 index 000000000..98bc3f4fd --- /dev/null +++ b/soc-ai/schema/models.go @@ -0,0 +1,61 @@ +package schema + +type AlertGPTDetails struct { + AlertID string `json:"alert_id,omitempty"` + Severity int `json:"severity,omitempty"` + TagRulesApplied string `json:"tag_rules_applied,omitempty"` + SeverityLabel string `json:"severity_label,omitempty"` + Notes string `json:"notes,omitempty"` + DataType string `json:"data_type,omitempty"` + Description string `json:"description,omitempty"` + StatusLabel string `json:"status_label,omitempty"` + Tactic string `json:"tactic,omitempty"` + Tags string `json:"tags,omitempty"` + Reference string `json:"reference,omitempty"` + Protocol string `json:"protocol,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + Solution string `json:"solution,omitempty"` + StatusObservation string `json:"status_observation,omitempty"` + Name string `json:"name,omitempty"` + IsIncident bool `json:"is_incident,omitempty"` + Category string `json:"category,omitempty"` + DataSource string `json:"data_source,omitempty"` + Logs string `json:"logs,omitempty"` + Status int `json:"status,omitempty"` + DestinationCountry string `json:"destination_country,omitempty"` + DestinationAccuracyRadius int `json:"destination_accuracy_radius,omitempty"` + DestinationCity string `json:"destination_city,omitempty"` + DestinationIP string `json:"-"` + DestinationPort int `json:"destination_port,omitempty"` + DestinationCountryCode string `json:"destination_country_code,omitempty"` + DestinationIsAnonymousProxy bool `json:"destination_is_anonymous_proxy,omitempty"` + DestinationHost string `json:"destination_host,omitempty"` + DestinationCoordinates string `json:"destination_coordinates,omitempty"` + DestinationIsSatelliteProvider bool `json:"destination_is_satellite_provider,omitempty"` + DestinationAso string `json:"destination_aso,omitempty"` + DestinationAsn int `json:"destination_asn,omitempty"` + DestinationUser string `json:"-"` + SourceCountry string `json:"source_country,omitempty"` + SourceAccuracyRadius int `json:"source_accuracy_radius,omitempty"` + SourceCity string `json:"source_city,omitempty"` + SourceIP string `json:"-"` + SourceCoordinates string `json:"source_coordinates,omitempty"` + SourcePort int `json:"source_port,omitempty"` + SourceCountryCode string `json:"source_country_code,omitempty"` + SourceIsAnonymousProxy bool `json:"source_is_anonymous_proxy,omitempty"` + SourceIsSatelliteProvider bool `json:"source_is_satellite_provider,omitempty"` + SourceHost string `json:"source_host,omitempty"` + SourceAso string `json:"source_aso,omitempty"` + SourceAsn int `json:"source_asn,omitempty"` + SourceUser string `json:"-"` + IncidentCreatedBy string `json:"incident_created_by,omitempty"` + IncidentObservation string `json:"incident_observation,omitempty"` + IncidentSource string `json:"incident_source,omitempty"` + IncidentCreationDate string `json:"incident_creation_date,omitempty"` + IncidentName string `json:"incident_name,omitempty"` + IncidentID string `json:"incident_id,omitempty"` + GPTTimestamp string `json:"gpt_timestamp,omitempty"` + GPTClassification string `json:"gpt_classification,omitempty"` + GPTReasoning string `json:"gpt_reasoning,omitempty"` + GPTNextSteps string `json:"gpt_next_steps,omitempty"` +} diff --git a/soc-ai/schema/schema.go b/soc-ai/schema/schema.go new file mode 100644 index 000000000..08f59b9a4 --- /dev/null +++ b/soc-ai/schema/schema.go @@ -0,0 +1,195 @@ +package schema + +type SearchResult struct{} + +type SeachRequest struct{} + +type SearchDetailsRequest []struct { + Field string `json:"field"` + Operator string `json:"operator"` + Value string `json:"value"` +} + +type GPTRequest struct { + Model string `json:"model"` + Messages []GPTMessage `json:"messages"` +} + +type GPTMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type GPTResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Created int `json:"created"` + Model string `json:"model"` + Choices []GPTChoice `json:"choices"` + Usage GPTUsage `json:"usage"` + SystemFingerprint string `json:"system_fingerprint"` +} + +type GPTChoice struct { + Index int `json:"index"` + Message GPTMessage `json:"message"` + LogProbs string `json:"logprobs"` + FinishReason string `json:"finish_reason"` +} + +type GPTUsage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` +} + +type GPTAlertResponse struct { + Timestamp string `json:"@timestamp,omitempty"` + Status string `json:"status,omitempty"` + Severity int `json:"severity,omitempty"` + Category string `json:"category,omitempty"` + AlertName string `json:"alertName,omitempty"` + ActivityID string `json:"activityId,omitempty"` + Classification string `json:"classification,omitempty"` + Reasoning []string `json:"reasoning,omitempty"` + NextSteps []NextStep `json:"nextSteps,omitempty"` +} + +type NextStep struct { + Step int `json:"step"` + Action string `json:"action"` + Details string `json:"details"` +} + +type ChangeAlertStatus struct { + AlertIDs []string `json:"alertIds"` + StatusObservation string `json:"statusObservation"` + Status int `json:"status"` +} + +type CreateNewIncidentRequest struct { + IncidentName string `json:"incidentName"` + IncidentDescription string `json:"incidentDescription"` + IncidentAssignedTo string `json:"incidentAssignedTo"` + AlertList AlertList `json:"alertList"` +} + +type AlertList []struct { + AlertID string `json:"alertId"` + AlertName string `json:"alertName"` + AlertStatus int `json:"alertStatus"` + AlertSeverity int `json:"alertSeverity"` +} + +type IncidentResp struct { + ID int `json:"id"` + IncidentName string `json:"incidentName"` + IncidentDescription string `json:"incidentDescription"` + IncidentStatus string `json:"incidentStatus"` + IncidentAssignedTo string `json:"incidentAssignedTo"` + IncidentSeverity int `json:"incidentSeverity"` + IncidentCreatedDate string `json:"incidentCreatedDate"` + IncidentSolution string `json:"incidentSolution"` +} + +type AddNewAlertToIncidentRequest struct { + IncidenId int `json:"incidentId"` + AlertList AlertList `json:"alertList"` +} + +type AlertDetails []Alert + +type Alert struct { + ID string `json:"id"` + Severity int `json:"severity"` + TagRulesApplied string `json:"TagRulesApplied,omitempty"` + SeverityLabel string `json:"severityLabel"` + Notes string `json:"notes"` + DataType string `json:"dataType"` + Description string `json:"description"` + StatusLabel string `json:"statusLabel"` + Tactic string `json:"tactic"` + Tags []string `json:"tags"` + Reference []string `json:"reference"` + Protocol string `json:"protocol"` + Timestamp string `json:"@timestamp"` + Solution string `json:"solution"` + StatusObservation string `json:"statusObservation"` + Name string `json:"name"` + IsIncident bool `json:"isIncident"` + Category string `json:"category"` + DataSource string `json:"dataSource"` + Logs []string `json:"logs"` + Status int `json:"status"` + Destination Destination `json:"destination"` + Source Source `json:"source"` + IncidentDetails IncidentDetail `json:"incidentDetail"` +} + +type Destination struct { + Country string `json:"country"` + AccuracyRadius int `json:"accuracyRadius"` + City string `json:"city"` + IP string `json:"ip,omitempty"` + Port int `json:"port"` + CountryCode string `json:"countryCode"` + IsAnonymousProxy bool `json:"isAnonymousProxy"` + Host string `json:"host,omitempty"` + Coordinates []float64 `json:"coordinates"` + IsSatelliteProvider bool `json:"isSatelliteProvider"` + Aso string `json:"aso"` + Asn int `json:"asn"` + User string `json:"user"` +} + +type Source struct { + Country string `json:"country"` + AccuracyRadius int `json:"accuracyRadius"` + City string `json:"city"` + IP string `json:"ip,omitempty"` + Coordinates []float64 `json:"coordinates"` + Port int `json:"port"` + CountryCode string `json:"countryCode"` + IsAnonymousProxy bool `json:"isAnonymousProxy"` + IsSatelliteProvider bool `json:"isSatelliteProvider"` + Host string `json:"host,omitempty"` + Aso string `json:"aso"` + Asn int `json:"asn"` + User string `json:"user"` +} + +type IncidentDetail struct { + CreatedBy string `json:"createdBy"` + Observation string `json:"observation"` + Source string `json:"source"` + CreationDate string `json:"creationDate"` + IncidentName string `json:"incidentName,omitempty"` + IncidentID string `json:"incidentId,omitempty"` +} + +type UpdateDocRequest struct { + Query Query `json:"query"` + Script Script `json:"script"` +} + +type Query struct { + Bool `json:"bool"` +} + +type Bool struct { + Must []Must `json:"must"` +} + +type Must struct { + Match Match `json:"match"` +} + +type Match struct { + ActivityID string `json:"activityId"` +} + +type Script struct { + Source string `json:"source"` + Lang string `json:"lang"` + Params GPTAlertResponse `json:"params"` +} diff --git a/soc-ai/utils/check.go b/soc-ai/utils/check.go new file mode 100644 index 000000000..dd0f48632 --- /dev/null +++ b/soc-ai/utils/check.go @@ -0,0 +1,41 @@ +package utils + +import ( + "fmt" + "net/http" + "time" +) + +func ConnectionChecker(url string) error { + checkConn := func() error { + if err := checkConnection(url); err != nil { + return fmt.Errorf("connection failed: %v", err) + } + return nil + } + + if err := Logger.InfiniteRetryIfXError(checkConn, "connection failed"); err != nil { + return err + } + + return nil +} + +func checkConnection(url string) error { + client := &http.Client{ + Timeout: 30 * time.Second, + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return err + } + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + return nil +} diff --git a/soc-ai/utils/convert.go b/soc-ai/utils/convert.go new file mode 100644 index 000000000..caf07fc16 --- /dev/null +++ b/soc-ai/utils/convert.go @@ -0,0 +1,25 @@ +package utils + +import ( + "encoding/json" + "fmt" +) + +func ConvertFromStructToJsonString(alert interface{}) (string, error) { + bytes, err := json.Marshal(alert) + if err != nil { + return "", fmt.Errorf("error marshalling alert: %v", err) + } + + return string(bytes), nil +} + +func ConvertFromJsonToStruct[responseType any](jsonString string) (responseType, error) { + var response responseType + err := json.Unmarshal([]byte(jsonString), &response) + if err != nil { + return *new(responseType), fmt.Errorf("error unmarshalling GPT response: %v", err) + } + + return response, nil +} diff --git a/soc-ai/utils/env.go b/soc-ai/utils/env.go new file mode 100644 index 000000000..99979fcfd --- /dev/null +++ b/soc-ai/utils/env.go @@ -0,0 +1,21 @@ +package utils + +import ( + "log" + "os" +) + +func Getenv(key string, isMandatory bool) string { + value, defined := os.LookupEnv(key) + if !defined { + if isMandatory { + log.Fatalf("Error loading environment variable: %s: environment variable does not exist\n", key) + } else { + return "" + } + } + if (value == "" || value == " ") && isMandatory { + log.Fatalf("Error loading environment variable: %s: empty environment variable\n", key) + } + return value +} diff --git a/soc-ai/utils/files.go b/soc-ai/utils/files.go new file mode 100644 index 000000000..d483f886d --- /dev/null +++ b/soc-ai/utils/files.go @@ -0,0 +1,35 @@ +package utils + +import ( + "fmt" + "os" + "path/filepath" +) + +func GetMyPath() (string, error) { + ex, err := os.Executable() + if err != nil { + return "", err + } + exPath := filepath.Dir(ex) + return exPath, nil +} + +// CreatePathIfNotExist creates a specific path if not exist +func CreatePathIfNotExist(path string) error { + if _, err := os.Stat(path); os.IsNotExist(err) { + if err := os.Mkdir(path, 0755); err != nil { + return fmt.Errorf("error creating path: %v", err) + } + } else if err != nil { + return fmt.Errorf("error checking path: %v", err) + } + return nil +} + +func CheckIfPathExist(path string) bool { + if _, err := os.Stat(path); os.IsNotExist(err) { + return false + } + return true +} diff --git a/soc-ai/utils/logger.go b/soc-ai/utils/logger.go new file mode 100644 index 000000000..4332e7f5a --- /dev/null +++ b/soc-ai/utils/logger.go @@ -0,0 +1,31 @@ +package utils + +import ( + "log" + "os" + "strconv" + + "github.com/threatwinds/logger" +) + +var Logger *logger.Logger + +func init() { + lenv := os.Getenv("LOG_LEVEL") + var level int + var err error + + if lenv != "" && lenv != " " { + level, err = strconv.Atoi(lenv) + if err != nil { + log.Fatalln(err) + } + } else { + level = 200 + } + + Logger = logger.NewLogger(&logger.Config{ + Format: "text", + Level: level, + }) +} diff --git a/soc-ai/utils/request.go b/soc-ai/utils/request.go new file mode 100644 index 000000000..8ae06dc58 --- /dev/null +++ b/soc-ai/utils/request.go @@ -0,0 +1,63 @@ +package utils + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "time" +) + +func DoParseReq[response any](url string, data []byte, method string, headers map[string]string, timeoutInSec int) (response, int, error) { + body, status, err := DoReq(url, data, method, headers, timeoutInSec) + if err != nil { + return *new(response), status, fmt.Errorf("error reading response body: %v", err) + } + + var result response + + err = json.Unmarshal(body, &result) + if err != nil { + return *new(response), http.StatusInternalServerError, fmt.Errorf("error decoding response: %v", err) + } + + if status != http.StatusAccepted && status != http.StatusOK { + return result, status, fmt.Errorf("status code '%d' received '%s' while sending '%s'", status, body, data) + } + + return result, status, nil +} + +func DoReq(url string, data []byte, method string, headers map[string]string, timeoutInSec int) ([]byte, int, error) { + req, err := http.NewRequest(method, url, bytes.NewBuffer(data)) + if err != nil { + return nil, http.StatusInternalServerError, fmt.Errorf("error creating request: %v", err) + } + + for k, v := range headers { + req.Header.Add(k, v) + } + + client := &http.Client{ + Timeout: time.Duration(timeoutInSec) * time.Second, + } + + resp, err := client.Do(req) + if err != nil { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + return nil, http.StatusInternalServerError, fmt.Errorf("request timed out: %v: %s", err, data) + } + return nil, http.StatusInternalServerError, fmt.Errorf("error performing request: %v: %s", err, data) + } + + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, http.StatusInternalServerError, fmt.Errorf("error reading response body: %v", err) + } + + return body, resp.StatusCode, nil +} From 4a7c81928900f388e1c39c9bf7ca1a763b3a44df Mon Sep 17 00:00:00 2001 From: Yorjander Hernandez Vergara Date: Tue, 22 Apr 2025 14:21:18 -0400 Subject: [PATCH 21/22] Update UTMStack Version --- CHANGELOG.md | 16 ++++++++-------- version.yml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa1c862da..b786f2d72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ -# UTMStack 10.7.3 Release Notes --- Implemented backend support for filtering compliance reports based on active integrations, optimizing query performance and data retrieval. --- Introduced new compliance reports aligned with the PCI DSS standard to expand auditing capabilities. --- Added support for creating and updating tag-based rules with dynamic conditions. - -### Bug Fixes --- Improved exception handling in `automaticReview` to prevent the process from stopping due to errors, ensuring the system continues evaluating alerts even if a specific rule fails. --- Improved operator selection for more accurate and consistent filtering. \ No newline at end of file +# UTMStack 10.8.0 Release Notes +- Updated Soc-AI models and released the code as open source. +- Added the ability for users to choose which model to use with Soc-AI. +- Enhanced the prompt sent to OpenAI by including additional contextual details. +- Added support for RedHat; UTMStack can now be installed on both Ubuntu and RedHat. +- Improved log delivery from ARM-based agents on Windows, now sending native system logs. +- Added support for macOS ARM64; agents can now be installed on that platform. +- Improved agent information displayed in the Sources panel, providing more accurate OS details and agent versions. \ No newline at end of file diff --git a/version.yml b/version.yml index 21c159ed9..40b613fa8 100644 --- a/version.yml +++ b/version.yml @@ -1 +1 @@ -version: 10.7.3 \ No newline at end of file +version: 10.8.0 \ No newline at end of file From a13850b0d5b5b07ce6183aa7044aeae020c7f095 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Tue, 22 Apr 2025 13:22:53 -0500 Subject: [PATCH 22/22] feat(module-integration): add SOC AI model selection field --- ...418001_add_options_module_group_config.xml | 51 +++++-------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml b/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml index 9df2d404a..b1831ff2d 100644 --- a/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml +++ b/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml @@ -16,8 +16,8 @@ conf_description, conf_data_type, conf_required, - conf_options) - VALUES ( + conf_options + ) VALUES ( default, 1, 'utmstack.socai.model', @@ -27,49 +27,22 @@ 'select', true, '[ - { "value": "o3-mini", "label": "O3 Mini Model" }, - { "value": "o3-mini-2025-01-31", "label": "O3 Mini Model (2025-01-31)" }, - { "value": "o1", "label": "O1 Model" }, - { "value": "o1-2024-12-17", "label": "O1 Model (2024-12-17)" }, - { "value": "o1-preview", "label": "O1 Preview Model" }, - { "value": "o1-preview-2024-09-12", "label": "O1 Preview Model (2024-09-12)" }, - { "value": "o1-mini", "label": "O1 Mini Model" }, - { "value": "o1-mini-2024-09-12", "label": "O1 Mini Model (2024-09-12)" }, + { "value": "gpt-4.1", "label": "GPT-4.1 Model" }, + { "value": "gpt-4.1-mini", "label": "GPT-4.1 Mini Model" }, + { "value": "gpt-4.1-nano", "label": "GPT-4.1 Nano Model" }, { "value": "gpt-4o", "label": "GPT-4 Omni Model" }, - { "value": "gpt-4o-2024-11-20", "label": "GPT-4 Omni Model (2024-11-20)" }, - { "value": "gpt-4o-2024-08-06", "label": "GPT-4 Omni Model (2024-08-06)" }, - { "value": "gpt-4o-2024-05-13", "label": "GPT-4 Omni Model (2024-05-13)" }, - { "value": "gpt-4o-audio-preview", "label": "GPT-4 Omni Audio Preview" }, - { "value": "gpt-4o-audio-preview-2024-10-01", "label": "GPT-4 Omni Audio Preview (2024-10-01)" }, - { "value": "gpt-4o-audio-preview-2024-12-17", "label": "GPT-4 Omni Audio Preview (2024-12-17)" }, - { "value": "gpt-4o-mini-audio-preview", "label": "GPT-4 Omni Mini Audio Preview" }, - { "value": "gpt-4o-mini-audio-preview-2024-12-17", "label": "GPT-4 Omni Mini Audio Preview (2024-12-17)" }, - { "value": "gpt-4o-search-preview", "label": "GPT-4 Omni Search Preview" }, - { "value": "gpt-4o-mini-search-preview", "label": "GPT-4 Omni Mini Search Preview" }, - { "value": "gpt-4o-search-preview-2025-03-11", "label": "GPT-4 Omni Search Preview (2025-03-11)" }, - { "value": "gpt-4o-mini-search-preview-2025-03-11", "label": "GPT-4 Omni Mini Search Preview (2025-03-11)" }, - { "value": "chatgpt-4o-latest", "label": "ChatGPT-4 Omni Latest Model" }, { "value": "gpt-4o-mini", "label": "GPT-4 Omni Mini Model" }, - { "value": "gpt-4o-mini-2024-07-18", "label": "GPT-4 Omni Mini Model (2024-07-18)" }, { "value": "gpt-4-turbo", "label": "GPT-4 Turbo Model" }, - { "value": "gpt-4-turbo-2024-04-09", "label": "GPT-4 Turbo Model (2024-04-09)" }, + { "value": "gpt-4-0614", "label": "GPT-4 Model (0614)" }, { "value": "gpt-4-0125-preview", "label": "GPT-4 Model (0125 Preview)" }, - { "value": "gpt-4-turbo-preview", "label": "GPT-4 Turbo Model (Preview)" }, - { "value": "gpt-4-1106-preview", "label": "GPT-4 Model (1106 Preview)" }, - { "value": "gpt-4-vision-preview", "label": "GPT-4 Vision Model Preview" }, - { "value": "gpt-4", "label": "GPT-4 Model" }, - { "value": "gpt-4-0314", "label": "GPT-4 Model (0314)" }, - { "value": "gpt-4-0613", "label": "GPT-4 Model (0613)" }, - { "value": "gpt-4-32k", "label": "GPT-4 Model (32K)" }, - { "value": "gpt-4-32k-0314", "label": "GPT-4 Model (32K 0314)" }, - { "value": "gpt-4-32k-0613", "label": "GPT-4 Model (32K 0613)" }, { "value": "gpt-3.5-turbo", "label": "GPT-3.5 Turbo Model" }, - { "value": "gpt-3.5-turbo-16k", "label": "GPT-3.5 Turbo Model (16K)" }, - { "value": "gpt-3.5-turbo-0301", "label": "GPT-3.5 Turbo Model (0301)" }, - { "value": "gpt-3.5-turbo-0613", "label": "GPT-3.5 Turbo Model (0613)" }, + { "value": "gpt-3.5-turbo-instruct", "label": "GPT-3.5 Turbo Instruct Model" }, { "value": "gpt-3.5-turbo-1106", "label": "GPT-3.5 Turbo Model (1106)" }, - { "value": "gpt-3.5-turbo-0125", "label": "GPT-3.5 Turbo Model (0125)" }, - { "value": "gpt-3.5-turbo-16k-0613", "label": "GPT-3.5 Turbo Model (16K 0613)" } + { "value": "o1", "label": "O1 Model" }, + { "value": "o1-pro", "label": "O1 Pro Model" }, + { "value": "o3", "label": "O3 Model" }, + { "value": "o3-mini", "label": "O3 Mini Model" }, + { "value": "o4-mini", "label": "O4 Mini Model" } ]' );