From e48a97bb4bcf25fb717ca1faddc4e1e541e58689 Mon Sep 17 00:00:00 2001 From: Yadian Llada Lopez Date: Wed, 16 Apr 2025 08:54:08 -0400 Subject: [PATCH 01/11] feat: Improve installer to add RHELD support --- installer/cloud.go | 17 +++---- installer/docker.go | 64 +++++++++++++++++-------- installer/main.go | 9 ++-- installer/master.go | 16 +++---- installer/network.go | 79 +++++++++++++++++++++---------- installer/nginx.go | 50 ++++++++++++++----- installer/system.go | 10 +++- installer/templates/interfaces.go | 13 ++++- installer/templates/proxy.go | 53 ++++++++++++++++++++- installer/tools.go | 17 +++++-- installer/utils/requirements.go | 11 +++-- 11 files changed, 250 insertions(+), 89 deletions(-) diff --git a/installer/cloud.go b/installer/cloud.go index 5cb8548a2..46e1f97e4 100644 --- a/installer/cloud.go +++ b/installer/cloud.go @@ -9,7 +9,8 @@ import ( "github.com/utmstack/UTMStack/installer/utils" ) -func Cloud(c *types.Config, update bool) error { +func Cloud(c *types.Config, update bool, distro string) error { + if err := utils.CheckCPU(2); err != nil { return err } @@ -43,7 +44,7 @@ func Cloud(c *types.Config, update bool) error { if utils.GetLock(2, stack.LocksDir) { fmt.Println("Preparing system to run UTMStack") - if err := PrepareSystem(); err != nil { + if err := PrepareSystem(distro); err != nil { return err } @@ -72,11 +73,11 @@ func Cloud(c *types.Config, update bool) error { return err } - if err := InstallVlan(); err != nil { + if err := InstallVlan(distro); err != nil { return err } - if err := ConfigureVLAN(iface); err != nil { + if err := ConfigureVLAN(iface, distro); err != nil { return err } @@ -101,7 +102,7 @@ func Cloud(c *types.Config, update bool) error { if utils.GetLock(3, stack.LocksDir) { fmt.Println("Installing Docker") - if err := InstallDocker(); err != nil { + if err := InstallDocker(distro); err != nil { return err } @@ -145,11 +146,11 @@ func Cloud(c *types.Config, update bool) error { if utils.GetLock(12, stack.LocksDir) || update { fmt.Println("Installing reverse proxy. This may take a while.") - if err := InstallNginx(); err != nil { + if err := InstallNginx(distro); err != nil { return err } - if err := ConfigureNginx(c, stack); err != nil { + if err := ConfigureNginx(c, stack, distro); err != nil { return err } @@ -162,7 +163,7 @@ func Cloud(c *types.Config, update bool) error { if utils.GetLock(5, stack.LocksDir) { fmt.Println("Installing Administration Tools") - if err := InstallTools(); err != nil { + if err := InstallTools(distro); err != nil { return err } diff --git a/installer/docker.go b/installer/docker.go index 437d6d21a..278db7e04 100644 --- a/installer/docker.go +++ b/installer/docker.go @@ -9,33 +9,57 @@ import ( "github.com/utmstack/UTMStack/installer/utils" ) -func InstallDocker() error { - env := []string{"DEBIAN_FRONTEND=noninteractive"} +func InstallDocker(distro string) error { + switch distro { + case "ubuntu": + env := []string{"DEBIAN_FRONTEND=noninteractive"} - if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "apt-transport-https", "ca-certificates", "curl", "gnupg-agent", "software-properties-common"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "apt-transport-https", "ca-certificates", "curl", "gnupg-agent", "software-properties-common"); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "sh", "-c", "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "sh", "-c", "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -"); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "sh", "-c", `add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"`); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "sh", "-c", `add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"`); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "docker-ce", "docker-ce-cli", "containerd.io", "docker-compose"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "docker-ce", "docker-ce-cli", "containerd.io", "docker-compose"); err != nil { + return err + } + case "redhat": + env := []string{"DNF_YUM_AUTO_YES=1"} + + if err := utils.RunEnvCmd(env, "dnf", "install", "-y", "dnf-plugins-core", "ca-certificates", "curl"); err != nil { + return err + } + + if err := utils.RunEnvCmd(env, "dnf", "config-manager", "--add-repo", "https://download.docker.com/linux/centos/docker-ce.repo"); err != nil { + return err + } + if err := utils.RunEnvCmd(env, "dnf", "makecache"); err != nil { + return err + } + + if err := utils.RunEnvCmd(env, "dnf", "install", "-y", "docker-ce", "docker-ce-cli", "containerd.io", "docker-compose-plugin", "docker-buildx-plugin"); err != nil { + return err + } + + if err := utils.RunEnvCmd(env, "systemctl", "enable", "--now", "docker"); err != nil { + return err + } + } return nil } diff --git a/installer/main.go b/installer/main.go index 6a3464649..fe5316376 100644 --- a/installer/main.go +++ b/installer/main.go @@ -4,8 +4,8 @@ import ( "fmt" "os" - "github.com/utmstack/UTMStack/installer/utils" "github.com/utmstack/UTMStack/installer/types" + "github.com/utmstack/UTMStack/installer/utils" ) func main() { @@ -22,7 +22,8 @@ func main() { } } - if err := utils.CheckDistro("ubuntu"); err != nil { + distro, err := utils.CheckDistro() + if err != nil { fmt.Println(err) os.Exit(1) } @@ -77,13 +78,13 @@ func main() { switch config.ServerType { case "aio": - err := Master(config) + err := Master(config, distro) if err != nil { fmt.Println(err) os.Exit(1) } case "cloud": - err := Cloud(config, update) + err := Cloud(config, update, distro) if err != nil { fmt.Println(err) os.Exit(1) diff --git a/installer/master.go b/installer/master.go index 3e6643eb5..792002cfb 100644 --- a/installer/master.go +++ b/installer/master.go @@ -9,7 +9,7 @@ import ( "github.com/utmstack/UTMStack/installer/utils" ) -func Master(c *types.Config) error { +func Master(c *types.Config, distro string) error { if err := utils.CheckCPU(2); err != nil { return err } @@ -43,7 +43,7 @@ func Master(c *types.Config) error { if utils.GetLock(2, stack.LocksDir) { fmt.Println("Preparing system to run UTMStack") - if err := PrepareSystem(); err != nil { + if err := PrepareSystem(distro); err != nil { return err } @@ -72,11 +72,11 @@ func Master(c *types.Config) error { return err } - if err := InstallVlan(); err != nil { + if err := InstallVlan(distro); err != nil { return err } - if err := ConfigureVLAN(iface); err != nil { + if err := ConfigureVLAN(iface, distro); err != nil { return err } @@ -101,7 +101,7 @@ func Master(c *types.Config) error { if utils.GetLock(3, stack.LocksDir) { fmt.Println("Installing Docker") - if err := InstallDocker(); err != nil { + if err := InstallDocker(distro); err != nil { return err } @@ -138,11 +138,11 @@ func Master(c *types.Config) error { fmt.Println("Installing reverse proxy. This may take a while.") - if err := InstallNginx(); err != nil { + if err := InstallNginx(distro); err != nil { return err } - if err := ConfigureNginx(c, stack); err != nil { + if err := ConfigureNginx(c, stack, distro); err != nil { return err } @@ -150,7 +150,7 @@ func Master(c *types.Config) error { if utils.GetLock(5, stack.LocksDir) { fmt.Println("Installing Administration Tools") - if err := InstallTools(); err != nil { + if err := InstallTools(distro); err != nil { return err } diff --git a/installer/network.go b/installer/network.go index 381f7bb0b..d82486eba 100644 --- a/installer/network.go +++ b/installer/network.go @@ -40,40 +40,71 @@ func checkRenderer() (string, error) { return "networkd", nil } -func ConfigureVLAN(mainIface string) error { - renderer, err := checkRenderer() - if err != nil { - return err - } - +func ConfigureVLAN(mainIface string, distro string) error { c := Vlan{ - Renderer: renderer, - Iface: mainIface, + Iface: mainIface, } - log.Println("Generating vlan config") - err = utils.GenerateConfig(c, templates.Vlan, path.Join("/etc", "netplan", "99-vlan.yaml")) - if err != nil { - return err - } + switch distro { + case "ubuntu": + renderer, err := checkRenderer() + if err != nil { + return err + } + c.Renderer = renderer - log.Println("Applying vlan config") - if err := utils.RunCmd("netplan", "apply"); err != nil { - return err - } + log.Println("Generating vlan config") + err = utils.GenerateConfig(c, templates.VlanUbuntu, path.Join("/etc", "netplan", "99-vlan.yaml")) + if err != nil { + return err + } + + log.Println("Applying vlan config") + if err := utils.RunCmd("netplan", "apply"); err != nil { + return err + } + + case "redhat": + err := utils.GenerateConfig(c, templates.VlanRedHat, "/etc/sysconfig/network-scripts/ifcfg-vlan10") + if err != nil { + return err + } + + if err := utils.RunCmd("modprobe", "8021q"); err != nil { + return err + } + + if err := os.WriteFile("/etc/modules-load.d/8021q.conf", []byte("8021q\n"), 0644); err != nil { + return err + } + + if err := utils.RunCmd("systemctl", "restart", "NetworkManager"); err != nil { + return err + } + } return nil } -func InstallVlan() error { - env := []string{"DEBIAN_FRONTEND=noninteractive"} +func InstallVlan(distro string) error { + switch distro { + case "ubuntu": + env := []string{"DEBIAN_FRONTEND=noninteractive"} - if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "update", "-y"); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "vlan"); err != nil { - return err + if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "vlan"); err != nil { + return err + } + + case "redhat": + env := []string{"DNF_YUM_AUTO_YES=1"} + + if err := utils.RunEnvCmd(env, "dnf", "update", "-y"); err != nil { + return err + } } return nil diff --git a/installer/nginx.go b/installer/nginx.go index 6cddc9cf6..3691a5b70 100644 --- a/installer/nginx.go +++ b/installer/nginx.go @@ -8,21 +8,30 @@ import ( "github.com/utmstack/UTMStack/installer/utils" ) -func InstallNginx() error { - env := []string{"DEBIAN_FRONTEND=noninteractive"} +func InstallNginx(distro string) error { + switch distro { + case "ubuntu": + env := []string{"DEBIAN_FRONTEND=noninteractive"} - if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "update"); err != nil { + return err + } - if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "nginx"); err != nil { - return err - } + if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "nginx"); err != nil { + return err + } + + case "redhat": + env := []string{"DNF_YUM_AUTO_YES=1"} + if err := utils.RunEnvCmd(env, "dnf", "install", "-y", "nginx"); err != nil { + return err + } + } return nil } -func ConfigureNginx(conf *types.Config, stack *types.StackConfig) error { +func ConfigureNginx(conf *types.Config, stack *types.StackConfig, distro string) error { c := types.NginxConfig{ SharedKey: conf.InternalKey, } @@ -32,9 +41,26 @@ func ConfigureNginx(conf *types.Config, stack *types.StackConfig) error { return err } - err = utils.GenerateConfig(c, templates.Proxy, path.Join("/", "etc", "nginx", "sites-available", "default")) - if err != nil { - return err + switch distro { + case "ubuntu": + err = utils.GenerateConfig(c, templates.ProxyUbuntu, path.Join("/", "etc", "nginx", "sites-available", "default")) + if err != nil { + return err + } + + case "redhat": + err = utils.GenerateConfig(c, templates.ProxyRHEL, path.Join("/", "etc", "nginx", "nginx.conf")) + if err != nil { + return err + } + + if err := utils.RunCmd("chcon", "-R", "-t", "httpd_sys_content_t", "/utmstack/cert/"); err != nil { + return err + } + + if err := utils.RunCmd("setsebool", "-P", "httpd_read_user_content", "1"); err != nil { + return err + } } if err := utils.RunCmd("systemctl", "restart", "nginx"); err != nil { diff --git a/installer/system.go b/installer/system.go index 10123a13a..00fb3e974 100644 --- a/installer/system.go +++ b/installer/system.go @@ -8,7 +8,15 @@ import ( "github.com/utmstack/UTMStack/installer/utils" ) -func PrepareSystem() error { +func PrepareSystem(distro string) error { + if distro == "redhat" { + if err := utils.RunCmd("systemctl", "disable", "firewalld"); err != nil { + return fmt.Errorf("failed to disable firewalld: %v", err) + } + if err := utils.RunCmd("systemctl", "stop", "firewalld"); err != nil { + return fmt.Errorf("failed to stop firewalld: %v", err) + } + } sysctl := []string{ "vm.max_map_count=262144", "net.ipv6.conf.all.disable_ipv6=1", diff --git a/installer/templates/interfaces.go b/installer/templates/interfaces.go index c22a2f11d..56fec7cb2 100644 --- a/installer/templates/interfaces.go +++ b/installer/templates/interfaces.go @@ -1,6 +1,6 @@ package templates -const Vlan string = ` +const VlanUbuntu string = ` network: version: 2 renderer: {{ .Renderer }} @@ -11,3 +11,14 @@ network: link: {{ .Iface }} addresses: [10.21.199.3/24] ` +const VlanRedHat string = ` +DEVICE=vlan10 +TYPE=Vlan +VLAN_ID=10 +PHYSDEV={{ .Iface }} +BOOTPROTO=none +IPADDR=10.21.199.3 +PREFIX=24 +VLAN=yes +ONBOOT=yes +` diff --git a/installer/templates/proxy.go b/installer/templates/proxy.go index 8170cf87e..9a50b75ad 100644 --- a/installer/templates/proxy.go +++ b/installer/templates/proxy.go @@ -1,6 +1,6 @@ package templates -const Proxy string = `server { +const ProxyUbuntu string = `server { listen 80 default_server; server_name _; return 301 https://$host$request_uri; @@ -34,4 +34,53 @@ server { client_max_body_size 200M; client_body_buffer_size 200M; -}` \ No newline at end of file +}` + +const ProxyRHEL string = `worker_processes auto; + +events { + worker_connections 2048; +} + +http { + include mime.types; + default_type application/octet-stream; + + keepalive_timeout 65; + +server { + listen 80 default_server; + server_name _; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + server_name _; + + set $utmstack http://127.0.0.1:10001; + set $shared_key {{.SharedKey}}; + + location / { + proxy_pass $utmstack; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header x-shared-key $shared_key; + add_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_read_timeout 900; + } + + ssl_certificate /utmstack/cert/utm.crt; + ssl_certificate_key /utmstack/cert/utm.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH'; + ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + chunked_transfer_encoding on; + + client_max_body_size 200M; + client_body_buffer_size 200M; + } +}` diff --git a/installer/tools.go b/installer/tools.go index a3a283981..aca35db06 100644 --- a/installer/tools.go +++ b/installer/tools.go @@ -2,11 +2,18 @@ package main import "github.com/utmstack/UTMStack/installer/utils" -func InstallTools() error { - env := []string{"DEBIAN_FRONTEND=noninteractive"} - err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "cockpit") - if err != nil { - return err +func InstallTools(distro string) error { + switch distro { + case "ubuntu": + env := []string{"DEBIAN_FRONTEND=noninteractive"} + if err := utils.RunEnvCmd(env, "apt-get", "install", "-y", "cockpit"); err != nil { + return err + } + case "redhat": + env := []string{"DNF_YUM_AUTO_YES=1"} + if err := utils.RunEnvCmd(env, "dnf", "install", "-y", "cockpit"); err != nil { + return err + } } return nil diff --git a/installer/utils/requirements.go b/installer/utils/requirements.go index 10826cc7d..f6206b1af 100644 --- a/installer/utils/requirements.go +++ b/installer/utils/requirements.go @@ -45,10 +45,13 @@ func CheckCPU(cores int) error { return nil } -func CheckDistro(distro string) error { +func CheckDistro() (string, error) { info, _ := host.Info() - if info.Platform != distro { - return fmt.Errorf("your Linux distribution (%s) is not %s", info.Platform, distro) + + distro := info.Platform + if distro != "ubuntu" && distro != "redhat" { + return "", fmt.Errorf("your Linux distribution (%s) is not supported. Supported distributions are: %s, %s", + distro, "ubuntu", "redhat") } - return nil + return distro, nil } From 91977d87be8405803bcf4f986a0a84ad8cd5d793 Mon Sep 17 00:00:00 2001 From: Yadian Llada Lopez Date: Wed, 16 Apr 2025 09:12:03 -0400 Subject: [PATCH 02/11] fix: Update windows ARM collector --- agent/collectors/windows_arm64.go | 459 ++++++++++++++++++------------ 1 file changed, 282 insertions(+), 177 deletions(-) diff --git a/agent/collectors/windows_arm64.go b/agent/collectors/windows_arm64.go index a71bd68e7..155f1a4e2 100644 --- a/agent/collectors/windows_arm64.go +++ b/agent/collectors/windows_arm64.go @@ -4,215 +4,320 @@ package collectors import ( + "encoding/json" + "encoding/xml" + "fmt" + "log" "os" - "os/exec" - "path/filepath" + "os/signal" + "strconv" "strings" - "time" + "syscall" + "unsafe" + "github.com/threatwinds/go-sdk/plugins" + "github.com/threatwinds/validations" "github.com/utmstack/UTMStack/agent/config" "github.com/utmstack/UTMStack/agent/logservice" - - "github.com/threatwinds/validations" "github.com/utmstack/UTMStack/agent/utils" + "golang.org/x/sys/windows" ) -type Windows struct{} +type Event struct { + XMLName xml.Name `xml:"Event"` + System SystemData `xml:"System"` + EventData []*EventData `xml:"EventData>Data"` +} -func getCollectorsInstances() []Collector { - var collectors []Collector - collectors = append(collectors, Windows{}) - return collectors +type EventData struct { + Key string `xml:"Name,attr"` + Value string `xml:",chardata"` +} + +type ProviderData struct { + ProviderName string `xml:"Name,attr"` + ProviderGUID string `xml:"Guid,attr"` } -const PowerShellScript = ` -<# -.SYNOPSIS - Collects Windows Application, System, and Security logs from the last 5 minutes, then prints them to the console in a compact, single-line JSON format, - emulating the field structure that Winlogbeat typically produces. - -.DESCRIPTION - 1. Retrieves the last 5 minutes of Windows logs (Application, System, Security) using FilterHashtable (no post-fetch filtering). - 2. Maps event properties to a schema similar to Winlogbeat's, including: - - @timestamp - - message - - event.code - - event.provider - - event.kind - - winlog fields (e.g. record_id, channel, activity_id, etc.) - 3. Prints each log record as a single-line JSON object with no indentation/extra spacing. - 4. If no logs are found, the script produces no output at all. -#> - -# Suppress any runtime errors that would clutter the console. -$ErrorActionPreference = 'SilentlyContinue' - -# Calculate the start time for filtering -$startTime = (Get-Date).AddSeconds(-30) - -# Retrieve logs with filter hashtable -$applicationLogs = Get-WinEvent -FilterHashtable @{ LogName='Application'; StartTime=$startTime } -$systemLogs = Get-WinEvent -FilterHashtable @{ LogName='System'; StartTime=$startTime } -$securityLogs = Get-WinEvent -FilterHashtable @{ LogName='Security'; StartTime=$startTime } - -# Safeguard against null results -if (-not $applicationLogs) { $applicationLogs = @() } -if (-not $systemLogs) { $systemLogs = @() } -if (-not $securityLogs) { $securityLogs = @() } - -# Combine them -$recentLogs = $applicationLogs + $systemLogs + $securityLogs - -# If no logs are found, produce no output at all -if (-not $recentLogs) { - return -} - -# Function to convert the raw Properties array to a dictionary-like object under winlog.event_data -function Convert-PropertiesToEventData { - param([Object[]] $Properties) - - # If nothing is there, return an empty hashtable - if (-not $Properties) { return @{} } - - # Winlogbeat places custom fields under winlog.event_data. - # Typically, it tries to parse known keys, but we'll do a simple best-effort approach: - # We'll create paramN = pairs for each array index. - $eventData = [ordered]@{} - - for ($i = 0; $i -lt $Properties.Count; $i++) { - $value = $Properties[$i].Value - - # If the property is itself an object with nested fields, we can flatten or store as-is. - # We'll store as-is for clarity. - # We'll name them param1, param2, param3,... unless you'd like more specific field logic. - $paramName = "param$($i+1)" - - $eventData[$paramName] = $value - } - - return $eventData -} - -# Transform each event into a structure emulating Winlogbeat -foreach ($rawEvent in $recentLogs) { - # Convert TimeCreated to a universal ISO8601 string - $timestamp = $rawEvent.TimeCreated.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') - - # Build the top-level document - $doc = [ordered]@{ - # Matches Winlogbeat's typical top-level timestamp field - '@timestamp' = $timestamp - - # The main message content from the event - 'message' = $rawEvent.Message - - # "event" block: minimal example - 'event' = [ordered]@{ - 'code' = $rawEvent.Id # event_id in Winlogbeat is typically a string or numeric - 'provider' = $rawEvent.ProviderName - 'kind' = 'event' - 'created' = $timestamp # or you could omit if desired - } - - # "winlog" block: tries to mirror Winlogbeat's structure for Windows - 'winlog' = [ordered]@{ - 'record_id' = $rawEvent.RecordId - 'computer_name' = $rawEvent.MachineName - 'channel' = $rawEvent.LogName - 'provider_name' = $rawEvent.ProviderName - 'provider_guid' = $rawEvent.ProviderId - 'process' = [ordered]@{ - 'pid' = $rawEvent.ProcessId - 'thread' = @{ - 'id' = $rawEvent.ThreadId - } - } - 'event_id' = $rawEvent.Id - 'version' = $rawEvent.Version - 'activity_id' = $rawEvent.ActivityId - 'related_activity_id'= $rawEvent.RelatedActivityId - 'task' = $rawEvent.TaskDisplayName - 'opcode' = $rawEvent.OpcodeDisplayName - 'keywords' = $rawEvent.KeywordsDisplayNames - 'time_created' = $timestamp - # Convert "Properties" into a dictionary for event_data - 'event_data' = Convert-PropertiesToEventData $rawEvent.Properties - } - } - - # Convert our object to JSON (with no extra formatting). - $json = $doc | ConvertTo-Json -Depth 20 - - # Remove all newlines and indentation for a single-line representation - $compactJson = $json -replace '(\r?\n\s*)+', '' - - # Output the line - Write-Output $compactJson -} -` +type TimeCreatedData struct { + SystemTime string `xml:"SystemTime,attr"` +} -func (w Windows) Install() error { - return nil +type CorrelationData struct { + ActivityID string `xml:"ActivityID,attr"` } -func (w Windows) SendSystemLogs() { - path := utils.GetMyPath() - collectorPath := filepath.Join(path, "collector.ps1") +type ExecutionData struct { + ProcessID int `xml:"ProcessID,attr"` + ThreadID int `xml:"ThreadID,attr"` +} + +type SecurityData struct{} + +type SystemData struct { + Provider ProviderData `xml:"Provider"` + EventID int `xml:"EventID"` + Version int `xml:"Version"` + Level int `xml:"Level"` + Task int `xml:"Task"` + Opcode int `xml:"Opcode"` + Keywords string `xml:"Keywords"` + TimeCreated TimeCreatedData `xml:"TimeCreated"` + EventRecordID int64 `xml:"EventRecordID"` + Correlation CorrelationData `xml:"Correlation"` + Execution ExecutionData `xml:"Execution"` + Channel string `xml:"Channel"` + Computer string `xml:"Computer"` + Security SecurityData `xml:"Security"` +} + +type EventSubscription struct { + Channel string + Query string + Errors chan error + Callback func(event *Event) + winAPIHandle windows.Handle +} - err := os.WriteFile(collectorPath, []byte(PowerShellScript), 0644) +const ( + EvtSubscribeToFutureEvents = 1 + evtSubscribeActionError = 0 + evtSubscribeActionDeliver = 1 + evtRenderEventXML = 1 +) + +var ( + modwevtapi = windows.NewLazySystemDLL("wevtapi.dll") + procEvtSubscribe = modwevtapi.NewProc("EvtSubscribe") + procEvtRender = modwevtapi.NewProc("EvtRender") + procEvtClose = modwevtapi.NewProc("EvtClose") +) + +func (evtSub *EventSubscription) Create() error { + if evtSub.winAPIHandle != 0 { + return fmt.Errorf("windows_events: subscription has already been created") + } + + winChannel, err := windows.UTF16PtrFromString(evtSub.Channel) if err != nil { - _ = utils.Logger.ErrorF("error writing powershell script: %v", err) - return + return fmt.Errorf("windows_events: invalid channel name: %s", err) } - cmd := exec.Command("Powershell.exe", "-Command", `"Set-ExecutionPolicy -ExecutionPolicy Unrestricted -Scope CurrentUser -Force"`) - err = cmd.Run() + winQuery, err := windows.UTF16PtrFromString(evtSub.Query) if err != nil { - _ = utils.Logger.ErrorF("error setting powershell execution policy: %v", err) - return + return fmt.Errorf("windows_events: invalid query: %s", err) } - for { - select { - case <-time.After(30 * time.Second): - go func() { - cmd := exec.Command("Powershell.exe", "-File", collectorPath) + callback := syscall.NewCallback(evtSub.winAPICallback) - output, err := cmd.Output() - if err != nil { - _ = utils.Logger.ErrorF("error executing powershell script: %v", err) - return - } + log.Printf("Debug - Subscribing to channel: %s", evtSub.Channel) - utils.Logger.LogF(100, "output: %s", string(output)) + handle, _, err := procEvtSubscribe.Call( + 0, + 0, + uintptr(unsafe.Pointer(winChannel)), + uintptr(unsafe.Pointer(winQuery)), + 0, + 0, + callback, + uintptr(EvtSubscribeToFutureEvents), + ) - logLines := strings.Split(string(output), "\n") + if handle == 0 { + return fmt.Errorf("windows_events: failed to subscribe to events: %v", err) + } - validatedLogs := make([]string, 0, len(logLines)) + evtSub.winAPIHandle = windows.Handle(handle) + return nil +} - for _, logLine := range logLines { - validatedLog, _, err := validations.ValidateString(logLine, false) - if err != nil { - utils.Logger.LogF(100, "error validating log: %s: %v", logLine, err) - continue - } +func (evtSub *EventSubscription) Close() error { + if evtSub.winAPIHandle == 0 { + return fmt.Errorf("windows_events: no active subscription to close") + } + ret, _, err := procEvtClose.Call(uintptr(evtSub.winAPIHandle)) + if ret == 0 { + return fmt.Errorf("windows_events: error closing handle: %s", err) + } + evtSub.winAPIHandle = 0 + return nil +} - validatedLogs = append(validatedLogs, validatedLog) +func (evtSub *EventSubscription) winAPICallback(action, userContext, event uintptr) uintptr { + switch action { + case evtSubscribeActionError: + evtSub.Errors <- fmt.Errorf("windows_events: error in callback, code: %x", uint16(event)) + case evtSubscribeActionDeliver: + bufferSize := uint32(4096) + for { + renderSpace := make([]uint16, bufferSize/2) + bufferUsed := uint32(0) + propertyCount := uint32(0) + ret, _, err := procEvtRender.Call( + 0, + event, + evtRenderEventXML, + uintptr(bufferSize), + uintptr(unsafe.Pointer(&renderSpace[0])), + uintptr(unsafe.Pointer(&bufferUsed)), + uintptr(unsafe.Pointer(&propertyCount)), + ) + if ret == 0 { + if err == windows.ERROR_INSUFFICIENT_BUFFER { + bufferSize *= 2 + continue } + evtSub.Errors <- fmt.Errorf("windows_events: failed to render event: %w", err) + return 0 + } + xmlStr := windows.UTF16ToString(renderSpace) + xmlStr = cleanXML(xmlStr) + + dataParsed := new(Event) + if err := xml.Unmarshal([]byte(xmlStr), dataParsed); err != nil { + evtSub.Errors <- fmt.Errorf("windows_events: failed to parse XML: %s", err) + } else { + evtSub.Callback(dataParsed) + } + break + } + default: + evtSub.Errors <- fmt.Errorf("windows_events: unsupported action in callback: %x", uint16(action)) + } + return 0 +} - logservice.LogQueue <- logservice.LogPipe{ - Src: string(config.DataTypeWindowsAgent), - Logs: validatedLogs, - } - }() +func cleanXML(xmlStr string) string { + xmlStr = strings.TrimSpace(xmlStr) + if idx := strings.Index(xmlStr, " 0 { + xmlStr = xmlStr[idx:] + } + xmlStr = strings.Map(func(r rune) rune { + if r < 32 && r != '\n' && r != '\r' && r != '\t' { + return -1 } + return r + }, xmlStr) + return xmlStr +} + +type Windows struct{} + +func getCollectorsInstances() []Collector { + var collectors []Collector + collectors = append(collectors, Windows{}) + return collectors +} + +func (w Windows) SendSystemLogs() { + errorsChan := make(chan error, 10) + + callback := func(event *Event) { + eventJSON, err := convertEventToJSON(event) + if err != nil { + utils.Logger.ErrorF("error converting event to JSON: %v", err) + return + } + host, err := os.Hostname() + if err != nil { + utils.Logger.ErrorF("error getting hostname: %v", err) + host = "unknown" + } + validatedLog, _, err := validations.ValidateString(eventJSON, false) + if err != nil { + utils.Logger.LogF(100, "error validating log: %s: %v", eventJSON, err) + return + } + logservice.LogQueue <- &plugins.Log{ + DataType: string(config.DataTypeWindowsAgent), + DataSource: host, + Raw: validatedLog, + } + } + + channels := []string{"Security", "Application", "System"} + var subscriptions []*EventSubscription + + for _, channel := range channels { + sub := &EventSubscription{ + Channel: channel, + Query: "*", + Errors: errorsChan, + Callback: callback, + } + if err := sub.Create(); err != nil { + utils.Logger.ErrorF("Error subscribing to channel %s: %s", channel, err) + continue + } + subscriptions = append(subscriptions, sub) + utils.Logger.LogF(100, "Subscribed to channel: %s", channel) } + + go func() { + for err := range errorsChan { + utils.Logger.ErrorF("Subscription error: %s", err) + } + }() + + exitChan := make(chan os.Signal, 1) + signal.Notify(exitChan, os.Interrupt) + <-exitChan + utils.Logger.LogF(100, "Interrupt received, closing subscriptions...") + for _, sub := range subscriptions { + if err := sub.Close(); err != nil { + utils.Logger.ErrorF("Error closing subscription for %s: %v", sub.Channel, err) + } + } + utils.Logger.LogF(100, "Agent finished successfully.") +} + +func convertEventToJSON(event *Event) (string, error) { + eventMap := map[string]interface{}{ + "timestamp": event.System.TimeCreated.SystemTime, + "providerName": event.System.Provider.ProviderName, + "providerGuid": event.System.Provider.ProviderGUID, + "eventCode": event.System.EventID, + "version": event.System.Version, + "level": event.System.Level, + "task": event.System.Task, + "opcode": event.System.Opcode, + "keywords": event.System.Keywords, + "timeCreated": event.System.TimeCreated.SystemTime, + "recordId": event.System.EventRecordID, + "correlation": event.System.Correlation, + "execution": event.System.Execution, + "channel": event.System.Channel, + "computer": event.System.Computer, + "data": make(map[string]interface{}), + } + + dataMap := eventMap["data"].(map[string]interface{}) + for _, data := range event.EventData { + if strings.HasPrefix(data.Value, "0x") { + if val, err := strconv.ParseInt(data.Value[2:], 16, 64); err == nil { + dataMap[data.Key] = val + continue + } + } + if data.Key != "" { + value := strings.TrimSpace(data.Value) + if value != "" { + dataMap[data.Key] = value + } + } + } + + jsonBytes, err := json.Marshal(eventMap) + if err != nil { + return "", err + } + return string(jsonBytes), nil +} + +func (w Windows) Install() error { + return nil } func (w Windows) Uninstall() error { - path := utils.GetMyPath() - collectorPath := filepath.Join(path, "collector.ps1") - err := os.Remove(collectorPath) - return err + return nil } From 435cfcadbf0855377cfeceae3d3bd70d5175c5e8 Mon Sep 17 00:00:00 2001 From: Osmany Montero Date: Wed, 16 Apr 2025 19:00:47 +0200 Subject: [PATCH 03/11] wip --- agent/collectors/collectors.go | 4 +- agent/collectors/filebeat_amd64.go | 8 +- agent/collectors/macos_arm64.go | 94 +++++++++++++ agent/collectors/windows_amd64.go | 5 +- agent/collectors/windows_arm64.go | 2 +- agent/config/const.go | 1 + agent/config/macos_arm64.go | 10 ++ .../config/{win_amd64.go => windows_amd64.go} | 0 .../config/{win_arm64.go => windows_arm64.go} | 0 agent/docs/tasks.md | 133 ++++++++++++++++++ agent/logs/utmstack_agent.log | 1 + agent/main.go | 11 +- agent/serv/clean-old.go | 27 ++-- agent/serv/run.go | 2 + agent/updates/dependencies.go | 4 + agent/utils/services.go | 7 +- 16 files changed, 281 insertions(+), 28 deletions(-) create mode 100644 agent/collectors/macos_arm64.go create mode 100644 agent/config/macos_arm64.go rename agent/config/{win_amd64.go => windows_amd64.go} (100%) rename agent/config/{win_arm64.go => windows_arm64.go} (100%) create mode 100644 agent/docs/tasks.md create mode 100644 agent/logs/utmstack_agent.log diff --git a/agent/collectors/collectors.go b/agent/collectors/collectors.go index d49cadfd1..753a655ff 100644 --- a/agent/collectors/collectors.go +++ b/agent/collectors/collectors.go @@ -13,7 +13,7 @@ type CollectorConfig struct { type Collector interface { Install() error - SendSystemLogs() + SendLogs() Uninstall() error } @@ -35,7 +35,7 @@ func InstallCollectors() error { func LogsReader() { collectors := getCollectorsInstances() for _, collector := range collectors { - go collector.SendSystemLogs() + go collector.SendLogs() } } diff --git a/agent/collectors/filebeat_amd64.go b/agent/collectors/filebeat_amd64.go index 290cf64a7..700e02b8d 100644 --- a/agent/collectors/filebeat_amd64.go +++ b/agent/collectors/filebeat_amd64.go @@ -104,7 +104,7 @@ func (f Filebeat) Install() error { return nil } -func (f Filebeat) SendSystemLogs() { +func (f Filebeat) SendLogs() { logLinesChan := make(chan []string) path := utils.GetMyPath() filebLogPath := filepath.Join(path, "beats", "filebeat", "logs") @@ -112,12 +112,16 @@ func (f Filebeat) SendSystemLogs() { parser := parser.GetParser("beats") go utils.WatchFolder("modulescollector", filebLogPath, logLinesChan, config.BatchCapacity) - for logLine := range logLinesChan { + + for { + logLine := <-logLinesChan + beatsData, err := parser.ProcessData(logLine) if err != nil { utils.Logger.ErrorF("error processing beats data: %v", err) continue } + for typ, logB := range beatsData { logservice.LogQueue <- logservice.LogPipe{ Src: typ, diff --git a/agent/collectors/macos_arm64.go b/agent/collectors/macos_arm64.go new file mode 100644 index 000000000..36296a5c5 --- /dev/null +++ b/agent/collectors/macos_arm64.go @@ -0,0 +1,94 @@ +//go:build darwin && arm64 +// +build darwin,arm64 + +package collectors + +import ( + "bufio" + "github.com/threatwinds/validations" + "github.com/utmstack/UTMStack/agent/config" + "github.com/utmstack/UTMStack/agent/logservice" + "github.com/utmstack/UTMStack/agent/utils" + "os/exec" + "path/filepath" +) + +type Darwin struct{} + +func (d Darwin) Install() error { + return nil +} + +func getCollectorsInstances() []Collector { + var collectors []Collector + collectors = append(collectors, Darwin{}) + return collectors +} + +func (d Darwin) SendLogs() { + path := utils.GetMyPath() + collectorPath := filepath.Join(path, "utmstack-collector-mac") + + cmd := exec.Command(collectorPath) + + stdout, err := cmd.StdoutPipe() + if err != nil { + _ = utils.Logger.ErrorF("error creating stdout pipe: %v", err) + return + } + + stderr, err := cmd.StderrPipe() + if err != nil { + _ = utils.Logger.ErrorF("error creating stderr pipe: %v", err) + return + } + + if err := cmd.Start(); err != nil { + _ = utils.Logger.ErrorF("error starting macOS collector: %v", err) + return + } + + go func() { + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + logLine := scanner.Text() + + utils.Logger.LogF(100, "output: %s", logLine) + + validatedLog, _, err := validations.ValidateString(logLine, false) + if err != nil { + utils.Logger.ErrorF("error validating log: %s: %v", logLine, err) + continue + } + + logservice.LogQueue <- logservice.LogPipe{ + Src: string(config.DataTypeMacOSAgent), + Logs: []string{validatedLog}, + } + } + + if err := scanner.Err(); err != nil { + _ = utils.Logger.ErrorF("error reading stdout: %v", err) + } + }() + + go func() { + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + errLine := scanner.Text() + _ = utils.Logger.ErrorF("collector error: %s", errLine) + } + + if err := scanner.Err(); err != nil { + _ = utils.Logger.ErrorF("error reading stderr: %v", err) + } + }() + + if err := cmd.Wait(); err != nil { + _ = utils.Logger.ErrorF("macOS collector process ended with error: %v", err) + } +} + +func (d Darwin) Uninstall() error { + return nil +} diff --git a/agent/collectors/windows_amd64.go b/agent/collectors/windows_amd64.go index c95a1429f..b65b565e7 100644 --- a/agent/collectors/windows_amd64.go +++ b/agent/collectors/windows_amd64.go @@ -70,13 +70,14 @@ func (w Windows) Install() error { return nil } -func (w Windows) SendSystemLogs() { +func (w Windows) SendLogs() { logLinesChan := make(chan []string) path := utils.GetMyPath() winbLogPath := filepath.Join(path, "beats", "winlogbeat", "logs") go utils.WatchFolder("windowscollector", winbLogPath, logLinesChan, config.BatchCapacity) - for logLine := range logLinesChan { + for { + logLine := <-logLinesChan validatedLogs := []string{} for _, log := range logLine { validatedLog, _, err := validations.ValidateString(log, false) diff --git a/agent/collectors/windows_arm64.go b/agent/collectors/windows_arm64.go index a71bd68e7..7f0ba8f2b 100644 --- a/agent/collectors/windows_arm64.go +++ b/agent/collectors/windows_arm64.go @@ -156,7 +156,7 @@ func (w Windows) Install() error { return nil } -func (w Windows) SendSystemLogs() { +func (w Windows) SendLogs() { path := utils.GetMyPath() collectorPath := filepath.Join(path, "collector.ps1") diff --git a/agent/config/const.go b/agent/config/const.go index 287b4842e..40632244f 100644 --- a/agent/config/const.go +++ b/agent/config/const.go @@ -45,6 +45,7 @@ var ( // RedlineServName = "UTMStackRedline" // BatchToSend = 5 + DataTypeMacOSAgent DataType = "macos_agent" DataTypeWindowsAgent DataType = "beats_windows_agent" DataTypeSyslog DataType = "syslog" DataTypeVmware DataType = "vmware" diff --git a/agent/config/macos_arm64.go b/agent/config/macos_arm64.go new file mode 100644 index 000000000..615b84ac0 --- /dev/null +++ b/agent/config/macos_arm64.go @@ -0,0 +1,10 @@ +//go:build darwin && arm64 +// +build darwin,arm64 + +package config + +var ( + UpdaterSelf = "utmstack_updater_self_arm64%s" + ServiceFile = "utmstack_agent_service_arm64%s" + DependFiles = []string{} +) diff --git a/agent/config/win_amd64.go b/agent/config/windows_amd64.go similarity index 100% rename from agent/config/win_amd64.go rename to agent/config/windows_amd64.go diff --git a/agent/config/win_arm64.go b/agent/config/windows_arm64.go similarity index 100% rename from agent/config/win_arm64.go rename to agent/config/windows_arm64.go diff --git a/agent/docs/tasks.md b/agent/docs/tasks.md new file mode 100644 index 000000000..a748637c0 --- /dev/null +++ b/agent/docs/tasks.md @@ -0,0 +1,133 @@ +# UTMStack Agent Improvement Tasks + +This document contains a prioritized list of tasks for improving the UTMStack Agent codebase. Each task is marked with a checkbox that can be checked off when completed. + +## Architecture Improvements + +1. [ ] Implement a more modular plugin architecture to make it easier to add new collectors and data sources +2. [ ] Create a unified configuration management system that centralizes all configuration options +3. [ ] Implement a proper dependency injection system to improve testability and reduce tight coupling +4. [ ] Refactor the service management code to use a common interface across all platforms +5. [ ] Implement a more robust error handling and recovery mechanism for all collectors +6. [ ] Create a standardized logging framework with configurable log levels +7. [ ] Implement a metrics collection system to monitor agent performance +8. [ ] Design a more efficient log batching and transmission system to reduce resource usage +9. [ ] Implement a circuit breaker pattern for external service connections +10. [ ] Create a formal API versioning strategy for communication with the server + +## Code Quality Improvements + +11. [ ] Add comprehensive unit tests for all packages (current coverage appears low) +12. [ ] Implement integration tests for collector functionality +13. [ ] Add proper documentation for all exported functions, types, and interfaces +14. [ ] Standardize error handling across the codebase +15. [ ] Refactor long functions (like in main.go) into smaller, more focused functions +16. [ ] Remove commented-out code in config/const.go and other files +17. [ ] Implement consistent naming conventions across the codebase +18. [ ] Add context support for all long-running operations +19. [ ] Implement proper resource cleanup in error cases +20. [ ] Add validation for all user inputs and configuration values + +## Security Improvements + +21. [ ] Implement secure storage for sensitive configuration (beyond basic encryption) +22. [ ] Add TLS certificate validation by default (currently optional) +23. [ ] Implement proper authentication for all API endpoints +24. [ ] Add integrity verification for downloaded dependencies +25. [ ] Implement secure logging practices (masking sensitive data) +26. [ ] Add rate limiting for authentication attempts +27. [ ] Implement proper permission checking for file operations +28. [ ] Add security scanning in the CI/CD pipeline +29. [ ] Implement secure coding practices training for developers +30. [ ] Create a security incident response plan + +## Performance Improvements + +31. [ ] Profile the application to identify performance bottlenecks +32. [ ] Optimize log collection and processing for high-volume environments +33. [ ] Implement more efficient data structures for log storage +34. [ ] Add caching for frequently accessed configuration +35. [ ] Optimize file operations to reduce disk I/O +36. [ ] Implement connection pooling for network operations +37. [ ] Reduce memory usage in log processing pipelines +38. [ ] Optimize CPU usage during idle periods +39. [ ] Implement more efficient serialization/deserialization +40. [ ] Add performance benchmarks to CI/CD pipeline + +## Platform Support Improvements + +41. [ ] Standardize platform-specific code to reduce duplication +42. [ ] Improve macOS arm64 support (recently added) +43. [ ] Add support for additional Linux distributions +44. [ ] Improve Windows arm64 support +45. [ ] Create a unified build system for all platforms +46. [ ] Implement automated testing on all supported platforms +47. [ ] Add containerization support for easier deployment +48. [ ] Create platform-specific installation documentation +49. [ ] Implement feature parity across all supported platforms +50. [ ] Add support for cloud-native environments + +## User Experience Improvements + +51. [ ] Improve installation process with better user feedback +52. [ ] Create a web-based configuration interface +53. [ ] Implement a status dashboard for monitoring agent health +54. [ ] Add better error messages for common failure scenarios +55. [ ] Improve logging with more context and clarity +56. [ ] Create user-friendly documentation with examples +57. [ ] Implement a troubleshooting guide for common issues +58. [ ] Add a command-line interface for common administrative tasks +59. [ ] Implement a self-diagnostic tool for agent issues +60. [ ] Create a user feedback mechanism + +## Operational Improvements + +61. [ ] Implement a more robust update mechanism with rollback capability +62. [ ] Add health checks for all components +63. [ ] Implement proper service lifecycle management +64. [ ] Create automated deployment scripts for common environments +65. [ ] Implement configuration validation before applying changes +66. [ ] Add support for configuration templates for common scenarios +67. [ ] Implement a backup and restore mechanism for agent configuration +68. [ ] Create operational runbooks for common maintenance tasks +69. [ ] Implement proper service dependencies management +70. [ ] Add support for centralized configuration management + +## Technical Debt Reduction + +71. [ ] Refactor the collector implementation to reduce code duplication +72. [ ] Update deprecated dependencies and APIs +73. [ ] Remove unused code and dependencies +74. [ ] Consolidate duplicate functionality across packages +75. [ ] Implement consistent error handling patterns +76. [ ] Refactor configuration handling to use a single approach +77. [ ] Improve code organization with better package structure +78. [ ] Standardize file naming conventions +79. [ ] Implement a code style guide and enforce with linters +80. [ ] Refactor platform-specific code to improve maintainability + +## Documentation Improvements + +81. [ ] Create comprehensive API documentation +82. [ ] Document the architecture and design decisions +83. [ ] Create developer onboarding documentation +84. [ ] Document the build and release process +85. [ ] Create user guides for all features +86. [ ] Document troubleshooting procedures +87. [ ] Create diagrams for system architecture and data flow +88. [ ] Document security considerations and best practices +89. [ ] Create change management documentation +90. [ ] Document performance tuning recommendations + +## Future Enhancements + +91. [ ] Implement support for additional data sources and integrations +92. [ ] Add machine learning capabilities for anomaly detection +93. [ ] Implement real-time alerting based on log patterns +94. [ ] Create a plugin marketplace for community contributions +95. [ ] Implement advanced data correlation features +96. [ ] Add support for custom log parsing rules +97. [ ] Implement advanced visualization capabilities +98. [ ] Add support for compliance reporting +99. [ ] Implement predictive analytics for security events +100. [ ] Create an ecosystem of complementary tools and integrations \ No newline at end of file diff --git a/agent/logs/utmstack_agent.log b/agent/logs/utmstack_agent.log new file mode 100644 index 000000000..602489253 --- /dev/null +++ b/agent/logs/utmstack_agent.log @@ -0,0 +1 @@ +2025-04-16T13:37:14.854518Z ERROR f18ed04e-83e4-49d7-b82b-b96385b8b0bd /Users/osmany/Projects/UTMStack/agent/serv/service.go 36 error getting config: error reading config file: open /Users/osmany/Projects/UTMStack/agent/config.yml: no such file or directory diff --git a/agent/main.go b/agent/main.go index c733479d6..57bf702d4 100644 --- a/agent/main.go +++ b/agent/main.go @@ -25,6 +25,7 @@ func main() { fmt.Println("Error checking if service is installed: ", err) os.Exit(1) } + if arg != "install" && !isInstalled { fmt.Println("UTMStackAgent service is not installed") os.Exit(1) @@ -62,14 +63,17 @@ func main() { fmt.Println("\nError registering agent: ", err) os.Exit(1) } + if err = config.SaveConfig(cnf); err != nil { fmt.Println("\nError saving config: ", err) os.Exit(1) } + if err = modules.ConfigureCollectorFirstTime(); err != nil { fmt.Println("\nError configuring collector: ", err) os.Exit(1) } + if err = collectors.InstallCollectors(); err != nil { fmt.Println("\nError installing collectors: ", err) os.Exit(1) @@ -80,7 +84,6 @@ func main() { serv.InstallService() fmt.Println("[OK]") fmt.Println("UTMStackAgent service installed correctly") - case "enable-integration", "disable-integration": fmt.Println("Changing integration status ...") integration := os.Args[2] @@ -93,7 +96,6 @@ func main() { } fmt.Printf("Action %s %s %s correctly in port %s\n", arg, integration, proto, port) time.Sleep(5 * time.Second) - case "change-port": fmt.Println("Changing integration port ...") integration := os.Args[2] @@ -107,7 +109,6 @@ func main() { } fmt.Printf("Port changed correctly from %s to %s\n", old, port) time.Sleep(5 * time.Second) - case "uninstall": fmt.Print("Uninstalling UTMStackAgent service ...") @@ -125,10 +126,10 @@ func main() { os.Remove(config.ConfigurationFile) serv.UninstallService() - fmt.Println("[OK]") + fmt.Println("UTMStackAgent service uninstalled correctly") - os.Exit(1) + os.Exit(0) case "help": Help() default: diff --git a/agent/serv/clean-old.go b/agent/serv/clean-old.go index bbe3a53a3..0814da03f 100644 --- a/agent/serv/clean-old.go +++ b/agent/serv/clean-old.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "github.com/utmstack/UTMStack/agent/config" "github.com/utmstack/UTMStack/agent/utils" @@ -14,57 +15,59 @@ func CleanOldServices(cnf *config.Config) { isUpdaterINstalled, err := utils.CheckIfServiceIsInstalled("UTMStackUpdater") if err != nil { - utils.Logger.LogF(100, "Error checking if service is installed: %v", err) + utils.Logger.LogF(100, "error checking if service is installed: %v", err) } if isUpdaterINstalled { oldVersion = true err = utils.StopService("UTMStackUpdater") if err != nil { - utils.Logger.LogF(100, "Error stopping service: %v", err) + utils.Logger.LogF(100, "error stopping service: %v", err) } err = utils.UninstallService("UTMStackUpdater") if err != nil { - utils.Logger.LogF(100, "Error uninstalling service: %v", err) + utils.Logger.LogF(100, "error uninstalling service: %v", err) } } isRedlineInstalled, err := utils.CheckIfServiceIsInstalled("UTMStackRedline") if err != nil { - utils.Logger.LogF(100, "Error checking if service is installed: %v", err) + utils.Logger.LogF(100, "error checking if service is installed: %v", err) } if isRedlineInstalled { oldVersion = true err = utils.StopService("UTMStackRedline") if err != nil { - utils.Logger.LogF(100, "Error stopping service: %v", err) + utils.Logger.LogF(100, "error stopping service: %v", err) } err = utils.UninstallService("UTMStackRedline") if err != nil { - utils.Logger.LogF(100, "Error uninstalling service: %v", err) + utils.Logger.LogF(100, "error uninstalling service: %v", err) } } if oldVersion { - utils.Logger.Info("Old version of agent found, downloading new version") + utils.Logger.Info("old version of agent found, downloading new version") headers := map[string]string{ "key": cnf.AgentKey, "id": fmt.Sprintf("%v", cnf.AgentID), "type": "agent", } - if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, cnf.Server, config.DependenciesPort, fmt.Sprintf(config.UpdaterSelf, "")), headers, fmt.Sprintf(config.UpdaterSelf, "_new"), utils.GetMyPath(), cnf.SkipCertValidation); err != nil { - utils.Logger.LogF(100, "error downloading updater: %v", err) - return + if runtime.GOOS != "darwin" { + if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, cnf.Server, config.DependenciesPort, fmt.Sprintf(config.UpdaterSelf, "")), headers, fmt.Sprintf(config.UpdaterSelf, "_new"), utils.GetMyPath(), cnf.SkipCertValidation); err != nil { + utils.Logger.LogF(100, "error downloading updater: %v", err) + return + } } oldFilePath := filepath.Join(utils.GetMyPath(), fmt.Sprintf(config.UpdaterSelf, "")) newFilePath := filepath.Join(utils.GetMyPath(), fmt.Sprintf(config.UpdaterSelf, "_new")) - utils.Logger.LogF(100, "Renaming %s to %s", newFilePath, oldFilePath) + utils.Logger.LogF(100, "renaming %s to %s", newFilePath, oldFilePath) err := os.Remove(oldFilePath) if err != nil { utils.Logger.LogF(100, "error removing old updater: %v", err) @@ -74,6 +77,6 @@ func CleanOldServices(cnf *config.Config) { utils.Logger.LogF(100, "error renaming updater: %v", err) } } else { - utils.Logger.LogF(100, "No old version of agent found") + utils.Logger.LogF(100, "no old version of agent found") } } diff --git a/agent/serv/run.go b/agent/serv/run.go index d7d864905..b737d6c1e 100644 --- a/agent/serv/run.go +++ b/agent/serv/run.go @@ -8,10 +8,12 @@ import ( func RunService() { svcConfig := GetConfigServ() p := new(program) + newService, err := service.New(p, svcConfig) if err != nil { utils.Logger.Fatal("error creating new service: %v", err) } + err = newService.Run() if err != nil { utils.Logger.Fatal("error running new service: %v", err) diff --git a/agent/updates/dependencies.go b/agent/updates/dependencies.go index a740f3551..a23752828 100644 --- a/agent/updates/dependencies.go +++ b/agent/updates/dependencies.go @@ -12,6 +12,10 @@ import ( ) func DownloadFirstDependencies(address string, authKey string, insecure bool) error { + if runtime.GOOS == "darwin" { + return nil + } + headers := map[string]string{"connection-key": authKey} if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, address, config.DependenciesPort, "version.json"), headers, "version.json", utils.GetMyPath(), insecure); err != nil { diff --git a/agent/utils/services.go b/agent/utils/services.go index f3faf1191..0757f4b16 100644 --- a/agent/utils/services.go +++ b/agent/utils/services.go @@ -53,14 +53,13 @@ func CheckIfServiceIsInstalled(serv string) (bool, error) { err = Execute("sc", path, "query", serv) case "linux": err = Execute("systemctl", path, "status", serv) + case "darwin": + return true, nil default: return false, fmt.Errorf("operative system unknown") } - if err != nil { - return false, nil - } - return true, nil + return err == nil, err } func CreateLinuxService(serviceName string, execStart string) error { From 756066cca91df58e1aeef2fa8254a7730b400f2f Mon Sep 17 00:00:00 2001 From: Yadian Llada Lopez Date: Wed, 16 Apr 2025 15:34:08 -0400 Subject: [PATCH 04/11] feat: Add SELinux configuration for RedHat systems --- installer/system.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/installer/system.go b/installer/system.go index 00fb3e974..e6aa3c95a 100644 --- a/installer/system.go +++ b/installer/system.go @@ -10,6 +10,14 @@ import ( func PrepareSystem(distro string) error { if distro == "redhat" { + if err := utils.RunCmd("setenforce", "0"); err != nil { + return fmt.Errorf("failed to disable SELinux immediately: %v", err) + } + + if err := utils.RunCmd("sed", "-i", "s/^SELINUX=.*/SELINUX=disabled/", "/etc/selinux/config"); err != nil { + return fmt.Errorf("failed to configure permanent SELinux setting: %v", err) + } + if err := utils.RunCmd("systemctl", "disable", "firewalld"); err != nil { return fmt.Errorf("failed to disable firewalld: %v", err) } From 4e505277337cf3d9564502069de8abfe72eb11a5 Mon Sep 17 00:00:00 2001 From: Yorjander Hernandez Vergara Date: Fri, 18 Apr 2025 12:36:12 -0400 Subject: [PATCH 05/11] fix interface agent problem --- agent/collectors/windows_arm64.go | 15 ++++----------- agent/go.mod | 2 +- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/agent/collectors/windows_arm64.go b/agent/collectors/windows_arm64.go index 155f1a4e2..cea5dbda0 100644 --- a/agent/collectors/windows_arm64.go +++ b/agent/collectors/windows_arm64.go @@ -15,7 +15,6 @@ import ( "syscall" "unsafe" - "github.com/threatwinds/go-sdk/plugins" "github.com/threatwinds/validations" "github.com/utmstack/UTMStack/agent/config" "github.com/utmstack/UTMStack/agent/logservice" @@ -209,7 +208,7 @@ func getCollectorsInstances() []Collector { return collectors } -func (w Windows) SendSystemLogs() { +func (w Windows) SendLogs() { errorsChan := make(chan error, 10) callback := func(event *Event) { @@ -218,20 +217,14 @@ func (w Windows) SendSystemLogs() { utils.Logger.ErrorF("error converting event to JSON: %v", err) return } - host, err := os.Hostname() - if err != nil { - utils.Logger.ErrorF("error getting hostname: %v", err) - host = "unknown" - } validatedLog, _, err := validations.ValidateString(eventJSON, false) if err != nil { utils.Logger.LogF(100, "error validating log: %s: %v", eventJSON, err) return } - logservice.LogQueue <- &plugins.Log{ - DataType: string(config.DataTypeWindowsAgent), - DataSource: host, - Raw: validatedLog, + logservice.LogQueue <- logservice.LogPipe{ + Src: string(config.DataTypeWindowsAgent), + Logs: []string{validatedLog}, } } diff --git a/agent/go.mod b/agent/go.mod index e25ed3f7a..ca414eba0 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -12,6 +12,7 @@ require ( github.com/tehmaze/netflow v0.0.0-20240303214733-8c13bb004068 github.com/threatwinds/logger v1.2.1 github.com/threatwinds/validations v1.0.9 + golang.org/x/sys v0.31.0 google.golang.org/grpc v1.71.0 google.golang.org/protobuf v1.36.5 gopkg.in/yaml.v2 v2.4.0 @@ -44,7 +45,6 @@ require ( golang.org/x/arch v0.9.0 // indirect golang.org/x/crypto v0.36.0 // indirect golang.org/x/net v0.37.0 // indirect - golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect From 7b909e91da4999145df696f4b360497cfd3d6e21 Mon Sep 17 00:00:00 2001 From: Yorjander Hernandez Vergara Date: Fri, 18 Apr 2025 12:41:03 -0400 Subject: [PATCH 06/11] add bad gateway page --- installer/nginx.go | 6 +++ installer/templates/nginx.go | 97 ++++++++++++++++++++++++++++++++++++ installer/templates/proxy.go | 12 +++++ installer/utils/os.go | 4 +- 4 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 installer/templates/nginx.go diff --git a/installer/nginx.go b/installer/nginx.go index 3691a5b70..8fc0feea3 100644 --- a/installer/nginx.go +++ b/installer/nginx.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "path" "github.com/utmstack/UTMStack/installer/templates" @@ -41,6 +42,11 @@ func ConfigureNginx(conf *types.Config, stack *types.StackConfig, distro string) return err } + err = utils.WriteToFile(path.Join("/", "etc", "nginx", "html", "custom_502.html"), templates.NginxCustomBadGateway) + if err != nil { + fmt.Printf("Error writing custom 502 page: %v\n", err) + } + switch distro { case "ubuntu": err = utils.GenerateConfig(c, templates.ProxyUbuntu, path.Join("/", "etc", "nginx", "sites-available", "default")) diff --git a/installer/templates/nginx.go b/installer/templates/nginx.go new file mode 100644 index 000000000..e08e46ff1 --- /dev/null +++ b/installer/templates/nginx.go @@ -0,0 +1,97 @@ +package templates + +const NginxCustomBadGateway string = ` + + + + UTMStack - Maintenance + + + + +
+
+ +

We're getting things ready

+

+ UTMStack is currently under maintenance.
Please check back in a few minutes. +

+
+ © UTMStack ©Copyright 2025. All Rights Reserved. +
+
+ + + ` diff --git a/installer/templates/proxy.go b/installer/templates/proxy.go index 9a50b75ad..07dfb4bb0 100644 --- a/installer/templates/proxy.go +++ b/installer/templates/proxy.go @@ -24,6 +24,12 @@ server { proxy_read_timeout 900; } + error_page 502 /custom_502.html; + location = /custom_502.html { + root /etc/nginx/html; + internal; + } + ssl_certificate /utmstack/cert/utm.crt; ssl_certificate_key /utmstack/cert/utm.key; ssl_protocols TLSv1.2 TLSv1.3; @@ -72,6 +78,12 @@ server { proxy_read_timeout 900; } + error_page 502 /custom_502.html; + location = /custom_502.html { + root /etc/nginx/html; + internal; + } + ssl_certificate /utmstack/cert/utm.crt; ssl_certificate_key /utmstack/cert/utm.key; ssl_protocols TLSv1.2 TLSv1.3; diff --git a/installer/utils/os.go b/installer/utils/os.go index b17fd4460..d46d26f6d 100644 --- a/installer/utils/os.go +++ b/installer/utils/os.go @@ -48,7 +48,7 @@ func CreatePathIfNotExist(path string) error { return nil } -func writeToFile(fileName string, body string) error { +func WriteToFile(fileName string, body string) error { file, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.ModePerm) if err != nil { @@ -67,7 +67,7 @@ func WriteYAML(url string, data interface{}) error { return err } - err = writeToFile(url, string(config[:])) + err = WriteToFile(url, string(config[:])) if err != nil { return err } From 1173856fa100d0ad5e2619eaca94b0865defc245 Mon Sep 17 00:00:00 2001 From: Yorjander Hernandez Vergara Date: Sat, 19 Apr 2025 01:15:24 -0400 Subject: [PATCH 07/11] complete macos agent --- agent/agent/incident_response.go | 2 +- agent/collectors/macos_arm64.go | 7 +++--- agent/config/const.go | 4 +-- agent/config/macos_arm64.go | 4 +-- agent/modules/modules.go | 2 +- agent/self/config/macos_arm64.go | 8 ++++++ agent/self/utils/services.go | 34 +++++++++++++++++++++---- agent/updates/dependencies.go | 8 ++---- agent/updates/update.go | 2 +- agent/utils/address.go | 43 +++++++++++++++++++++++++------- agent/utils/os.go | 2 +- agent/utils/services.go | 13 ++++++++-- 12 files changed, 95 insertions(+), 34 deletions(-) create mode 100644 agent/self/config/macos_arm64.go diff --git a/agent/agent/incident_response.go b/agent/agent/incident_response.go index 2ad06f481..478a68102 100644 --- a/agent/agent/incident_response.go +++ b/agent/agent/incident_response.go @@ -119,7 +119,7 @@ func commandProcessor(path string, stream AgentService_AgentStreamClient, cnf *c switch runtime.GOOS { case "windows": result, errB = utils.ExecuteWithResult("cmd.exe", path, "/C", commandPair[0]) - case "linux": + case "linux", "darwin": result, errB = utils.ExecuteWithResult("sh", path, "-c", commandPair[0]) default: utils.Logger.Fatal("unsupported operating system: %s", runtime.GOOS) diff --git a/agent/collectors/macos_arm64.go b/agent/collectors/macos_arm64.go index 36296a5c5..44febabe2 100644 --- a/agent/collectors/macos_arm64.go +++ b/agent/collectors/macos_arm64.go @@ -5,12 +5,13 @@ package collectors import ( "bufio" + "os/exec" + "path/filepath" + "github.com/threatwinds/validations" "github.com/utmstack/UTMStack/agent/config" "github.com/utmstack/UTMStack/agent/logservice" "github.com/utmstack/UTMStack/agent/utils" - "os/exec" - "path/filepath" ) type Darwin struct{} @@ -62,7 +63,7 @@ func (d Darwin) SendLogs() { } logservice.LogQueue <- logservice.LogPipe{ - Src: string(config.DataTypeMacOSAgent), + Src: string(config.DataTypeMacOs), Logs: []string{validatedLog}, } } diff --git a/agent/config/const.go b/agent/config/const.go index 40632244f..550e41197 100644 --- a/agent/config/const.go +++ b/agent/config/const.go @@ -45,7 +45,6 @@ var ( // RedlineServName = "UTMStackRedline" // BatchToSend = 5 - DataTypeMacOSAgent DataType = "macos_agent" DataTypeWindowsAgent DataType = "beats_windows_agent" DataTypeSyslog DataType = "syslog" DataTypeVmware DataType = "vmware" @@ -100,7 +99,6 @@ var ( DataTypeSonicwall: {UDP: "7009", TCP: "7009"}, DataTypeDeceptivebytes: {UDP: "7010", TCP: "7010"}, DataTypeSentinelOne: {UDP: "7012", TCP: "7012"}, - DataTypeMacOs: {UDP: "7015", TCP: "7015"}, DataTypeAix: {UDP: "7016", TCP: "7016"}, DataTypePfsense: {UDP: "7017", TCP: "7017"}, DataTypeFortiweb: {UDP: "7018", TCP: "7018"}, @@ -117,7 +115,7 @@ func GetMessageFormated(host string, msg string) string { func ValidateModuleType(typ string) string { switch DataType(typ) { case DataTypeSyslog, DataTypeVmware, DataTypeEset, DataTypeKaspersky, DataTypeFortinet, DataTypePaloalto, - DataTypeMikrotik, DataTypeSophosXG, DataTypeSonicwall, DataTypeSentinelOne, DataTypeCiscoGeneric, DataTypeMacOs, + DataTypeMikrotik, DataTypeSophosXG, DataTypeSonicwall, DataTypeSentinelOne, DataTypeCiscoGeneric, DataTypeDeceptivebytes, DataTypeAix, DataTypePfsense, DataTypeFortiweb: return "syslog" case DataTypeNetflow: diff --git a/agent/config/macos_arm64.go b/agent/config/macos_arm64.go index 615b84ac0..35533b1ac 100644 --- a/agent/config/macos_arm64.go +++ b/agent/config/macos_arm64.go @@ -4,7 +4,7 @@ package config var ( - UpdaterSelf = "utmstack_updater_self_arm64%s" - ServiceFile = "utmstack_agent_service_arm64%s" + UpdaterSelf = "utmstack_updater_self%s" + ServiceFile = "utmstack_agent_service%s" DependFiles = []string{} ) diff --git a/agent/modules/modules.go b/agent/modules/modules.go index 0ce4eb491..508f2cecc 100644 --- a/agent/modules/modules.go +++ b/agent/modules/modules.go @@ -55,7 +55,7 @@ func StartModules() { if index == -1 { newModule := GetModule(intType) if newModule == nil { - utils.Logger.ErrorF("error getting module %s", intType) + utils.Logger.LogF(100, "error getting module %s", intType) continue } moCache = append(moCache, newModule) diff --git a/agent/self/config/macos_arm64.go b/agent/self/config/macos_arm64.go new file mode 100644 index 000000000..edff9f770 --- /dev/null +++ b/agent/self/config/macos_arm64.go @@ -0,0 +1,8 @@ +//go:build darwin && arm64 +// +build darwin,arm64 + +package config + +var ( + ServiceFile = "utmstack_agent_service%s" +) diff --git a/agent/self/utils/services.go b/agent/self/utils/services.go index b8f273306..b6009d27f 100644 --- a/agent/self/utils/services.go +++ b/agent/self/utils/services.go @@ -16,6 +16,8 @@ func CheckIfServiceIsActive(serv string) (bool, error) { output, errB = ExecuteWithResult("sc", path, "query", serv) case "linux": output, errB = ExecuteWithResult("systemctl", path, "is-active", serv) + case "darwin": + output, errB = ExecuteWithResult("launchctl", path, "list", serv) default: return false, fmt.Errorf("unknown operating system") } @@ -25,13 +27,18 @@ func CheckIfServiceIsActive(serv string) (bool, error) { } serviceStatus := strings.ToLower(strings.TrimSpace(output)) - if runtime.GOOS == "linux" { - return serviceStatus == "active", nil - } else if runtime.GOOS == "windows" { + + switch runtime.GOOS { + case "windows": return strings.Contains(serviceStatus, "running"), nil + case "linux": + return serviceStatus == "active", nil + case "darwin": + // launchctl list returns a JSON-ish block or error.If the service is listed, it's running + return true, nil + default: + return false, fmt.Errorf("unsupported operating system") } - - return false, fmt.Errorf("unsupported operating system") } func RestartService(serv string) error { @@ -66,6 +73,18 @@ func RestartService(serv string) error { return fmt.Errorf("error starting service: %v", err) } } + case "darwin": + plistPath := fmt.Sprintf("/Library/LaunchDaemons/%s.plist", serv) + + if isRunning { + if err := Execute("launchctl", path, "remove", serv); err != nil { + return fmt.Errorf("error stopping macOS service: %v", err) + } + } + + if err := Execute("launchctl", path, "load", plistPath); err != nil { + return fmt.Errorf("error starting macOS service: %v", err) + } } return nil } @@ -83,6 +102,11 @@ func StopService(name string) error { if err != nil { return fmt.Errorf("error stoping service: %v", err) } + case "darwin": + err := Execute("launchctl", path, "remove", name) + if err != nil { + return fmt.Errorf("error stopping macOS service: %v", err) + } } return nil } diff --git a/agent/updates/dependencies.go b/agent/updates/dependencies.go index a23752828..56e59707e 100644 --- a/agent/updates/dependencies.go +++ b/agent/updates/dependencies.go @@ -12,10 +12,6 @@ import ( ) func DownloadFirstDependencies(address string, authKey string, insecure bool) error { - if runtime.GOOS == "darwin" { - return nil - } - headers := map[string]string{"connection-key": authKey} if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, address, config.DependenciesPort, "version.json"), headers, "version.json", utils.GetMyPath(), insecure); err != nil { @@ -44,7 +40,7 @@ func handleDependenciesPostDownload(dependencies []string) error { return fmt.Errorf("error unzipping dependencies: %v", err) } - if runtime.GOOS == "linux" { + if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { if err := utils.Execute("chmod", utils.GetMyPath(), "-R", "777", fmt.Sprintf(config.UpdaterSelf, "")); err != nil { return fmt.Errorf("error executing chmod on %s: %v", fmt.Sprintf(config.UpdaterSelf, ""), err) } @@ -53,7 +49,7 @@ func handleDependenciesPostDownload(dependencies []string) error { if err := os.Remove(filepath.Join(utils.GetMyPath(), file)); err != nil { return fmt.Errorf("error removing file %s: %v", file, err) } - } else if runtime.GOOS == "linux" { + } else if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { if err := utils.Execute("chmod", utils.GetMyPath(), "-R", "777", file); err != nil { return fmt.Errorf("error executing chmod on %s: %v", file, err) } diff --git a/agent/updates/update.go b/agent/updates/update.go index f6ada2f47..e35b09add 100644 --- a/agent/updates/update.go +++ b/agent/updates/update.go @@ -60,7 +60,7 @@ func UpdateDependencies(cnf *config.Config) { continue } - if runtime.GOOS == "linux" { + if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { if err = utils.Execute("chmod", utils.GetMyPath(), "-R", "777", filepath.Join(utils.GetMyPath(), fmt.Sprintf(config.ServiceFile, "_new"))); err != nil { utils.Logger.ErrorF("error executing chmod: %v", err) } diff --git a/agent/utils/address.go b/agent/utils/address.go index ddf0a79db..2c634f808 100644 --- a/agent/utils/address.go +++ b/agent/utils/address.go @@ -3,22 +3,47 @@ package utils import ( "errors" "net" + "strings" ) func GetIPAddress() (string, error) { - addrs, err := net.InterfaceAddrs() - if err != nil { - return "", err + conn, err := net.Dial("udp", "8.8.8.8:80") + if err == nil { + defer conn.Close() + if localAddr, ok := conn.LocalAddr().(*net.UDPAddr); ok { + if ip := localAddr.IP.To4(); ip != nil && !ip.IsLoopback() { + return ip.String(), nil + } + } } - for _, addr := range addrs { - ipNet, ok := addr.(*net.IPNet) - if ok && !ipNet.IP.IsLoopback() { - if ipNet.IP.To4() != nil { - return ipNet.IP.String(), nil + return "", errors.New("failed to get IP address") +} + +func CleanIPAddresses(input string) string { + var clean []string + + addresses := strings.Split(input, ",") + for _, addr := range addresses { + addr = strings.TrimSpace(addr) + + ip := net.ParseIP(addr) + if ip == nil { + if ipNet, _, err := net.ParseCIDR(addr); err == nil { + ip = ipNet } } + + if ip == nil { + continue + } + + if ip.IsLoopback() { + continue + } + + clean = append(clean, ip.String()) } - return "", errors.New("failed to get IP address") + return strings.Join(clean, ",") } diff --git a/agent/utils/os.go b/agent/utils/os.go index 2338880ad..6998f6646 100644 --- a/agent/utils/os.go +++ b/agent/utils/os.go @@ -58,7 +58,7 @@ func GetOsInfo() (*OSInfo, error) { aliases = append(aliases, "") } info.Aliases = strings.Join(aliases, ",") - info.Addresses = strings.Join(hostInfo.Info().IPs, ",") + info.Addresses = CleanIPAddresses(strings.Join(hostInfo.Info().IPs, ",")) return &info, nil } diff --git a/agent/utils/services.go b/agent/utils/services.go index 0757f4b16..dc5565402 100644 --- a/agent/utils/services.go +++ b/agent/utils/services.go @@ -19,6 +19,11 @@ func StopService(name string) error { if err != nil { return fmt.Errorf("error stoping service: %v", err) } + case "darwin": + err := Execute("launchctl", path, "remove", name) + if err != nil { + return fmt.Errorf("error stopping macOS service: %v", err) + } } return nil } @@ -40,11 +45,15 @@ func UninstallService(name string) error { if err != nil { return fmt.Errorf("error uninstalling service: %v", err) } + case "darwin": + Execute("launchctl", path, "remove", name) + Execute("rm", "/Library/LaunchDaemons/"+name+".plist") + Execute("rm", "/Users/"+os.Getenv("USER")+"/Library/LaunchAgents/"+name+".plist") + } return nil } -// CheckIfServiceIsInstalled checks if a service is installed func CheckIfServiceIsInstalled(serv string) (bool, error) { path := GetMyPath() var err error @@ -54,7 +63,7 @@ func CheckIfServiceIsInstalled(serv string) (bool, error) { case "linux": err = Execute("systemctl", path, "status", serv) case "darwin": - return true, nil + err = Execute("launchctl", path, "list", serv) default: return false, fmt.Errorf("operative system unknown") } From 4ff0be533163e7e3c4701d0de62a1601fc4a1dd6 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Mon, 21 Apr 2025 09:02:32 -0500 Subject: [PATCH 08/11] feat(module-integration): add SOC AI model selection field --- .../UtmModuleGroupConfiguration.java | 11 +++++ .../factory/impl/ModuleSocAi.java | 29 +++++++++++ .../types/ModuleConfigurationKey.java | 18 +++++++ ...418001_add_options_module_group_config.xml | 49 +++++++++++++++++++ .../resources/config/liquibase/master.xml | 3 ++ 5 files changed, 110 insertions(+) create mode 100644 backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml diff --git a/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java b/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java index 2e42bfaca..19d2b7f3c 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java @@ -46,6 +46,9 @@ public class UtmModuleGroupConfiguration implements Serializable { @Column(name = "conf_required", nullable = false) private Boolean confRequired; + @Column(name = "conf_options", nullable = false) + private String confOptions; + @JsonIgnore @ManyToOne @JoinColumn(name = "group_id", insertable = false, updatable = false) @@ -134,4 +137,12 @@ public UtmModuleGroup getModuleGroup() { public void setModuleGroup(UtmModuleGroup moduleGroup) { this.moduleGroup = moduleGroup; } + + public String getConfOptions() { + return confOptions; + } + + public void setConfOptions(String confOptions) { + this.confOptions = confOptions; + } } diff --git a/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java b/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java index ddb9d8d44..694a9349d 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java @@ -50,6 +50,7 @@ public List getConfigurationKeys(Long groupId) throws Ex .withConfDataType("password") .withConfRequired(true) .build()); + keys.add(ModuleConfigurationKey.builder() .withGroupId(groupId) .withConfKey("utmstack.socai.incidentCreation") @@ -58,6 +59,7 @@ public List getConfigurationKeys(Long groupId) throws Ex .withConfDataType("bool") .withConfRequired(false) .build()); + keys.add(ModuleConfigurationKey.builder() .withGroupId(groupId) .withConfKey("utmstack.socai.changeAlertStatus") @@ -68,6 +70,33 @@ public List getConfigurationKeys(Long groupId) throws Ex .withConfRequired(false) .build()); + keys.add(ModuleConfigurationKey.builder() + .withGroupId(groupId) + .withConfKey("utmstack.socai.model") + .withConfName("Select AI Model") + .withConfDescription("Choose the AI model that SOC AI will use to analyze alerts.") + .withConfDataType("select") + .withConfRequired(true) + .withConfOptions( + "[" + + "{\"value\": \"gpt-4\", \"label\": \"GPT-4\"}," + + "{\"value\": \"gpt-4-0613\", \"label\": \"GPT-4 (0613)\"}," + + "{\"value\": \"gpt-4-32k\", \"label\": \"GPT-4 32K\"}," + + "{\"value\": \"gpt-4-32k-0613\", \"label\": \"GPT-4 32K (0613)\"}," + + "{\"value\": \"gpt-4-turbo\", \"label\": \"GPT-4 Turbo\"}," + + "{\"value\": \"gpt-4o\", \"label\": \"GPT-4 Omni\"}," + + "{\"value\": \"gpt-4o-mini\", \"label\": \"GPT-4 Omni Mini\"}," + + "{\"value\": \"gpt-4.1\", \"label\": \"GPT-4.1\"}," + + "{\"value\": \"gpt-4.1-mini\", \"label\": \"GPT-4.1 Mini\"}," + + "{\"value\": \"gpt-4.1-nano\", \"label\": \"GPT-4.1 Nano\"}," + + "{\"value\": \"gpt-3.5-turbo\", \"label\": \"GPT-3.5 Turbo\"}," + + "{\"value\": \"gpt-3.5-turbo-0613\", \"label\": \"GPT-3.5 Turbo (0613)\"}," + + "{\"value\": \"gpt-3.5-turbo-16k\", \"label\": \"GPT-3.5 Turbo 16K\"}," + + "{\"value\": \"gpt-3.5-turbo-16k-0613\", \"label\": \"GPT-3.5 Turbo 16K (0613)\"}" + + "]" + ) + .build()); + return keys; } } diff --git a/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java b/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java index 765b67cf3..1e546f795 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java @@ -6,6 +6,8 @@ public class ModuleConfigurationKey { private String confName; private String confDescription; private String confDataType; + private String confOptions; + private Boolean confRequired; private ModuleConfigurationKey() { @@ -59,6 +61,14 @@ public void setConfRequired(Boolean confRequired) { this.confRequired = confRequired; } + public String getConfOptions() { + return confOptions; + } + + public void setConfOptions(String confOptions) { + this.confOptions = confOptions; + } + public static Builder builder() { return new Builder(); } @@ -71,6 +81,8 @@ public static class Builder { private String confDataType; private Boolean confRequired; + private String confOptions; + public Builder withGroupId(Long groupId) { this.groupId = groupId; return this; @@ -101,6 +113,11 @@ public Builder withConfRequired(Boolean confRequired) { return this; } + public Builder withConfOptions(String confOptions) { + this.confOptions = confOptions; + return this; + } + public ModuleConfigurationKey build() { ModuleConfigurationKey key = new ModuleConfigurationKey(); key.setGroupId(groupId); @@ -109,6 +126,7 @@ public ModuleConfigurationKey build() { key.setConfDescription(confDescription); key.setConfDataType(confDataType); key.setConfRequired(confRequired); + key.setConfOptions(confOptions); return key; } } diff --git a/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml b/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml new file mode 100644 index 000000000..7ff458451 --- /dev/null +++ b/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml @@ -0,0 +1,49 @@ + + + + + + ALTER TABLE utm_module_group_configuration ADD COLUMN conf_options TEXT; + INSERT INTO public.utm_module_group_configuration ( + id, + group_id, + conf_key, + conf_value, + conf_name, + conf_description, + conf_data_type, + conf_required, + conf_options + ) VALUES ( + default, + 1, + 'utmstack.socai.model', + 'gpt-4-turbo', + 'Select AI Model', + 'Choose the AI model that SOC AI will use to analyze alerts. Models differ in capability, performance, and cost.', + 'select', + true, + '[ + { "value": "gpt-4", "label": "GPT-4 (Default)" }, + { "value": "gpt-4-0613", "label": "GPT-4 (0613)" }, + { "value": "gpt-4-32k", "label": "GPT-4 32K" }, + { "value": "gpt-4-32k-0613", "label": "GPT-4 32K (0613)" }, + { "value": "gpt-4-turbo", "label": "GPT-4 Turbo" }, + { "value": "gpt-4o", "label": "GPT-4 Omni" }, + { "value": "gpt-4o-mini", "label": "GPT-4 Omni Mini" }, + { "value": "gpt-4.1", "label": "GPT-4.1" }, + { "value": "gpt-4.1-mini", "label": "GPT-4.1 Mini" }, + { "value": "gpt-4.1-nano", "label": "GPT-4.1 Nano" }, + { "value": "gpt-3.5-turbo", "label": "GPT-3.5 Turbo" }, + { "value": "gpt-3.5-turbo-0613", "label": "GPT-3.5 Turbo (0613)" }, + { "value": "gpt-3.5-turbo-16k", "label": "GPT-3.5 Turbo 16K" }, + { "value": "gpt-3.5-turbo-16k-0613", "label": "GPT-3.5 Turbo 16K (0613)" } + ]' + ); + + + + \ No newline at end of file diff --git a/backend/src/main/resources/config/liquibase/master.xml b/backend/src/main/resources/config/liquibase/master.xml index 1642ad210..e4508e78c 100644 --- a/backend/src/main/resources/config/liquibase/master.xml +++ b/backend/src/main/resources/config/liquibase/master.xml @@ -89,4 +89,7 @@ + + + From aa7290859398d3056d27144d2989b00f49344544 Mon Sep 17 00:00:00 2001 From: Osmany Montero Date: Wed, 16 Apr 2025 19:00:47 +0200 Subject: [PATCH 09/11] wip --- agent/config/const.go | 1 + agent/updates/dependencies.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/agent/config/const.go b/agent/config/const.go index 550e41197..6e41c4ae7 100644 --- a/agent/config/const.go +++ b/agent/config/const.go @@ -45,6 +45,7 @@ var ( // RedlineServName = "UTMStackRedline" // BatchToSend = 5 + DataTypeMacOSAgent DataType = "macos_agent" DataTypeWindowsAgent DataType = "beats_windows_agent" DataTypeSyslog DataType = "syslog" DataTypeVmware DataType = "vmware" diff --git a/agent/updates/dependencies.go b/agent/updates/dependencies.go index 56e59707e..9ec17c6a8 100644 --- a/agent/updates/dependencies.go +++ b/agent/updates/dependencies.go @@ -12,6 +12,10 @@ import ( ) func DownloadFirstDependencies(address string, authKey string, insecure bool) error { + if runtime.GOOS == "darwin" { + return nil + } + headers := map[string]string{"connection-key": authKey} if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, address, config.DependenciesPort, "version.json"), headers, "version.json", utils.GetMyPath(), insecure); err != nil { From ad97c50a8f6b063af794127e74cc201c0fb92b3c Mon Sep 17 00:00:00 2001 From: Yorjander Hernandez Vergara Date: Sat, 19 Apr 2025 01:15:24 -0400 Subject: [PATCH 10/11] complete macos agent --- agent/config/const.go | 1 - agent/updates/dependencies.go | 4 ---- 2 files changed, 5 deletions(-) diff --git a/agent/config/const.go b/agent/config/const.go index 6e41c4ae7..550e41197 100644 --- a/agent/config/const.go +++ b/agent/config/const.go @@ -45,7 +45,6 @@ var ( // RedlineServName = "UTMStackRedline" // BatchToSend = 5 - DataTypeMacOSAgent DataType = "macos_agent" DataTypeWindowsAgent DataType = "beats_windows_agent" DataTypeSyslog DataType = "syslog" DataTypeVmware DataType = "vmware" diff --git a/agent/updates/dependencies.go b/agent/updates/dependencies.go index 9ec17c6a8..56e59707e 100644 --- a/agent/updates/dependencies.go +++ b/agent/updates/dependencies.go @@ -12,10 +12,6 @@ import ( ) func DownloadFirstDependencies(address string, authKey string, insecure bool) error { - if runtime.GOOS == "darwin" { - return nil - } - headers := map[string]string{"connection-key": authKey} if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, address, config.DependenciesPort, "version.json"), headers, "version.json", utils.GetMyPath(), insecure); err != nil { From e189f5d6aa0a9d3681363cc948c23d8b1dd21879 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Mon, 21 Apr 2025 09:02:32 -0500 Subject: [PATCH 11/11] feat(module-integration): add SOC AI model selection field --- .../UtmModuleGroupConfiguration.java | 11 +++++ .../factory/impl/ModuleSocAi.java | 29 +++++++++++ .../types/ModuleConfigurationKey.java | 18 +++++++ ...418001_add_options_module_group_config.xml | 49 +++++++++++++++++++ .../resources/config/liquibase/master.xml | 3 ++ 5 files changed, 110 insertions(+) create mode 100644 backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml diff --git a/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java b/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java index 2e42bfaca..19d2b7f3c 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_modules/UtmModuleGroupConfiguration.java @@ -46,6 +46,9 @@ public class UtmModuleGroupConfiguration implements Serializable { @Column(name = "conf_required", nullable = false) private Boolean confRequired; + @Column(name = "conf_options", nullable = false) + private String confOptions; + @JsonIgnore @ManyToOne @JoinColumn(name = "group_id", insertable = false, updatable = false) @@ -134,4 +137,12 @@ public UtmModuleGroup getModuleGroup() { public void setModuleGroup(UtmModuleGroup moduleGroup) { this.moduleGroup = moduleGroup; } + + public String getConfOptions() { + return confOptions; + } + + public void setConfOptions(String confOptions) { + this.confOptions = confOptions; + } } diff --git a/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java b/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java index ddb9d8d44..694a9349d 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSocAi.java @@ -50,6 +50,7 @@ public List getConfigurationKeys(Long groupId) throws Ex .withConfDataType("password") .withConfRequired(true) .build()); + keys.add(ModuleConfigurationKey.builder() .withGroupId(groupId) .withConfKey("utmstack.socai.incidentCreation") @@ -58,6 +59,7 @@ public List getConfigurationKeys(Long groupId) throws Ex .withConfDataType("bool") .withConfRequired(false) .build()); + keys.add(ModuleConfigurationKey.builder() .withGroupId(groupId) .withConfKey("utmstack.socai.changeAlertStatus") @@ -68,6 +70,33 @@ public List getConfigurationKeys(Long groupId) throws Ex .withConfRequired(false) .build()); + keys.add(ModuleConfigurationKey.builder() + .withGroupId(groupId) + .withConfKey("utmstack.socai.model") + .withConfName("Select AI Model") + .withConfDescription("Choose the AI model that SOC AI will use to analyze alerts.") + .withConfDataType("select") + .withConfRequired(true) + .withConfOptions( + "[" + + "{\"value\": \"gpt-4\", \"label\": \"GPT-4\"}," + + "{\"value\": \"gpt-4-0613\", \"label\": \"GPT-4 (0613)\"}," + + "{\"value\": \"gpt-4-32k\", \"label\": \"GPT-4 32K\"}," + + "{\"value\": \"gpt-4-32k-0613\", \"label\": \"GPT-4 32K (0613)\"}," + + "{\"value\": \"gpt-4-turbo\", \"label\": \"GPT-4 Turbo\"}," + + "{\"value\": \"gpt-4o\", \"label\": \"GPT-4 Omni\"}," + + "{\"value\": \"gpt-4o-mini\", \"label\": \"GPT-4 Omni Mini\"}," + + "{\"value\": \"gpt-4.1\", \"label\": \"GPT-4.1\"}," + + "{\"value\": \"gpt-4.1-mini\", \"label\": \"GPT-4.1 Mini\"}," + + "{\"value\": \"gpt-4.1-nano\", \"label\": \"GPT-4.1 Nano\"}," + + "{\"value\": \"gpt-3.5-turbo\", \"label\": \"GPT-3.5 Turbo\"}," + + "{\"value\": \"gpt-3.5-turbo-0613\", \"label\": \"GPT-3.5 Turbo (0613)\"}," + + "{\"value\": \"gpt-3.5-turbo-16k\", \"label\": \"GPT-3.5 Turbo 16K\"}," + + "{\"value\": \"gpt-3.5-turbo-16k-0613\", \"label\": \"GPT-3.5 Turbo 16K (0613)\"}" + + "]" + ) + .build()); + return keys; } } diff --git a/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java b/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java index 765b67cf3..1e546f795 100644 --- a/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java +++ b/backend/src/main/java/com/park/utmstack/domain/application_modules/types/ModuleConfigurationKey.java @@ -6,6 +6,8 @@ public class ModuleConfigurationKey { private String confName; private String confDescription; private String confDataType; + private String confOptions; + private Boolean confRequired; private ModuleConfigurationKey() { @@ -59,6 +61,14 @@ public void setConfRequired(Boolean confRequired) { this.confRequired = confRequired; } + public String getConfOptions() { + return confOptions; + } + + public void setConfOptions(String confOptions) { + this.confOptions = confOptions; + } + public static Builder builder() { return new Builder(); } @@ -71,6 +81,8 @@ public static class Builder { private String confDataType; private Boolean confRequired; + private String confOptions; + public Builder withGroupId(Long groupId) { this.groupId = groupId; return this; @@ -101,6 +113,11 @@ public Builder withConfRequired(Boolean confRequired) { return this; } + public Builder withConfOptions(String confOptions) { + this.confOptions = confOptions; + return this; + } + public ModuleConfigurationKey build() { ModuleConfigurationKey key = new ModuleConfigurationKey(); key.setGroupId(groupId); @@ -109,6 +126,7 @@ public ModuleConfigurationKey build() { key.setConfDescription(confDescription); key.setConfDataType(confDataType); key.setConfRequired(confRequired); + key.setConfOptions(confOptions); return key; } } diff --git a/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml b/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml new file mode 100644 index 000000000..7ff458451 --- /dev/null +++ b/backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml @@ -0,0 +1,49 @@ + + + + + + ALTER TABLE utm_module_group_configuration ADD COLUMN conf_options TEXT; + INSERT INTO public.utm_module_group_configuration ( + id, + group_id, + conf_key, + conf_value, + conf_name, + conf_description, + conf_data_type, + conf_required, + conf_options + ) VALUES ( + default, + 1, + 'utmstack.socai.model', + 'gpt-4-turbo', + 'Select AI Model', + 'Choose the AI model that SOC AI will use to analyze alerts. Models differ in capability, performance, and cost.', + 'select', + true, + '[ + { "value": "gpt-4", "label": "GPT-4 (Default)" }, + { "value": "gpt-4-0613", "label": "GPT-4 (0613)" }, + { "value": "gpt-4-32k", "label": "GPT-4 32K" }, + { "value": "gpt-4-32k-0613", "label": "GPT-4 32K (0613)" }, + { "value": "gpt-4-turbo", "label": "GPT-4 Turbo" }, + { "value": "gpt-4o", "label": "GPT-4 Omni" }, + { "value": "gpt-4o-mini", "label": "GPT-4 Omni Mini" }, + { "value": "gpt-4.1", "label": "GPT-4.1" }, + { "value": "gpt-4.1-mini", "label": "GPT-4.1 Mini" }, + { "value": "gpt-4.1-nano", "label": "GPT-4.1 Nano" }, + { "value": "gpt-3.5-turbo", "label": "GPT-3.5 Turbo" }, + { "value": "gpt-3.5-turbo-0613", "label": "GPT-3.5 Turbo (0613)" }, + { "value": "gpt-3.5-turbo-16k", "label": "GPT-3.5 Turbo 16K" }, + { "value": "gpt-3.5-turbo-16k-0613", "label": "GPT-3.5 Turbo 16K (0613)" } + ]' + ); + + + + \ No newline at end of file diff --git a/backend/src/main/resources/config/liquibase/master.xml b/backend/src/main/resources/config/liquibase/master.xml index 1642ad210..e4508e78c 100644 --- a/backend/src/main/resources/config/liquibase/master.xml +++ b/backend/src/main/resources/config/liquibase/master.xml @@ -89,4 +89,7 @@ + + +