From bdaa5b0d5fe0bb79f7d0170806b904fd8895bc2a Mon Sep 17 00:00:00 2001 From: Osmany Montero Date: Wed, 19 Mar 2025 15:43:36 +0000 Subject: [PATCH 1/4] Remove unused utility functions for cleaner codebase Eliminated unused functions across utils files, such as `DoReq`, `DetectLinuxFamily`, `IsPortUsed`, and `IncrementReconnectTime`. This cleanup reduces clutter and improves maintainability by removing unused or unnecessary methods. --- agent/utils/address.go | 9 ------- agent/utils/delay.go | 8 ------ agent/utils/os.go | 24 ----------------- agent/utils/req.go | 58 ------------------------------------------ 4 files changed, 99 deletions(-) delete mode 100644 agent/utils/req.go diff --git a/agent/utils/address.go b/agent/utils/address.go index a2c86cdd4..ddf0a79db 100644 --- a/agent/utils/address.go +++ b/agent/utils/address.go @@ -22,12 +22,3 @@ func GetIPAddress() (string, error) { return "", errors.New("failed to get IP address") } - -func IsPortUsed(proto string, port string) bool { - conn, err := net.Listen(proto, ":"+port) - if err != nil { - return true - } - conn.Close() - return false -} diff --git a/agent/utils/delay.go b/agent/utils/delay.go index fa4c1f5a6..e3000ea71 100644 --- a/agent/utils/delay.go +++ b/agent/utils/delay.go @@ -9,11 +9,3 @@ func IncrementReconnectDelay(delay time.Duration, maxReconnectDelay time.Duratio } return delay } - -func IncrementReconnectTime(currentTime time.Duration, delay time.Duration, maxReconnectTime time.Duration) time.Duration { - currentTime = currentTime + delay - if currentTime >= maxReconnectTime { - currentTime = maxReconnectTime - } - return currentTime -} diff --git a/agent/utils/os.go b/agent/utils/os.go index a50bb725b..2338880ad 100644 --- a/agent/utils/os.go +++ b/agent/utils/os.go @@ -3,7 +3,6 @@ package utils import ( "fmt" "os" - "os/exec" "os/user" "strconv" "strings" @@ -11,29 +10,6 @@ import ( "github.com/elastic/go-sysinfo" ) -func DetectLinuxFamily() (string, error) { - var pmCommands map[string]string = map[string]string{ - "debian": "apt list", - "rhel": "yum list", - } - - for dist, command := range pmCommands { - cmd := strings.Split(command, " ") - var err error - - if len(cmd) > 1 { - _, err = exec.Command(cmd[0], cmd[1:]...).Output() - } else { - _, err = exec.Command(cmd[0]).Output() - } - - if err == nil { - return dist, nil - } - } - return "", fmt.Errorf("unknown distribution") -} - type OSInfo struct { Hostname string OsType string diff --git a/agent/utils/req.go b/agent/utils/req.go deleted file mode 100644 index 9fe861f46..000000000 --- a/agent/utils/req.go +++ /dev/null @@ -1,58 +0,0 @@ -package utils - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "path/filepath" -) - -func DoReq[response any](url string, data []byte, method string, headers map[string]string, skipTlsVerification bool) (response, int, error) { - var result response - - req, err := http.NewRequest(method, url, bytes.NewBuffer(data)) - if err != nil { - return result, http.StatusInternalServerError, err - } - - for k, v := range headers { - req.Header.Add(k, v) - } - - client := &http.Client{} - if !skipTlsVerification { - tlsConfig, err := LoadHTTPTLSCredentials(filepath.Join(GetMyPath(), "certs", "utm.crt")) - if err != nil { - return result, http.StatusInternalServerError, fmt.Errorf("failed to load TLS credentials: %v", err) - } - client.Transport = &http.Transport{ - TLSClientConfig: tlsConfig, - } - } - - resp, err := client.Do(req) - if err != nil { - return result, http.StatusInternalServerError, err - } - defer func() { - _ = resp.Body.Close() - }() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return result, http.StatusInternalServerError, err - } - - if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK { - return result, resp.StatusCode, fmt.Errorf("while sending request to %s received status code: %d and response body: %s", url, resp.StatusCode, body) - } - - err = json.Unmarshal(body, &result) - if err != nil { - return result, http.StatusInternalServerError, err - } - - return result, resp.StatusCode, nil -} From d45f1ffe6eee8fdc246b60c20ec49b43769e8552 Mon Sep 17 00:00:00 2001 From: Osmany Montero Date: Wed, 19 Mar 2025 15:44:01 +0000 Subject: [PATCH 2/4] Simplify Filebeat setup by removing redundant checks. Removed unnecessary detection of Linux families before executing Filebeat setup commands. This change streamlines the process by directly executing the required commands without conditional branching, improving code readability and maintainability. --- agent/collectors/filebeat.go | 47 +++++++++++++++--------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/agent/collectors/filebeat.go b/agent/collectors/filebeat.go index 6bca22e12..290cf64a7 100644 --- a/agent/collectors/filebeat.go +++ b/agent/collectors/filebeat.go @@ -74,36 +74,29 @@ func (f Filebeat) Install() error { return fmt.Errorf("error reloading daemon: %v", err) } - family, err := utils.DetectLinuxFamily() + err := utils.Execute("systemctl", filebLogPath, "enable", config.ModulesServName) if err != nil { - return err + return fmt.Errorf("%s", err) } - if family == "debian" || family == "rhel" { - err := utils.Execute("systemctl", filebLogPath, "enable", config.ModulesServName) - if err != nil { - return fmt.Errorf("%s", err) - } - - err = utils.Execute("systemctl", filebLogPath, "start", config.ModulesServName) - if err != nil { - return fmt.Errorf("%s", err) - } - - err = utils.Execute("./filebeat", filebLogPath, "modules", "enable", "system") - if err != nil { - return fmt.Errorf("%s", err) - } - - err = utils.Execute("sed", filepath.Join(filebLogPath, "modules.d"), "-i", "s/enabled: false/enabled: true/g", "system.yml") - if err != nil { - return fmt.Errorf("%s", err) - } - - err = utils.Execute("systemctl", filebLogPath, "restart", config.ModulesServName) - if err != nil { - return fmt.Errorf("%s", err) - } + err = utils.Execute("systemctl", filebLogPath, "start", config.ModulesServName) + if err != nil { + return fmt.Errorf("%s", err) + } + + err = utils.Execute("./filebeat", filebLogPath, "modules", "enable", "system") + if err != nil { + return fmt.Errorf("%s", err) + } + + err = utils.Execute("sed", filepath.Join(filebLogPath, "modules.d"), "-i", "s/enabled: false/enabled: true/g", "system.yml") + if err != nil { + return fmt.Errorf("%s", err) + } + + err = utils.Execute("systemctl", filebLogPath, "restart", config.ModulesServName) + if err != nil { + return fmt.Errorf("%s", err) } } } From 824dbc4f8adf1df75284c23144bee04a5a72619f Mon Sep 17 00:00:00 2001 From: Osmany Montero Date: Wed, 19 Mar 2025 15:44:58 +0000 Subject: [PATCH 3/4] Refactor Windows log collectors with architecture-specific logic Replaced `Winlogbeat` with a unified `Windows` type and added distinct implementations for amd64 and arm64 architectures. Introduced a dedicated ARM64 log collector using PowerShell and selective inclusion of Filebeat for AMD64. This ensures better compatibility and maintains architecture-specific functionality. --- agent/collectors/collectors.go | 6 +- .../{winlogbeat.go => windows_amd64.go} | 11 +- agent/collectors/windows_arm64.go | 126 ++++++++++++++++++ 3 files changed, 137 insertions(+), 6 deletions(-) rename agent/collectors/{winlogbeat.go => windows_amd64.go} (94%) create mode 100644 agent/collectors/windows_arm64.go diff --git a/agent/collectors/collectors.go b/agent/collectors/collectors.go index 865a5df6a..2a7ef909a 100644 --- a/agent/collectors/collectors.go +++ b/agent/collectors/collectors.go @@ -22,8 +22,10 @@ func getCollectorsInstances() []Collector { var collectors []Collector switch runtime.GOOS { case "windows": - collectors = append(collectors, Winlogbeat{}) - collectors = append(collectors, Filebeat{}) + collectors = append(collectors, Windows{}) + if runtime.GOARCH == "amd64" { + collectors = append(collectors, Filebeat{}) + } case "linux": collectors = append(collectors, Filebeat{}) } diff --git a/agent/collectors/winlogbeat.go b/agent/collectors/windows_amd64.go similarity index 94% rename from agent/collectors/winlogbeat.go rename to agent/collectors/windows_amd64.go index d39fbf1e6..477d3f064 100644 --- a/agent/collectors/winlogbeat.go +++ b/agent/collectors/windows_amd64.go @@ -1,3 +1,6 @@ +//go:build windows && amd64 +// +build windows,amd64 + package collectors import ( @@ -10,9 +13,9 @@ import ( "github.com/utmstack/UTMStack/agent/utils" ) -type Winlogbeat struct{} +type Windows struct{} -func (w Winlogbeat) Install() error { +func (w Windows) Install() error { path := utils.GetMyPath() winlogPath := filepath.Join(path, "beats", "winlogbeat") @@ -59,7 +62,7 @@ func (w Winlogbeat) Install() error { return nil } -func (w Winlogbeat) SendSystemLogs() { +func (w Windows) SendSystemLogs() { logLinesChan := make(chan []string) path := utils.GetMyPath() winbLogPath := filepath.Join(path, "beats", "winlogbeat", "logs") @@ -82,7 +85,7 @@ func (w Winlogbeat) SendSystemLogs() { } } -func (w Winlogbeat) Uninstall() error { +func (w Windows) Uninstall() error { if isInstalled, err := utils.CheckIfServiceIsInstalled(config.WinServName); err != nil { return fmt.Errorf("error checking if %s is running: %v", config.WinServName, err) } else if isInstalled { diff --git a/agent/collectors/windows_arm64.go b/agent/collectors/windows_arm64.go new file mode 100644 index 000000000..edfa21277 --- /dev/null +++ b/agent/collectors/windows_arm64.go @@ -0,0 +1,126 @@ +//go:build windows && arm64 +// +build windows,arm64 + +package collectors + +import ( + "github.com/utmstack/UTMStack/agent/config" + "github.com/utmstack/UTMStack/agent/logservice" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/threatwinds/validations" + "github.com/utmstack/UTMStack/agent/utils" +) + +type Windows struct{} + +const PowerShellScript = ` +<# +.SYNOPSIS + Collects Windows Application, System, and Security logs from the last 5 minutes and prints each log as a minimal single-line JSON object. + If no logs are found, no output is produced. + +.DESCRIPTION + 1. Retrieves the last 5 minutes of Windows logs (Application, System, Security) using FilterHashtable (no post-fetch filtering). + 2. Transforms TimeCreated into a human-readable format. + 3. Prints each log record as a single-line JSON object with no indentation or extra spacing. +#> + +# Suppress any script errors to avoid cluttering the console output +$ErrorActionPreference = 'SilentlyContinue' + +# Calculate the start time for filtering +$startTime = (Get-Date).AddSeconds(-30) + +# Retrieve logs with filter hashtable +$applicationLogs = Get-WinEvent -FilterHashtable @{ LogName='Application'; StartTime=$startTime } +$systemLogs = Get-WinEvent -FilterHashtable @{ LogName='System'; StartTime=$startTime } +$securityLogs = Get-WinEvent -FilterHashtable @{ LogName='Security'; StartTime=$startTime } + +# Ensure null collections are treated as empty +if (-not $applicationLogs) { $applicationLogs = @() } +if (-not $systemLogs) { $systemLogs = @() } +if (-not $securityLogs) { $securityLogs = @() } + +# Combine them +$recentLogs = $applicationLogs + $systemLogs + $securityLogs + +# If no logs are found, produce no output +if (-not $recentLogs) { + return +} + +# Transform TimeCreated and output as single-line JSON +$processedLogs = $recentLogs | ForEach-Object { + $item = $_ | Select-Object * + $item | Add-Member -MemberType NoteProperty -Name TimeCreated -Value ($_.TimeCreated.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')) -Force + $item +} + +# Print each log record as a single-line, compact JSON object +foreach ($log in $processedLogs) { + # Convert to JSON with normal indentation + $json = $log | ConvertTo-Json -Depth 20 + # Remove all newlines and indentation + $compactJson = $json -replace '(\r?\n\s*)+', '' + # Print to console (standard output) without additional formatting + Write-Output $compactJson +} +` + +func (w Windows) Install() error { + path := utils.GetMyPath() + collectorPath := filepath.Join(path, "collector.ps1") + err := os.WriteFile(collectorPath, []byte(PowerShellScript), 0644) + return err +} + +func (w Windows) SendSystemLogs() { + path := utils.GetMyPath() + collectorPath := filepath.Join(path, "collector.ps1") + + for { + select { + case <-time.After(30 * time.Second): + go func() { + cmd := exec.Command("Powershell.exe", "-File", collectorPath) + + output, err := cmd.Output() + if err != nil { + utils.Logger.ErrorF("error executing powershell script: %v", err) + return + } + + logLines := strings.Split(string(output), "\n") + + validatedLogs := make([]string, 0, len(logLines)) + + for logLine := range logLines { + validatedLog, _, err := validations.ValidateString(logLine, false) + if err != nil { + utils.Logger.ErrorF("error validating log: %s: %v", logLine, err) + continue + } + + validatedLogs = append(validatedLogs, validatedLog) + } + + logservice.LogQueue <- logservice.LogPipe{ + Src: string(config.DataTypeWindowsAgent), + Logs: validatedLogs, + } + }() + } + } +} + +func (w Windows) Uninstall() error { + path := utils.GetMyPath() + collectorPath := filepath.Join(path, "collector.ps1") + err := os.Remove(collectorPath) + return err +} From 2f8366844c46cae724fc1061767f95adcf904cb3 Mon Sep 17 00:00:00 2001 From: Osmany Montero Date: Wed, 19 Mar 2025 20:12:12 +0000 Subject: [PATCH 4/4] Enhance PowerShell script for structured log output Formatted Windows logs to align with Winlogbeat schema, including detailed event metadata and structured fields such as @timestamp, event, and winlog. Added support for processing Properties into key-value pairs under winlog.event_data and ensured consistent single-line JSON output. Improved robustness and clarity by handling null values and suppressing runtime errors. --- agent/collectors/windows_arm64.go | 129 +++++++++++++++++++++++------- 1 file changed, 101 insertions(+), 28 deletions(-) diff --git a/agent/collectors/windows_arm64.go b/agent/collectors/windows_arm64.go index edfa21277..20a5fd3e1 100644 --- a/agent/collectors/windows_arm64.go +++ b/agent/collectors/windows_arm64.go @@ -21,16 +21,23 @@ type Windows struct{} const PowerShellScript = ` <# .SYNOPSIS - Collects Windows Application, System, and Security logs from the last 5 minutes and prints each log as a minimal single-line JSON object. - If no logs are found, no output is produced. + Collects Windows Application, System, and Security logs from the last 5 minutes, then prints them to the console in a compact, single-line JSON format, + emulating the field structure that Winlogbeat typically produces. .DESCRIPTION - 1. Retrieves the last 5 minutes of Windows logs (Application, System, Security) using FilterHashtable (no post-fetch filtering). - 2. Transforms TimeCreated into a human-readable format. - 3. Prints each log record as a single-line JSON object with no indentation or extra spacing. + 1. Retrieves the last 5 minutes of Windows logs (Application, System, Security) using FilterHashtable (no post-fetch filtering). + 2. Maps event properties to a schema similar to Winlogbeat's, including: + - @timestamp + - message + - event.code + - event.provider + - event.kind + - winlog fields (e.g. record_id, channel, activity_id, etc.) + 3. Prints each log record as a single-line JSON object with no indentation/extra spacing. + 4. If no logs are found, the script produces no output at all. #> -# Suppress any script errors to avoid cluttering the console output +# Suppress any runtime errors that would clutter the console. $ErrorActionPreference = 'SilentlyContinue' # Calculate the start time for filtering @@ -38,37 +45,103 @@ $startTime = (Get-Date).AddSeconds(-30) # Retrieve logs with filter hashtable $applicationLogs = Get-WinEvent -FilterHashtable @{ LogName='Application'; StartTime=$startTime } -$systemLogs = Get-WinEvent -FilterHashtable @{ LogName='System'; StartTime=$startTime } -$securityLogs = Get-WinEvent -FilterHashtable @{ LogName='Security'; StartTime=$startTime } +$systemLogs = Get-WinEvent -FilterHashtable @{ LogName='System'; StartTime=$startTime } +$securityLogs = Get-WinEvent -FilterHashtable @{ LogName='Security'; StartTime=$startTime } -# Ensure null collections are treated as empty +# Safeguard against null results if (-not $applicationLogs) { $applicationLogs = @() } -if (-not $systemLogs) { $systemLogs = @() } -if (-not $securityLogs) { $securityLogs = @() } +if (-not $systemLogs) { $systemLogs = @() } +if (-not $securityLogs) { $securityLogs = @() } # Combine them $recentLogs = $applicationLogs + $systemLogs + $securityLogs -# If no logs are found, produce no output +# If no logs are found, produce no output at all if (-not $recentLogs) { - return + return } -# Transform TimeCreated and output as single-line JSON -$processedLogs = $recentLogs | ForEach-Object { - $item = $_ | Select-Object * - $item | Add-Member -MemberType NoteProperty -Name TimeCreated -Value ($_.TimeCreated.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')) -Force - $item +# Function to convert the raw Properties array to a dictionary-like object under winlog.event_data +function Convert-PropertiesToEventData { + param([Object[]] $Properties) + + # If nothing is there, return an empty hashtable + if (-not $Properties) { return @{} } + + # Winlogbeat places custom fields under winlog.event_data. + # Typically, it tries to parse known keys, but we'll do a simple best-effort approach: + # We'll create paramN = pairs for each array index. + $eventData = [ordered]@{} + + for ($i = 0; $i -lt $Properties.Count; $i++) { + $value = $Properties[$i].Value + + # If the property is itself an object with nested fields, we can flatten or store as-is. + # We'll store as-is for clarity. + # We'll name them param1, param2, param3,... unless you'd like more specific field logic. + $paramName = "param$($i+1)" + + $eventData[$paramName] = $value + } + + return $eventData } -# Print each log record as a single-line, compact JSON object -foreach ($log in $processedLogs) { - # Convert to JSON with normal indentation - $json = $log | ConvertTo-Json -Depth 20 - # Remove all newlines and indentation - $compactJson = $json -replace '(\r?\n\s*)+', '' - # Print to console (standard output) without additional formatting - Write-Output $compactJson +# Transform each event into a structure emulating Winlogbeat +foreach ($rawEvent in $recentLogs) { + # Convert TimeCreated to a universal ISO8601 string + $timestamp = $rawEvent.TimeCreated.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + + # Build the top-level document + $doc = [ordered]@{ + # Matches Winlogbeat's typical top-level timestamp field + '@timestamp' = $timestamp + + # The main message content from the event + 'message' = $rawEvent.Message + + # "event" block: minimal example + 'event' = [ordered]@{ + 'code' = $rawEvent.Id # event_id in Winlogbeat is typically a string or numeric + 'provider' = $rawEvent.ProviderName + 'kind' = 'event' + 'created' = $timestamp # or you could omit if desired + } + + # "winlog" block: tries to mirror Winlogbeat's structure for Windows + 'winlog' = [ordered]@{ + 'record_id' = $rawEvent.RecordId + 'computer_name' = $rawEvent.MachineName + 'channel' = $rawEvent.LogName + 'provider_name' = $rawEvent.ProviderName + 'provider_guid' = $rawEvent.ProviderId + 'process' = [ordered]@{ + 'pid' = $rawEvent.ProcessId + 'thread' = @{ + 'id' = $rawEvent.ThreadId + } + } + 'event_id' = $rawEvent.Id + 'version' = $rawEvent.Version + 'activity_id' = $rawEvent.ActivityId + 'related_activity_id'= $rawEvent.RelatedActivityId + 'task' = $rawEvent.TaskDisplayName + 'opcode' = $rawEvent.OpcodeDisplayName + 'keywords' = $rawEvent.KeywordsDisplayNames + 'time_created' = $timestamp + # Convert "Properties" into a dictionary for event_data + 'event_data' = Convert-PropertiesToEventData $rawEvent.Properties + } + } + + # Convert our object to JSON (with no extra formatting). + $json = $doc | ConvertTo-Json -Depth 20 + + # Remove all newlines and indentation for a single-line representation + $compactJson = $json -replace '(\r?\n\s*)+', '' + + # Output the line + Write-Output $compactJson } ` @@ -91,7 +164,7 @@ func (w Windows) SendSystemLogs() { output, err := cmd.Output() if err != nil { - utils.Logger.ErrorF("error executing powershell script: %v", err) + _ = utils.Logger.ErrorF("error executing powershell script: %v", err) return } @@ -102,7 +175,7 @@ func (w Windows) SendSystemLogs() { for logLine := range logLines { validatedLog, _, err := validations.ValidateString(logLine, false) if err != nil { - utils.Logger.ErrorF("error validating log: %s: %v", logLine, err) + _ = utils.Logger.ErrorF("error validating log: %s: %v", logLine, err) continue }