Bugfix/10.8.0/macos

.github/workflows/principal-multi-env.yml

@@ -88,7 +88,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix: 
-        service: ['aws', 'backend', 'correlation', 'frontend', 'bitdefender', 'mutate', 'office365', 'log-auth-proxy', 'sophos', 'user-auditor', 'web-pdf'] 
+        service: ['aws', 'backend', 'correlation', 'frontend', 'bitdefender', 'mutate', 'office365', 'log-auth-proxy', 'soc-ai', 'sophos', 'user-auditor', 'web-pdf'] 
     uses: ./.github/workflows/used-runner.yml
     with:
       microservice: ${{ matrix.service }}

.github/workflows/used-runner.yml

@@ -22,7 +22,7 @@ jobs:
         id: get_tech
         run: |
           folder_changed="${{inputs.microservice}}"
-          if [[ "$folder_changed" == "aws" || "$folder_changed" == "correlation" || "$folder_changed" == "bitdefender" || "$folder_changed" == "office365" || "$folder_changed" == "sophos" || "$folder_changed" == "log-auth-proxy" ]]; then
+          if [[ "$folder_changed" == "aws" || "$folder_changed" == "correlation" || "$folder_changed" == "bitdefender" || "$folder_changed" == "office365" || "$folder_changed" == "soc-ai" || "$folder_changed" == "sophos" || "$folder_changed" == "log-auth-proxy" ]]; then
             tech="golang"
           elif [[ "$folder_changed" == "backend" ]]; then
             tech="java-11"

CHANGELOG.md

@@ -1,8 +1,8 @@
-# UTMStack 10.7.3 Release Notes
--- Implemented backend support for filtering compliance reports based on active integrations, optimizing query performance and data retrieval.
--- Introduced new compliance reports aligned with the PCI DSS standard to expand auditing capabilities.
--- Added support for creating and updating tag-based rules with dynamic conditions.
-
-### Bug Fixes
--- Improved exception handling in `automaticReview` to prevent the process from stopping due to errors, ensuring the system continues evaluating alerts even if a specific rule fails.
--- Improved operator selection for more accurate and consistent filtering.
+# UTMStack 10.8.0 Release Notes
+- Updated Soc-AI models and released the code as open source.
+- Added the ability for users to choose which model to use with Soc-AI.
+- Enhanced the prompt sent to OpenAI by including additional contextual details.
+- Added support for RedHat; UTMStack can now be installed on both Ubuntu and RedHat.
+- Improved log delivery from ARM-based agents on Windows, now sending native system logs.
+- Added support for macOS ARM64; agents can now be installed on that platform.
+- Improved agent information displayed in the Sources panel, providing more accurate OS details and agent versions.

backend/src/main/resources/config/liquibase/changelog/20250418001_add_options_module_group_config.xml

@@ -27,20 +27,22 @@
                          'select',
                          true,
                          '[
-                           { "value": "gpt-4", "label": "GPT-4 (Default)" },
-                           { "value": "gpt-4-0613", "label": "GPT-4 (0613)" },
-                           { "value": "gpt-4-32k", "label": "GPT-4 32K" },
-                           { "value": "gpt-4-32k-0613", "label": "GPT-4 32K (0613)" },
-                           { "value": "gpt-4-turbo", "label": "GPT-4 Turbo" },
-                           { "value": "gpt-4o", "label": "GPT-4 Omni" },
-                           { "value": "gpt-4o-mini", "label": "GPT-4 Omni Mini" },
-                           { "value": "gpt-4.1", "label": "GPT-4.1" },
-                           { "value": "gpt-4.1-mini", "label": "GPT-4.1 Mini" },
-                           { "value": "gpt-4.1-nano", "label": "GPT-4.1 Nano" },
-                           { "value": "gpt-3.5-turbo", "label": "GPT-3.5 Turbo" },
-                           { "value": "gpt-3.5-turbo-0613", "label": "GPT-3.5 Turbo (0613)" },
-                           { "value": "gpt-3.5-turbo-16k", "label": "GPT-3.5 Turbo 16K" },
-                           { "value": "gpt-3.5-turbo-16k-0613", "label": "GPT-3.5 Turbo 16K (0613)" }
+                           { "value": "gpt-4.1", "label": "GPT-4.1 Model" },
+                           { "value": "gpt-4.1-mini", "label": "GPT-4.1 Mini Model" },
+                           { "value": "gpt-4.1-nano", "label": "GPT-4.1 Nano Model" },
+                           { "value": "gpt-4o", "label": "GPT-4 Omni Model" },
+                           { "value": "gpt-4o-mini", "label": "GPT-4 Omni Mini Model" },
+                           { "value": "gpt-4-turbo", "label": "GPT-4 Turbo Model" },
+                           { "value": "gpt-4-0614", "label": "GPT-4 Model (0614)" },
+                           { "value": "gpt-4-0125-preview", "label": "GPT-4 Model (0125 Preview)" },
+                           { "value": "gpt-3.5-turbo", "label": "GPT-3.5 Turbo Model" },
+                           { "value": "gpt-3.5-turbo-instruct", "label": "GPT-3.5 Turbo Instruct Model" },
+                           { "value": "gpt-3.5-turbo-1106", "label": "GPT-3.5 Turbo Model (1106)" },
+                           { "value": "o1", "label": "O1 Model" },
+                           { "value": "o1-pro", "label": "O1 Pro Model" },
+                           { "value": "o3", "label": "O3 Model" },
+                           { "value": "o3-mini", "label": "O3 Mini Model" },
+                           { "value": "o4-mini", "label": "O4 Mini Model" }
                          ]'
                      );
 

correlation/cache/operators.go

@@ -88,7 +88,7 @@ func compare(operator, val1, val2 string) bool {
 		return !lowerEqual(val1, val2)
 	case "contains":
 		return contain(val1, val2)
-	case "not contain":
+	case "not contain", "not contains":
 		return !contain(val1, val2)
 	case "in":
 		return in(val1, val2)

frontend/src/app/shared/components/utm/util/utm-agent-detail/utm-agent-detail.component.html

@@ -56,9 +56,13 @@
     <app-assets-apply-note *ngIf="asset" [asset]="asset" class="ml-2"></app-assets-apply-note>
   </div>
   <div class="agent-details w-100 d-flex justify-content-start mb-2" *ngIf="agent.version">
-    <span class="text-blue-800 font-weight-light has-minimum-width">OS Version:</span>&nbsp;
+    <span class="text-blue-800 font-weight-light has-minimum-width">Agent Version:</span>&nbsp;
     {{agent.version}}
   </div>
+  <div class="agent-details w-100 d-flex justify-content-start mb-2" *ngIf="agent.osMajorVersion && agent.osMinorVersion">
+    <span class="text-blue-800 font-weight-light has-minimum-width">OS Version:</span>&nbsp;
+    {{agent.osMajorVersion + '.' + agent.osMinorVersion}}
+  </div>
   <div class="agent-details w-100 d-flex justify-content-start mb-2" *ngIf="agent.lastSeen">
     <span class="text-blue-800 font-weight-light has-minimum-width">Last seen:</span>&nbsp;
     {{agent.lastSeen}}

installer/types/compose.go

@@ -507,7 +507,7 @@ func (c *Compose) Populate(conf *Config, stack *StackConfig) *Compose {
 
 	socAIMem := stack.ServiceResources["socai"].AssignedMemory
 	c.Services["socai"] = Service{
-		Image: utils.Str("ghcr.io/utmstack/soc-ai/soc-ai:" + conf.Branch),
+		Image: utils.Str("ghcr.io/utmstack/utmstack/soc-ai:" + conf.Branch),
 		DependsOn: []string{
 			"node1",
 			"backend",

installer/utils/os.go

@@ -49,6 +49,11 @@ func CreatePathIfNotExist(path string) error {
 }
 
 func WriteToFile(fileName string, body string) error {
+	filePath := filepath.Dir(fileName)
+	if err := CreatePathIfNotExist(filePath); err != nil {
+		return fmt.Errorf("error creating directory for file %s: %v", fileName, err)
+	}
+
 	file, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.ModePerm)
 
 	if err != nil {

soc-ai/Dockerfile

@@ -0,0 +1,13 @@
+FROM ubuntu:24.04
+
+COPY soc-ai /app/
+
+RUN apt-get update && \
+    apt-get install -y ca-certificates jq wget && \
+    update-ca-certificates && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+EXPOSE 8080
+
+CMD ["/app/soc-ai"]

soc-ai/configurations/config.go

@@ -0,0 +1,71 @@
+package configurations
+
+import (
+	"time"
+
+	UTMStackConfigurationClient "github.com/utmstack/config-client-go"
+	"github.com/utmstack/config-client-go/enum"
+	"github.com/utmstack/soc-ai/utils"
+)
+
+var (
+	gptConfig GPTConfig
+)
+
+type GPTConfig struct {
+	APIKey                    string
+	ChangeAlertStatus         bool
+	AutomaticIncidentCreation bool
+	Model                     string
+	ModuleActive              bool
+}
+
+func GetGPTConfig() *GPTConfig {
+	return &gptConfig
+}
+
+func UpdateGPTConfigurations() {
+	intKey := GetInternalKey()
+	panelServ := GetPanelServiceName()
+	client := UTMStackConfigurationClient.NewUTMClient(intKey, panelServ)
+
+	for {
+		if err := utils.ConnectionChecker(GPT_API_ENDPOINT); err != nil {
+			utils.Logger.ErrorF("Failed to establish internet connection: %v", err)
+		}
+
+		tempModuleConfig, err := client.GetUTMConfig(enum.SOCAI)
+		if err != nil && err.Error() != "" && err.Error() != " " {
+			utils.Logger.LogF(100, "Error while getting GPT configuration: %v", err)
+			time.Sleep(TIME_FOR_GET_CONFIG * time.Second)
+			continue
+		}
+
+		gptConfig.ModuleActive = tempModuleConfig.ModuleActive
+
+		if gptConfig.ModuleActive && tempModuleConfig != nil && len(tempModuleConfig.ConfigurationGroups) > 0 {
+			for _, config := range tempModuleConfig.ConfigurationGroups[0].Configurations {
+				switch config.ConfKey {
+				case "utmstack.socai.key":
+					if config.ConfValue != "" && config.ConfValue != " " {
+						gptConfig.APIKey = config.ConfValue
+					}
+				case "utmstack.socai.incidentCreation":
+					if config.ConfValue != "" && config.ConfValue != " " {
+						gptConfig.AutomaticIncidentCreation = config.ConfValue == "true"
+					}
+				case "utmstack.socai.changeAlertStatus":
+					if config.ConfValue != "" && config.ConfValue != " " {
+						gptConfig.ChangeAlertStatus = config.ConfValue == "true"
+					}
+				case "utmstack.socai.model":
+					if config.ConfValue != "" && config.ConfValue != " " {
+						gptConfig.Model = config.ConfValue
+					}
+				}
+			}
+		}
+
+		time.Sleep(TIME_FOR_GET_CONFIG * time.Second)
+	}
+}

soc-ai/configurations/const.go

@@ -0,0 +1,89 @@
+package configurations
+
+import (
+	"path/filepath"
+
+	"github.com/utmstack/soc-ai/utils"
+)
+
+const (
+	SOC_AI_SERVER_PORT                  = "8080"
+	SOC_AI_SERVER_ENDPOINT              = "/process"
+	API_ALERT_ENDPOINT                  = "/api/elasticsearch/search"
+	API_ALERT_STATUS_ENDPOINT           = "/api/utm-alerts/status"
+	API_INCIDENT_ENDPOINT               = "/api/utm-incidents"
+	API_INCIDENT_ADD_NEW_ALERT_ENDPOINT = "/api/utm-incidents/add-alerts"
+	API_ALERT_COMPLETED_STATUS_CODE     = 5
+	API_ALERT_INFO_PARAMS               = "?page=0&size=25&top=10000&indexPattern="
+	ELASTIC_DOC_ENDPOINT                = "/_doc/"
+	ELASTIC_UPDATE_BY_QUERY_ENDPOINT    = "/_update_by_query"
+	ALERT_INDEX_PATTERN                 = "alert-*"
+	LOGS_INDEX_PATTERN                  = "log-*"
+	SOC_AI_INDEX                        = "soc-ai"
+	GPT_API_ENDPOINT                    = "https://api.openai.com/v1/chat/completions"
+	TIME_FOR_GET_CONFIG                 = 10
+	CLEANER_DELAY                       = 10
+	MAX_ATTEMPS_TO_GPT                  = 3
+	GPT_RESPONSE_TOKENS                 = 10
+	HTTP_GPT_TIMEOUT                    = 90
+	HTTP_TIMEOUT                        = 30
+	LOGS_SEPARATOR                      = "[utm-logs-separator]"
+)
+
+var (
+	AllowedGPTModels = map[string]int{
+		"gpt-4.1":                1047576,
+		"gpt-4.1-mini":           1047576,
+		"gpt-4.1-nano":           1047576,
+		"gpt-4o":                 128000,
+		"gpt-4o-mini":            128000,
+		"gpt-4-turbo":            128000,
+		"gpt-4-0614":             8192,
+		"gpt-4-0125-preview":     128000,
+		"gpt-3.5-turbo":          16385,
+		"gpt-3.5-turbo-instruct": 4096,
+		"gpt-3.5-turbo-1106":     16385,
+		"o1":                     200000,
+		"o1-pro":                 200000,
+		"o3":                     200000,
+		"o3-mini":                200000,
+		"o4-mini":                200000,
+		// "gpt-4-0314":             8192, // Removed 2024-06-13
+		// "gpt-4-1106-preview":     128000, // Removed 2024-12-06
+	}
+)
+
+type SensitivePattern struct {
+	Regexp    string
+	FakeValue string
+}
+
+var (
+	SensitivePatterns = map[string]SensitivePattern{
+		"email": {Regexp: `([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})`, FakeValue: "[email protected]"},
+		//"ipv4":  `(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)`,
+	}
+	GPT_INSTRUCTION    = "You are an expert security engineer. Perform a deep analysis of an alert created by a SIEM and the logs related to it. Determine if the alert could be an actual potential threat or not and explain why. Provide a description that shows a deep understanding of the alert based on a deep analysis of its logs and estimate the risk to the systems affected. Classify the alert in the following manner: if the alert information is sufficient to determine that the security, availability, confidentiality, or integrity of the systems has being compromised, then classify it as \"possible incident\". If the alert does not pose a security risk to the organization or has no security relevance, classify it as \"possible false positive\". If the alert does not pose an imminent risk to the systems, requires no urgent action from an administrator, or requires not urgent review by an administrator, it should be classified as a \"standard alert\". You will also provide context-specific instructions for remediation, mitigation, or further investigation, related to the alert and logs analyzed. Your answer should be provided using the following JSON format and the total number of characters in your answer must not exceed 1500 words. Your entire answer must be inside this json format. {\"activity_id\":\"<activity_id>\",\"classification\":\"<classification>\",\"reasoning\":[\"<deep_reasoning>\"],\"nextSteps\":[{\"step\":1,\"action\":\"<action_1>\",\"details\":\"<action_1_details>\"},{\"step\":2,\"action\":\"<action_2>\",\"details\":\"<action_2_details>\"},{\"step\":3,\"action\":\"<action_3>\"]}Ensure that your entire answer adheres to the provided JSON format. The response should be valid JSON syntax and schema."
+	GPT_FALSE_POSITIVE = "This alert is categorized as a potential false positive due to two key factors. Firstly, it originates from an automated system, which may occasionally produce alerts without direct human validation. Additionally, the absence of any correlated logs further raises suspicion, as a genuine incident typically leaves a trail of relevant log entries. Hence, the combination of its system-generated nature and the lack of associated logs suggests a likelihood of being a false positive rather than a genuine security incident."
+)
+
+func GetInternalKey() string {
+	return utils.Getenv("INTERNAL_KEY", true)
+}
+
+func GetPanelServiceName() string {
+	return utils.Getenv("PANEL_SERV_NAME", true)
+}
+
+func GetOpenSearchHost() string {
+	return "http://" + utils.Getenv("OPENSEARCH_HOST", true)
+}
+
+func GetOpenSearchPort() string {
+	return utils.Getenv("OPENSEARCH_PORT", true)
+}
+
+func GetAlertsDBPath() string {
+	path, _ := utils.GetMyPath()
+	return filepath.Join(path, "database", "alerts.sqlite3")
+}

soc-ai/elastic/alerts.go

@@ -0,0 +1,67 @@
+package elastic
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/utmstack/soc-ai/configurations"
+	"github.com/utmstack/soc-ai/schema"
+	"github.com/utmstack/soc-ai/utils"
+)
+
+func GetAlertsInfo(id string) (schema.Alert, error) {
+	result, err := ElasticSearch(configurations.ALERT_INDEX_PATTERN, "id", id)
+	if err != nil {
+		return schema.Alert{}, fmt.Errorf("error while getting alert %s info: %v", id, err)
+	}
+
+	var alertDetails schema.AlertDetails
+	err = json.Unmarshal(result, &alertDetails)
+	if err != nil {
+		return schema.Alert{}, fmt.Errorf("error decoding response: %v", err)
+	}
+
+	if len(alertDetails) == 0 {
+		return schema.Alert{}, fmt.Errorf("no alert found for id")
+	}
+
+	if len(alertDetails[0].Logs) > 0 {
+		var logs []string
+		if len(alertDetails[0].Logs) > 3 {
+			logs = alertDetails[0].Logs[:3]
+		} else {
+			logs = alertDetails[0].Logs
+		}
+
+		for i, log := range logs {
+			resp, err := ElasticSearch(configurations.LOGS_INDEX_PATTERN, "id", log)
+			if err != nil {
+				continue
+			}
+			alertDetails[0].Logs[i] = string(resp)
+		}
+	}
+	return alertDetails[0], nil
+}
+
+func ChangeAlertStatus(id string, status int, observations string) error {
+	url := configurations.GetPanelServiceName() + configurations.API_ALERT_STATUS_ENDPOINT
+	headers := map[string]string{
+		"Content-Type":     "application/json",
+		"Utm-Internal-Key": configurations.GetInternalKey(),
+	}
+
+	body := schema.ChangeAlertStatus{AlertIDs: []string{id}, Status: status, StatusObservation: observations}
+	bodyBytes, err := json.Marshal(body)
+	if err != nil {
+		return fmt.Errorf("error marshalling body: %v", err)
+	}
+
+	resp, statusCode, err := utils.DoReq(url, bodyBytes, "POST", headers, configurations.HTTP_TIMEOUT)
+	if err != nil || statusCode != http.StatusOK {
+		return fmt.Errorf("error while doing request: %v, status: %d, response: %v", err, statusCode, string(resp))
+	}
+
+	return nil
+}