diff --git a/.github/workflows/principal-multi-env.yml b/.github/workflows/principal-multi-env.yml index 5cbeedfdc..73beeacf9 100644 --- a/.github/workflows/principal-multi-env.yml +++ b/.github/workflows/principal-multi-env.yml @@ -54,9 +54,9 @@ jobs: $env:GOARCH = "amd64" go build -o utmstack_agent_service.exe -v . signtool sign /fd SHA256 /tr http://timestamp.digicert.com /td SHA256 /f "${{ vars.SIGN_CERT }}" /csp "eToken Base Cryptographic Provider" /k "[{{${{ secrets.SIGN_KEY }}}}]=${{ secrets.SIGN_CONTAINER }}" "utmstack_agent_service.exe" - # $env:GOARCH = "arm64" - # go build -o utmstack_agent_service_arm64.exe -v . - # signtool sign /fd SHA256 /tr http://timestamp.digicert.com /td SHA256 /f "${{ vars.SIGN_CERT }}" /csp "eToken Base Cryptographic Provider" /k "[{{${{ secrets.SIGN_KEY }}}}]=${{ secrets.SIGN_CONTAINER }}" "utmstack_agent_service_arm64.exe" + $env:GOARCH = "arm64" + go build -o utmstack_agent_service_arm64.exe -v . + signtool sign /fd SHA256 /tr http://timestamp.digicert.com /td SHA256 /f "${{ vars.SIGN_CERT }}" /csp "eToken Base Cryptographic Provider" /k "[{{${{ secrets.SIGN_KEY }}}}]=${{ secrets.SIGN_CONTAINER }}" "utmstack_agent_service_arm64.exe" echo "Agent build completed" @@ -74,7 +74,7 @@ jobs: Copy-Item -Path "${{ github.workspace }}/agent/utmstack_agent_service" -Destination "./dependencies/agent/" # Copy-Item -Path "${{ github.workspace }}/agent/utmstack_agent_service_arm64" -Destination "./dependencies/agent/" Copy-Item -Path "${{ github.workspace }}/agent/utmstack_agent_service.exe" -Destination "./dependencies/agent/" - # Copy-Item -Path "${{ github.workspace }}/agent/utmstack_agent_service_arm64.exe" -Destination "./dependencies/agent/" + Copy-Item -Path "${{ github.workspace }}/agent/utmstack_agent_service_arm64.exe" -Destination "./dependencies/agent/" Copy-Item -Path "${{ github.workspace }}/agent/version.json" -Destination "./dependencies/agent/" docker build -t ghcr.io/utmstack/utmstack/agent-manager:${{ needs.setup_deployment.outputs.env_version }} . diff --git a/CHANGELOG.md b/CHANGELOG.md index 1992b0f16..e4b1ddbdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# UTMStack 10.6.3 Release Notes +# UTMStack 10.7.0 Release Notes ## New Features and Improvements - **Agent & Collector Dependencies**: agents and collectors now fetch their dependencies from the **agent-manager**, improving consistency and centralizing dependency management. @@ -7,3 +7,9 @@ - **Agent Service Cleanup**: removed unnecessary services to streamline the system and reduce overhead. - **Error Recovery**: enhanced the agent's ability to recover from certain data streaming errors when interacting with the agent-manager, improving stability and fault tolerance. + +- **Debug Mode for Agents**: Added a debug mode for agents, allowing better troubleshooting and logging for debugging purposes. + +- **Certificate Verification Improvements**: Improved certificate verification in agents to enhance security and prevent connection issues. + +- **Windows ARM64 Agent Support**: Added support for a Windows ARM64 agent, expanding compatibility to more architectures. \ No newline at end of file diff --git a/agent/agent/incident_response.go b/agent/agent/incident_response.go index 77cec794e..2ad06f481 100644 --- a/agent/agent/incident_response.go +++ b/agent/agent/incident_response.go @@ -35,6 +35,8 @@ func IncidentResponseStream(cnf *config.Config, ctx context.Context) { if !connErrMsgWritten { utils.Logger.ErrorF("failed to start AgentStream: %v", err) connErrMsgWritten = true + } else { + utils.Logger.LogF(100, "failed to start AgentStream: %v", err) } time.Sleep(timeToSleep) continue @@ -46,6 +48,7 @@ func IncidentResponseStream(cnf *config.Config, ctx context.Context) { in, err := stream.Recv() if err != nil { if strings.Contains(err.Error(), "EOF") { + utils.Logger.LogF(100, "error receiving command from server: %v", err) time.Sleep(timeToSleep) break } @@ -54,6 +57,8 @@ func IncidentResponseStream(cnf *config.Config, ctx context.Context) { if !errorLogged { utils.Logger.ErrorF("error receiving command from server: %v", err) errorLogged = true + } else { + utils.Logger.LogF(100, "error receiving command from server: %v", err) } time.Sleep(timeToSleep) break @@ -61,6 +66,8 @@ func IncidentResponseStream(cnf *config.Config, ctx context.Context) { if !errorLogged { utils.Logger.ErrorF("error receiving command from server: %v", err) errorLogged = true + } else { + utils.Logger.LogF(100, "error receiving command from server: %v", err) } time.Sleep(timeToSleep) continue @@ -72,6 +79,7 @@ func IncidentResponseStream(cnf *config.Config, ctx context.Context) { err = commandProcessor(path, stream, cnf, []string{msg.Command.Command, in.GetCommand().CmdId}) if err != nil { if strings.Contains(err.Error(), "EOF") { + utils.Logger.LogF(100, "error sending result to server: %v", err) time.Sleep(timeToSleep) break } @@ -80,6 +88,8 @@ func IncidentResponseStream(cnf *config.Config, ctx context.Context) { if !errorLogged { utils.Logger.ErrorF("error sending result to server: %v", err) errorLogged = true + } else { + utils.Logger.LogF(100, "error sending result to server: %v", err) } time.Sleep(timeToSleep) break @@ -87,6 +97,8 @@ func IncidentResponseStream(cnf *config.Config, ctx context.Context) { if !errorLogged { utils.Logger.ErrorF("error sending result to server: %v", err) errorLogged = true + } else { + utils.Logger.LogF(100, "error sending result to server: %v", err) } time.Sleep(timeToSleep) continue @@ -102,7 +114,7 @@ func commandProcessor(path string, stream AgentService_AgentStreamClient, cnf *c var result string var errB bool - utils.Logger.Info("Received command: %s", commandPair[0]) + utils.Logger.LogF(100, "Received command: %s", commandPair[0]) switch runtime.GOOS { case "windows": @@ -116,7 +128,7 @@ func commandProcessor(path string, stream AgentService_AgentStreamClient, cnf *c if errB { utils.Logger.ErrorF("error executing command %s: %s", commandPair[0], result) } else { - utils.Logger.Info("Result when executing the command %s: %s", commandPair[0], result) + utils.Logger.LogF(100, "Result when executing the command %s: %s", commandPair[0], result) } if err := stream.Send(&BidirectionalStream{ @@ -126,7 +138,7 @@ func commandProcessor(path string, stream AgentService_AgentStreamClient, cnf *c }); err != nil { return err } else { - utils.Logger.Info("Result sent to server successfully!!!") + utils.Logger.LogF(100, "Result sent to server successfully!!!") } return nil } diff --git a/agent/agent/ping_imp.go b/agent/agent/ping_imp.go index 4dea548ef..40b5081cc 100644 --- a/agent/agent/ping_imp.go +++ b/agent/agent/ping_imp.go @@ -26,6 +26,8 @@ func StartPing(cnf *config.Config, ctx context.Context) { if !connErrMsgWritten { utils.Logger.ErrorF("error connecting to Agent Manager: %v", err) connErrMsgWritten = true + } else { + utils.Logger.LogF(100, "error connecting to Agent Manager: %v", err) } time.Sleep(timeToSleep) continue @@ -37,11 +39,14 @@ func StartPing(cnf *config.Config, ctx context.Context) { if !connErrMsgWritten { utils.Logger.ErrorF("failed to start Ping Stream: %v", err) connErrMsgWritten = true + } else { + utils.Logger.LogF(100, "failed to start Ping Stream: %v", err) } time.Sleep(timeToSleep) continue } + utils.Logger.LogF(100, "Ping Stream started") connErrMsgWritten = false ticker := time.NewTicker(pingInterval) @@ -50,6 +55,7 @@ func StartPing(cnf *config.Config, ctx context.Context) { err := stream.Send(&PingRequest{Type: ConnectorType_AGENT}) if err != nil { if strings.Contains(err.Error(), "EOF") { + utils.Logger.LogF(100, "error sending Ping request: %v", err) time.Sleep(timeToSleep) break } @@ -58,6 +64,8 @@ func StartPing(cnf *config.Config, ctx context.Context) { if !errorLogged { utils.Logger.ErrorF("error sending Ping request: %v", err) errorLogged = true + } else { + utils.Logger.LogF(100, "error sending Ping request: %v", err) } time.Sleep(timeToSleep) break @@ -65,6 +73,8 @@ func StartPing(cnf *config.Config, ctx context.Context) { if !errorLogged { utils.Logger.ErrorF("error sending Ping request: %v", err) errorLogged = true + } else { + utils.Logger.LogF(100, "error sending Ping request: %v", err) } time.Sleep(timeToSleep) continue @@ -72,6 +82,7 @@ func StartPing(cnf *config.Config, ctx context.Context) { } errorLogged = false + utils.Logger.LogF(100, "Ping request sent") } ticker.Stop() diff --git a/agent/agent/register.go b/agent/agent/register.go index b69de591e..8e434f137 100644 --- a/agent/agent/register.go +++ b/agent/agent/register.go @@ -60,5 +60,7 @@ func RegisterAgent(cnf *config.Config, UTMKey string) error { cnf.AgentID = uint(response.Id) cnf.AgentKey = response.Key + utils.Logger.LogF(100, "Agent registered with ID: %v", cnf.AgentID) + return nil } diff --git a/agent/agent/update.go b/agent/agent/update.go index 7fca7450e..4ea6aa2fb 100644 --- a/agent/agent/update.go +++ b/agent/agent/update.go @@ -24,9 +24,13 @@ func UpdateAgent(cnf *config.Config, ctx context.Context) error { } version := models.Version{} - err = utils.ReadJson(config.VersionPath, &version) - if err != nil { - utils.Logger.Fatal("error reading version file: %v", err) + if utils.CheckIfPathExist(config.VersionPath) { + err = utils.ReadJson(config.VersionPath, &version) + if err != nil { + utils.Logger.Fatal("error reading version file: %v", err) + } + } else { + version.Version = "10.6.0" } request := &AgentRequest{ diff --git a/agent/collectors/collectors.go b/agent/collectors/collectors.go index 2a7ef909a..d49cadfd1 100644 --- a/agent/collectors/collectors.go +++ b/agent/collectors/collectors.go @@ -2,7 +2,6 @@ package collectors import ( "fmt" - "runtime" "github.com/utmstack/UTMStack/agent/utils" ) @@ -18,21 +17,6 @@ type Collector interface { Uninstall() error } -func getCollectorsInstances() []Collector { - var collectors []Collector - switch runtime.GOOS { - case "windows": - collectors = append(collectors, Windows{}) - if runtime.GOARCH == "amd64" { - collectors = append(collectors, Filebeat{}) - } - case "linux": - collectors = append(collectors, Filebeat{}) - } - - return collectors -} - func InstallCollectors() error { collectors := getCollectorsInstances() @@ -43,7 +27,7 @@ func InstallCollectors() error { } } - utils.Logger.Info("collector installed correctly") + utils.Logger.LogF(100, "collectors installed correctly") return nil } @@ -65,6 +49,6 @@ func UninstallCollectors() error { } } - utils.Logger.Info("collectors uninstalled correctly") + utils.Logger.LogF(100, "collectors uninstalled correctly") return nil } diff --git a/agent/collectors/filebeat.go b/agent/collectors/filebeat_amd64.go similarity index 100% rename from agent/collectors/filebeat.go rename to agent/collectors/filebeat_amd64.go diff --git a/agent/collectors/linux_amd64.go b/agent/collectors/linux_amd64.go new file mode 100644 index 000000000..fb50593f2 --- /dev/null +++ b/agent/collectors/linux_amd64.go @@ -0,0 +1,11 @@ +//go:build linux && amd64 +// +build linux,amd64 + +package collectors + +func getCollectorsInstances() []Collector { + var collectors []Collector + collectors = append(collectors, Filebeat{}) + + return collectors +} diff --git a/agent/collectors/windows_amd64.go b/agent/collectors/windows_amd64.go index 477d3f064..c95a1429f 100644 --- a/agent/collectors/windows_amd64.go +++ b/agent/collectors/windows_amd64.go @@ -15,6 +15,14 @@ import ( type Windows struct{} +func getCollectorsInstances() []Collector { + var collectors []Collector + collectors = append(collectors, Windows{}) + collectors = append(collectors, Filebeat{}) + + return collectors +} + func (w Windows) Install() error { path := utils.GetMyPath() diff --git a/agent/collectors/windows_arm64.go b/agent/collectors/windows_arm64.go index 20a5fd3e1..47e526cad 100644 --- a/agent/collectors/windows_arm64.go +++ b/agent/collectors/windows_arm64.go @@ -4,20 +4,27 @@ package collectors import ( - "github.com/utmstack/UTMStack/agent/config" - "github.com/utmstack/UTMStack/agent/logservice" "os" "os/exec" "path/filepath" "strings" "time" + "github.com/utmstack/UTMStack/agent/config" + "github.com/utmstack/UTMStack/agent/logservice" + "github.com/threatwinds/validations" "github.com/utmstack/UTMStack/agent/utils" ) type Windows struct{} +func getCollectorsInstances() []Collector { + var collectors []Collector + collectors = append(collectors, Windows{}) + return collectors +} + const PowerShellScript = ` <# .SYNOPSIS @@ -172,10 +179,10 @@ func (w Windows) SendSystemLogs() { validatedLogs := make([]string, 0, len(logLines)) - for logLine := range logLines { + for _, logLine := range logLines { validatedLog, _, err := validations.ValidateString(logLine, false) if err != nil { - _ = utils.Logger.ErrorF("error validating log: %s: %v", logLine, err) + utils.Logger.LogF(100, "error validating log: %s: %v", logLine, err) continue } diff --git a/agent/config/const.go b/agent/config/const.go index 51ce80aae..287b4842e 100644 --- a/agent/config/const.go +++ b/agent/config/const.go @@ -32,7 +32,6 @@ var ( PortRangeMax = "9000" RetentionConfigFile = filepath.Join(utils.GetMyPath(), "retention.json") LogsDBFile = filepath.Join(utils.GetMyPath(), "logs_process", "logs.db") - CertPath = filepath.Join(utils.GetMyPath(), "certs", "utm.crt") VersionPath = filepath.Join(utils.GetMyPath(), "version.json") MESSAGE_HEADER = "utm_stack_agent_ds" BatchCapacity = 100 @@ -46,46 +45,46 @@ var ( // RedlineServName = "UTMStackRedline" // BatchToSend = 5 - DataTypeWindowsAgent DataType = "wineventlog" + DataTypeWindowsAgent DataType = "beats_windows_agent" DataTypeSyslog DataType = "syslog" - DataTypeVmware DataType = "vmware-esxi" - DataTypeLinuxAgent DataType = "linux" - DataTypeEset DataType = "antivirus-esmc-eset" - DataTypeKaspersky DataType = "antivirus-kaspersky" - DataTypeTraefikModule DataType = "traefik" - DataTypeMongodbModule DataType = "mongodb" - DataTypeMysqlModule DataType = "mysql" - DataTypePostgresqlModule DataType = "postgresql" - DataTypeRedisModule DataType = "redis" - DataTypeElasticsearchModule DataType = "elasticsearch" - DataTypeKafkaModule DataType = "kafka" - DataTypeKibanaModule DataType = "kibana" - DataTypeLogstashModule DataType = "logstash" - DataTypeCiscoAsa DataType = "firewall-cisco-asa" - DataTypeCiscoMeraki DataType = "firewall-meraki" - DataTypeFortinet DataType = "firewall-fortigate-traffic" - DataTypePaloalto DataType = "firewall-paloalto" - DataTypeMikrotik DataType = "firewall-mikrotik" - DataTypeCiscoFirepower DataType = "firewall-cisco-firepower" - DataTypeSophosXG DataType = "firewall-sophos-xg" - DataTypeCiscoSwitch DataType = "cisco-switch" - DataTypeSonicwall DataType = "firewall-sonicwall" - DataTypeNatsModule DataType = "nats" - DataTypeDeceptivebytes DataType = "deceptive-bytes" - DataTypeOsqueryModule DataType = "osquery" - DataTypeLinuxAuditdModule DataType = "auditd" - DataTypeHaproxyModule DataType = "haproxy" - DataTypeNginxModule DataType = "nginx" - DataTypeIisModule DataType = "iis" - DataTypeApacheModule DataType = "apache" - DataTypeSentinelOne DataType = "antivirus-sentinel-one" + DataTypeVmware DataType = "vmware" + DataTypeLinuxAgent DataType = "beats_linux_agent" + DataTypeEset DataType = "antivirus_eset" + DataTypeKaspersky DataType = "antivirus_kaspersky" + DataTypeTraefikModule DataType = "beats_traefik_module" + DataTypeMongodbModule DataType = "beats_mongodb_module" + DataTypeMysqlModule DataType = "beats_mysql_module" + DataTypePostgresqlModule DataType = "beats_postgresql_module" + DataTypeRedisModule DataType = "beats_redis_module" + DataTypeElasticsearchModule DataType = "beats_elasticsearch_module" + DataTypeKafkaModule DataType = "beats_kafka_module" + DataTypeKibanaModule DataType = "beats_kibana_module" + DataTypeLogstashModule DataType = "beats_logstash_module" + DataTypeCiscoAsa DataType = "cisco_asa" + DataTypeCiscoMeraki DataType = "cisco_meraki" + DataTypeFortinet DataType = "firewall_fortinet" + DataTypePaloalto DataType = "firewall_paloalto" + DataTypeMikrotik DataType = "firewall_mikrotik" + DataTypeCiscoFirepower DataType = "cisco_firepower" + DataTypeSophosXG DataType = "firewall_sophos" + DataTypeCiscoSwitch DataType = "cisco_switch" + DataTypeSonicwall DataType = "firewall_sonicwall" + DataTypeNatsModule DataType = "beats_nats_module" + DataTypeDeceptivebytes DataType = "antivirus_deceptivebytes" + DataTypeOsqueryModule DataType = "beats_osquery_module" + DataTypeLinuxAuditdModule DataType = "beats_auditd_module" + DataTypeHaproxyModule DataType = "beats_haproxy_module" + DataTypeNginxModule DataType = "beats_nginx_module" + DataTypeIisModule DataType = "beats_iis_module" + DataTypeApacheModule DataType = "beats_apache_module" + DataTypeSentinelOne DataType = "antivirus_sentinel_one" DataTypeCiscoGeneric DataType = "cisco" - DataTypeMacOs DataType = "macos" + DataTypeMacOs DataType = "macos_logs" DataTypeGeneric DataType = "generic" DataTypeNetflow DataType = "netflow" - DataTypeAix DataType = "ibm-aix" - DataTypePfsense DataType = "firewall-pfsense" - DataTypeFortiweb DataType = "firewall-fortiweb" + DataTypeAix DataType = "ibm_aix" + DataTypePfsense DataType = "firewall_pfsense" + DataTypeFortiweb DataType = "firewall_fortiweb" ProtoPorts = map[DataType]ProtoPort{ DataTypeSyslog: {UDP: "7014", TCP: "7014"}, diff --git a/agent/conn/conn.go b/agent/conn/conn.go index e045ca020..35e6a2c26 100644 --- a/agent/conn/conn.go +++ b/agent/conn/conn.go @@ -88,21 +88,10 @@ func connectToServer(addrs, port string, skip bool) (*grpc.ClientConn, error) { return nil, fmt.Errorf("failed to connect to Server: %v", err) } - dialOptions := []grpc.DialOption{ + conn, err = grpc.NewClient( + serverAddress, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMessageSize)), - } - - if !skip { - tlsCredentials, err := utils.LoadGRPCTLSCredentials(config.CertPath) - if err != nil { - return nil, fmt.Errorf("failed to load TLS credentials: %v", err) - } - dialOptions = append(dialOptions, grpc.WithTransportCredentials(tlsCredentials)) - } else { - dialOptions = append(dialOptions, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{}))) - } - - conn, err = grpc.NewClient(serverAddress, dialOptions...) + grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: skip}))) if err != nil { connectionAttemps++ utils.Logger.ErrorF("error connecting to Server, trying again in %.0f seconds", reconnectDelay.Seconds()) diff --git a/agent/logservice/processor.go b/agent/logservice/processor.go index 009081132..fa76adda8 100644 --- a/agent/logservice/processor.go +++ b/agent/logservice/processor.go @@ -71,16 +71,16 @@ func (l *LogProcessor) ProcessLogs(cnf *config.Config, ctx context.Context) { client := NewLogServiceClient(connection) l.connErrWritten = false - l.processLogs(client, ctxEof, cancelEof) + l.processLogs(client, ctx, ctxEof, cancelEof) } } -func (l *LogProcessor) processLogs(client LogServiceClient, ctx context.Context, cancel context.CancelFunc) { +func (l *LogProcessor) processLogs(client LogServiceClient, ctx context.Context, ctxEof context.Context, cancel context.CancelFunc) { invalidKeyCounter := 0 for { select { - case <-ctx.Done(): + case <-ctxEof.Done(): utils.Logger.Info("LogProcessor: Context done, exiting...") return case newLog := <-LogQueue: diff --git a/agent/main.go b/agent/main.go index cdd84a7c7..6a4ab80ed 100644 --- a/agent/main.go +++ b/agent/main.go @@ -3,7 +3,6 @@ package main import ( "fmt" "os" - "path/filepath" "time" pb "github.com/utmstack/UTMStack/agent/agent" @@ -50,20 +49,6 @@ func main() { } fmt.Println("[OK]") - fmt.Print("Creating certificates ... ") - certsPath := filepath.Join(utils.GetMyPath(), "certs") - err = utils.CreatePathIfNotExist(certsPath) - if err != nil { - fmt.Println("\nError creating certs path: ", err) - os.Exit(1) - } - err = utils.GenerateCerts(certsPath) - if err != nil { - fmt.Println("\nError generating certs: ", err) - os.Exit(1) - } - fmt.Println("[OK]") - fmt.Print("Downloading dependencies ... ") if err := updates.DownloadFirstDependencies(cnf.Server, utmKey, cnf.SkipCertValidation); err != nil { fmt.Println("\nError downloading dependencies: ", err) diff --git a/agent/rsrc_windows_arm64.syso b/agent/rsrc_windows_arm64.syso new file mode 100644 index 000000000..edf10285e Binary files /dev/null and b/agent/rsrc_windows_arm64.syso differ diff --git a/agent/self/rsrc_windows_arm64.syso b/agent/self/rsrc_windows_arm64.syso new file mode 100644 index 000000000..edf10285e Binary files /dev/null and b/agent/self/rsrc_windows_arm64.syso differ diff --git a/agent/serv/clean-old.go b/agent/serv/clean-old.go index 489edd9b1..bbe3a53a3 100644 --- a/agent/serv/clean-old.go +++ b/agent/serv/clean-old.go @@ -3,6 +3,7 @@ package serv import ( "fmt" "os" + "path/filepath" "github.com/utmstack/UTMStack/agent/config" "github.com/utmstack/UTMStack/agent/utils" @@ -13,41 +14,42 @@ func CleanOldServices(cnf *config.Config) { isUpdaterINstalled, err := utils.CheckIfServiceIsInstalled("UTMStackUpdater") if err != nil { - utils.Logger.ErrorF("Error checking if service is installed: %v", err) + utils.Logger.LogF(100, "Error checking if service is installed: %v", err) } if isUpdaterINstalled { oldVersion = true err = utils.StopService("UTMStackUpdater") if err != nil { - utils.Logger.ErrorF("Error stopping service: %v", err) + utils.Logger.LogF(100, "Error stopping service: %v", err) } err = utils.UninstallService("UTMStackUpdater") if err != nil { - utils.Logger.ErrorF("Error uninstalling service: %v", err) + utils.Logger.LogF(100, "Error uninstalling service: %v", err) } } isRedlineInstalled, err := utils.CheckIfServiceIsInstalled("UTMStackRedline") if err != nil { - utils.Logger.ErrorF("Error checking if service is installed: %v", err) + utils.Logger.LogF(100, "Error checking if service is installed: %v", err) } if isRedlineInstalled { oldVersion = true err = utils.StopService("UTMStackRedline") if err != nil { - utils.Logger.ErrorF("Error stopping service: %v", err) + utils.Logger.LogF(100, "Error stopping service: %v", err) } err = utils.UninstallService("UTMStackRedline") if err != nil { - utils.Logger.ErrorF("Error uninstalling service: %v", err) + utils.Logger.LogF(100, "Error uninstalling service: %v", err) } } if oldVersion { + utils.Logger.Info("Old version of agent found, downloading new version") headers := map[string]string{ "key": cnf.AgentKey, "id": fmt.Sprintf("%v", cnf.AgentID), @@ -55,11 +57,23 @@ func CleanOldServices(cnf *config.Config) { } if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, cnf.Server, config.DependenciesPort, fmt.Sprintf(config.UpdaterSelf, "")), headers, fmt.Sprintf(config.UpdaterSelf, "_new"), utils.GetMyPath(), cnf.SkipCertValidation); err != nil { - utils.Logger.ErrorF("error downloading updater: %v", err) + utils.Logger.LogF(100, "error downloading updater: %v", err) return } - os.Remove(fmt.Sprintf(config.UpdaterSelf, "")) - os.Rename(fmt.Sprintf(config.UpdaterSelf, "_new"), fmt.Sprintf(config.UpdaterSelf, "")) + oldFilePath := filepath.Join(utils.GetMyPath(), fmt.Sprintf(config.UpdaterSelf, "")) + newFilePath := filepath.Join(utils.GetMyPath(), fmt.Sprintf(config.UpdaterSelf, "_new")) + + utils.Logger.LogF(100, "Renaming %s to %s", newFilePath, oldFilePath) + err := os.Remove(oldFilePath) + if err != nil { + utils.Logger.LogF(100, "error removing old updater: %v", err) + } + err = os.Rename(newFilePath, oldFilePath) + if err != nil { + utils.Logger.LogF(100, "error renaming updater: %v", err) + } + } else { + utils.Logger.LogF(100, "No old version of agent found") } } diff --git a/agent/updates/update.go b/agent/updates/update.go index 4b4312c8c..f6ada2f47 100644 --- a/agent/updates/update.go +++ b/agent/updates/update.go @@ -48,7 +48,7 @@ func UpdateDependencies(cnf *config.Config) { if newVersion.Version != currentVersion.Version { utils.Logger.Info("New version of agent found: %s", newVersion.Version) - if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, cnf.Server, fmt.Sprintf(config.ServiceFile, "")), headers, fmt.Sprintf(config.ServiceFile, "_new"), utils.GetMyPath(), cnf.SkipCertValidation); err != nil { + if err := utils.DownloadFile(fmt.Sprintf(config.DependUrl, cnf.Server, config.DependenciesPort, fmt.Sprintf(config.ServiceFile, "")), headers, fmt.Sprintf(config.ServiceFile, "_new"), utils.GetMyPath(), cnf.SkipCertValidation); err != nil { utils.Logger.ErrorF("error downloading agent: %v", err) continue } @@ -59,7 +59,6 @@ func UpdateDependencies(cnf *config.Config) { utils.Logger.ErrorF("error writing version file: %v", err) continue } - os.Remove(filepath.Join(utils.GetMyPath(), "version_new.json")) if runtime.GOOS == "linux" { if err = utils.Execute("chmod", utils.GetMyPath(), "-R", "777", filepath.Join(utils.GetMyPath(), fmt.Sprintf(config.ServiceFile, "_new"))); err != nil { @@ -69,5 +68,7 @@ func UpdateDependencies(cnf *config.Config) { utils.Execute(fmt.Sprintf(config.UpdaterSelf, ""), utils.GetMyPath()) } + + os.Remove(filepath.Join(utils.GetMyPath(), "version_new.json")) } } diff --git a/agent/utils/certs.go b/agent/utils/certs.go deleted file mode 100644 index 85d078654..000000000 --- a/agent/utils/certs.go +++ /dev/null @@ -1,136 +0,0 @@ -package utils - -import ( - "bytes" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "math/big" - "net" - "os" - "path/filepath" - "time" -) - -func GenerateCerts(folder string) error { - ca := &x509.Certificate{ - SerialNumber: big.NewInt(2019), - Subject: pkix.Name{ - Organization: []string{"UTMStack LLC"}, - Country: []string{"US"}, - Province: []string{"FL"}, - Locality: []string{"Coral Springs"}, - }, - NotBefore: time.Now(), - NotAfter: time.Now().AddDate(10, 0, 0), - IsCA: true, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, - BasicConstraintsValid: true, - } - - caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return err - } - - caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey) - if err != nil { - return err - } - - caPEM := new(bytes.Buffer) - pem.Encode(caPEM, &pem.Block{ - Type: "CERTIFICATE", - Bytes: caBytes, - }) - - caPrivKeyPEM := new(bytes.Buffer) - pem.Encode(caPrivKeyPEM, &pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(caPrivKey), - }) - - caCertFile, err := os.Create(filepath.Join(folder, "ca.crt")) - if err != nil { - return err - } - defer caCertFile.Close() - _, err = caCertFile.WriteString(caPEM.String()) - if err != nil { - return err - } - - caKeyFile, err := os.Create(filepath.Join(folder, "ca.key")) - if err != nil { - return err - } - defer caKeyFile.Close() - _, err = caKeyFile.WriteString(caPrivKeyPEM.String()) - if err != nil { - return err - } - - cert := &x509.Certificate{ - SerialNumber: big.NewInt(1658), - Subject: pkix.Name{ - Organization: []string{"UTMStack LLC"}, - Country: []string{"US"}, - Province: []string{"FL"}, - Locality: []string{"Coral Springs"}, - }, - IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, - DNSNames: []string{"localhost"}, - NotBefore: time.Now(), - NotAfter: time.Now().AddDate(10, 0, 0), - SubjectKeyId: []byte{1, 2, 3, 4, 6}, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, - KeyUsage: x509.KeyUsageDigitalSignature, - } - - certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return err - } - - certBytes, err := x509.CreateCertificate(rand.Reader, cert, ca, &certPrivKey.PublicKey, caPrivKey) - if err != nil { - return err - } - - certPEM := new(bytes.Buffer) - pem.Encode(certPEM, &pem.Block{ - Type: "CERTIFICATE", - Bytes: certBytes, - }) - - certPrivKeyPEM := new(bytes.Buffer) - pem.Encode(certPrivKeyPEM, &pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), - }) - - crtFile, err := os.Create(filepath.Join(folder, "utm.crt")) - if err != nil { - return err - } - defer crtFile.Close() - _, err = crtFile.WriteString(certPEM.String()) - if err != nil { - return err - } - - keyFile, err := os.Create(filepath.Join(folder, "utm.key")) - if err != nil { - return err - } - defer keyFile.Close() - _, err = keyFile.WriteString(certPrivKeyPEM.String()) - if err != nil { - return err - } - - return nil -} diff --git a/agent/utils/download.go b/agent/utils/download.go index ec350c51e..055c44881 100644 --- a/agent/utils/download.go +++ b/agent/utils/download.go @@ -1,6 +1,7 @@ package utils import ( + "crypto/tls" "fmt" "io" "net/http" @@ -8,7 +9,7 @@ import ( "path/filepath" ) -func DownloadFile(url string, headers map[string]string, fileName string, path string, skipTls bool) error { +func DownloadFile(url string, headers map[string]string, fileName string, path string, skipTlsVerification bool) error { req, err := http.NewRequest("GET", url, nil) if err != nil { return fmt.Errorf("error creating new request: %v", err) @@ -18,14 +19,8 @@ func DownloadFile(url string, headers map[string]string, fileName string, path s } client := &http.Client{} - if !skipTls { - tlsConfig, err := LoadHTTPTLSCredentials(filepath.Join(GetMyPath(), "certs", "utm.crt")) - if err != nil { - return fmt.Errorf("failed to load TLS credentials: %v", err) - } - client.Transport = &http.Transport{ - TLSClientConfig: tlsConfig, - } + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTlsVerification}, } resp, err := client.Do(req) diff --git a/agent/utils/logger.go b/agent/utils/logger.go index 1ae428c49..a93ac7455 100644 --- a/agent/utils/logger.go +++ b/agent/utils/logger.go @@ -1,6 +1,7 @@ package utils import ( + "path/filepath" "sync" "github.com/threatwinds/logger" @@ -9,12 +10,35 @@ import ( var ( Logger *logger.Logger loggerOnceInstance sync.Once + logLevelConfigFile = filepath.Join(GetMyPath(), "log_level.yml") + LogLevelMap = map[string]int{ + "debug": 100, + "info": 200, + "notice": 300, + "warning": 400, + "error": 500, + "critical": 502, + "alert": 509, + } ) +type LogLevels struct { + Level string `yaml:"level"` +} + func InitLogger(filename string) { + logLevel := LogLevels{} + err := ReadYAML(logLevelConfigFile, &logLevel) + if err != nil { + logLevel.Level = "info" + } + logLevelInt := 200 + if val, ok := LogLevelMap[logLevel.Level]; ok { + logLevelInt = val + } loggerOnceInstance.Do(func() { Logger = logger.NewLogger( - &logger.Config{Format: "text", Level: 200, Output: filename, Retries: 3, Wait: 5}, + &logger.Config{Format: "text", Level: logLevelInt, Output: filename, Retries: 3, Wait: 5}, ) }) } diff --git a/agent/utils/req.go b/agent/utils/req.go new file mode 100644 index 000000000..d0ed278dd --- /dev/null +++ b/agent/utils/req.go @@ -0,0 +1,52 @@ +package utils + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" +) + +func DoReq[response any](url string, data []byte, method string, headers map[string]string, skipTlsVerification bool) (response, int, error) { + var result response + + req, err := http.NewRequest(method, url, bytes.NewBuffer(data)) + if err != nil { + return result, http.StatusInternalServerError, err + } + + for k, v := range headers { + req.Header.Add(k, v) + } + + client := &http.Client{} + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTlsVerification}, + } + + resp, err := client.Do(req) + if err != nil { + return result, http.StatusInternalServerError, err + } + defer func() { + _ = resp.Body.Close() + }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return result, http.StatusInternalServerError, err + } + + if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK { + return result, resp.StatusCode, fmt.Errorf("while sending request to %s received status code: %d and response body: %s", url, resp.StatusCode, body) + } + + err = json.Unmarshal(body, &result) + if err != nil { + return result, http.StatusInternalServerError, err + } + + return result, resp.StatusCode, nil +} diff --git a/installer/types/compose.go b/installer/types/compose.go index 2be5383f2..5dbd6a0a3 100644 --- a/installer/types/compose.go +++ b/installer/types/compose.go @@ -153,6 +153,7 @@ func (c *Compose) Populate(conf *Config, stack *StackConfig) *Compose { }, Ports: []string{ "9000:50051", + "9001:8080", }, Environment: []string{ "DB_PATH=/data/utmstack.db", diff --git a/version.yml b/version.yml index aa72feb11..0c72b9627 100644 --- a/version.yml +++ b/version.yml @@ -1 +1 @@ -version: 10.6.3 \ No newline at end of file +version: 10.7.0