diff --git a/backend/src/main/resources/config/liquibase/changelog/20250414001_updating_sophos_filter.xml b/backend/src/main/resources/config/liquibase/changelog/20250414001_updating_sophos_filter.xml new file mode 100644 index 000000000..9b0753ddd --- /dev/null +++ b/backend/src/main/resources/config/liquibase/changelog/20250414001_updating_sophos_filter.xml @@ -0,0 +1,1163 @@ + + + + + + + "message" + terminator => "" + } + + #Looking for datasource generated by an agent and parse original message + if [message]=~/\[utm_stack_agent_ds=(.+)\]-(.+)/ { + grok { + match => { + "message" => [ "\[utm_stack_agent_ds=%{DATA:dataSource}\]-%{GREEDYDATA:original_log_message}" ] + } + } + } + if [original_log_message] { + mutate { + update => { "message" => "%{[original_log_message]}" } + } + } + + if ![dataType] { + #First, search for common fields and parse them + grok { + match => { + "message" => [ + # Old format XG + "device=%{QUOTEDSTRING:device} date=%{DATA:syslog_timestamp} time=%{TIME:xg_time} timezone=%{DATA:xg_timezone} device_name=%{QUOTEDSTRING:device_name} device_id=%{DATA:device_id} log_id=%{DATA:log_id} log_type=%{QUOTEDSTRING:log_type} log_component=%{QUOTEDSTRING:log_component} %{GREEDYDATA:msg}", + "%{GREEDYDATA}device=%{QUOTEDSTRING:device} date=%{DATA:syslog_timestamp} time=%{TIME:xg_time} timezone=%{DATA:xg_timezone} device_name=%{QUOTEDSTRING:device_name} device_id=%{DATA:device_id} log_id=%{DATA:log_id} log_type=%{QUOTEDSTRING:log_type} log_component=%{QUOTEDSTRING:log_component} %{GREEDYDATA:msg}", + + # New format with ISO 8601 timestamp XGS + "device_name=%{QUOTEDSTRING:device} timestamp=%{QUOTEDSTRING:timestamp} device_model=%{QUOTEDSTRING:device_model} .*? log_type=%{QUOTEDSTRING:log_type} log_component=%{QUOTEDSTRING:log_component} %{GREEDYDATA:msg}", + "%{GREEDYDATA}device_name=%{QUOTEDSTRING:device} timestamp=%{QUOTEDSTRING:timestamp} device_model=%{QUOTEDSTRING:device_model} .*? log_type=%{QUOTEDSTRING:log_type} log_component=%{QUOTEDSTRING:log_component} %{GREEDYDATA:msg}", + + # Common fields for the log_type="WAF" format + "messageid=%{QUOTEDSTRING:messageid} log_type=%{QUOTEDSTRING:log_type} log_component=%{QUOTEDSTRING:log_component} %{GREEDYDATA:msg}", + "%{GREEDYDATA}messageid=%{QUOTEDSTRING:messageid} log_type=%{QUOTEDSTRING:log_type} log_component=%{QUOTEDSTRING:log_component} %{GREEDYDATA:msg}" + ] + } + } + # Other needed fields available in the new format + if ![syslog_timestamp] { + grok { + match => { + "message" => [ + "%{GREEDYDATA} device_serial_id=%{QUOTEDSTRING:device_serial_id} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "message" => [ + "%{GREEDYDATA} log_id=(?:%{QUOTEDSTRING:log_id}|%{DATA:log_id}) %{GREEDYDATA}" + ] + } + } + } + #Replacing string quotation for common fields + mutate { + gsub => ["device", ''"'', ""] + gsub => ["device_name", ''"'', ""] + gsub => ["log_type", ''"'', ""] + gsub => ["log_component", ''"'', ""] + gsub => ["device_model", ''"'', ""] + gsub => ["log_id", ''"'', ""] + gsub => ["device_serial_id", ''"'', ""] + gsub => ["messageid", ''"'', ""] + } + if [log_type] and ([log_type] == "Firewall" or [log_type] == "Content Filtering" or [log_type] == "Event" + or [log_type] == "WAF" or [log_type] == "System Health" or [log_type] == "IDP" + or [log_type] == "ATP" or [log_type] == "EATP" or ([log_type] == "Antivirus" or [log_type] == "Anti-Virus") + or [log_type] == "Anti-spam" or [log_type] == "Anti-Spam" + or [log_type] == "Heartbeat" or [log_type] == "Sandbox" + or [log_type] == "Wireless protection" or [log_type] == "Wireless Protection" + or [log_type] == "System health" or [log_type] == "Content filtering" + or [log_type] == "SSL" ) { + #Other fields needed for rules in Correlation Engine + if [msg] { + grok { + match => { + "msg" => [ + "status=%{QUOTEDSTRING:status} %{GREEDYDATA}", + "%{GREEDYDATA} status=%{QUOTEDSTRING:status} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "log_subtype=%{QUOTEDSTRING:log_subtype} %{GREEDYDATA}", + "%{GREEDYDATA} log_subtype=%{QUOTEDSTRING:log_subtype} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "priority=%{WORD:priority} %{GREEDYDATA}", + "%{GREEDYDATA} priority=%{WORD:priority} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (sent_bytes|bytes_sent)=%{INT:sent_bytes} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (recv_bytes|bytes_received|bytes_recv)=%{INT:recv_bytes} %{GREEDYDATA}" + ] + } + } + } + #Adding common fields to logx tree structure + mutate { + add_field => { "[dataType]" => "firewall-sophos-xg" } + rename => { "[device]" => "[logx][sophos][device]" } + rename => { "[device_name]" => "[logx][sophos][device_name]" } + rename => { "[device_id]" => "sophosDataSource" } + rename => { "[log_id]" => "[logx][sophos][log_id]" } + rename => { "[log_type]" => "[logx][sophos][log_type]" } + rename => { "[log_component]" => "[logx][sophos][log_component]" } + rename => { "[status]" => "[logx][sophos][status]" } + rename => { "[log_subtype]" => "[logx][sophos][log_subtype]" } + rename => { "[message]" => "[logx][sophos][message]" } + rename => { "[syslog_timestamp]" => "[logx][sophos][syslog_timestamp]" } + rename => { "[xg_time]" => "[logx][sophos][xg_time]" } + rename => { "[xg_timezone]" => "[logx][sophos][xg_timezone]" } + rename => { "[priority]" => "[logx][sophos][priority]" } + rename => { "[sent_bytes]" => "[logx][sophos][sent_bytes]" } + rename => { "[recv_bytes]" => "[logx][sophos][recv_bytes]" } + rename => { "[timestamp]" => "[logx][sophos][timestamp]" } + rename => { "[device_serial_id]" => "[logx][sophos][device_serial_id]" } + rename => { "[device_model]" => "[logx][sophos][device_model]" } + } + +#......................................................................# +#Generating dataSource field required by CurrelationRulesEngine +#Checks if sophosDataSource exists, if true, the dataSource field take its value, if not, take the agent dataSource value + if ([sophosDataSource]){ + if [dataSource] { + mutate { + update => { "dataSource" => "%{[sophosDataSource]}" } + } + } else { + mutate { + add_field => { "dataSource" => "%{sophosDataSource}" } + } + } + } +#Finally evaluates to the host variable if can not be calculated + if ![dataSource] { + mutate { + add_field => { "dataSource" => "%{host}" } + } + } + + if [logx][sophos][device] and [logx][sophos][device] == "SFW" { + if [msg] { + #Fields from Firewall log_type + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (src_ip|sourceip)=%{IP:src_ip} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} dst_ip=%{IP:dst_ip} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} protocol=%{QUOTEDSTRING:proto} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} src_port=%{NUMBER:src_port} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} dst_port=%{NUMBER:dst_port} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} application=%{QUOTEDSTRING:application} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (application_risk|app_risk)=%{NUMBER:application_risk} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (application_technology|app_technology)=%{QUOTEDSTRING:application_technology} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (application_name|app_name)=%{QUOTEDSTRING:application_name} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (application_category|app_category)=%{QUOTEDSTRING:application_category} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (user_name)=%{QUOTEDSTRING:user_name} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (src_country_code|src_country)=%{WORD:src_country_code} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (dst_country_code|dst_country)=%{WORD:dst_country_code} %{GREEDYDATA}" + ] + } + } + #1.3.7 + grok { + match => { + "msg" => [ + "%{GREEDYDATA} dst_mac=%{QUOTEDSTRING:dst_mac} %{GREEDYDATA}" + ] + } + } + #2.1.0 + #Fields from Firewall XGS log_type + grok { + match => { + "msg" => [ + "%{GREEDYDATA} fw_rule_name=%{QUOTEDSTRING:fw_rule_name} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} fw_rule_section=%{QUOTEDSTRING:fw_rule_section} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} nat_rule_id=%{QUOTEDSTRING:nat_rule_id} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} nat_rule_name=%{QUOTEDSTRING:nat_rule_name} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} fw_rule_type=%{QUOTEDSTRING:fw_rule_type} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} ether_type=%{QUOTEDSTRING:ether_type} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} in_interface=%{QUOTEDSTRING:in_interface} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} out_interface={QUOTEDSTRING:out_interface} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} src_zone_type={QUOTEDSTRING:src_zone_type} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} dst_zone_type={QUOTEDSTRING:dst_zone_type} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} dst_zone={QUOTEDSTRING:dst_zone} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} src_zone={QUOTEDSTRING:src_zone} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} con_event={QUOTEDSTRING:con_event} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} con_id={QUOTEDSTRING:con_id} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} hb_status={QUOTEDSTRING:hb_status} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} app_resolved_by={QUOTEDSTRING:app_resolved_by} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} app_is_cloud={QUOTEDSTRING:app_is_cloud} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} qualifier={QUOTEDSTRING:qualifier} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} in_display_interface={QUOTEDSTRING:in_display_interface} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} out_display_interface={QUOTEDSTRING:out_display_interface} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} log_occurrence={QUOTEDSTRING:log_occurrence} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} dst_country={QUOTEDSTRING:dst_country} %{GREEDYDATA}" + ] + } + } + + #New fields from Content Filtering + grok { + match => { + "msg" => [ + "%{GREEDYDATA} category=%{QUOTEDSTRING:category} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} category_type=%{QUOTEDSTRING:category_type} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (url|FTP_url)=%{NOTSPACE:url} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} download_file_name=%{QUOTEDSTRING:download_file_name} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} download_file_type=%{QUOTEDSTRING:download_file_type} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} upload_file_name=%{QUOTEDSTRING:upload_file_name} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} upload_file_type=%{QUOTEDSTRING:upload_file_type} %{GREEDYDATA}" + ] + } + } + + #New fields from Event log_type + grok { + match => { + "msg" => [ + "%{GREEDYDATA} reason=%{QUOTEDSTRING:reason} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} message=%{QUOTEDSTRING:event_message} %{GREEDYDATA}" + ] + } + } + #1.3.7 + grok { + match => { + "msg" => [ + "%{GREEDYDATA} src_mac=%{QUOTEDSTRING:src_mac} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (client_used|auth_client)=%{QUOTEDSTRING:client_used} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} gatewayname=%{QUOTEDSTRING:gateway_name} %{GREEDYDATA}" + ] + } + } + #Fields from WAF log_type + grok { + match => { + "msg" => [ + "%{GREEDYDATA} server=%{NOTSPACE:server} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} contenttype=%{QUOTEDSTRING:contenttype} %{GREEDYDATA}" + ] + } + } + #Fields from Antivirus log_type + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (status_code|httpstatus)=%{NUMBER:httpstatus} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (useragent)=%{QUOTEDSTRING:useragent} %{GREEDYDATA}" + ] + } + } + #New fields from 1.3.7 + grok { + match => { + "msg" => [ + "%{GREEDYDATA} severity=%{QUOTEDSTRING:severity} %{GREEDYDATA}", + "%{GREEDYDATA} severity=%{WORD:severity} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} malware=%{QUOTEDSTRING:malware_name} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (domain|domainname|src_domainname)=%{QUOTEDSTRING:domain} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} virus=%{QUOTEDSTRING:malware_name} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (user_group|user_gp|usergroupname)=%{QUOTEDSTRING:user_group} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (con_direction|FTP_direction)=%{QUOTEDSTRING:direction} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} file_path=%{QUOTEDSTRING:file_path} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (parent_app|parent_application)=%{QUOTEDSTRING:parent_application_name} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (parent_app_category|parent_application_category)=%{QUOTEDSTRING:parent_application_category} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (parent_app_risk|parent_application_risk)=%{QUOTEDSTRING:parent_application_risk} %{GREEDYDATA}" + ] + } + } + #Fields from IDP log_type + grok { + match => { + "msg" => [ + "%{GREEDYDATA} signature_msg=%{QUOTEDSTRING:event_message} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} classification=%{QUOTEDSTRING:classification} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} platform=%{QUOTEDSTRING:platform} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} target=%{QUOTEDSTRING:target} %{GREEDYDATA}" + ] + } + } + #Fields from ATP log_type + grok { + match => { + "msg" => [ + "%{GREEDYDATA} destinationip=%{IP:dst_ip} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} threatname=%{NOTSPACE:threatname} %{GREEDYDATA}" + ] + } + } + #1.3.7 + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (reported_user|login_user)=%{QUOTEDSTRING:login_user} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (process_user|proc_user)=%{QUOTEDSTRING:process_user} %{GREEDYDATA}" + ] + } + } + #Fields from Anti-Spam log_type + #1.3.7 + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (action|spamaction)=%{QUOTEDSTRING:action} %{GREEDYDATA}", + "%{GREEDYDATA} (action|spamaction)=%{WORD:action} %{GREEDYDATA}" + ] + } + } + #Fields from System Health log_type + grok { + match => { + "msg" => [ + "%{GREEDYDATA} users=%{NUMBER:users} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} display_interface=%{QUOTEDSTRING:display_interface} %{GREEDYDATA}" + ] + } + } + } + + mutate { + #Common Fields + gsub => ["[logx][sophos][status]", ''"'', ""] + gsub => ["[logx][sophos][log_subtype]", ''"'', ""] + gsub => ["[logx][sophos][xg_timezone]", ''"'', ""] + gsub => ["[logx][sophos][timestamp]", ''"'', ""] + + #Fields from Firewall log_type + gsub => ["src_ip", ''"'', ""] + gsub => ["src_port", ''"'', ""] + gsub => ["dst_ip", ''"'', ""] + gsub => ["dest_port", ''"'', ""] + gsub => ["proto", ''"'', ""] + gsub => ["application", ''"'', ""] + gsub => ["application_risk", ''"'', ""] + gsub => ["application_technology", ''"'', ""] + gsub => ["application_name", ''"'', ""] + gsub => ["application_category", ''"'', ""] + gsub => ["user_name", ''"'', ""] + gsub => ["src_country_code", ''"'', ""] + gsub => ["dst_country_code", ''"'', ""] + #1.3.7 + gsub => ["dst_mac", ''"'', ""] + #2.1.0 + gsub => ["fw_rule_name",''"'',""] + gsub => ["fw_rule_section",''"'',""] + gsub => ["nat_rule_id",''"'',""] + gsub => ["nat_rule_name",''"'',""] + gsub => ["fw_rule_type",''"'',""] + gsub => ["ether_type",''"'',""] + gsub => ["in_interface",''"'',""] + gsub => ["out_interface",''"'',""] + gsub => ["src_zone_type",''"'',""] + gsub => ["dst_zone_type",''"'',""] + gsub => ["dst_zone",''"'',""] + gsub => ["src_zone",''"'',""] + gsub => ["con_event",''"'',""] + gsub => ["con_id",''"'',""] + gsub => ["hb_status",''"'',""] + gsub => ["app_resolved_by",''"'',""] + gsub => ["app_is_cloud",''"'',""] + gsub => ["qualifier",''"'',""] + gsub => ["in_display_interface",''"'',""] + gsub => ["out_display_interface",''"'',""] + gsub => ["log_occurrence",''"'',""] + gsub => ["dst_country",''"'',""] + + #New fields from Content Filtering log_type + gsub => ["category", ''"'', ""] + gsub => ["category_type", ''"'', ""] + gsub => ["url", ''"'', ""] + gsub => ["download_file_name", ''"'', ""] + gsub => ["download_file_type", ''"'', ""] + gsub => ["upload_file_name", ''"'', ""] + gsub => ["upload_file_type", ''"'', ""] + + #New fields from Event log_type + gsub => ["reason", ''"'', ""] + gsub => ["event_message", ''"'', ""] + #1.3.7 + gsub => ["src_mac", ''"'', ""] + gsub => ["client_used", ''"'', ""] + gsub => ["src_host", ''"'', ""] + gsub => ["reported_ip", ''"'', ""] + gsub => ["updated_ip", ''"'', ""] + gsub => ["gateway_name", ''"'', ""] + + #Fields from WAF log_type + gsub => ["server", ''"'', ""] + gsub => ["contenttype", ''"'', ""] + gsub => ["useragent", ''"'', ""] + + #Fields from Antivirus log_type + #1.3.7 + gsub => ["severity", ''"'', ""] + gsub => ["malware_name", ''"'', ""] + gsub => ["domain", ''"'', ""] + gsub => ["user_group", ''"'', ""] + gsub => ["direction", ''"'', ""] + gsub => ["file_path", ''"'', ""] + gsub => ["parent_application_name", ''"'', ""] + gsub => ["parent_application_category", ''"'', ""] + gsub => ["parent_application_risk", ''"'', ""] + + #Fields from IDP log_type + gsub => ["classification", ''"'', ""] + gsub => ["platform", ''"'', ""] + gsub => ["target", ''"'', ""] + + #Fields from ATP log_type + gsub => ["threatname", ''"'', ""] + #1.3.7 + gsub => ["login_user", ''"'', ""] + gsub => ["process_user", ''"'', ""] + + #Fields from Anti-Spam log_type + #1.3.7 + gsub => ["action", ''"'', ""] + + #Fields from System Health log_type + gsub => ["users", ''"'', ""] + gsub => ["display_interface", ''"'', ""] + + } + + #Do specific mutations to unify field names across log_type + if ([logx][sophos][log_type] == "Antivirus" or [logx][sophos][log_type] == "Anti-Virus") + and [logx][sophos][log_component] == "FTP" { + if ([direction]) and ([direction] == "Upload" or [direction] == "upload") { + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (filename|file_name)=%{QUOTEDSTRING:upload_file_name} %{GREEDYDATA}" + ] + } + } + mutate { + gsub => ["upload_file_name", ''"'', ""] + } + + } else { + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (filename|file_name)=%{QUOTEDSTRING:download_file_name} %{GREEDYDATA}" + ] + } + } + mutate { + gsub => ["download_file_name", ''"'', ""] + } + } + } else if [logx][sophos][log_type] == "WAF" { + grok { + match => { + "msg" => [ + "%{GREEDYDATA} local_ip=%{QUOTEDSTRING:local_ip} %{GREEDYDATA}" + ] + } + } + mutate { + gsub => ["local_ip", ''"'', ""] + } + } else if [logx][sophos][log_type] == "Event" { + if ([logx][sophos][log_component] == "L2TP" or [logx][sophos][log_component] == "PPTP") { + grok { + match => { + "msg" => [ + "%{GREEDYDATA} localip=%{IP:src_ip} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} remotepeer=%{IP:dst_ip} %{GREEDYDATA}" + ] + } + } + mutate { + gsub => ["src_ip", ''"'', ""] + gsub => ["dst_ip", ''"'', ""] + } + } else if ([logx][sophos][log_component] == "DDNS" or [logx][sophos][log_component] == "DHCP Server") { + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (src_host|host|client_host_name|reported_host)=%{IPORHOST:src_host} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} client_physical_address=%{QUOTEDSTRING:src_mac} %{GREEDYDATA}" + ] + } + } + if ![src_ip] { + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (reported_ip|reportedip)=%{IPORHOST:src_ip} %{GREEDYDATA}", + "%{GREEDYDATA} (reported_ip|reportedip)=%{QUOTEDSTRING:src_ip} %{GREEDYDATA}" + ] + } + } + } + if ![src_ip] { + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (updatedip|updated_ip)=%{IPORHOST:src_ip} %{GREEDYDATA}", + "%{GREEDYDATA} (updatedip|updated_ip)=%{QUOTEDSTRING:src_ip} %{GREEDYDATA}" + ] + } + } + } + if ![src_ip] { + grok { + match => { + "msg" => [ + "%{GREEDYDATA} ipaddress=%{IPORHOST:src_ip} %{GREEDYDATA}", + "%{GREEDYDATA} ipaddress=%{QUOTEDSTRING:src_ip} %{GREEDYDATA}" + ] + } + } + } + if ![src_ip] { + grok { + match => { + "msg" => [ + "%{GREEDYDATA} leased_ip=%{IPORHOST:src_ip} %{GREEDYDATA}", + "%{GREEDYDATA} leased_ip=%{QUOTEDSTRING:src_ip} %{GREEDYDATA}" + ] + } + } + } + mutate { + gsub => ["src_ip", ''"'', ""] + gsub => ["src_mac", ''"'', ""] + gsub => ["src_host", ''"'', ""] + } + } else if [logx][sophos][log_component] == "Quarantine" { + grok { + match => { + "msg" => [ + "%{GREEDYDATA} from=%{QUOTEDSTRING:email_sender} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} to=%{QUOTEDSTRING:email_recipient} %{GREEDYDATA}" + ] + } + } + mutate { + gsub => ["email_sender", ''"'', ""] + gsub => ["email_recipient", ''"'', ""] + } + } + } else if ([logx][sophos][log_type] == "Anti-spam" or [logx][sophos][log_type] == "Anti-Spam") + or ([log_type] == "Antivirus" or [log_type] == "Anti-Virus") { + if ([logx][sophos][log_component] == "SMTP" or [logx][sophos][log_component] == "SMTPS" + or [logx][sophos][log_component] == "POP3" or [logx][sophos][log_component] == "IMAP4" + or [logx][sophos][log_component] == "IMAPS" or [logx][sophos][log_component] == "POPS") { + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (sender|from_email_address)=%{QUOTEDSTRING:email_sender} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (recipient|to_email_address)=%{QUOTEDSTRING:email_recipient} %{GREEDYDATA}" + ] + } + } + mutate { + gsub => ["email_sender", ''"'', ""] + gsub => ["email_recipient", ''"'', ""] + } + } + } else if [logx][sophos][log_type] == "Sandbox" { + grok { + match => { + "msg" => [ + "%{GREEDYDATA} source=%{QUOTEDSTRING:domain} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} (file_hash|sha1sum)=%{QUOTEDSTRING:file_hash} %{GREEDYDATA}" + ] + } + } + mutate { + gsub => ["file_hash", ''"'', ""] + gsub => ["domain", ''"'', ""] + } + } + + + #Set logx tree structure + mutate { + #Fields from Firewall log_type, adding to logx tree structure + rename => { "[src_ip]" => "[logx][sophos][src_ip]" } + rename => { "[src_port]" => "[logx][sophos][src_port]" } + rename => { "[dst_ip]" => "[logx][sophos][dest_ip]" } + rename => { "[dst_port]" => "[logx][sophos][dest_port]" } + rename => { "[proto]" => "[logx][sophos][proto]" } + rename => { "[application]" => "[logx][sophos][application]" } + rename => { "[application_risk]" => "[logx][sophos][application_risk]" } + rename => { "[application_technology]" => "[logx][sophos][application_technology]" } + rename => { "[application_name]" => "[logx][sophos][application_name]" } + rename => { "[application_category]" => "[logx][sophos][application_category]" } + rename => { "[user_name]" => "[logx][sophos][user_name]" } + rename => { "[src_country_code]" => "[logx][sophos][src_country_code]" } + rename => { "[dst_country_code]" => "[logx][sophos][dst_country_code]" } + #1.3.7 + rename => { "[dst_mac]" => "[logx][sophos][dst_mac]" } + #2.1.0 + #Fields from Firewall XGS log_type, adding to logx tree structure + rename => { "[fw_rule_name]" => "[logx][sophos][fw_rule_name]" } + rename => { "[fw_rule_section]" => "[logx][sophos][fw_rule_section]" } + rename => { "[nat_rule_id]" => "[logx][sophos][nat_rule_id]" } + rename => { "[nat_rule_name]" => "[logx][sophos][nat_rule_name]" } + rename => { "[fw_rule_type]" => "[logx][sophos][fw_rule_type]" } + rename => { "[ether_type]" => "[logx][sophos][ether_type]" } + rename => { "[out_interface]" => "[logx][sophos][out_interface]" } + rename => { "[in_interface]" => "[logx][sophos][in_interface]" } + rename => { "[src_zone_type]" => "[logx][sophos][src_zone_type]" } + rename => { "[dst_zone_type]" => "[logx][sophos][dst_zone_type]" } + rename => { "[dst_zone]" => "[logx][sophos][dst_zone]" } + rename => { "[src_zone]" => "[logx][sophos][src_zone]" } + rename => { "[con_event]" => "[logx][sophos][con_event]" } + rename => { "[con_id]" => "[logx][sophos][con_id]" } + rename => { "[hb_status]" => "[logx][sophos][hb_status]" } + rename => { "[app_resolved_by]" => "[logx][sophos][app_resolved_by]" } + rename => { "[app_is_cloud]" => "[logx][sophos][app_is_cloud]" } + rename => { "[qualifier]" => "[logx][sophos][qualifier]" } + rename => { "[in_display_interface]" => "[logx][sophos][in_display_interface]" } + rename => { "[out_display_interface]" => "[logx][sophos][out_display_interface]" } + rename => { "[log_occurrence]" => "[logx][sophos][log_occurrence]" } + rename => { "[dst_country]" => "[logx][sophos][dst_country]" } + + + #New fields from Content Filtering log_type + rename => { "[category]" => "[logx][sophos][category]" } + rename => { "[category_type]" => "[logx][sophos][category_type]" } + rename => { "[url]" => "[logx][sophos][url]" } + rename => { "[download_file_name]" => "[logx][sophos][download_file_name]" } + rename => { "[download_file_type]" => "[logx][sophos][download_file_type]" } + rename => { "[upload_file_name]" => "[logx][sophos][upload_file_name]" } + rename => { "[upload_file_type]" => "[logx][sophos][upload_file_type]" } + + #New fields from Event log_type + rename => { "[reason]" => "[logx][sophos][reason]" } + rename => { "[event_message]" => "[logx][sophos][event_message]" } + #1.3.7 + rename => { "[src_mac]" => "[logx][sophos][src_mac]" } + rename => { "[client_used]" => "[logx][sophos][client_used]" } + rename => { "[src_host]" => "[logx][sophos][src_host]" } + rename => { "[reported_ip]" => "[logx][sophos][reported_ip]" } + rename => { "[updated_ip]" => "[logx][sophos][updated_ip]" } + rename => { "[gateway_name]" => "[logx][sophos][gateway_name]" } + + #New fields from WAF log_type + rename => { "[server]" => "[logx][sophos][server]" } + rename => { "[httpstatus]" => "[logx][sophos][httpstatus]" } + rename => { "[contenttype]" => "[logx][sophos][contenttype]" } + rename => { "[useragent]" => "[logx][sophos][useragent]" } + #1.3.7 + rename => { "[local_ip]" => "[logx][sophos][local_ip]" } + + #New fields from Antivirus log_type + #1.3.7 + rename => { "[severity]" => "[logx][sophos][severity]" } + rename => { "[malware_name]" => "[logx][sophos][malware_name]" } + rename => { "[domain]" => "[logx][sophos][domain]" } + rename => { "[user_group]" => "[logx][sophos][user_group]" } + rename => { "[direction]" => "[logx][sophos][direction]" } + rename => { "[file_path]" => "[logx][sophos][file_path]" } + rename => { "[parent_application_name]" => "[logx][sophos][parent_application_name]" } + rename => { "[parent_application_category]" => "[logx][sophos][parent_application_category]" } + rename => { "[parent_application_risk]" => "[logx][sophos][parent_application_risk]" } + + #New fields from IDP log_type + rename => { "[classification]" => "[logx][sophos][classification]" } + rename => { "[platform]" => "[logx][sophos][platform]" } + rename => { "[target]" => "[logx][sophos][target]" } + + #New fields from ATP log_type + rename => { "[threatname]" => "[logx][sophos][threatname]" } + #1.3.7 + rename => { "[login_user]" => "[logx][sophos][login_user]" } + rename => { "[process_user]" => "[logx][sophos][process_user]" } + + #New fields from Anti-Spam log_type + #1.3.7 + rename => { "[email_sender]" => "[logx][sophos][email_sender]" } + rename => { "[email_recipient]" => "[logx][sophos][email_recipient]" } + rename => { "[action]" => "[logx][sophos][log_action]" } + + #New fields from System Health log_type + rename => { "[users]" => "[logx][sophos][users]" } + rename => { "[display_interface]" => "[logx][sophos][display_interface]" } + + #New fields from Sandbox log_type + #1.3.7 + rename => { "[file_hash]" => "[logx][sophos][file_hash]" } + + } + } + #Generating action field used by Correlation engine + if [logx][sophos][status] and [logx][sophos][status] =~/(Allow|Allow Session|Successful|Established|Connected|Success|Renew)/ { + mutate { + add_field => { "[logx][utm][action]" => "Success" } + } + } else if ([logx][sophos][status] and [logx][sophos][status] == "Interim") and + ([logx][sophos][sent_bytes] and [logx][sophos][sent_bytes] != "0") and ([logx][sophos][recv_bytes] and [logx][sophos][recv_bytes] != "0") { + mutate { + add_field => { "[logx][utm][action]" => "Success" } + } + } else if ([logx][sophos][log_subtype] and [logx][sophos][log_subtype] == "Allowed") and + (![logx][sophos][status] or [logx][sophos][status] == "") and + ([logx][sophos][sent_bytes] and [logx][sophos][sent_bytes] != "0") and + ([logx][sophos][recv_bytes] and [logx][sophos][recv_bytes] != "0") { + mutate { + add_field => { "[logx][utm][action]" => "Success" } + } + } else if [logx][sophos][log_component] and ([logx][sophos][log_component] == "FTP" or [logx][sophos][log_component] == "HTTP") and + ([logx][sophos][recv_bytes] and [logx][sophos][recv_bytes] != "0") { + mutate { + add_field => { "[logx][utm][action]" => "Success" } + } + } else if [logx][sophos][log_type] == "SSL" and ([logx][sophos][log_subtype] == "Decrypt" + or [logx][sophos][log_subtype] == "Do not decrypt" ) { + mutate { + add_field => { "[logx][utm][action]" => "Success" } + } + + } + #Finally, remove unnecessary fields + mutate { + remove_field => ["@version","path","tags","type","msg"] + } + + } + } + #Also, remove unwanted fields if the message not match with conditions + mutate { + remove_field => ["sophosDataSource","original_log_message","headers"] + } +}', + filter_version='2.1.0' + WHERE id=801; + + ]]> + + + diff --git a/backend/src/main/resources/config/liquibase/changelog/20250414002_updating_gcp_filter.xml b/backend/src/main/resources/config/liquibase/changelog/20250414002_updating_gcp_filter.xml new file mode 100644 index 000000000..d4a5f9a85 --- /dev/null +++ b/backend/src/main/resources/config/liquibase/changelog/20250414002_updating_gcp_filter.xml @@ -0,0 +1,113 @@ + + + + + + + "message" + } + } + if [resource][type] and [resource][labels][project_id] or [protoPayload][@type]{ + #Generating dataType and dataSource fields + mutate { + add_field => { + "dataType" => "google" + } + } + if ([@metadata][dataSource]) { + mutate { + #Add based on metadata + add_field => { + "dataSource" => "%{[@metadata][dataSource]}" + } + add_field => { + "[logx][tenant]" => "%{[@metadata][dataSource]}" + } + } + } + mutate { + #Non variable fields, without fields inside + rename => { "[logName]" => "[logx][google][logName]" } + rename => { "[timestamp]" => "[logx][google][timestamp]" } + rename => { "[receiveTimestamp]" => "[logx][google][receiveTimestamp]" } + rename => { "[severity]" => "[logx][google][severityLabel]" } + rename => { "[insertId]" => "[logx][google][insertId]" } + rename => { "[trace]" => "[logx][google][trace]" } + rename => { "[spanId]" => "[logx][google][spanId]" } + rename => { "[traceSampled]" => "[logx][google][traceSampled]" } + rename => { "[message]" => "[logx][google][message]" } + + #Non variable fields, with other fields inside + #httpRequest + rename => { "[httpRequest][requestMethod]" => "[logx][google][requestMethod]" } + rename => { "[httpRequest][requestUrl]" => "[logx][google][requestUrl]" } + rename => { "[httpRequest][requestSize]" => "[logx][google][requestSize]" } + rename => { "[httpRequest][status]" => "[logx][google][status]" } + rename => { "[httpRequest][responseSize]" => "[logx][google][responseSize]" } + rename => { "[httpRequest][userAgent]" => "[logx][google][userAgent]" } + rename => { "[httpRequest][serverIp]" => "[logx][google][dest_ip]" } + rename => { "[httpRequest][referer]" => "[logx][google][referer]" } + rename => { "[httpRequest][latency]" => "[logx][google][latency]" } + rename => { "[httpRequest][cacheLookup]" => "[logx][google][cacheLookup]" } + rename => { "[httpRequest][cacheHit]" => "[logx][google][cacheHit]" } + rename => { "[httpRequest][cacheValidatedWithOriginServer]" => "[logx][google][cacheValidatedWithOriginServer]" } + rename => { "[httpRequest][cacheFillBytes]" => "[logx][google][cacheFillBytes]" } + rename => { "[httpRequest][protocol]" => "[logx][google][proto]" } + #operation + rename => { "[operation][id]" => "[logx][google][operation_id]" } + rename => { "[operation][producer]" => "[logx][google][producer]" } + rename => { "[operation][first]" => "[logx][google][first]" } + rename => { "[operation][last]" => "[logx][google][last]" } + #sourceLocation + rename => { "[sourceLocation][file]" => "[logx][google][file]" } + rename => { "[sourceLocation][line]" => "[logx][google][line]" } + rename => { "[sourceLocation][function]" => "[logx][google][function]" } + + #Fields with variable content, stay the same as in the source + rename => { "[resource]" => "[logx][google][resource]" } + rename => { "[labels]" => "[logx][google][labels]" } + + #Union payload field, variants, stay the same as in the source + rename => { "[jsonPayload]" => "[logx][google][jsonPayload]" } + rename => { "[protoPayload]" => "[logx][google][protoPayload]" } + rename => { "[textPayload]" => "[logx][google][textPayload]" } + + } + #............................................................................................ + #Generate src_ip used in correlation engine + if [logx][google][protoPayload][requestMetadata][callerIp] { + mutate { + rename => { "[httpRequest][remoteIp]" => "[logx][google][remoteIp]" } + rename => { "[logx][google][protoPayload][requestMetadata][callerIp]" => "[logx][google][src_ip]" } + } + } else { + mutate { + rename => { "[httpRequest][remoteIp]" => "[logx][google][src_ip]" } + } + } + + #Finally remove unused fields + mutate { + remove_field => ["path","@version","httpRequest","operation","sourceLocation"] + } + } + }', + filter_version='1.3.0' + WHERE id=1301; + + ]]> + + + diff --git a/backend/src/main/resources/config/liquibase/master.xml b/backend/src/main/resources/config/liquibase/master.xml index 9de1c5f7b..1642ad210 100644 --- a/backend/src/main/resources/config/liquibase/master.xml +++ b/backend/src/main/resources/config/liquibase/master.xml @@ -85,4 +85,8 @@ + + + +